├── gui-tester ├── .gitignore ├── src │ └── main │ │ └── python │ │ ├── gui_interaction │ │ ├── __init__.py │ │ ├── load_json_image.py │ │ ├── model_plot_confidence.py │ │ ├── clusterer.py │ │ ├── user_model_plotter.py │ │ ├── model_plot.py │ │ ├── model_run.py │ │ ├── model_rest.py │ │ ├── model_stats.py │ │ ├── process_imglab_data.py │ │ ├── ngram.py │ │ ├── config.py.BACK │ │ ├── model_plot_truth.py │ │ ├── test_model.py │ │ ├── user_model.py │ │ ├── autoencoder.py │ │ ├── darknet-run.py │ │ ├── monkey-tester.py │ │ ├── data_plot.py │ │ ├── task_distributor.py │ │ ├── model_plot_correct.py │ │ ├── replay_tester.py │ │ ├── user_model_random_tester.py │ │ ├── user_model_tester.py │ │ ├── data_processor.py │ │ ├── data_visualiser.py │ │ ├── manual_tester.py │ │ ├── swing_test.py │ │ ├── swing_plot.py │ │ ├── model_info.py │ │ └── batch_data_processor.py │ │ └── __main__.py ├── .settings │ ├── org.eclipse.m2e.core.prefs │ ├── org.eclipse.core.resources.prefs │ └── org.eclipse.jdt.core.prefs ├── dependency-reduced-pom.xml └── pom.xml ├── show_training_tensorboard.sh ├── train.sh ├── run_img.sh ├── public └── app-annotated.png ├── .gitattributes ├── weights.zip ├── data └── data.names ├── .gitignore ├── requirements.txt ├── README.md └── scripts ├── plot_loss.R ├── tune_threshold.R ├── eval_backup.R ├── print_info.R ├── evaluate_models.R ├── process_results.R ├── process_results_users.R └── process_results_users_min.R /gui-tester/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /show_training_tensorboard.sh: -------------------------------------------------------------------------------- 1 | tensorboard --logdir logs/1 2 | -------------------------------------------------------------------------------- /train.sh: -------------------------------------------------------------------------------- 1 | python3 gui-tester/src/main/python/gui_interaction/model_train.py 2 | -------------------------------------------------------------------------------- /run_img.sh: -------------------------------------------------------------------------------- 1 | python3 gui-tester/src/main/python/gui_interaction/model_plot_confidence.py "$1" 2 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/__main__.py: -------------------------------------------------------------------------------- 1 | from gui_interaction import tensor_play 2 | 3 | tensor_play.run() -------------------------------------------------------------------------------- /public/app-annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasdeanwhite/GUIdance/HEAD/public/app-annotated.png -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | weights.tar.xz filter=lfs diff=lfs merge=lfs -text 2 | weights.zip filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /gui-tester/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /weights.zip: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8b6c77df593bf8ded4f6b62354af99e0abee84a5e7e01c2abab87b93f23c4f0b 3 | size 738954653 4 | -------------------------------------------------------------------------------- /data/data.names: -------------------------------------------------------------------------------- 1 | text_field 2 | button 3 | combo_box 4 | tree 5 | list 6 | scroll_bar 7 | menu_item 8 | menu 9 | toggle_button 10 | tabs 11 | slider 12 | -------------------------------------------------------------------------------- /gui-tester/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=UTF-8 3 | encoding//src/test/java=UTF-8 4 | encoding/=UTF-8 5 | -------------------------------------------------------------------------------- /gui-tester/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 3 | org.eclipse.jdt.core.compiler.compliance=1.6 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.6 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.class 2 | **/*.bmp 3 | **/*.png 4 | **/*.pdf 5 | .idea/* 6 | **.idea/** 7 | .idea 8 | *.iml 9 | **/*.iml 10 | target/* 11 | **/target/* 12 | **.log 13 | *.*~ 14 | **/*.*~ 15 | *.csv 16 | **/*.csv 17 | venv/ 18 | **/*/__pycache__ 19 | **/*/config.py 20 | weights 21 | *weights* 22 | .RData 23 | .Rhistory 24 | **/*.pyc 25 | output 26 | tom* 27 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/load_json_image.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | import sys 3 | from data_loader import load_image, load_raw_image, disable_transformation, convert_coords 4 | import json 5 | 6 | if __name__ == '__main__': 7 | 8 | disable_transformation() 9 | 10 | raw_img = load_raw_image(sys.argv[1]).tolist() 11 | 12 | print(json.dumps(raw_img).replace("]],", "]],\n")) 13 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.1 2 | astor==0.8.0 3 | gast==0.2.2 4 | google-pasta==0.1.7 5 | grpcio==1.21.1 6 | h5py==2.9.0 7 | Keras-Applications==1.0.8 8 | Keras-Preprocessing==1.1.0 9 | Markdown==3.1.1 10 | numpy==1.16.4 11 | opencv-python==4.1.0.25 12 | protobuf==3.8.0 13 | PyAutoGUI==0.9.45 14 | PyGetWindow==0.0.6 15 | PyMsgBox==1.0.7 16 | PyRect==0.1.4 17 | PyScreeze==0.1.21 18 | python3-xlib==0.15 19 | PyTweening==1.0.3 20 | tensorboard==1.14.0 21 | tensorflow==1.15.2 22 | tensorflow-estimator==1.14.0 23 | termcolor==1.1.0 24 | Werkzeug==0.15.4 25 | wrapt==1.11.2 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GuiWidgets # 2 | A technique to identify widgets in screenshots of GUIs. 3 | 4 | # Installation # 5 | - `git clone git@github.com:thomasdeanwhite/GUIdance.git` 6 | - Fetch weights using GitLFS: `git lfs pull` or download them from this repo 7 | - unzip weights zip file 8 | - Copy `gui-tester/src/main/python/gui_interaction/config.py.BACK` to `config.py` 9 | - Install dependencies 10 | - `pip3 install -r requirements.txt` 11 | - Change `gui-tester/src/main/python/gui_interaction/config.py` 12 | - `data_dir` and `output_dir` to directory of training input data and desired output directory 13 | 14 | # Running # 15 | 16 | To visualise bounding boxes, model_plot.py can be used: 17 | ``` 18 | gui-tester/src/main/python/gui_interaction/model_plot.py [path to image] 19 | ``` 20 | e.g. 21 | ``` 22 | python gui-tester/src/main/python/gui_interaction/model_plot.py /home/user/img.png 23 | ``` 24 | 25 | 26 | ![Annotated App](https://raw.githubusercontent.com/thomasdeanwhite/GUIdance/master/public/app-annotated.png) 27 | 28 | Ensure that the model weights are in the folder _weights_ in the directory you are running the python script from. 29 | 30 | If struggling with Git LFS, the weights file can be found here: https://github.com/thomasdeanwhite/GUIdance/blob/master/weights.zip 31 | -------------------------------------------------------------------------------- /scripts/plot_loss.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library("RColorBrewer") 6 | args = commandArgs(TRUE) 7 | 8 | load_data <- function(directory){ 9 | wd = getwd() 10 | setwd(directory) 11 | data <- readr::read_csv("training.csv") 12 | 13 | return(data) 14 | } 15 | 16 | setwd('/home/thomas/work/GUIdance') 17 | 18 | data <- load_data('/home/thomas/work/GUIdance') 19 | 20 | data = data %>% gather("variable", "value", loss, loss_position, loss_dimension, loss_obj, loss_class, precision, recall, mAP) 21 | 22 | # data = data[data$epoch > 0,] 23 | # data = data[data$loss != "mAP",] 24 | # data = data[data$loss != "precision",] 25 | # data = data[data$loss != "recall",] 26 | #data = data[data$dataset != "Real",] 27 | 28 | 29 | p = data %>% 30 | ggplot(aes(x=epoch, y=value, lty=dataset, color=dataset)) + 31 | geom_line() + 32 | #geom_smooth(method="lm", se=F) + 33 | labs(x="Epoch", 34 | y="", 35 | title=paste("Loss when training GUI recognition model")) + 36 | #scale_x_discrete() + 37 | #scale_y_log10() + 38 | theme_minimal() + 39 | facet_wrap(~variable, scales = "free") + 40 | theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 41 | scale_fill_brewer(palette="Set1") + 42 | scale_color_brewer(palette="Dark2") 43 | 44 | print(p) 45 | -------------------------------------------------------------------------------- /scripts/tune_threshold.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | 7 | 8 | args = commandArgs(TRUE) 9 | 10 | load_data <- function(directory){ 11 | wd = getwd() 12 | setwd(directory) 13 | data <- readr::read_csv("tuning.csv") 14 | return(data) 15 | } 16 | 17 | setwd('/home/thomas/work/GUIdance') 18 | 19 | data <- load_data('/home/thomas/work/GUIdance') 20 | 21 | p = data %>% 22 | ggplot(aes(x=threshold, y=val, color=var)) + 23 | geom_line() + 24 | #geom_smooth(method="lm", se=F) + 25 | #scale_y_log10() + 26 | labs(x="Confidence Threshold", 27 | y="", 28 | title=paste("")) + 29 | #scale_x_discrete() + 30 | #scale_y_log10() + 31 | theme_minimal() + 32 | theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 33 | scale_fill_brewer(palette="Set1") + 34 | scale_color_brewer(palette="Dark2") 35 | 36 | spread_data = data %>% spread(var, val) 37 | 38 | correlation = spread_data %>% 39 | ggplot(aes(y=true_positive_rate, x=false_positive_rate)) + 40 | geom_point() + 41 | geom_line() 42 | #geom_smooth(method="lm", se=F) + 43 | #scale_y_log10() + 44 | labs(x="False Positive Rate", 45 | y="True Positive Rate", 46 | title="Precision against Recall")+ 47 | #scale_x_discrete() + 48 | #scale_y_log10() + 49 | theme_minimal() + 50 | theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 51 | scale_fill_brewer(palette="Set1") + 52 | scale_color_brewer(palette="Dark2") 53 | 54 | print(p) 55 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_plot_confidence.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | from data_loader import load_image, load_raw_image, disable_transformation, convert_coords 13 | 14 | from model_plot import plot_boxes 15 | 16 | if __name__ == '__main__': 17 | 18 | disable_transformation() 19 | 20 | yolo = Yolo() 21 | 22 | yolo.create_network() 23 | 24 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 25 | 26 | yolo.prepare() 27 | 28 | config = tf.ConfigProto(device_count = {'GPU': 0}) 29 | with tf.Session(config=config) as sess: 30 | 31 | if not yolo.init_session(sess, model_file): 32 | print("Cannot find weights file.", file=sys.stderr) 33 | sys.exit(-1) 34 | 35 | yolo.set_training(False) 36 | 37 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 38 | images = np.array([load_image(sys.argv[1])]) 39 | 40 | img = images[0] 41 | 42 | #normalise data between 0 and 1 43 | imgs = np.array(images)/127.5-1 44 | 45 | boxes = sess.run(yolo.output, feed_dict={ 46 | yolo.x: imgs, 47 | yolo.anchors: anchors, 48 | }) 49 | 50 | raw_img = load_raw_image(sys.argv[1]) 51 | 52 | proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True).tolist()[0] 53 | 54 | proc_boxes.sort(key=lambda box: -box[5]) 55 | 56 | height, width = raw_img.shape[:2] 57 | 58 | proc_boxes = yolo.normalise_boxes(proc_boxes, width, height) 59 | 60 | print("Plotting Figures") 61 | for c in range(0,11): 62 | 63 | img = np.copy(raw_img) 64 | 65 | p_boxes = proc_boxes[:] 66 | 67 | plot_boxes(p_boxes, img, c/10, yolo) 68 | -------------------------------------------------------------------------------- /scripts/eval_backup.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | 7 | 8 | args = commandArgs(TRUE) 9 | 10 | load_data <- function(directory){ 11 | wd = getwd() 12 | setwd(directory) 13 | data <- readr::read_csv("validation.csv") 14 | return(data) 15 | } 16 | 17 | setwd('/home/thomas/work/GUIdance') 18 | 19 | data <- load_data('/home/thomas/work/GUIdance') 20 | 21 | p = data %>% 22 | ggplot(aes(x=var, y=val, color=dataset)) + 23 | geom_boxplot() + 24 | #geom_smooth(method="lm", se=F) + 25 | #scale_y_log10() + 26 | labs(x="IoU Threshold", 27 | y="", 28 | title=paste("")) + 29 | #scale_x_discrete() + 30 | #scale_y_log10() + 31 | theme_minimal() + 32 | theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 33 | scale_fill_brewer(palette="Set1") + 34 | scale_color_brewer(palette="Dark2") 35 | 36 | spread_data = data %>% spread(var, val) 37 | 38 | correlation = spread_data %>% 39 | ggplot(aes(x=recall, y=precision)) + 40 | geom_smooth() + 41 | #geom_smooth(method="lm", se=F) + 42 | #scale_y_log10() + 43 | labs(x="Recall", 44 | y="Precision", 45 | title="Precision against Recall")+ 46 | #scale_x_discrete() + 47 | #scale_y_log10() + 48 | theme_minimal() + 49 | theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 50 | scale_fill_brewer(palette="Set1") + 51 | scale_color_brewer(palette="Dark2") 52 | 53 | synthetic_data = spread_data %>% 54 | filter(dataset == "synthetic") 55 | 56 | real_data = spread_data %>% 57 | filter(dataset == "real") 58 | 59 | print(paste("recall", wilcox.test(synthetic_data$recall, real_data$recall, exact=FALSE)$p.value)) 60 | print(paste("precision", wilcox.test(synthetic_data$precision, real_data$precision, exact = FALSE)$p.value)) 61 | print(paste("mAP", wilcox.test(synthetic_data$mAP, real_data$mAP, exact = FALSE)$p.value)) 62 | print(paste("average_iou", wilcox.test(synthetic_data$average_iou, real_data$average_iou, exact = FALSE)$p.value)) 63 | 64 | print(mean(real_data$actual_boxes)) 65 | print(mean(synthetic_data$actual_boxes)) 66 | 67 | print(p) 68 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/clusterer.py: -------------------------------------------------------------------------------- 1 | 2 | import random 3 | import numpy as np 4 | import math 5 | 6 | class Clusterer(): 7 | 8 | def __init__(self): 9 | self.data = [] 10 | self.shape = [0, 0] 11 | 12 | def append_data(self, row): 13 | self.data.append(row) 14 | if self.shape[0] == 0: 15 | self.shape[1] = len(row) 16 | self.shape[0] += 1 17 | 18 | def clear_data(self): 19 | self.shape = [0, 0] 20 | del self.data 21 | self.data = [] 22 | 23 | def recommend_clusters(self): 24 | return int(min(self.shape[0], max(5, 3*math.sqrt(self.shape[0])))) 25 | 26 | def cluster(self, k, iterations): 27 | centroids = random.sample(self.data, k) 28 | 29 | data = np.tile(np.expand_dims(np.array(self.data), 0), [k, 1, 1]) 30 | 31 | i = iterations 32 | 33 | min = [] 34 | 35 | while i > 0: 36 | assignments = [] 37 | 38 | cent = np.expand_dims(np.array(centroids), 1) 39 | 40 | cent = np.tile( 41 | np.reshape(cent, [k, 1, self.shape[1]]), [1, self.shape[0], 1]) 42 | 43 | diff = np.sum(np.square(cent - data), axis=-1) 44 | 45 | min = np.argmin(diff, axis=0).tolist() 46 | 47 | for i in range(k): 48 | assignments.append([]) 49 | 50 | for j in range(len(min)): 51 | assignments[min[j]].append(self.data[j]) 52 | 53 | new_centroids = [] 54 | 55 | for i in range(len(assignments)): 56 | row = assignments[i] 57 | 58 | if len(row) == 0: 59 | row = [[0 for x in range(self.shape[1])]] 60 | new_c = np.mean(row, axis=0).tolist() 61 | new_centroids.append(new_c) 62 | 63 | new_cent = False 64 | 65 | for c in new_centroids: 66 | if not c in centroids: 67 | new_cent = True 68 | if not new_cent: 69 | break 70 | 71 | centroids = new_centroids 72 | 73 | i -= 1 74 | 75 | return centroids, min 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/user_model_plotter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import sys 4 | import os 5 | import time 6 | import subprocess 7 | import signal 8 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 9 | from clusterer import Clusterer 10 | from user_model import WindowModel, UserModel 11 | from ngram import Ngram 12 | import pickle 13 | import cv2 14 | import numpy as np 15 | 16 | 17 | def create_img(w_m, screenshot): 18 | w = screenshot.shape[1] 19 | h = screenshot.shape[0] 20 | s_copy = np.copy(screenshot) 21 | 22 | for cl in w_m.clusters["EventType.LEFT_DOWN"]: 23 | 24 | cv2.rectangle(s_copy, 25 | (int(cl[0]*w)-2, int(cl[1]*h)-2), 26 | (int(cl[0]*w)+2, int(cl[1]*h)+2), 27 | (0, 255, 0), -1, 8) 28 | 29 | return s_copy 30 | 31 | if __name__ == '__main__': 32 | wd = os.getcwd() 33 | 34 | if len(sys.argv) < 4: 35 | print("Must have filename argument!", file=sys.stderr) 36 | sys.exit(1) 37 | input_dir = sys.argv[1] 38 | file = input_dir + "/user_model.mdl" 39 | user_model = UserModel() 40 | 41 | output = "." 42 | 43 | if len(sys.argv) > 4: 44 | output = sys.argv[4] 45 | 46 | with open(file, "rb") as f: 47 | user_model = pickle.load(f) 48 | 49 | screenshot_file = sys.argv[2] 50 | 51 | screenshot = cv2.imread(screenshot_file) 52 | 53 | window_title = sys.argv[3] 54 | 55 | output += "/" + window_title 56 | 57 | w_m = user_model.get_window_model(window_title) 58 | 59 | if w_m == user_model.default_model: 60 | print("Window model not found for title: '" + window_title + "'.") 61 | screen_app_model = create_img(user_model.default_model, screenshot) 62 | 63 | cv2.imwrite(output + "-one-out.png", screen_app_model) 64 | else: 65 | 66 | screen_app_model = create_img(user_model.default_model, screenshot) 67 | screen_window_model = create_img(w_m, screenshot) 68 | 69 | cv2.imwrite(output + "-app.png", screen_app_model) 70 | cv2.imwrite(output + "-window.png", screen_window_model) 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_plot.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | from data_loader import load_image, load_raw_image, disable_transformation, convert_coords 13 | 14 | def plot_boxes(proc_boxes, raw_img, threshold, yolo): 15 | cfg.object_detection_threshold = threshold 16 | 17 | img = np.copy(raw_img) 18 | 19 | proc_boxes = yolo.prune_boxes(proc_boxes[:]) 20 | 21 | 22 | # proc_boxes = yolo.trim_overlapping_boxes(proc_boxes) 23 | 24 | img = yolo.plot_boxes(proc_boxes, img) 25 | 26 | cv2.imshow('image',img) 27 | cv2.waitKey(0) 28 | cv2.destroyAllWindows() 29 | 30 | 31 | if __name__ == '__main__': 32 | 33 | disable_transformation() 34 | 35 | yolo = Yolo() 36 | 37 | yolo.create_network() 38 | 39 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 40 | 41 | yolo.prepare() 42 | 43 | config = tf.ConfigProto(device_count = {'GPU': 0}) 44 | 45 | with tf.Session(config=config) as sess: 46 | 47 | if not yolo.init_session(sess, model_file): 48 | print("Cannot find weights file.", file=sys.stderr) 49 | sys.exit(-1) 50 | 51 | yolo.set_training(False) 52 | 53 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 54 | images = np.array([load_image(sys.argv[1])]) 55 | 56 | img = images[0] 57 | 58 | #normalise data between 0 and 1 59 | imgs = np.array(images)/127.5-1 60 | 61 | 62 | raw_img = load_raw_image(sys.argv[1]) 63 | 64 | boxes = sess.run(yolo.output, feed_dict={ 65 | yolo.x: imgs, 66 | yolo.anchors: anchors, 67 | }) 68 | 69 | height, width = raw_img.shape[:2] 70 | 71 | proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True).tolist()[0] 72 | 73 | # proc_boxes = yolo.restore_aspect_ratio(proc_boxes, width/height) 74 | 75 | raw_img = load_raw_image(sys.argv[1]) 76 | 77 | proc_boxes.sort(key=lambda box: -box[5]) 78 | 79 | print("Processing Boxes") 80 | 81 | proc_boxes = yolo.normalise_boxes(proc_boxes, width, height) 82 | 83 | plot_boxes(proc_boxes, raw_img, cfg.object_detection_threshold, yolo) 84 | 85 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_run.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | 13 | def load_file(files): 14 | images = [] 15 | 16 | for f in files: 17 | image = cv2.imread(f, 0) 18 | img_raw = cv2.imread(f) 19 | image = cv2.resize(image, (cfg.width, cfg.height)) 20 | image = np.reshape(image, [cfg.width, cfg.height, 1]) 21 | images.append([image, img_raw]) 22 | 23 | return images 24 | 25 | 26 | 27 | if __name__ == '__main__': 28 | 29 | with tf.device(cfg.gpu): 30 | 31 | 32 | 33 | tf.reset_default_graph() 34 | yolo = Yolo() 35 | 36 | yolo.create_network() 37 | #yolo.set_training(False) 38 | #yolo.create_training() 39 | 40 | learning_rate = tf.placeholder(tf.float64) 41 | learning_r = cfg.learning_rate_start 42 | 43 | saver = tf.train.Saver() 44 | 45 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 46 | 47 | #chkp.print_tensors_in_checkpoint_file(model_file, tensor_name='', all_tensors=True) 48 | 49 | config = tf.ConfigProto(allow_soft_placement = True) 50 | 51 | with tf.Session(config=config) as sess: 52 | 53 | init_op = tf.global_variables_initializer() 54 | model = sess.run(init_op) 55 | if os.path.isfile(os.getcwd() + "/" + cfg.weights_dir + "/checkpoint"): 56 | saver.restore(sess, model_file) 57 | print("Restored model") 58 | yolo.set_training(False) 59 | 60 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 61 | images = load_file(sys.argv[1:]) 62 | 63 | #normalise data between 0 and 1 64 | imgs = (np.array([row[0] for row in images])/255) 65 | 66 | boxes = sess.run(yolo.output, feed_dict={ 67 | yolo.x: imgs, 68 | yolo.anchors: anchors, 69 | }) 70 | 71 | proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True) 72 | 73 | 74 | for box in proc_boxes: 75 | cls = yolo.names[int(box[0])] 76 | 77 | hex = cls.encode('utf-8').hex()[0:6] 78 | 79 | color = tuple(int(hex[k:k+2], 16) for k in (0, 2 ,4)) 80 | 81 | print(cls, box[1], box[2], box[3], box[4], box[5]) -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_rest.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | from data_loader import load_image, load_raw_image, disable_transformation, convert_coords 13 | import cherrypy 14 | import json 15 | 16 | anchors = None 17 | yolo = None 18 | sess = None 19 | 20 | class RestPrediction (object): 21 | @cherrypy.expose 22 | @cherrypy.tools.json_out() 23 | @cherrypy.tools.json_in() 24 | def predictions(self): 25 | input = cherrypy.request.json 26 | img = input["image"] 27 | 28 | result = process_image(img) 29 | 30 | json_img = json.dumps(result) 31 | 32 | return json_img 33 | 34 | def neaten_boxes(proc_boxes, img): 35 | height, width = img.shape[:2] 36 | 37 | # plot boxes 38 | for box in proc_boxes: 39 | 40 | box[0] = yolo.names[int(box[0])] 41 | 42 | box[1] = max(int(width*box[1]), 0) 43 | box[2] = max(int(height*box[2]), 0) 44 | box[3] = int(width*box[3]) 45 | box[4] = int(height*box[4]) 46 | return proc_boxes 47 | 48 | def process_image(img_file): 49 | 50 | raw_img = load_raw_image(img_file) 51 | 52 | img = np.array([load_image(img_file)])[0] 53 | 54 | images = np.array([img]) 55 | 56 | #normalise data between 0 and 1 57 | imgs = np.array(images)/127.5-1 58 | 59 | boxes = sess.run(yolo.output, feed_dict={ 60 | yolo.x: imgs, 61 | yolo.anchors: anchors, 62 | }) 63 | 64 | height, width = raw_img.shape[:2] 65 | 66 | proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=False).tolist()[0] 67 | 68 | proc_boxes.sort(key=lambda box: -box[5]) 69 | 70 | print("Processing Boxes") 71 | 72 | proc_boxes = neaten_boxes(yolo.normalise_boxes(proc_boxes, width, height), raw_img) 73 | 74 | return proc_boxes 75 | 76 | if __name__ == '__main__': 77 | 78 | disable_transformation() 79 | 80 | yolo = Yolo() 81 | 82 | yolo.create_network() 83 | 84 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 85 | 86 | yolo.prepare() 87 | 88 | config = tf.ConfigProto(device_count = {'GPU': 0}) 89 | 90 | with tf.Session(config=config) as ses: 91 | 92 | sess = ses 93 | 94 | if not yolo.init_session(sess, model_file): 95 | print("Cannot find weights file.", file=sys.stderr) 96 | sys.exit(-1) 97 | 98 | yolo.set_training(False) 99 | 100 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 101 | 102 | cherrypy.quickstart(RestPrediction()) 103 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_stats.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | from cv2 import imread, resize 4 | import cv2 5 | import numpy as np 6 | import tensorflow as tf 7 | import sys 8 | import gc 9 | import math 10 | import random 11 | import os 12 | import pickle 13 | import re 14 | from data_loader import load_files 15 | 16 | debug = False 17 | 18 | last_epochs = 12 19 | 20 | tf.logging.set_verbosity(tf.logging.INFO) 21 | 22 | def count_params(): 23 | return np.sum([np.product([xi.value for xi in x.get_shape()]) for x in tf.global_variables()]) 24 | 25 | 26 | if __name__ == '__main__': 27 | 28 | tf.reset_default_graph() 29 | yolo = Yolo() 30 | 31 | yolo.create_network() 32 | #yolo.set_training(False) 33 | #yolo.create_training() 34 | 35 | learning_rate = tf.placeholder(tf.float64) 36 | learning_r = cfg.learning_rate_start 37 | 38 | saver = tf.train.Saver() 39 | 40 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 41 | 42 | #chkp.print_tensors_in_checkpoint_file(model_file, tensor_name='', all_tensors=True) 43 | 44 | gpu_options = tf.GPUOptions(allow_growth=True) 45 | 46 | config = tf.ConfigProto( 47 | device_count = {'GPU': 0} 48 | ) 49 | 50 | with tf.Session(config=config) as sess: 51 | 52 | model_file = "weights-full" + "/model.ckpt" 53 | 54 | init_op = tf.global_variables_initializer() 55 | 56 | print("Initialising Memory Values") 57 | model = sess.run(init_op) 58 | 59 | #print(tf.get_default_graph().as_graph_def()) 60 | 61 | if os.path.isfile(os.getcwd() + "/" + "weights-full" + "/checkpoint"): 62 | saver.restore(sess, model_file) 63 | print("Restored model") 64 | else: 65 | print("Training from scratch.") 66 | 67 | reader = tf.train.NewCheckpointReader("weights-full" + "/model.ckpt") 68 | 69 | print('\nCount the number of parameters in ckpt file(%s)' % cfg.weights_dir) 70 | param_map = reader.get_variable_to_shape_map() 71 | total_count = 0 72 | for k, v in param_map.items(): 73 | if 'Adam' not in k and 'global_step' not in k: 74 | temp = np.prod(v) 75 | total_count += temp 76 | print('%s: %s => %d' % (k, str(v), temp)) 77 | 78 | vars = tf.trainable_variables() 79 | print(vars) #some infos about variables... 80 | vars_vals = sess.run(vars) 81 | count = 0 82 | 83 | positive = 0 84 | for var, val in zip(vars, vars_vals): 85 | print("var: {}, value: {}".format(var.name, val)) 86 | count += val.size 87 | positive += np.sum((val >= 0).astype(int)) 88 | 89 | print('Total Param Count: %d' % total_count) 90 | print("Activations", positive, count) 91 | 92 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/process_imglab_data.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | import os 3 | import xmltodict 4 | from yolo import Yolo 5 | from shutil import copy 6 | 7 | if __name__ == '__main__': 8 | 9 | class_mapping = { # map annotated class to actual class names 10 | "textfield": "text_field", 11 | "textbox": "text_field", 12 | "button": "button", 13 | "combobox": "combo_box", 14 | "tree": "tree", 15 | "list": "list", 16 | "scrollbar": "scroll_bar", 17 | "menuitem": "menu_item", 18 | "menu": "menu", 19 | "togglebutton": "toggle_button", 20 | "tabs": "tabs", 21 | "slider": "slider", 22 | "menuww": "menu", 23 | 24 | } 25 | 26 | yolo = Yolo() 27 | 28 | real_folder = cfg.data_dir + "/../mac" 29 | unproc_folder = real_folder + "/unproc" 30 | files = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(unproc_folder + "/images")) for f in fn] 31 | 32 | print(files) 33 | 34 | label_files = [(unproc_folder + "/labels/" + x[x.rfind("/") + 1:-4] + ".xml") for x in files] 35 | 36 | for i in range(len(files)): 37 | img_exists = os.path.isfile(files[i]) 38 | labs_exists = os.path.isfile(label_files[i]) 39 | 40 | if img_exists and labs_exists: 41 | 42 | os.system("cp \"" + files[i] + "\" " + real_folder + "/data/images/" + str(i) + ".png") 43 | 44 | with open(real_folder + "/data/labels/" + str(i) + ".txt", "w+") as label_output_file: 45 | with open(label_files[i], "r") as fo: 46 | xml = xmltodict.parse(fo.read())['annotation'] 47 | 48 | size = xml['size'] 49 | 50 | width, height = (int(size['width']), int(size['height'])) 51 | 52 | objects = xml['object'] 53 | 54 | if not type(objects) == list: # only 1 widget 55 | objects = [objects] 56 | 57 | for o in range(len(objects)): 58 | obj = objects[o] 59 | #print(obj) 60 | clazz = obj['name'] 61 | 62 | if not clazz in class_mapping: 63 | print("Class",clazz,"not supported!") 64 | continue 65 | 66 | class_index = yolo.names.index(class_mapping[clazz]) 67 | 68 | box = obj['bndbox'] 69 | x1, y1 = (int(box['xmin']), int(box['ymin'])) 70 | x2, y2 = (int(box['xmax']), int(box['ymax'])) 71 | 72 | x, y = ((x1+x2)/2, (y1+y2)/2) 73 | w, h = (x2-x1, y2-y1) 74 | 75 | line = str(class_index) + " " + str(x/width) + " " + str(y/height) + " " + str(w/width) + " " + str(h/height) + "\n" 76 | 77 | label_output_file.write(line) 78 | 79 | print("Finished Exporting Files") 80 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/ngram.py: -------------------------------------------------------------------------------- 1 | class Ngram (): 2 | 3 | delimiter = "\n" 4 | 5 | def __init__(self, name): 6 | self.name = name 7 | self.children = [] 8 | self.count = 0 9 | self.probability = 0 10 | self.length = 0 11 | 12 | def set_count(self, count): 13 | self.count = count 14 | 15 | def increment_count(self): 16 | self.count += 1 17 | 18 | def add_child(self, child): 19 | self.children.append(child) 20 | 21 | def get_child(self, name): 22 | for ng in self.children: 23 | if ng.name == name: 24 | return ng 25 | 26 | return None 27 | 28 | def find_children(self, names): 29 | nms = names.split() # split on space 30 | 31 | node = self 32 | 33 | for n in nms: 34 | node = node.get_child(n) 35 | 36 | if node == None: 37 | break 38 | 39 | return node 40 | 41 | def find_parent(self, names): 42 | nms = names.split() # split on space 43 | 44 | node = self 45 | 46 | for n in nms: 47 | node = node.get_child(n) 48 | 49 | if node == None: 50 | node = self 51 | break 52 | 53 | return node 54 | 55 | def construct(self, data, length): 56 | words = data.split() 57 | 58 | if length > len(words): 59 | length = len(words)-1 60 | 61 | self.length = length 62 | 63 | for lower_lim in range(len(words)-length): 64 | 65 | word_string = words[lower_lim] 66 | 67 | n = self.find_children(word_string) 68 | 69 | if n is None: 70 | n = Ngram(words[lower_lim]) 71 | n.set_count(1) 72 | 73 | self.add_child(n) 74 | else: 75 | n.increment_count() 76 | 77 | node = n 78 | 79 | for upper_lim in range(1, length): 80 | word_string += " " + words[lower_lim+upper_lim] 81 | 82 | if self.delimiter in word_string: 83 | continue 84 | 85 | n2 = self.find_children(word_string) 86 | 87 | if n2 is None: 88 | n2 = Ngram(words[lower_lim+upper_lim]) 89 | n2.set_count(1) 90 | 91 | node.add_child(n2) 92 | else: 93 | n2.increment_count() 94 | 95 | node = n2 96 | 97 | def calculate_probabilities(self): 98 | 99 | total = 0 100 | 101 | for c in self.children: 102 | total += c.count 103 | 104 | for c in self.children: 105 | c.probability = c.count / total 106 | c.calculate_probabilities() 107 | 108 | 109 | 110 | def print_index(self, i): 111 | print(" "*i, self.name, "(", self.count, ")") 112 | 113 | for c in self.children: 114 | c.print_index(i+1) 115 | 116 | def print(self): 117 | self.print_index(0) -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/config.py.BACK: -------------------------------------------------------------------------------- 1 | # ------------------------ 2 | # User Specific | 3 | # ------------------------ 4 | data_dir = "./data" 5 | output_dir = "" 6 | 7 | 8 | # ------------------------ 9 | # YOLO variables | 10 | # ------------------------ 11 | 12 | # Generated anchors from GUI Image dataset using 13 | # K-Means clustering 14 | 15 | #anchors = [0.9817857382941845, 9.398278193098788, 16 | # 5.52115919597547, 6.985948423888699, 17 | # 0.9090057538352261, 1.94398118062752, 18 | # 2.4060115242354794, 0.7252024765267888, 19 | # 9.922079927099324, 1.5033104111199442] 20 | 21 | anchors = [5.689762815680707,7.089220613019247, 22 | 4.146250839026031,2.8451912727793127, 23 | 1.184713044693887,4.635493841471227, 24 | 6.401568727399865,0.5919843713657224, 25 | 1.0073469991938633,0.6949452764581193] 26 | 27 | # resolution of input image 28 | width = 416 29 | height = 416 30 | 31 | # Threshold to identify a bounding box as correct 32 | threshold = 0.1 33 | 34 | # Use Nvidia CUDA for training? 35 | cudnn_on_gpu = False 36 | 37 | # Shape of the grid (subdivisions of image to apply anchors to) 38 | grid_shape = [13, 13] 39 | 40 | # ------------------------ 41 | # input files | 42 | # ------------------------ 43 | 44 | # Relative to [data_dir] 45 | 46 | names_file = "data.names" 47 | train_file = "train.txt" 48 | validate_file = "validate.txt" 49 | test_file = "test.txt" 50 | 51 | images_dir = "images" 52 | labels_dir = "labels" 53 | 54 | 55 | # ------------------------ 56 | # training variables | 57 | # ------------------------ 58 | learning_rate_start = 0.0001 59 | learning_rate_min = 0.00001 60 | learning_rate_decay = 0.95 61 | momentum = 0.9 62 | object_detection_threshold = 0.12 63 | 64 | # maximum training epochs 65 | epochs = 100000 66 | 67 | # Amount of images to train on in parallel 68 | batch_size = 32 69 | 70 | # standard deviation of variables when initialised 71 | var_sd = 0.001 72 | 73 | brightness_probability = 0.1 74 | contrast_probability = 0.1 75 | invert_probability = 0.1 76 | 77 | brightness_var = 50 78 | contrast_var = 1 79 | 80 | 81 | # ------------------------ 82 | # output locations | 83 | # ------------------------ 84 | 85 | weights_dir = "weights" 86 | results_dir = "results" 87 | log_file = "loss.csv" 88 | 89 | # GPU to use (defaults to CPU) 90 | gpu = "/job:localhost/replica:0/task:0/device:CPU:0" 91 | # gpu = "/gpu:0" 92 | 93 | 94 | # ------------------------ 95 | # training weights | 96 | # ------------------------ 97 | 98 | coord_weight = 1.0 99 | obj_weight = 5.0 100 | noobj_weight = 1.0 101 | class_weight = 1.0 102 | 103 | # ------------------------ 104 | # testing properties | 105 | # ------------------------ 106 | window_name = "Firefox" 107 | 108 | enable_logging = False 109 | 110 | run_all_batches = True 111 | 112 | fullscreen = False 113 | resolution = [1920, 1080] 114 | 115 | # fallback approach is no GUI window exists (either "random" or "application") 116 | fallback = "application" 117 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_plot_truth.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import sys 6 | from data_loader import load_file_raw, disable_transformation 7 | 8 | disable_transformation() 9 | 10 | if __name__ == '__main__': 11 | 12 | yolo = Yolo() 13 | 14 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 15 | images, labels, obj_detect = load_file_raw(sys.argv[1]) 16 | 17 | img = images 18 | 19 | #normalise data between 0 and 1 20 | imgs = np.array(images)/127.5-1 21 | labels = np.array(labels) 22 | obj_detect = np.array(obj_detect) 23 | 24 | 25 | trim_overlap = True 26 | 27 | proc_boxes = np.reshape(labels, [169, -1])/cfg.grid_shape[0] 28 | classes = np.reshape(obj_detect, [169, -1]) 29 | 30 | proc_boxes = np.append(classes, proc_boxes, axis=1).tolist() 31 | 32 | i=0 33 | while i < len(proc_boxes): 34 | box = proc_boxes[i] 35 | x, y, w, h = (box[1],box[2],box[3],box[4]) 36 | box[1] = x - w/2 37 | box[2] = y - h/2 38 | box[3] = x + w/2 39 | box[4] = y + h/2 40 | i = i + 1 41 | 42 | 43 | for bc in range(len(proc_boxes)): 44 | height, width = img.shape[:2] 45 | 46 | box = proc_boxes[bc] 47 | 48 | cls = yolo.names[int(box[0])] 49 | 50 | hex = cls.encode('utf-8').hex()[0:6] 51 | 52 | color = tuple(int(int(hex[k:k+2], 16)*0.75) for k in (0, 2 ,4)) 53 | 54 | if (box[5]>0): 55 | print(box) 56 | 57 | x1 = max(int(width*box[1]), 0) 58 | y1 = max(int(height*box[2]), 0) 59 | x2 = int(width*box[3]) 60 | y2 = int(height*box[4]) 61 | 62 | cv2.rectangle(img, (x1, y1), 63 | (x2, y2), 64 | (color[0], color[1], color[2]), 5, 8) 65 | 66 | for bc in range(len(proc_boxes)): 67 | box = proc_boxes[bc] 68 | 69 | cls = yolo.names[int(box[0])] 70 | 71 | hex = cls.encode('utf-8').hex()[0:6] 72 | 73 | color = tuple(int(int(hex[k:k+2], 16)*0.75) for k in (0, 2 ,4)) 74 | 75 | if (box[5]>0): 76 | height, width = img.shape[:2] 77 | 78 | avg_col = (color[0] + color[1] + color[2]) / 3 79 | 80 | text_col = (255, 255, 255) 81 | 82 | if avg_col > 127: 83 | text_col = (0, 0, 0) 84 | 85 | x1 = max(int(width*box[1]), 0) 86 | y1 = max(int(height*box[2]), 0) 87 | x2 = int(width*box[3]) 88 | y2 = int(height*box[4]) 89 | 90 | cv2.rectangle(img, 91 | (x1-3, y1-23), 92 | (x1 + (len(cls)+1)*10, y1), 93 | (color[0], color[1], color[2]), -1, 8) 94 | 95 | cv2.putText(img, cls.upper(), 96 | (x1, y1-6), 97 | cv2.FONT_HERSHEY_PLAIN, 98 | 1, text_col, 1, lineType=cv2.LINE_AA) 99 | 100 | cv2.imshow('image',img) 101 | cv2.waitKey(0) 102 | cv2.destroyAllWindows() 103 | 104 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/test_model.py: -------------------------------------------------------------------------------- 1 | import data_loader as ten 2 | import config as cfg 3 | import unittest 4 | import model_test as tester 5 | 6 | class TestTensor(unittest.TestCase): 7 | 8 | # test creation of input from files ---------- 9 | def test_point_centre(self): 10 | self.assertAlmostEqual(1.5, ten.normalise_point(0.5, 3)[0]) 11 | 12 | def test_point_above(self): 13 | self.assertAlmostEqual(1.8, ten.normalise_point(0.6, 3)[0]) 14 | 15 | def test_point_boundary(self): 16 | self.assertAlmostEqual(2, ten.normalise_point(0.66666666, 3)[0]) 17 | 18 | def test_label_normalise(self): 19 | cfg.grid_shape = [10, 10] 20 | label = [0.1, 0.4, 0.3, 0.6, 1.0] 21 | label_norm, centres = ten.normalise_label(label) 22 | 23 | true_norm = [1, 4, 3, 6, 1] 24 | 25 | for i in range(len(label_norm)): 26 | self.assertAlmostEqual(true_norm[i], label_norm[i]) 27 | # as using factor of 10 for grid_shape, the centres are 28 | # equal to the position values! 29 | self.assertAlmostEqual(true_norm[0], centres[0]) 30 | self.assertAlmostEqual(true_norm[1], centres[1]) 31 | 32 | def test_coordinate_conversion_aspect_one(self): 33 | aspect = 1.0 34 | input = [0.1, 0.4, 0.2, 0.5] 35 | converted = tester.convert_coords(input[0], input[1], input[2], input[3], aspect) 36 | correct = [0.1, 0.4, 0.2, 0.5] 37 | 38 | for i in range(len(input)): 39 | self.assertAlmostEqual(correct[i], converted[i]) 40 | 41 | def test_coordinate_conversion_aspect_two(self): 42 | aspect = 2 43 | input = [0.1, 0.3, 0.2, 0.5] 44 | converted = tester.convert_coords(input[0], input[1], input[2], input[3], aspect) 45 | correct = [0.1, 0.1, 0.2, 1.0] 46 | 47 | for i in range(len(input)): 48 | self.assertAlmostEqual(correct[i], converted[i], msg="expected "+str(correct)+" but got "+str(converted)) 49 | 50 | 51 | def test_coordinate_conversion_aspect_two_border(self): 52 | aspect = 2 53 | input = [0.15, 0.25, 0.3, 0.1] 54 | converted = tester.convert_coords(input[0], input[1], input[2], input[3], aspect) 55 | correct = [0.15, 0.0, 0.3, 0.2] 56 | 57 | for i in range(len(input)): 58 | self.assertAlmostEqual(correct[i], converted[i], msg="expected "+str(correct)+" but got "+str(converted)) 59 | 60 | 61 | def test_coordinate_conversion_aspect_two(self): 62 | aspect = 0.5 63 | input = [0.3, 0.1, 0.2, 0.5] 64 | converted = tester.convert_coords(input[0], input[1], input[2], input[3], aspect) 65 | correct = [0.1, 0.1, 0.1, 0.5] 66 | 67 | for i in range(len(input)): 68 | self.assertAlmostEqual(correct[i], converted[i], msg="expected "+str(correct)+" but got "+str(converted)) 69 | 70 | 71 | def test_coordinate_conversion_aspect_half_border(self): 72 | aspect = 0.5 73 | input = [0.25, 0.25, 0.3, 0.1] 74 | converted = tester.convert_coords(input[0], input[1], input[2], input[3], aspect) 75 | correct = [0.0, 0.25, 0.15, 0.1] 76 | 77 | for i in range(len(input)): 78 | self.assertAlmostEqual(correct[i], converted[i], msg="expected "+str(correct)+" but got "+str(converted)) 79 | 80 | 81 | if __name__ == '__main__': 82 | unittest.main() -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/user_model.py: -------------------------------------------------------------------------------- 1 | from ngram import Ngram 2 | import random 3 | import re 4 | import time 5 | 6 | class UserModel(): 7 | 8 | default_chance = 0.05 9 | 10 | def __init__(self): 11 | self.window_models = {} 12 | self.default_model = None 13 | 14 | def set_default_model(self, default_model): 15 | self.default_model = default_model 16 | 17 | def add_window_model(self, window_name, window_model): 18 | self.window_models[window_name] = window_model 19 | 20 | def get_window_model(self, window_name): 21 | 22 | model = self.default_model 23 | if window_name in self.window_models and random.random() > self.default_chance: 24 | model = self.window_models[window_name] 25 | 26 | if model is None: 27 | raise ValueError("Window " + window_name + " cannot be found, has model equal to None, or default model is None!") 28 | return model 29 | 30 | 31 | class WindowModel(): 32 | 33 | exp = "{(\d+)}" 34 | 35 | def __init__(self, ngram, clusters, length=2): 36 | self.chain = MarkovChain(ngram, length) 37 | self.clusters = clusters 38 | 39 | def label_to_event_description(self, label): 40 | m = re.search(self.exp, label) 41 | if not m is None: # has clustered information! 42 | c = int(m.group(1)) 43 | nt = label.split("[")[0] 44 | metadata = self.clusters[nt][c] 45 | cluster = "" 46 | 47 | for i in metadata: 48 | cluster += str(i) + "," 49 | cluster = cluster[:-2] 50 | 51 | label = re.sub(self.exp, cluster, label) 52 | 53 | label = label.replace("[", "::<<") 54 | label = label.replace("]", ">>") 55 | return label 56 | 57 | def next(self): 58 | label = self.chain.next() 59 | 60 | if label is None: 61 | return None 62 | 63 | label = self.label_to_event_description(label) 64 | 65 | label = label.replace("EventType.", "") 66 | 67 | label = label.replace("::<<", "@" + str(time.time()) + "::<<") 68 | 69 | return label 70 | 71 | class MarkovChain(): 72 | def __init__(self, ngram, length): 73 | self.ngram = ngram 74 | self.sequence = "" 75 | if length > self.ngram.length: 76 | length = self.ngram.length 77 | self.length = length 78 | self.sparsity = 0 79 | 80 | def next(self): 81 | seqs = self.sequence.split() 82 | 83 | if len(seqs) >= self.length: 84 | seqs = seqs[-self.length+1:] 85 | 86 | self.sequence = " ".join(seqs) 87 | 88 | node = self.ngram.find_parent(self.sequence) 89 | 90 | p = random.random() 91 | 92 | if len(node.children) == 0: 93 | # sparse data 94 | if self.sparsity > 3: 95 | return None 96 | self.sparsity += 1 97 | self.sequence = "" 98 | return self.next() 99 | 100 | self.sparsity = 0 101 | 102 | next_node = node.children[0] 103 | for c in node.children: 104 | next_node = c 105 | p -= c.probability 106 | 107 | if p <= 0: 108 | break 109 | 110 | self.sequence += " " + next_node.name 111 | return next_node.name 112 | 113 | 114 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/autoencoder.py: -------------------------------------------------------------------------------- 1 | # These are all the modules we'll be using later. Make sure you can import them 2 | # before proceeding further. 3 | from __future__ import print_function 4 | import tensorflow as tf 5 | 6 | 7 | def weight_variable(shape): 8 | initial = tf.random_normal(shape, mean=0, stddev=0.05) 9 | return tf.Variable(initial) 10 | 11 | def bias_variable(shape): 12 | initial = tf.random_normal(shape, mean=0, stddev=0.05) 13 | return tf.Variable(initial) 14 | 15 | class AutoEncoder: 16 | learning_rate = 0.1 17 | epochs = 1000 18 | batch_size = 25 19 | input_features = 1024*1024 20 | image_size = 1024*1024 21 | network_width = 1024 22 | hidden_layers_n = [[2048, 2048, 1], [32, 32, 4]] 23 | # TODO: make this support a CNN 24 | variables = [] 25 | activation_functions = [] 26 | image = None 27 | hidden_layers = [None] 28 | output_layer = None 29 | minimal_layer = None 30 | minimal_layer_size = 0 31 | loss = None 32 | output_size = 0 33 | 34 | keep_prob = tf.placeholder(tf.float32) 35 | 36 | def leaky_relu(self, x, alpha): 37 | return tf.maximum(alpha*x,x) 38 | 39 | def dropout(self, x): 40 | return tf.nn.dropout(self.leaky_relu(x, 0.1), self.keep_prob) 41 | 42 | def lr_activation(self, x): 43 | return self.leaky_relu(x, 0.3) 44 | 45 | def __init__(self, image=tf.placeholder(tf.float32, [None, image_size])): 46 | self.image = image 47 | 48 | #self.activation_functions = [tf.nn.relu, self.dropout, 49 | # tf.nn.relu, tf.nn.sigmoid] 50 | 51 | self.activation_functions = [self.dropout, self.lr_activation]#tf.nn.tanh] 52 | 53 | hidden_layers_n = [self.image_size] 54 | hidden_layers_n.extend(self.hidden_layers_n) 55 | 56 | self.hidden_layers = [] 57 | last_layer = tf.reshape(self.image, self.hidden_layers_n[0]) 58 | last_size = 0 59 | 60 | input_stack = [] 61 | 62 | for i in range(0, len(hidden_layers_n)-1): 63 | weights = weight_variable([hidden_layers_n[i], hidden_layers_n[i+1]]) 64 | biases = bias_variable([hidden_layers_n[i+1]]) 65 | self.variables.append(weights) 66 | self.variables.append(biases) 67 | hidden_layer = self.activation_functions[len(self.hidden_layers)](tf.add(tf.matmul(last_layer, weights), biases)) 68 | last_layer = hidden_layer 69 | self.hidden_layers.append(hidden_layer) 70 | input_stack.append(hidden_layers_n[i]) 71 | last_size = hidden_layers_n[i+1] 72 | 73 | self.minimal_layer = last_layer 74 | self.minimal_layer_size = last_size 75 | 76 | while len(input_stack) > 0: 77 | next_layer = input_stack.pop() 78 | prev_layer = last_size 79 | weights = weight_variable([prev_layer, next_layer]) 80 | biases = bias_variable([next_layer]) 81 | self.variables.append(weights) 82 | self.variables.append(biases) 83 | hidden_layer = self.activation_functions[len(self.hidden_layers)](tf.add(tf.matmul(last_layer, weights), biases)) 84 | last_layer = hidden_layer 85 | self.hidden_layers.append(hidden_layer) 86 | last_size = next_layer 87 | 88 | last_layer = self.hidden_layers.pop() 89 | self.output_layer = last_layer 90 | self.output_size = last_size 91 | 92 | self.loss = tf.reduce_mean(tf.square(self.image - self.output_layer)) -------------------------------------------------------------------------------- /scripts/print_info.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library("RColorBrewer") 6 | library(ggthemes) 7 | args = commandArgs(TRUE) 8 | 9 | load_data <- function(directory, filename){ 10 | wd = getwd() 11 | setwd(directory) 12 | data <- readr::read_csv(filename) 13 | setwd(wd) 14 | 15 | return(data) 16 | } 17 | 18 | setwd('/home/thomas/work/GUIdance/pdfs') 19 | 20 | data <- load_data('/home/thomas/work/GUIdance', "img_hist.csv") 21 | 22 | data$pixel_value = data$pixel_value * 8.5 23 | data$quantity = sqrt(data$quantity) 24 | p_data = data 25 | 26 | p = data %>% 27 | ggplot(aes(pixel_value, quantity)) + 28 | #geom_boxplot() + 29 | geom_bar(aes(x=pixel_value, y=quantity), color="grey40", alpha=0.2, stat="identity") + 30 | #geom_smooth(method="lm", se=F) + 31 | labs(x="Pixel Value", 32 | y="sqrt(Quantity)", 33 | title=paste("Pixel Histogram for dataset images")) + 34 | #scale_x_discrete() + 35 | #scale_y_log10() + 36 | facet_wrap(~dataset, scales = "free") + 37 | theme_minimal() 38 | #theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 39 | 40 | print(p) 41 | 42 | ggsave("hist.png", p, height=4, width=6, dpi=150) 43 | 44 | data <- load_data('/home/thomas/work/GUIdance', "label_heat.csv") 45 | 46 | p = data %>% ggplot(aes(x,y)) + 47 | geom_raster(aes(fill=density)) + 48 | facet_wrap(class~dataset, ncol=6) + 49 | scale_fill_gradient(low = "#0000FF", high = "#FF0000", na.value = "#00FF00") + 50 | scale_y_reverse() 51 | 52 | ggsave("heatmap.png", p, height=10, width=10, dpi=150) 53 | 54 | data <- load_data('/home/thomas/work/GUIdance', "class_count.csv") 55 | 56 | p = data %>% ggplot(aes(dataset, count)) + 57 | geom_boxplot() + 58 | facet_wrap(~class) + 59 | theme(axis.text.x = element_text(angle = 45, hjust = 1)) + 60 | theme_minimal() + 61 | ylab("Proportion") + 62 | stat_summary(fun.y=mean, colour="darkred", geom="point", shape=18, size=3, show.legend = FALSE) + 63 | stat_summary(fun.y=mean, colour="red", geom="text", show.legend = FALSE, 64 | vjust=-0.7, aes( label=round(..y.., digits=3))) 65 | 66 | ggsave("class_count.png", p, height=10, width=10, dpi=150) 67 | 68 | data <- load_data('/home/thomas/work/GUIdance', "label_dims.csv") 69 | 70 | p = data %>% filter(dimension != "area") %>% 71 | ggplot(aes(interaction(dimension, dataset), value, fill=dimension)) + 72 | geom_boxplot() + 73 | theme_minimal() + 74 | facet_wrap(~class, scales="free") + 75 | stat_summary(fun.y=mean, colour="darkred", geom="point", shape=18, size=3, show.legend = FALSE) + 76 | stat_summary(fun.y=mean, colour="red", geom="text", show.legend = FALSE, 77 | vjust=-0.7, aes( label=round(..y.., digits=3))) + 78 | theme(axis.text.x = element_text(angle = 45, hjust = 1)) 79 | 80 | 81 | ggsave("label_dims.png", p, height=10, width=10, dpi=150) 82 | 83 | data <- load_data('/home/thomas/work/GUIdance', "white_space.csv") 84 | 85 | p = data %>% 86 | ggplot(aes(area, widget_count, color=dataset)) + 87 | #geom_boxplot() + 88 | #geom_bar(color="grey40", alpha=0.2, stat="identity") + 89 | geom_point()+ 90 | #geom_smooth(method="lm", se=F) + 91 | labs(x="Image Area", 92 | y="Widget Count", 93 | title="Widget Count against Image Area") + 94 | #scale_x_discrete() + 95 | #scale_y_log10() + 96 | #facet_wrap(~dataset, scales = "free") + 97 | theme_minimal() + 98 | scale_color_grey(start=0, end=0.7) 99 | #theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 100 | 101 | ggsave("widget_quant.png", p, height=10, width=10, dpi=150) 102 | 103 | p = data %>% 104 | ggplot(aes(area, widget_area/area, color=dataset)) + 105 | #geom_boxplot() + 106 | #geom_bar(color="grey40", alpha=0.2, stat="identity") + 107 | geom_point()+ 108 | #geom_smooth(method="lm", se=F) + 109 | labs(x="Image Area", 110 | y="Widget Area Proportion", 111 | title="Widget Area against Image Area") + 112 | #scale_x_discrete() + 113 | #scale_y_log10() + 114 | #facet_wrap(~dataset, scales = "free") + 115 | theme_minimal() + 116 | scale_color_grey(start=0, end=0.7) 117 | #theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 118 | 119 | ggsave("widget_area.png", p, height=5, width=5, dpi=150) 120 | 121 | print(p) 122 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/darknet-run.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | import pyautogui 12 | from operator import itemgetter 13 | import time 14 | import subprocess 15 | import Xlib 16 | 17 | def get_window_size(window_name): 18 | display = Xlib.display.Display() 19 | root = display.screen().root 20 | 21 | windowIDs = root.get_full_property(display.intern_atom('_NET_CLIENT_LIST'), Xlib.X.AnyPropertyType).value 22 | wid = 0 23 | win = None 24 | for windowID in windowIDs: 25 | window = display.create_resource_object('window', windowID) 26 | name = window.get_wm_name() # Title 27 | if isinstance(name, str) and window_name in name: 28 | wid = windowID 29 | win = window 30 | #prop = window.get_full_property(display.intern_atom('_NET_WM_PID'), Xlib.X.AnyPropertyType) 31 | #pid = prop.value[0] # PID 32 | break; 33 | 34 | geom = win.get_geometry() 35 | 36 | app_x, app_y, app_w, app_h = (geom.x, geom.y, geom.width, geom.height) 37 | 38 | parent_win = win.query_tree().parent 39 | 40 | while parent_win != 0: 41 | #print(parent_win) 42 | p_geom = parent_win.get_geometry() 43 | app_x += p_geom.x 44 | app_y += p_geom.y 45 | parent_win = parent_win.query_tree().parent 46 | return app_x, app_y, app_w, app_h 47 | 48 | if __name__ == '__main__': 49 | 50 | program_name = "Firefox" 51 | 52 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 53 | 54 | start_time = time.time() 55 | 56 | yolo = Yolo() 57 | 58 | os.chdir(cfg.darknet_location) 59 | 60 | while (time.time() - start_time < 300): 61 | os.system("gnome-screenshot --file=/tmp/current_screen.png") 62 | 63 | image = cv2.imread("/tmp/current_screen.png") 64 | 65 | image = image[app_y:app_y+app_h, app_x:app_x+app_w] 66 | 67 | cv2.imwrite("/tmp/current_screen.png", image) 68 | 69 | proc_boxes =[[]] 70 | 71 | command = "export DISPLAY='';./darknet detector test cfg/gui.data cfg/yolo-gui.2.0.cfg" \ 72 | " backup/yolo-gui_9000.weights /tmp/current_screen.png -thresh 0" 73 | 74 | boxes_string = subprocess.run(command, stdout=subprocess.PIPE, shell=True).stdout.decode('utf-8') 75 | 76 | boxes_string_arr = boxes_string.split("\n") 77 | 78 | proc_boxes = [] 79 | 80 | for line in boxes_string_arr[1:]: 81 | eles = line.split(" ") 82 | 83 | if (len(eles) == 6): 84 | proc_boxes.append([int(eles[0]), 85 | float(eles[1]), 86 | float(eles[2]), 87 | float(eles[3]), 88 | float(eles[4]), 89 | float(eles[5])]) 90 | 91 | 92 | 93 | 94 | for box_num in range(20): 95 | 96 | highest_conf = proc_boxes[0][5] 97 | best_box = proc_boxes[0] 98 | for b in proc_boxes: 99 | if (b[5] > highest_conf): 100 | highest_conf = b[5] 101 | best_box = b 102 | b[5] = 0 103 | 104 | x = int(max(app_x+10, min(app_x + app_w - 10, app_x + (best_box[1]*app_w)))) 105 | y = int(max(app_y+10, min(app_y + app_h - 10, app_y + (best_box[2]*app_h)))) 106 | 107 | # x = best_box[1] 108 | # y = best_box[2] 109 | 110 | pyautogui.moveTo(x, y) 111 | 112 | # handle widget type: 113 | widget = yolo.names[int(best_box[0])] 114 | 115 | print("interacting with", widget) 116 | print("at position:", "(", x, y, ")") 117 | 118 | 119 | if widget == "button" or widget == "combo_box" or widget == "list" or widget == "tree" or \ 120 | widget == "scroll_bar" or widget == "tabs" or widget == "menu" or widget == "menu_item": 121 | pyautogui.click(x, y) 122 | elif widget == "text_field": 123 | pyautogui.click(x, y) 124 | pyautogui.typewrite('Hello world!', interval=0.01) 125 | else: 126 | print(widget, "unrecognised") 127 | 128 | 129 | -------------------------------------------------------------------------------- /gui-tester/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | gui 5 | com.thomasdeanwhite 6 | 0.0.1-SNAPSHOT 7 | 8 | 4.0.0 9 | com.thomasdeanwhite 10 | gui-tester 11 | Gui Tester Master 12 | 0.0.1-SNAPSHOT 13 | http://maven.apache.org 14 | 15 | 16 | 17 | maven-surefire-plugin 18 | 2.19 19 | 20 | 21 | org.junit.platform 22 | junit-platform-surefire-provider 23 | RELEASE 24 | 25 | 26 | org.junit.jupiter 27 | junit-jupiter-engine 28 | RELEASE 29 | 30 | 31 | org.junit.vintage 32 | junit-vintage-engine 33 | RELEASE 34 | 35 | 36 | 37 | 1 38 | true 39 | 40 | java.library.path 41 | ${project.basedir}/lib 42 | 43 | 1 44 | 45 | 46 | 47 | org.jacoco 48 | jacoco-maven-plugin 49 | 0.7.6.201602180812 50 | 51 | 52 | prepare-agent 53 | 54 | prepare-agent 55 | 56 | 57 | 58 | 59 | 60 | org.eluder.coveralls 61 | coveralls-maven-plugin 62 | 4.3.0 63 | 64 | 65 | 66 | 67 | 68 | central 69 | http://central.maven.org/maven2/ 70 | 71 | 72 | 73 | 74 | org.junit.platform 75 | junit-platform-launcher 76 | RELEASE 77 | test 78 | 79 | 80 | junit-platform-engine 81 | org.junit.platform 82 | 83 | 84 | 85 | 86 | org.junit.jupiter 87 | junit-jupiter-engine 88 | RELEASE 89 | test 90 | 91 | 92 | junit-jupiter-api 93 | org.junit.jupiter 94 | 95 | 96 | junit-platform-engine 97 | org.junit.platform 98 | 99 | 100 | 101 | 102 | org.junit.vintage 103 | junit-vintage-engine 104 | RELEASE 105 | test 106 | 107 | 108 | junit-platform-engine 109 | org.junit.platform 110 | 111 | 112 | 113 | 114 | 115 | UTF-8 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/monkey-tester.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | import pyautogui 12 | from operator import itemgetter 13 | import time 14 | import subprocess 15 | import Xlib 16 | from pynput import keyboard 17 | import signal 18 | import timeit 19 | from test_helper import get_window_size, is_running, screenshot, perform_interaction 20 | 21 | sub_window = False 22 | 23 | running = True 24 | 25 | debug = False 26 | 27 | quit_counter = 3 28 | 29 | working_dir = "" 30 | aut_command = "" 31 | process_id = -1 32 | 33 | pyautogui.FAILSAFE = False # disables the fail-safe 34 | 35 | def on_release(key): 36 | global running, quit_counter 37 | if key == keyboard.Key.f1: 38 | quit_counter -= 1 39 | if quit_counter == 0: 40 | running = False 41 | print("[Random] Killing tester.") 42 | 43 | def kill_old_process(): 44 | global process_id 45 | 46 | if process_id != -1: 47 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 48 | process_id = -1 49 | 50 | def start_aut(): 51 | global working_dir, aut_command, process_id 52 | 53 | if aut_command != "": 54 | 55 | print("[Random] Starting AUT") 56 | 57 | kill_old_process() 58 | 59 | os.chdir(working_dir) 60 | 61 | with open('stdout.log', "a+") as outfile: 62 | with open('stderr.log', "a+") as errfile: 63 | outfile.write("\n" + str(time.time()) + "\n") 64 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 65 | 66 | time.sleep(10) 67 | else: 68 | print("[Random] Could not find AUT to start!") 69 | 70 | 71 | def generate_input_string(): 72 | if random.random() < 0.5: 73 | return "Hello World!" 74 | else: 75 | return str(random.randint(-10000, 10000)) 76 | 77 | if __name__ == '__main__': 78 | # Collect events until released 79 | with keyboard.Listener(on_release=on_release) as listener: 80 | if len(sys.argv) > 1: 81 | cfg.window_name = sys.argv[1] 82 | 83 | if len(sys.argv) > 3: 84 | working_dir = sys.argv[2] 85 | aut_command = sys.argv[3] 86 | 87 | start_aut() 88 | 89 | print("[Random] Starting") 90 | 91 | start_time = time.time() 92 | 93 | runtime = cfg.test_time 94 | 95 | actions = 0 96 | 97 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 98 | 99 | with open(csv_file, "w+") as p_f: 100 | p_f.write("time,actions,technique,iteration_time,window_name\n") 101 | 102 | while is_running(start_time, runtime, actions, running): 103 | time.sleep(1) 104 | iteration_time = time.time() 105 | 106 | exec_time = time.time() - start_time 107 | 108 | os.system('killall "firefox"') 109 | 110 | box = [0, 0.5, 0.5, 0.5, 0.5] 111 | 112 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 113 | 114 | counter = 0 115 | 116 | while app_w == 0: 117 | 118 | start_aut() 119 | counter += 1 120 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 121 | if counter >= 3: 122 | print("[Detection] Couldn't find application window!") 123 | break 124 | 125 | if not is_running(start_time, runtime, actions, running): 126 | break 127 | 128 | image = screenshot() 129 | 130 | perform_interaction(box, app_x, app_y, app_w, app_h) 131 | 132 | end_iteration_time = time.time() 133 | if debug: 134 | print("[Random] Iteration Time:", end_iteration_time - iteration_time) 135 | 136 | # write test info 137 | actions += 1 138 | 139 | with open(csv_file, "a+") as p_f: 140 | # TODO: Write python functions for click, type, etc 141 | p_f.write(str(exec_time) + "," + str(actions) + ",random," + str(iteration_time) + "," + cfg.window_name + "\n") 142 | # if sub_window: 143 | # if random.random() < 0.05: 144 | # pyautogui.press('escape') 145 | 146 | kill_old_process() 147 | 148 | time.sleep(20) 149 | print("[Random] Finished testing!") 150 | 151 | 152 | -------------------------------------------------------------------------------- /scripts/evaluate_models.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | library(ggthemes) 7 | library(viridis) 8 | library(Hmisc) 9 | 10 | args = commandArgs(TRUE) 11 | 12 | load_data <- function(directory, file){ 13 | wd = getwd() 14 | setwd(directory) 15 | data <- readr::read_csv(file) 16 | return(data) 17 | } 18 | 19 | setwd('/home/thomas/work/GUIdance') 20 | 21 | data <- load_data('/home/thomas/work/GUIdance', 'validation-iou.csv') 22 | data2 <- load_data('/home/thomas/work/GUIdance', 'validation-centre.csv') 23 | 24 | data = data %>% mutate(Metric="IoU") 25 | data2 = data2 %>% mutate(Metric="Centre") 26 | 27 | data = bind_rows(data, data2) 28 | 29 | data$class = "Widget" 30 | 31 | average_precision = data[FALSE,] 32 | 33 | classes = unique(data$class) 34 | datasets = unique(data$dataset) 35 | 36 | t_data = data 37 | 38 | t_data$Sensitivity = capitalize(t_data$sensitivity) 39 | 40 | colscale <- c("Synthetic"="#FFAA99", "Ubuntu"="#99AAFF", 41 | "Mac"="#FFFFAA") 42 | fillscale <- scale_fill_manual(name="Dataset", values=colscale) 43 | 44 | dataset_a = t_data %>% mutate(Dataset=dataset) 45 | dataset_r = dataset_a %>% filter(dataset == "real") %>% mutate(Dataset="Ubuntu") 46 | dataset_s = dataset_a %>% filter(dataset == "synthetic") %>% mutate(Dataset="Synthetic") 47 | dataset_m = dataset_a %>% filter(dataset == "mac") %>% mutate(Dataset="Mac") 48 | t_data = rbind(dataset_r, dataset_s, dataset_m) 49 | 50 | t_data = t_data[t_data$Metric == "IoU",] 51 | 52 | t_data$Dataset = factor(t_data$Dataset, c("Ubuntu", "Mac", "Synthetic")) 53 | 54 | p = t_data %>% filter(dataset == "real" | dataset == "synthetic") %>% 55 | ggplot(aes(x=Sensitivity, y=value, fill=Dataset)) + 56 | geom_boxplot() + 57 | labs(x="", 58 | y="", 59 | title="") + 60 | #facet_wrap(~Metric) + 61 | theme_bw() + 62 | #theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 63 | scale_fill_viridis(discrete=TRUE, option="viridis", begin=0.5) + 64 | theme(legend.position="bottom", 65 | legend.margin=margin(t=-0.7, r=0, b=0.5, l=0, unit="cm"), 66 | plot.margin = unit(x = c(0, 0, -0.2, 0), units = "cm")) 67 | 68 | print(median((t_data %>% filter(dataset == "real", sensitivity == "precision", Metric == "Centre"))$value)) 69 | print(median((t_data %>% filter(dataset == "real", sensitivity == "recall", Metric == "Centre"))$value)) 70 | 71 | print(median((t_data %>% filter(dataset == "real", sensitivity == "precision", Metric == "IoU"))$value)) 72 | print(median((t_data %>% filter(dataset == "real", sensitivity == "recall", Metric == "IoU"))$value)) 73 | 74 | ggsave("rq1.pdf", p, width = 3, height=2.5) 75 | ggsave("rq1.png", p, width = 3, height=2.5) 76 | 77 | 78 | p = t_data %>% filter(dataset == "mac" | dataset == "synthetic") %>% 79 | #filter(iou_threshold == 0.5) %>% 80 | #filter(correct==1) %>% 81 | #group_by(size_cat) %>% 82 | ggplot(aes(x=Sensitivity, y=value, fill=Dataset)) + 83 | geom_boxplot() + 84 | #geom_bar(position = "dodge", stat = "summary", fun.y="sum") + 85 | #geom_smooth(method="lm", se=F) + 86 | #scale_y_log10() + 87 | labs(x="", 88 | y="", 89 | title="") + 90 | #scale_x_discrete() + 91 | #scale_y_log10 92 | #facet_wrap(~Metric) + 93 | theme_bw() + 94 | #facet_wrap(busy_cat~size_cat) + 95 | #theme(axis.text.x = element_text(angle = 45, hjust = 1))+ 96 | scale_fill_viridis(discrete=TRUE, option="viridis", begin=0.75) + 97 | theme(legend.position="bottom", 98 | legend.margin=margin(t=-0.7, r=0, b=0.5, l=0, unit="cm"), 99 | plot.margin = unit(x = c(0, 0, -0.2, 0), units = "cm")) 100 | #scale_fill_brewer(palette="Set1") + 101 | #scale_color_brewer(palette="BrBG") 102 | 103 | print(median((t_data %>% filter(dataset == "mac", sensitivity == "precision", Metric == "Centre"))$value)) 104 | print(median((t_data %>% filter(dataset == "mac", sensitivity == "recall", Metric == "Centre"))$value)) 105 | 106 | 107 | print(median((t_data %>% filter(dataset == "mac", sensitivity == "precision", Metric == "IoU"))$value)) 108 | print(median((t_data %>% filter(dataset == "mac", sensitivity == "recall", Metric == "IoU"))$value)) 109 | 110 | ggsave("rq2.pdf", p, width = 3, height=2.5) 111 | 112 | print(p) 113 | 114 | 115 | # data <- load_data('/home/thomas/work/GUIdance', 'confusion.csv') 116 | # 117 | # #data = data[!data$dataset == "real",] 118 | # 119 | # #data = data %>% filter(predicted_class != "menu_item", actual_class != "menu_item") 120 | # 121 | # data = data %>% mutate(dataset = capitalize(dataset)) 122 | # 123 | # data = data %>% group_by(dataset, predicted_class) %>% 124 | # mutate(percent = (quantity/(sum(quantity)+1))) 125 | # 126 | # p = data %>% ggplot(aes(predicted_class, actual_class)) + 127 | # geom_tile(aes(fill=percent)) + 128 | # labs(x="Predicted Class", 129 | # y="Actual Class", 130 | # title="") + 131 | # theme_bw() + 132 | # theme(axis.text.x = element_text(angle = 45, hjust = 1), legend.position = "none") + 133 | # scale_fill_gradient(low = "#000077", high = "#FF9900", na.value = "#00FF00") + 134 | # facet_wrap(~dataset) 135 | # 136 | # ggsave("confusion.pdf", p, height=3, width=8) 137 | # 138 | # print(p) 139 | 140 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/data_plot.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | from data_loader import load_files, load_raw_image, disable_transformation 13 | import data_loader 14 | import re 15 | 16 | def convert_coords(x, y, w, h, aspect): 17 | if aspect > 1: # width is bigger than height 18 | h = h * aspect 19 | y = 0.5 + ((y - 0.5)*aspect) 20 | elif aspect < 1: 21 | w = w / aspect 22 | x = 0.5 + ((x - 0.5)/aspect) 23 | 24 | return x, y, w, h 25 | 26 | if __name__ == '__main__': 27 | 28 | data_loader.debug = True 29 | 30 | training_file = cfg.data_dir + "/train-25000.txt" 31 | 32 | pattern = re.compile(".*\/([0-9]+).*") 33 | 34 | training_images = [] 35 | 36 | real_images = [] 37 | 38 | with open(training_file, "r") as tfile: 39 | for l in tfile: 40 | file_num = int(pattern.findall(l)[-1]) 41 | training_images.append(l.strip()) 42 | 43 | training_images = [f.replace("/home/thomas/work/GuiImages/public", "/data/acp15tdw/data") for f in training_images] 44 | 45 | while len(training_images) < 100000: 46 | training_images = training_images + training_images 47 | 48 | training_images = training_images[:100000] 49 | 50 | valid_file = cfg.data_dir + "/" + cfg.validate_file 51 | 52 | valid_images = [] 53 | 54 | with open(valid_file, "r") as tfile: 55 | for l in tfile: 56 | valid_images.append(l.strip()) 57 | 58 | random.shuffle(valid_images) 59 | valid_images = valid_images[:500] 60 | 61 | training_file = cfg.data_dir + "/../backup/data/train.txt" 62 | 63 | with open(training_file, "r") as tfile: 64 | for l in tfile: 65 | 66 | file_num = int(pattern.findall(l)[-1]) 67 | 68 | if file_num <= 243: 69 | real_images.append(l.strip()) 70 | 71 | 72 | 73 | valid_file = cfg.data_dir + "/../backup/data/validate.txt" 74 | 75 | with open(valid_file, "r") as tfile: 76 | for l in tfile: 77 | file_num = int(pattern.findall(l)[-1]) 78 | 79 | if file_num <= 243: 80 | real_images.append(l.strip()) 81 | 82 | valid_file = cfg.data_dir + "/../backup/data/test.txt" 83 | 84 | with open(valid_file, "r") as tfile: 85 | for l in tfile: 86 | file_num = int(pattern.findall(l)[-1]) 87 | 88 | if file_num <= 243: 89 | real_images.append(l.strip()) 90 | 91 | real_images = [f.replace("/data/acp15tdw", "/data/acp15tdw/backup") for f in real_images] 92 | 93 | real_images_manual = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(cfg.data_dir + "/../real/data/images")) for f in fn] 94 | 95 | for ri in real_images_manual: 96 | real_images.append(ri) 97 | 98 | # random.seed(cfg.random_seed) 99 | # random.shuffle(real_images) 100 | #training_images = real_images[:100] 101 | 102 | #real_images = real_images[100:] 103 | 104 | print("Found", len(real_images), "real GUI screenshots.") 105 | 106 | #valid_images = random.sample(valid_images, cfg.batch_size) 107 | 108 | tf.reset_default_graph() 109 | 110 | yolo = Yolo() 111 | 112 | yolo.create_network() 113 | 114 | yolo.set_training(True) 115 | 116 | yolo.create_training() 117 | 118 | global_step = tf.placeholder(tf.int32) 119 | batches = math.ceil(len(training_images)/cfg.batch_size) if cfg.run_all_batches else 1 120 | 121 | 122 | learning_rate = tf.train.exponential_decay(0.01, global_step, 123 | 1, 0.9, staircase=True) 124 | #learning_rate = tf.placeholder(tf.float64) 125 | #learning_r = cfg.learning_rate_start 126 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 127 | 128 | with tf.control_dependencies(update_ops): 129 | yolo.set_update_ops(update_ops) 130 | 131 | train_step = tf.train.AdamOptimizer(5e-5). \ 132 | minimize(yolo.loss) 133 | 134 | saver = tf.train.Saver() 135 | 136 | model_file = cfg.weights_dir + "/model.ckpt" 137 | 138 | valid_batches = math.ceil(len(valid_images)/cfg.batch_size) if cfg.run_all_batches else 1 139 | 140 | real_batches = math.ceil(len(real_images)/cfg.batch_size) if cfg.run_all_batches else 1 141 | 142 | if (real_batches < 1): 143 | real_batches = 1 144 | 145 | config = tf.ConfigProto( 146 | device_count = {'GPU': 0} 147 | ) 148 | 149 | with tf.Session(config=config) as sess: 150 | 151 | init_op = tf.global_variables_initializer() 152 | 153 | print("Initialising Memory Values") 154 | model = sess.run(init_op) 155 | 156 | random.shuffle(valid_images) 157 | 158 | random.shuffle(training_images) 159 | yolo.set_training(False) 160 | 161 | losses = [0, 0, 0, 0, 0, 0, 0, 0] 162 | 163 | for j in range(valid_batches): 164 | gc.collect() 165 | 166 | lower_index = j*cfg.batch_size 167 | upper_index = min(len(valid_images), ((j+1)*cfg.batch_size)) 168 | 169 | v_imgs, v_labels, v_obj_detection = load_files( 170 | valid_images[lower_index:upper_index]) 171 | 172 | v_imgs = (np.array(v_imgs)/127.5)-1 173 | 174 | v_labels = np.array(v_labels) 175 | 176 | v_obj_detection = np.array(v_obj_detection) -------------------------------------------------------------------------------- /gui-tester/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 4.0.0 7 | 8 | com.thomasdeanwhite 9 | gui 10 | 0.0.1-SNAPSHOT 11 | 12 | com.thomasdeanwhite 13 | gui-tester 14 | 0.0.1-SNAPSHOT 15 | Gui Tester Master 16 | http://maven.apache.org 17 | 18 | UTF-8 19 | 20 | 21 | 22 | 23 | central 24 | http://central.maven.org/maven2/ 25 | 26 | 27 | 28 | 29 | 30 | 31 | com.1stleg 32 | jnativehook 33 | 2.0.2 34 | 35 | 36 | 37 | org.seleniumhq.selenium 38 | selenium-java 39 | 3.4.0 40 | 41 | 42 | 43 | com.google.code.gson 44 | gson 45 | 2.3.1 46 | 47 | 48 | 49 | commons-cli 50 | commons-cli 51 | 1.2 52 | 53 | 54 | 55 | nz.ac.waikato.cms.weka 56 | weka-dev 57 | 3.9.1 58 | 59 | 60 | 61 | commons-codec 62 | commons-codec 63 | 1.10 64 | 65 | 66 | 67 | org.junit.platform 68 | junit-platform-launcher 69 | RELEASE 70 | test 71 | 72 | 73 | org.junit.jupiter 74 | junit-jupiter-engine 75 | RELEASE 76 | test 77 | 78 | 79 | org.junit.vintage 80 | junit-vintage-engine 81 | RELEASE 82 | test 83 | 84 | 85 | com.scythe 86 | scythe 87 | 0.0.1-DEV 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | org.apache.maven.plugins 96 | maven-surefire-plugin 97 | 2.19 98 | 99 | 100 | org.junit.platform 101 | junit-platform-surefire-provider 102 | RELEASE 103 | 104 | 105 | 106 | org.junit.jupiter 107 | junit-jupiter-engine 108 | RELEASE 109 | 110 | 111 | 112 | org.junit.vintage 113 | junit-vintage-engine 114 | RELEASE 115 | 116 | 117 | 118 | 119 | 1 120 | true 121 | 122 | java.library.path 123 | ${project.basedir}/lib 124 | 125 | 1 126 | 127 | 128 | 129 | 130 | 131 | org.jacoco 132 | jacoco-maven-plugin 133 | 0.7.6.201602180812 134 | 135 | 136 | prepare-agent 137 | 138 | prepare-agent 139 | 140 | 141 | 142 | 143 | 144 | 145 | org.eluder.coveralls 146 | coveralls-maven-plugin 147 | 4.3.0 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/task_distributor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | import os 3 | import sys 4 | import random 5 | from fpdf import FPDF 6 | 7 | 8 | apps = [("00", "bellmanzadeh"), 9 | ("02", "JabRef"), 10 | ("03", "Java Fabled Lands"), 11 | ("05", "ordrumbox"), 12 | ("11", "Dietetics"), 13 | ("12", "Minesweeper"), 14 | ("13", "SQuiz"), 15 | ("14", "blackjack"), 16 | ("15", "UPM"), 17 | ("16", "Simple Calculator")] 18 | 19 | line_height = 7 20 | 21 | if __name__ == '__main__': 22 | 23 | wd = os.getcwd() 24 | 25 | if len(sys.argv) < 4: 26 | print("Must include three parameters: input task location, output task location, number of users.", file=sys.stderr) 27 | sys.exit(1) 28 | 29 | input_dir = sys.argv[1] 30 | output_dir = sys.argv[2] 31 | num_users = int(sys.argv[3]) 32 | 33 | for u in range(num_users): 34 | applications = apps[:] 35 | random.shuffle(applications) 36 | 37 | user_out = output_dir + "/" + str(u+1) 38 | 39 | if not os.path.isdir(user_out): 40 | os.makedirs(user_out, exist_ok=True) 41 | try: 42 | os.mkdir(user_out) 43 | except OSError as e: 44 | #folder exists 45 | pass 46 | 47 | pdf = FPDF(orientation="P", unit="mm", format="A4") 48 | 49 | pdf.add_page() 50 | 51 | pdf.set_font("Courier", style="u", size=32) 52 | 53 | pdf.cell(200, 32, txt="GUI Interaction Observations", ln=1, align="C") 54 | pdf.set_font("Courier", size=32) 55 | pdf.cell(200, 32, txt="Participant " + str(u+1), ln=1, align="C") 56 | 57 | pdf.add_page() 58 | pdf.cell(200, 32, txt="Information", ln=1, align="C") 59 | 60 | pdf.set_font("Courier", size=14) 61 | pdf.write(line_height, "You will be interacting with each application twice, once to familiarise " + 62 | "yourself with the application and again to perform a set of tasks.\n\nWhen a set of " + 63 | "tasks is complete, hit ESCAPE three (3) times to finish interaction " + 64 | "with that application. You may take a break between each application. DO NOT press " + 65 | "ESCAPE during the application warm up phase, the application will automatically close " + 66 | "when this phase is finished.\n\nPlease give the application 15 seconds to load before " + 67 | "you start interacting with it.\n\nIf file input/output is required, please use " + 68 | "/home/thomas/tmp to save and load files.\n\nYour home directory is located at: " + 69 | user_out + "/") 70 | 71 | counter = 0 72 | 73 | for app in applications: 74 | 75 | a = app[0] 76 | name = app[1] 77 | script_name = name.replace(" ", "_") 78 | 79 | counter += 1 80 | 81 | pre = [] 82 | tasks = [] 83 | post = [] 84 | 85 | with open(input_dir + "/" + a + ".txt") as f: 86 | for line in f: 87 | c = line.strip() 88 | 89 | if len(c) == 0: 90 | continue 91 | 92 | if c.startswith("##"): 93 | post.append(c[2:]) 94 | elif c.startswith("#"): 95 | pre.append(c[1:]) 96 | else: 97 | tasks.append(c) 98 | 99 | random.shuffle(tasks) 100 | 101 | pdf.add_page() 102 | 103 | pdf.set_font("Courier", style="u", size=32) 104 | 105 | pdf.cell(200, 32, txt="Application {}".format(counter), ln=1, align="c") 106 | 107 | pdf.set_font("Courier", size=32) 108 | 109 | pdf.cell(200, 32, txt=name, ln=1, align="c") 110 | 111 | pdf.set_font("Courier", size=24) 112 | 113 | if len(pre) > 0: 114 | contents = "Information" 115 | pdf.cell(200, 18, txt=contents, ln=1, align="c") 116 | pdf.set_font("Courier", size=14) 117 | contents = "" 118 | for c in pre: 119 | contents += "- " + c + "\n" 120 | 121 | pdf.write(line_height, contents) 122 | else : 123 | pdf.set_font("Courier", size=14) 124 | pdf.write(line_height, "No information is required for this application.\n") 125 | 126 | pdf.set_font("Courier", size=14) 127 | pdf.cell(200, 10, txt="Please run {}.sh from the user directory.".format(script_name), ln=1, align="c") 128 | pdf.cell(200, 10, txt="Click \"Run\".", ln=1, align="c") 129 | 130 | 131 | 132 | pdf.set_font("Courier", size=24) 133 | 134 | pdf.cell(200, 18, txt="3 Minute Warm Up", ln=1, align="c") 135 | 136 | pdf.add_page() 137 | 138 | contents = "Application {}: {}".format(counter, name) 139 | 140 | pdf.set_font("Courier", size=24) 141 | pdf.cell(200, 18, txt=contents, ln=1, align="c") 142 | 143 | 144 | 145 | contents = "Tasks:" 146 | pdf.set_font("Courier", size=24) 147 | pdf.cell(200, 18, txt=contents, ln=1, align="c") 148 | contents = "" 149 | for c in tasks: 150 | contents += "[ ] - " + c + "\n" 151 | pdf.set_font("Courier", size=14) 152 | pdf.write(line_height, contents) 153 | 154 | if len(post) > 0: 155 | pdf.set_font("Courier", size=24) 156 | pdf.cell(200, 18, txt="Finally/Optional:", ln=1, align="c") 157 | contents = "" 158 | for c in post: 159 | contents += "[ ] - " + c + "\n" 160 | pdf.set_font("Courier", size=14) 161 | pdf.write(line_height, contents) 162 | 163 | with open(user_out + "/" + script_name + ".sh", "w+") as f: 164 | f.write("#!/bin/bash\nbash ~/t2s.sh \"Starting warmup for " + name + "\"\n") 165 | f.write("cd ../..\n./run_manual.sh {} user-models/participant-{}/warmup 00 180\n".format(a, u+1)) 166 | f.write("bash ~/t2s.sh \"Warm up complete. Starting tasks for " + name + "\"\n") 167 | f.write("./run_manual.sh {} user-models/participant-{}/tasks 00\n".format(a, u+1)) 168 | f.write("bash ~/t2s.sh \"Data gathering for {}: complete!".format(name) + "\"\n") 169 | 170 | 171 | pdf.output(user_out + "/participant-" + str(u+1) + "-tasks.pdf") 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_plot_correct.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | from tensorflow.python.tools import inspect_checkpoint as chkp 12 | from data_loader import load_files, disable_transformation, load_raw_image, convert_coords 13 | 14 | disable_transformation() 15 | 16 | 17 | 18 | if __name__ == '__main__': 19 | 20 | tf.reset_default_graph() 21 | yolo = Yolo() 22 | 23 | yolo.create_network() 24 | #yolo.set_training(False) 25 | yolo.create_training() 26 | 27 | learning_rate = tf.placeholder(tf.float64) 28 | learning_r = cfg.learning_rate_start 29 | 30 | saver = tf.train.Saver() 31 | 32 | model_file = os.getcwd() + "/" + cfg.weights_dir + "/model.ckpt" 33 | 34 | #chkp.print_tensors_in_checkpoint_file(model_file, tensor_name='', all_tensors=True) 35 | 36 | gpu_options = tf.GPUOptions(allow_growth=True) 37 | 38 | config = tf.ConfigProto( 39 | device_count = {'GPU': 0} 40 | ) 41 | 42 | with tf.Session(config=config) as sess: 43 | 44 | init_op = tf.global_variables_initializer() 45 | model = sess.run(init_op) 46 | if os.path.isfile(os.getcwd() + "/" + cfg.weights_dir + "/checkpoint"): 47 | saver.restore(sess, model_file) 48 | print("Restored model") 49 | yolo.set_training(False) 50 | 51 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 52 | images, labels, obj_detect = load_files([sys.argv[1]]) 53 | 54 | img = images[0] 55 | 56 | #normalise data between 0 and 1 57 | imgs = np.array(images)/127.5-1 58 | labels = np.array(labels)/ cfg.grid_shape[0] 59 | obj_detect = np.array(obj_detect) 60 | 61 | conf_thresh = 0.1 62 | 63 | boxes, correct, iou = sess.run([yolo.output, yolo.matches, yolo.best_iou], feed_dict={ 64 | yolo.x: imgs, 65 | yolo.anchors: anchors, 66 | yolo.train_bounding_boxes: labels, 67 | yolo.train_object_recognition: obj_detect, 68 | yolo.iou_threshold: 0.5, 69 | yolo.object_detection_threshold: conf_thresh 70 | }) 71 | 72 | proc_correct = yolo.convert_correct_to_list(correct)[0] 73 | # proc_iou = yolo.convert_correct_to_list(np.reshape(iou, [-1, cfg.grid_shape[0], cfg.grid_shape[1]]))[0] 74 | proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True) 75 | 76 | labels_classes = np.append(np.expand_dims(obj_detect, axis=-1), labels, axis=-1) 77 | 78 | proc_boxes, iou_max = yolo.calculate_max_iou(proc_boxes, np.reshape(labels_classes, [labels.shape[0], -1, 6])) 79 | 80 | proc_boxes = proc_boxes.tolist()[0] 81 | 82 | 83 | img = load_raw_image(sys.argv[1]) 84 | 85 | height, width = img.shape[:2] 86 | 87 | trim_overlap = True 88 | 89 | i=0 90 | while i < len(proc_boxes): 91 | box = proc_boxes[i] 92 | 93 | box[1:5] = convert_coords(box[1], box[2], box[3], box[4], width/height) 94 | 95 | x, y, w, h = (box[1],box[2],box[3],box[4]) 96 | box[1] = x - w/2 97 | box[2] = y - h/2 98 | box[3] = x + w/2 99 | box[4] = y + h/2 100 | i = i + 1 101 | i = 0 102 | 103 | correct_q = 0 104 | predicted_boxes = 0 105 | 106 | for bc in range(len(proc_boxes)): 107 | box = proc_boxes[bc] 108 | if box[5] < conf_thresh: 109 | continue 110 | cor = proc_correct[bc] 111 | height, width = img.shape[:2] 112 | 113 | cls = yolo.names[int(box[0])] 114 | 115 | hex = cls.encode('utf-8').hex()[0:6] 116 | 117 | color = tuple(int(int(hex[k:k+2], 16)*0.75) for k in (0, 2 ,4)) 118 | 119 | # if box[6] > 0.3: 120 | # correct_q += 1 121 | # color = [255, 255, 255] 122 | # if box[6] < 0.3: 123 | # continue 124 | 125 | if (box[5]>cfg.object_detection_threshold): 126 | print(box) 127 | 128 | predicted_boxes += 1 129 | 130 | x1 = max(int(width*box[1]), 0) 131 | y1 = max(int(height*box[2]), 0) 132 | x2 = int(width*box[3]) 133 | y2 = int(height*box[4]) 134 | 135 | cv2.rectangle(img, (x1, y1), 136 | (x2, y2), 137 | (color[0], color[1], color[2]), 1+int(5*box[5]), 8) 138 | 139 | for bc in range(len(proc_boxes)): 140 | box = proc_boxes[bc] 141 | if box[5] < conf_thresh: 142 | continue 143 | 144 | cor = proc_correct[bc] 145 | cls = yolo.names[int(box[0])] 146 | 147 | hex = cls.encode('utf-8').hex()[0:6] 148 | color = tuple(int(int(hex[k:k+2], 16)*0.75) for k in (0, 2 ,4)) 149 | # if box[6] > 0.3: 150 | # color = [255, 255, 255] 151 | # if box[6] < 0.3: 152 | # continue 153 | 154 | if (box[5]>cfg.object_detection_threshold): 155 | height, width = img.shape[:2] 156 | 157 | avg_col = (color[0] + color[1] + color[2])/3 158 | 159 | text_col = (255, 255, 255) 160 | 161 | 162 | if avg_col > 127: 163 | text_col = (0, 0, 0) 164 | 165 | x1 = max(int(width*box[1]), 0) 166 | y1 = max(int(height*box[2]), 0) 167 | x2 = int(width*box[3]) 168 | y2 = int(height*box[4]) 169 | 170 | cv2.rectangle(img, 171 | (x1-3, y1-23), 172 | (x1 + (5 + len(cls)+10)*10, y1), 173 | (color[0], color[1], color[2]), -1, 8) 174 | 175 | cv2.putText(img, cls + " conf:" + str(round(box[5]*100)) + " iou:" + str(round(box[6]*100)), 176 | (x1-3, y1-3), 177 | cv2.FONT_HERSHEY_PLAIN, 178 | 1, text_col, 1, lineType=cv2.LINE_AA) 179 | 180 | actual_labels = np.sum((labels[..., 4]>0).astype(int)) 181 | 182 | print(predicted_boxes, ":", actual_labels) 183 | 184 | predicted_boxes = predicted_boxes if predicted_boxes > 0 else 1 185 | actual_labels = actual_labels if actual_labels > 0 else 1 186 | 187 | print(correct_q, "correct predictions. Precision:", correct_q / (predicted_boxes), "Recall:", 188 | correct_q / actual_labels) 189 | 190 | print(iou_max) 191 | 192 | cv2.imshow('image',img) 193 | cv2.waitKey(0) 194 | cv2.destroyAllWindows() 195 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/replay_tester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import config as cfg 4 | from yolo import Yolo 5 | import cv2 6 | import numpy as np 7 | import tensorflow as tf 8 | import sys 9 | import gc 10 | import math 11 | import random 12 | import os 13 | import pyautogui 14 | import time 15 | import subprocess 16 | import Xlib 17 | from pynput import keyboard 18 | import signal 19 | import timeit 20 | from pynput.mouse import Listener, Button 21 | from test_helper import get_window_size_focus, screenshot, perform_interaction, is_running 22 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor 23 | 24 | sub_window = False 25 | 26 | pyautogui.FAILSAFE = False # disables the fail-safe 27 | 28 | running = True 29 | 30 | debug = False 31 | 32 | quit_counter = 3 33 | 34 | working_dir = "" 35 | aut_command = "" 36 | process_id = -1 37 | 38 | def on_release(key): 39 | global running, quit_counter 40 | 41 | if key == keyboard.Key.f1: 42 | quit_counter -= 1 43 | if quit_counter == 0: 44 | running = False 45 | print("[Replay] Killing tester.") 46 | 47 | def kill_old_process(): 48 | global process_id 49 | 50 | if process_id != -1: 51 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 52 | process_id = -1 53 | 54 | def start_aut(): 55 | global working_dir, aut_command, process_id, delay 56 | 57 | if aut_command != "": 58 | 59 | print("[Replay] Starting AUT") 60 | 61 | kill_old_process() 62 | 63 | os.chdir(working_dir) 64 | 65 | with open('stdout.log', "a+") as outfile: 66 | with open('stderr.log', "a+") as errfile: 67 | outfile.write("\n" + str(time.time()) + "\n") 68 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 69 | 70 | time.sleep(10) 71 | delay += 10 72 | else: 73 | print("[Replay] Could not find AUT to start!") 74 | 75 | mouse_pos = (0, 0) 76 | 77 | events = [] 78 | 79 | update_window = True 80 | 81 | def mouse_move(x, y): 82 | global mouse_pos 83 | mouse_pos = (x, y) 84 | 85 | 86 | def mouse_click(x, y, button, pressed): 87 | global events 88 | events.append(MouseEvent(time.time(), 0 if button == Button.left else 89 | 1 if button == Button.right else 2, x, y, pressed=pressed)) 90 | 91 | 92 | 93 | def mouse_scroll(x, y, dx, dy): 94 | global events 95 | if events[-1].get_event_type() != EventType.MOUSE_WHEEL: 96 | events.append(ScrollEvent(time.time(), dx, dy)) 97 | else: 98 | events[-1].displace(x, y) 99 | 100 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 101 | 102 | window_event = default_window_event 103 | 104 | delay = 0 105 | 106 | def run_events(x, y, time): 107 | global window_event, events, default_window_event, delay, update_window 108 | if len(events) == 0: 109 | print("[Replay] Test finished!") 110 | return False 111 | 112 | event = events[0] 113 | 114 | while event.should_perform_event(time - delay): 115 | if isinstance(event, WindowChangeEvent): 116 | window_event = event 117 | print("[Replay] Changed expected window! (%s)" % window_event.wm_name) 118 | update_window = True 119 | 120 | if not window_event == default_window_event: 121 | if isinstance(event, MouseEvent): 122 | dx = x-window_event.position[0] 123 | dy = y-window_event.position[1] 124 | event.displace(dx, dy) 125 | elif isinstance(event, ScrollEvent): 126 | dx = x-window_event.position[0] 127 | dy = y-window_event.position[1] 128 | event.displace(0, 0, dx, dy) 129 | 130 | event.perform() 131 | events.pop(0) 132 | event = events[0] 133 | return True 134 | 135 | if __name__ == '__main__': 136 | # Collect events until released 137 | wd = os.getcwd() 138 | 139 | if len(sys.argv) < 4: 140 | print("Must have filename argument!", file=sys.stderr) 141 | sys.exit(1) 142 | input_dir = sys.argv[4] 143 | file = input_dir + "/" + "events.evnt" 144 | 145 | event_constructor = EventConstructor() 146 | 147 | with open(file, "r") as f: 148 | for line in f: 149 | events.append(event_constructor.construct(line)) 150 | 151 | events = sorted(events, key=lambda x : x.timestamp) 152 | prev_start_time = events[0].timestamp 153 | for i in range(len(events)): 154 | events[i].timestamp -= prev_start_time 155 | 156 | with keyboard.Listener(on_release=on_release) as klistener: 157 | cfg.window_name = sys.argv[1] 158 | 159 | working_dir = sys.argv[2] 160 | aut_command = sys.argv[3] 161 | 162 | start_aut() 163 | 164 | print("[Replay] Starting") 165 | 166 | for event in events: 167 | print(event) 168 | 169 | start_time = time.time() 170 | 171 | runtime = cfg.test_time 172 | 173 | actions = 0 174 | 175 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 176 | 177 | with open(csv_file, "w+") as p_f: 178 | p_f.write("time,actions,technique,iteration_time,window_name\n") 179 | 180 | while is_running(start_time, runtime, actions, running): 181 | 182 | iteration_time = time.time() 183 | 184 | exec_time = time.time() - start_time 185 | 186 | #os.system('wmctrl -c "firefox"') 187 | 188 | app_x, app_y, app_w, app_h = get_window_size_focus(window_event.get_title(), focus=False) 189 | 190 | while app_w == 0: 191 | exec_time = time.time() - start_time 192 | 193 | if not window_event == default_window_event: 194 | print("[Replay] Couldn't find window", window_event.wm_name ) 195 | kill_old_process() 196 | 197 | start_aut() 198 | 199 | title = window_event.get_title() 200 | 201 | app_x, app_y, app_w, app_h = get_window_size_focus(title, focus=False) 202 | 203 | update_window = False 204 | 205 | if not run_events(app_x, app_y, exec_time): 206 | break 207 | 208 | if not is_running(start_time, runtime+delay, actions, running): 209 | print("[Replay] Couldn't find application window!") 210 | break 211 | 212 | 213 | if not is_running(start_time, runtime+delay, actions, running): 214 | break 215 | 216 | if not run_events(app_x, app_y, exec_time): 217 | break 218 | 219 | with open(csv_file, "a+") as p_f: 220 | # TODO: Write python functions for click, type, etc 221 | p_f.write(str(exec_time) + "," + str(actions) + ",Replay," + str(iteration_time) + "," + cfg.window_name + "\n") 222 | kill_old_process() 223 | 224 | time.sleep(5) 225 | print("[Replay] Finished testing!") 226 | 227 | os.chdir(wd) 228 | 229 | 230 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/user_model_random_tester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import config as cfg 4 | import sys 5 | import os 6 | import time 7 | import subprocess 8 | from pynput import keyboard 9 | import signal 10 | from test_helper import get_window_size_focus, screenshot, perform_interaction, is_running, get_focused_window 11 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 12 | from user_model import UserModel 13 | import pickle 14 | import pyautogui 15 | import numpy as np 16 | from threading import Thread 17 | import cv2 18 | 19 | sub_window = False 20 | 21 | running = True 22 | 23 | debug = False 24 | 25 | quit_counter = 3 26 | 27 | working_dir = "" 28 | aut_command = "" 29 | process_id = -1 30 | pyautogui.PAUSE = 0 31 | 32 | pyautogui.FAILSAFE = False # disables the fail-safe 33 | seeding_key = False 34 | 35 | out_folder = "" 36 | 37 | def capture_screen(): 38 | global app_x, app_y, app_w, app_h, exec_time, out_folder 39 | 40 | time.sleep(1) 41 | 42 | img_folder = out_folder + "/images" 43 | 44 | img_out = img_folder + "/"+ window_event.filename_string() 45 | 46 | counter = int(exec_time/30) 47 | 48 | if not (os.path.isfile(img_out + str(counter) + ".png")): 49 | image = screenshot() 50 | 51 | if not image is None: 52 | 53 | if not cfg.fullscreen: 54 | image = image[app_y:app_y+app_h, app_x:app_x+app_w] 55 | 56 | image = np.array(image) 57 | 58 | 59 | if not os.path.isdir(img_folder): 60 | os.makedirs(img_folder, exist_ok=True) 61 | try: 62 | os.mkdir(img_folder) 63 | except OSError as e: 64 | #folder exists 65 | pass 66 | 67 | if not (os.path.isfile(img_out + str(counter) + ".png")): 68 | cv2.imwrite(img_out + str(counter) + ".png", image) 69 | 70 | def on_release(key): 71 | global running, quit_counter, seeding_key 72 | 73 | if key == keyboard.Key.esc and not seeding_key: 74 | quit_counter -= 1 75 | if quit_counter == 0: 76 | running = False 77 | print("[User Model Replay] Killing tester.") 78 | 79 | def kill_old_process(): 80 | global process_id 81 | 82 | if process_id != -1: 83 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 84 | process_id = -1 85 | 86 | def start_aut(): 87 | global working_dir, aut_command, process_id, delay 88 | 89 | if aut_command != "": 90 | 91 | print("[User Model Replay] Starting AUT") 92 | 93 | kill_old_process() 94 | 95 | os.chdir(working_dir) 96 | 97 | with open('stdout.log', "a+") as outfile: 98 | with open('stderr.log', "a+") as errfile: 99 | outfile.write("\n" + str(time.time()) + "\n") 100 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 101 | 102 | time.sleep(10) 103 | delay += 10 104 | else: 105 | print("[User Model Replay] Could not find AUT to start!") 106 | 107 | events = [] 108 | 109 | update_window = True 110 | 111 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 112 | 113 | window_event = default_window_event 114 | 115 | delay = 0 116 | 117 | if __name__ == '__main__': 118 | # Collect events until released 119 | 120 | with keyboard.Listener(on_release=on_release) as klistener: 121 | 122 | wd = os.getcwd() 123 | 124 | event_constructor = EventConstructor() 125 | 126 | if len(sys.argv) < 4: 127 | print("Must have filename argument!", file=sys.stderr) 128 | sys.exit(1) 129 | input_dir = sys.argv[4] 130 | file = input_dir + "/user_model.mdl" 131 | user_model = UserModel() 132 | with open(file, "rb") as f: 133 | user_model = pickle.load(f) 134 | 135 | output_dir = input_dir + "/tests/" + str(int(time.time())) 136 | 137 | out_folder = output_dir 138 | 139 | if not os.path.isdir(output_dir): 140 | os.makedirs(output_dir, exist_ok=True) 141 | try: 142 | os.mkdir(output_dir) 143 | except OSError as e: 144 | #folder exists 145 | pass 146 | 147 | with open(output_dir + "/test.log", "w+") as f: 148 | f.write("") 149 | 150 | cfg.window_name = sys.argv[1] 151 | 152 | working_dir = sys.argv[2] 153 | aut_command = sys.argv[3] 154 | 155 | start_aut() 156 | 157 | print("[User Model Replay] Starting") 158 | 159 | for event in events: 160 | print(event) 161 | 162 | start_time = time.time() 163 | 164 | runtime = cfg.test_time 165 | 166 | actions = 0 167 | 168 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 169 | 170 | with open(csv_file, "w+") as p_f: 171 | p_f.write("time,actions,technique,iteration_time,window_name\n") 172 | 173 | last_event = Event(0, EventType.NONE) 174 | 175 | while is_running(start_time, runtime, actions, running): 176 | 177 | 178 | 179 | iteration_time = time.time() 180 | 181 | exec_time = time.time() - start_time 182 | 183 | #os.system('wmctrl -c "firefox"') 184 | 185 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 186 | 187 | no_focus = 0 188 | 189 | while app_w == 0: 190 | 191 | time.sleep(1) 192 | 193 | if no_focus >= 20: 194 | kill_old_process() 195 | 196 | start_aut() 197 | 198 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 199 | if not is_running(start_time, runtime, actions, running): 200 | print("[User Model Replay] Couldn't find application window!") 201 | break 202 | 203 | no_focus += 1 204 | 205 | wm_class = "(" + w_class[0] + "," + w_class[1] + ")" 206 | 207 | if w_name != window_event.wm_name or wm_class != window_event.wm_class: 208 | window_event = WindowChangeEvent(time.time(), w_name, w_class, (app_x, app_y), 209 | (app_w, app_h)) 210 | os.chdir(wd) 211 | with open(output_dir + "/test.log", "a+") as f: 212 | f.write(window_event.hashcode() + "\n") 213 | 214 | print(window_event.hashcode()) 215 | 216 | event = event_constructor.random_event() 217 | 218 | event.change_window(window_event) 219 | seeding_key = True 220 | event.perform() 221 | seeding_key = False 222 | time.sleep(event.get_delay()) 223 | 224 | t = Thread(target=capture_screen) 225 | t.start() 226 | 227 | with open(output_dir + "/test.log", "a+") as f: 228 | f.write(event.hashcode() + "\n") 229 | 230 | actions += event.action_count(last_event) 231 | 232 | last_event = event 233 | 234 | os.chdir(wd) 235 | with open(csv_file, "a+") as p_f: 236 | # TODO: Write python functions for click, type, etc 237 | p_f.write(str(exec_time) + "," + str(actions) + ",UserModelGenerator," + str(iteration_time) + "," + cfg.window_name + "\n") 238 | 239 | for k in KeyEvent.keys_down: 240 | pyautogui.keyUp(k) 241 | 242 | kill_old_process() 243 | 244 | time.sleep(5) 245 | print("[User Model Replay] Finished testing! (" + str(actions) + " actions)") 246 | 247 | os.chdir(wd) 248 | 249 | 250 | -------------------------------------------------------------------------------- /scripts/process_results.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | library(ggthemes) 7 | library(xtable) 8 | library(effsize) 9 | library(viridis) 10 | 11 | setwd('/home/thomas/experiments/test_experiments') 12 | 13 | dir = "output" 14 | random_name = "random" 15 | detection_name = "detection" 16 | truth_name = "truth" 17 | user_name = "user-models" 18 | 19 | p_random_name = "Random" 20 | p_detection_name = "Pred" 21 | p_truth_name = "API" 22 | p_user_name = "User Model" 23 | 24 | load_file <- function(file){ 25 | filename = paste(getwd(), dir, file, sep="/") 26 | cat(paste("\r", filename)) 27 | row = readr::read_csv(filename) 28 | info=strsplit(file, "/") 29 | app = info[[1]][3] 30 | if (app == "Java_FireLands"){ 31 | app = "Java_FabledLands" 32 | } 33 | row = mutate(row, ITERATION=info[[1]][2]) 34 | row = mutate(row, APPLICATION=app) 35 | row = mutate(row, FILE=file) 36 | row = mutate(row, TECHNIQUE=info[[1]][1]) 37 | return(row) 38 | } 39 | test_file = "test.txt" 40 | 41 | images_dir = "images" 42 | labels_dir = "labels" 43 | 44 | 45 | # ------------------------ 46 | 47 | load_data <- function(){ 48 | if (file.exists("gui_data.RData")){ 49 | load("gui_data.RData") 50 | } else { 51 | pattern="coverage.csv" 52 | temp = list.files(path = dir, pattern=pattern, recursive = TRUE) 53 | r_data = bind_rows(lapply(temp, load_file)) 54 | } 55 | #data = mutate(data, BRANCH_COVERAGE=BRANCH_COVERED/(BRANCH_MISSED+BRANCH_COVERED+1)) 56 | #r_data$TECHNIQUE = factor(data$TECHNIQUE, levels=c(random_name, detection_name, truth_name, user_name)) 57 | return(data) 58 | } 59 | 60 | 61 | process_data <- function(data){ 62 | 63 | data = data %>% 64 | mutate(BRANCH_COVERED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_COVERED, BRANCH_COVERED), 65 | BRANCH_MISSED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_MISSED, BRANCH_MISSED)) 66 | 67 | #data$APPLICATION = substr(data$APPLICATION, 0, 8) 68 | 69 | #data[(data$BRANCH_COVERED + data$BRANCH_MISSED)==0,]$BRANCH_COVERED = 1 70 | 71 | #data$BRANCH_COVERAGE=data$BRANCH_COVERED/(data$BRANCH_COVERED+data$BRANCH_MISSED) 72 | 73 | p_data = data %>% 74 | group_by(ITERATION, APPLICATION, TECHNIQUE) %>% 75 | summarise(BRANCHES_COVERED_TOTAL=sum(BRANCH_COVERED), 76 | BRANCHES_MISSED_TOTAL=sum(BRANCH_MISSED), 77 | BRANCHES_TOTAL=sum(BRANCH_COVERED+BRANCH_MISSED), 78 | LINES_TOTAL=sum(LINE_COVERED+LINE_MISSED), 79 | BRANCH_COVERAGE=sum(BRANCH_COVERED)) 80 | 81 | 82 | apps = unique(p_data$APPLICATION) 83 | 84 | for (a in apps){ 85 | mb = max((p_data %>% filter(APPLICATION == a))$BRANCHES_TOTAL) 86 | p_data = p_data %>% mutate(BRANCH_COVERAGE=if_else(APPLICATION == a, BRANCH_COVERAGE/mb, as.double(BRANCH_COVERAGE)), 87 | BRANCHES_TOTAL=if_else(APPLICATION == a, mb, BRANCHES_TOTAL)) 88 | } 89 | 90 | p_data = p_data %>% filter(BRANCH_COVERAGE!=0) 91 | 92 | p_data = p_data %>% mutate(Technique = TECHNIQUE) 93 | p_data = p_data %>% mutate(Application = APPLICATION) 94 | 95 | p_data_r = p_data %>% filter(TECHNIQUE==random_name) %>% mutate(Technique = p_random_name) 96 | p_data_d = p_data %>% filter(TECHNIQUE==detection_name) %>% mutate(Technique = p_detection_name) 97 | p_data_t = p_data %>% filter(TECHNIQUE==truth_name) %>% mutate(Technique = p_truth_name) 98 | p_data_u = p_data %>% filter(TECHNIQUE==user_name) %>% mutate(Technique = p_user_name) 99 | 100 | 101 | p_data = rbind(p_data_r, p_data_d, p_data_t, p_data_u) 102 | 103 | p_data$Technique = factor(p_data$Technique, c(p_random_name, p_detection_name, p_truth_name, p_user_name)) 104 | 105 | #p_data = p_data %>% mutate(BRANCH_COVERAGE=BRANCHES_COVERED_TOTAL/(BRANCHES_COVERED_TOTAL+BRANCHES_MISSED_TOTAL)) 106 | 107 | return(p_data) 108 | } 109 | 110 | get_table_row <- function(){ 111 | names = c('Application', 'Detection Cov.', 'Random Cov.', 'P(Random)', 'A12(Random)', 'Truth Cov.', 'P(Truth)', 'A12(Truth)') 112 | d = data.frame(matrix(ncol = length(names), nrow = 0), stringsAsFactors=FALSE) 113 | colnames(d) <- names 114 | 115 | 116 | return(d) 117 | } 118 | 119 | r3dp <- function(x){ 120 | if (is.numeric(x)){ 121 | if (is.nan(x)){ 122 | return("1.000") 123 | } 124 | if(x < 0.001){ 125 | return("$<$0.001") 126 | } 127 | return(format(round(x, 3), nsmall=3)) 128 | } else { 129 | return(gsub("_", "-", x)) 130 | } 131 | } 132 | 133 | bold <- function(x) {paste('{\\textbf{',r3dp(x),'}}', sep ='')} 134 | 135 | get_table <- function(data){ 136 | apps = unique(data$APPLICATION) 137 | 138 | table_data = get_table_row() 139 | 140 | a_db = 0 141 | a_rb = 0 142 | a_drp= 0 143 | a_dra= 0 144 | a_tb= 0 145 | a_dtp= 0 146 | a_dta= 0 147 | 148 | a12_random = c() 149 | a12_truth = c() 150 | i = 0 151 | 152 | for (a in apps){ 153 | i = i + 1 154 | print(a) 155 | 156 | style = r3dp 157 | 158 | rand = data %>% filter(APPLICATION == a, TECHNIQUE == random_name) 159 | det = data %>% filter(APPLICATION == a, TECHNIQUE == detection_name) 160 | tru = data %>% filter(APPLICATION == a, TECHNIQUE == truth_name) 161 | 162 | pv = wilcox.test(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 163 | pvt = wilcox.test(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 164 | style_t = r3dp 165 | 166 | if (!is.nan(pv) && pv < 0.05){ 167 | style = bold 168 | } 169 | 170 | if (!is.nan(pvt) && pvt < 0.05){ 171 | style_t = bold 172 | } 173 | 174 | a_rb = a_rb + mean(rand$BRANCH_COVERAGE) 175 | a_db = a_db + mean(det$BRANCH_COVERAGE) 176 | a_tb = a_tb + mean(tru$BRANCH_COVERAGE) 177 | if (is.nan(pv)){ 178 | a_drp = a_drp + 1 179 | } else { 180 | a_drp = a_drp + pv 181 | } 182 | 183 | if (is.nan(pvt)){ 184 | a_dtp = a_dtp + 1 185 | } else { 186 | a_dtp = a_dtp + pvt 187 | } 188 | a_dra = a_dra + VD.A(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate 189 | a_dta = a_dta + VD.A(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate 190 | 191 | a12_random <- c(a12_random, a_dra) 192 | a12_truth <- c(a12_truth, a_dta) 193 | 194 | 195 | row = get_table_row() 196 | row[1,] = list(style(a), r3dp(mean(det$BRANCH_COVERAGE)), style(mean(rand$BRANCH_COVERAGE)), 197 | style(pv), style(VD.A(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate), style_t(mean(tru$BRANCH_COVERAGE)), style_t(pvt), style_t(VD.A(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate)) 198 | 199 | table_data = rbind(table_data, row) 200 | } 201 | 202 | row = get_table_row() 203 | row[1,] = list("Mean", r3dp(a_db / length(apps)), r3dp(a_rb / length(apps)), 204 | r3dp(a_drp / length(apps)), r3dp(a_dra / length(apps)), r3dp(a_tb / length(apps)), 205 | r3dp(a_dtp / length(apps)), r3dp(a_dta / length(apps))) 206 | 207 | table_data = rbind(table_data, row) 208 | 209 | table_data = table_data %>% 210 | mutate_each(funs(if(is.double(.)) as.double(.) else .)) 211 | 212 | return(xtable(table_data, caption="Branch Coverage of Random against Object Detection. \\textbf{Bold} indicates significance. ")) 213 | } 214 | print("Loading data...") 215 | r_data = load_data() 216 | print("Processing data...") 217 | data = process_data(r_data) 218 | 219 | box_plots <- function(data, file){ 220 | # colscale <- c("Random"="#8D8B97", p_detection_name="#B3BEAD", 221 | # p_truth_name="#FFE6CB") 222 | # fillscale <- scale_fill_manual(name="Technique", values=colscale) 223 | 224 | p = data %>% ggplot(aes(x=Technique, y=BRANCH_COVERAGE, fill=Technique)) + 225 | geom_boxplot() + 226 | xlab("Application") + 227 | ylab("Branch Coverage") + 228 | #fillscale + 229 | #scale_y_log10() + 230 | theme_bw() + 231 | theme(legend.position = "none") + 232 | facet_wrap(~APPLICATION, scales = "free", ncol=5) + 233 | scale_fill_viridis(discrete=TRUE, option="viridis", begin=0.5) 234 | 235 | 236 | ggsave(file, p, width = 9, height=6) 237 | 238 | return(p) 239 | } 240 | 241 | p = box_plots(data, "rq3.pdf") 242 | 243 | #data_rq4 = data %>% filter(grepl("detection", TECHNIQUE) | grepl("truth", TECHNIQUE)) 244 | 245 | #p = box_plots(data_rq4, "rq4.pdf") 246 | 247 | print(p) 248 | 249 | table = get_table(data) 250 | 251 | print(table, sanitize.text.function = function(x) x, include.rownames=FALSE) 252 | 253 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/user_model_tester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import config as cfg 4 | import sys 5 | import os 6 | import time 7 | import subprocess 8 | from pynput import keyboard 9 | import signal 10 | from test_helper import get_window_size_focus, screenshot, perform_interaction, is_running, get_focused_window 11 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 12 | from user_model import UserModel 13 | import pickle 14 | import pyautogui 15 | import numpy as np 16 | from threading import Thread 17 | import cv2 18 | 19 | sub_window = False 20 | 21 | running = True 22 | 23 | debug = False 24 | 25 | quit_counter = 3 26 | 27 | working_dir = "" 28 | aut_command = "" 29 | process_id = -1 30 | pyautogui.PAUSE = 0 31 | 32 | pyautogui.FAILSAFE = False # disables the fail-safe 33 | seeding_key = False 34 | 35 | ignore_windows = True 36 | 37 | out_folder = "" 38 | 39 | def capture_screen(): 40 | global app_x, app_y, app_w, app_h, exec_time, out_folder 41 | 42 | time.sleep(1) 43 | 44 | img_folder = out_folder + "/images" 45 | 46 | img_out = img_folder + "/"+ window_event.filename_string() 47 | 48 | counter = int(exec_time/5) 49 | 50 | if not (os.path.isfile(img_out + str(counter) + ".png")): 51 | image = screenshot() 52 | 53 | if not image is None: 54 | 55 | if not cfg.fullscreen: 56 | image = image[app_y:app_y+app_h, app_x:app_x+app_w] 57 | 58 | image = np.array(image) 59 | 60 | 61 | if not os.path.isdir(img_folder): 62 | os.makedirs(img_folder, exist_ok=True) 63 | try: 64 | os.mkdir(img_folder) 65 | except OSError as e: 66 | #folder exists 67 | pass 68 | 69 | if not (os.path.isfile(img_out + str(counter) + ".png")): 70 | cv2.imwrite(img_out + str(counter) + ".png", image) 71 | 72 | def on_release(key): 73 | global running, quit_counter, seeding_key 74 | 75 | if key == keyboard.Key.esc and not seeding_key: 76 | quit_counter -= 1 77 | if quit_counter == 0: 78 | running = False 79 | print("[User Model Replay] Killing tester.") 80 | 81 | def kill_old_process(): 82 | global process_id 83 | 84 | if process_id != -1: 85 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 86 | process_id = -1 87 | 88 | def start_aut(): 89 | global working_dir, aut_command, process_id, delay 90 | 91 | if aut_command != "": 92 | 93 | print("[User Model Replay] Starting AUT") 94 | 95 | kill_old_process() 96 | 97 | os.chdir(working_dir) 98 | 99 | with open('stdout.log', "a+") as outfile: 100 | with open('stderr.log', "a+") as errfile: 101 | outfile.write("\n" + str(time.time()) + "\n") 102 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 103 | 104 | time.sleep(10) 105 | delay += 10 106 | else: 107 | print("[User Model Replay] Could not find AUT to start!") 108 | 109 | events = [] 110 | 111 | update_window = True 112 | 113 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 114 | 115 | window_event = default_window_event 116 | 117 | delay = 0 118 | 119 | if __name__ == '__main__': 120 | # Collect events until released 121 | 122 | with keyboard.Listener(on_release=on_release) as klistener: 123 | 124 | wd = os.getcwd() 125 | 126 | event_constructor = EventConstructor() 127 | 128 | if len(sys.argv) < 4: 129 | print("Must have filename argument!", file=sys.stderr) 130 | sys.exit(1) 131 | input_dir = sys.argv[4] 132 | file = input_dir + "/user_model.mdl" 133 | user_model = UserModel() 134 | with open(file, "rb") as f: 135 | user_model = pickle.load(f) 136 | 137 | output_dir = input_dir + "/tests/" + str(int(time.time())) 138 | 139 | out_folder = output_dir 140 | 141 | if not os.path.isdir(output_dir): 142 | os.makedirs(output_dir, exist_ok=True) 143 | try: 144 | os.mkdir(output_dir) 145 | except OSError as e: 146 | #folder exists 147 | pass 148 | 149 | with open(output_dir + "/test.log", "w+") as f: 150 | f.write("") 151 | 152 | cfg.window_name = sys.argv[1] 153 | 154 | working_dir = sys.argv[2] 155 | aut_command = sys.argv[3] 156 | 157 | start_aut() 158 | 159 | print("[User Model Replay] Starting") 160 | 161 | for event in events: 162 | print(event) 163 | 164 | start_time = time.time() 165 | 166 | runtime = cfg.test_time 167 | 168 | actions = 0 169 | 170 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 171 | 172 | with open(csv_file, "w+") as p_f: 173 | p_f.write("time,actions,technique,iteration_time,window_name\n") 174 | 175 | last_event = Event(0, EventType.NONE) 176 | 177 | while is_running(start_time, runtime, actions, running): 178 | 179 | iteration_time = time.time() 180 | 181 | exec_time = time.time() - start_time 182 | 183 | #os.system('wmctrl -c "firefox"') 184 | 185 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 186 | 187 | no_focus = 0 188 | 189 | while app_w == 0 or w_class is None: 190 | 191 | time.sleep(1) 192 | 193 | if no_focus >= 20: 194 | kill_old_process() 195 | 196 | start_aut() 197 | 198 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 199 | if not is_running(start_time, runtime, actions, running): 200 | print("[User Model Replay] Couldn't find application window!") 201 | break 202 | 203 | no_focus += 1 204 | 205 | wm_class = "(" + w_class[0] + "," + w_class[1] + ")" 206 | 207 | if w_name != window_event.wm_name or wm_class != window_event.wm_class: 208 | window_event = WindowChangeEvent(time.time(), w_name, w_class, (app_x, app_y), 209 | (app_w, app_h)) 210 | os.chdir(wd) 211 | with open(output_dir + "/test.log", "a+") as f: 212 | f.write(window_event.hashcode() + "\n") 213 | 214 | print(window_event.hashcode()) 215 | 216 | window_model = user_model.get_window_model(window_event.wm_name) 217 | 218 | if window_model == None or (cfg.fallback == "random" and window_model == user_model.default_model): 219 | 220 | event = event_constructor.random_event() 221 | 222 | event.change_window(window_event) 223 | seeding_key = True 224 | event.perform() 225 | seeding_key = False 226 | time.sleep(event.get_delay()) 227 | 228 | t = Thread(target=capture_screen) 229 | t.start() 230 | 231 | with open(output_dir + "/test.log", "a+") as f: 232 | f.write(event.hashcode() + "\n") 233 | 234 | actions += event.action_count(last_event) 235 | else: 236 | 237 | if ignore_windows: 238 | window_model = user_model.default_model 239 | 240 | event_desc = window_model.next() 241 | 242 | if event_desc != "WINDOW_CHANGE": 243 | 244 | event = event_constructor.construct(event_desc) 245 | 246 | event.change_window(window_event) 247 | seeding_key = True 248 | event.perform() 249 | seeding_key = False 250 | time.sleep(event.get_delay()) 251 | 252 | t = Thread(target=capture_screen) 253 | t.start() 254 | 255 | with open(output_dir + "/test.log", "a+") as f: 256 | f.write(event.hashcode() + "\n") 257 | 258 | actions += event.action_count(last_event) 259 | 260 | last_event = event 261 | 262 | 263 | os.chdir(wd) 264 | with open(csv_file, "a+") as p_f: 265 | # TODO: Write python functions for click, type, etc 266 | p_f.write(str(exec_time) + "," + str(actions) + ",UserModelGenerator," + str(iteration_time) + "," + cfg.window_name + "\n") 267 | 268 | for k in KeyEvent.keys_down: 269 | pyautogui.keyUp(k) 270 | 271 | kill_old_process() 272 | 273 | time.sleep(5) 274 | print("[User Model Replay] Finished testing! (" + str(actions) + " actions)") 275 | 276 | os.chdir(wd) 277 | 278 | 279 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/data_processor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import sys 4 | import os 5 | import time 6 | import subprocess 7 | from pynput import keyboard 8 | import signal 9 | from pynput.mouse import Listener, Button 10 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 11 | from clusterer import Clusterer 12 | from user_model import WindowModel, UserModel 13 | from ngram import Ngram 14 | import pickle 15 | 16 | sub_window = False 17 | 18 | normalise = True 19 | 20 | running = True 21 | 22 | debug = False 23 | 24 | quit_counter = 3 25 | 26 | working_dir = "" 27 | aut_command = "" 28 | process_id = -1 29 | 30 | def on_release(key): 31 | global running, quit_counter 32 | 33 | if key == keyboard.Key.f1: 34 | quit_counter -= 1 35 | if quit_counter == 0: 36 | running = False 37 | print("[Replay] Killing tester.") 38 | 39 | def kill_old_process(): 40 | global process_id 41 | 42 | if process_id != -1: 43 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 44 | process_id = -1 45 | 46 | def start_aut(): 47 | global working_dir, aut_command, process_id, delay 48 | 49 | if aut_command != "": 50 | 51 | print("[Replay] Starting AUT") 52 | 53 | kill_old_process() 54 | 55 | os.chdir(working_dir) 56 | 57 | with open('stdout.log', "a+") as outfile: 58 | with open('stderr.log', "a+") as errfile: 59 | outfile.write("\n" + str(time.time()) + "\n") 60 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 61 | 62 | time.sleep(10) 63 | delay += 10 64 | else: 65 | print("[Replay] Could not find AUT to start!") 66 | 67 | mouse_pos = (0, 0) 68 | 69 | events = [] 70 | 71 | update_window = True 72 | 73 | def mouse_move(x, y): 74 | global mouse_pos 75 | mouse_pos = (x, y) 76 | 77 | 78 | def mouse_click(x, y, button, pressed): 79 | global events 80 | events.append(MouseEvent(time.time(), 0 if button == Button.left else 81 | 1 if button == Button.right else 2, x, y, pressed=pressed)) 82 | 83 | 84 | 85 | def mouse_scroll(x, y, dx, dy): 86 | global events 87 | if events[-1].get_event_type() != EventType.MOUSE_WHEEL: 88 | events.append(ScrollEvent(time.time(), dx, dy)) 89 | else: 90 | events[-1].displace(x, y) 91 | 92 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 93 | 94 | window_event = default_window_event 95 | 96 | windows = [] 97 | 98 | delay = 0 99 | 100 | def run_events(x, y, time): 101 | global window_event, events, default_window_event, delay, update_window 102 | if len(events) == 0: 103 | print("[Replay] Test finished!") 104 | return False 105 | 106 | event = events[0] 107 | 108 | while event.should_perform_event(time - delay): 109 | if isinstance(event, WindowChangeEvent): 110 | window_event = event 111 | print("[Replay] Changed expected window! (%s)" % window_event.wm_name) 112 | update_window = True 113 | 114 | if not window_event == default_window_event: 115 | if isinstance(event, MouseEvent): 116 | dx = x-window_event.position[0] 117 | dy = y-window_event.position[1] 118 | event.displace(dx, dy) 119 | elif isinstance(event, ScrollEvent): 120 | dx = x-window_event.position[0] 121 | dy = y-window_event.position[1] 122 | event.displace(0, 0, dx, dy) 123 | 124 | event.perform() 125 | events.pop(0) 126 | event = events[0] 127 | return True 128 | 129 | 130 | def gen_window_model(window_event, proc_events, clusterer=Clusterer()): 131 | global default_window_event 132 | if window_event == default_window_event.wm_name: 133 | return None 134 | 135 | assignments = {} 136 | clustered_events = {} 137 | 138 | for et in proc_events[window_event]: 139 | clusterer.clear_data() 140 | 141 | if et is EventType.NONE: 142 | continue 143 | 144 | try: 145 | for e in proc_events[window_event][et]: 146 | f = e.get_features() 147 | if len(f) == 0: 148 | break 149 | clusterer.append_data(f) 150 | if clusterer.shape[1] == 0: 151 | continue 152 | centroids, assigns = clusterer.cluster(clusterer.recommend_clusters(), 10) 153 | 154 | clustered_events[str(et)] = centroids 155 | 156 | for i in range(len(proc_events[window_event][et])): 157 | assignments[proc_events[window_event][et][i]] = assigns[i] 158 | 159 | except NotImplementedError as e: 160 | print(e) 161 | pass 162 | 163 | ngram = Ngram("") 164 | 165 | clustered_windowed_events = windowed_events[window_event][:] 166 | 167 | for i in range(len(clustered_windowed_events)): 168 | we = clustered_windowed_events[i] 169 | name = str(we.event_type) 170 | 171 | id = we.get_identifier() 172 | if not id is None: 173 | name += "[" + id + "]" 174 | 175 | if we in assignments: 176 | assignment = "{" + str(assignments[we]) + "}" 177 | if "{cluster}" in name: 178 | name = name.replace("{cluster}", assignment) 179 | else: 180 | name += "[" + assignment + "]" 181 | 182 | clustered_windowed_events[i] = name 183 | 184 | sequence = " ".join(clustered_windowed_events).replace("EventType.NONE", ngram.delimiter) 185 | ngram.construct(sequence, 5) 186 | 187 | ngram.calculate_probabilities() 188 | 189 | window_model = WindowModel(ngram, clustered_events) 190 | 191 | return window_model 192 | 193 | 194 | if __name__ == '__main__': 195 | # Collect events until released 196 | wd = os.getcwd() 197 | 198 | if len(sys.argv) < 2: 199 | print("Must have filename argument!", file=sys.stderr) 200 | sys.exit(1) 201 | input_dirs = sys.argv[1].split(":") 202 | output_dir = input_dirs[0] 203 | 204 | if len(sys.argv) > 2: 205 | output_dir = sys.argv[2] 206 | 207 | event_constructor = EventConstructor() 208 | 209 | for f in input_dirs: 210 | file = f + "/" + "events.evnt" 211 | 212 | 213 | 214 | with open(file, "r") as f: 215 | for line in f: 216 | events.append(event_constructor.construct(line)) 217 | events.append(Event(events[-1].timestamp+1, EventType.NONE)) 218 | 219 | events = sorted(events, key=lambda x : x.timestamp) 220 | 221 | proc_events = {} 222 | 223 | windowed_events = {} 224 | 225 | for e in events: 226 | 227 | key = window_event.filename_string() 228 | 229 | if isinstance(e, WindowChangeEvent): 230 | window_event = e 231 | windows.append(e) 232 | 233 | if not key in proc_events: 234 | proc_events[key] = {} 235 | 236 | if not e.event_type in proc_events[key]: 237 | proc_events[key][e.event_type] = [] 238 | 239 | if not None in proc_events: 240 | proc_events[None] = {} 241 | 242 | if not e.event_type in proc_events[None]: 243 | proc_events[None][e.event_type] = [] 244 | 245 | 246 | if normalise and not window_event == default_window_event: 247 | e.change_window(window_event) 248 | 249 | proc_events[key][e.event_type].append(e) 250 | proc_events[None][e.event_type].append(e) 251 | 252 | if not isinstance(e, WindowChangeEvent) or window_event.filename_string() != key: 253 | 254 | if not None in windowed_events: 255 | windowed_events[None] = [] 256 | 257 | windowed_events[None].append(e) 258 | 259 | if not key in windowed_events: 260 | windowed_events[key] = [] 261 | 262 | windowed_events[key].append(e) 263 | 264 | user_model = UserModel() 265 | 266 | for window_event in proc_events: 267 | 268 | 269 | window_model = gen_window_model(window_event, proc_events) 270 | 271 | if window_model is None: 272 | continue 273 | 274 | user_model.add_window_model(window_event, window_model) 275 | 276 | default_model = gen_window_model(None, proc_events) 277 | 278 | user_model.set_default_model(default_model) 279 | 280 | for i in range(100): 281 | print("Gen:", default_model.next()) 282 | 283 | if not os.path.isdir(output_dir): 284 | os.makedirs(output_dir, exist_ok=True) 285 | try: 286 | os.mkdir(output_dir) 287 | except OSError as e: 288 | #folder exists 289 | pass 290 | 291 | with open(output_dir + "/user_model.mdl", "bw+") as f: 292 | pickle.dump(user_model, f) 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/data_visualiser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import sys 4 | import os 5 | import time 6 | import subprocess 7 | from pynput import keyboard 8 | import signal 9 | from pynput.mouse import Listener, Button 10 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 11 | from clusterer import Clusterer 12 | from user_model import WindowModel, UserModel 13 | from ngram import Ngram 14 | import pickle 15 | 16 | sub_window = False 17 | 18 | normalise = True 19 | 20 | running = True 21 | 22 | debug = False 23 | 24 | quit_counter = 3 25 | 26 | working_dir = "" 27 | aut_command = "" 28 | process_id = -1 29 | 30 | def on_release(key): 31 | global running, quit_counter 32 | 33 | if key == keyboard.Key.f1: 34 | quit_counter -= 1 35 | if quit_counter == 0: 36 | running = False 37 | print("[Replay] Killing tester.") 38 | 39 | def kill_old_process(): 40 | global process_id 41 | 42 | if process_id != -1: 43 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 44 | process_id = -1 45 | 46 | def start_aut(): 47 | global working_dir, aut_command, process_id, delay 48 | 49 | if aut_command != "": 50 | 51 | print("[Replay] Starting AUT") 52 | 53 | kill_old_process() 54 | 55 | os.chdir(working_dir) 56 | 57 | with open('stdout.log', "a+") as outfile: 58 | with open('stderr.log', "a+") as errfile: 59 | outfile.write("\n" + str(time.time()) + "\n") 60 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 61 | 62 | time.sleep(10) 63 | delay += 10 64 | else: 65 | print("[Replay] Could not find AUT to start!") 66 | 67 | mouse_pos = (0, 0) 68 | 69 | events = [] 70 | 71 | update_window = True 72 | 73 | def mouse_move(x, y): 74 | global mouse_pos 75 | mouse_pos = (x, y) 76 | 77 | 78 | def mouse_click(x, y, button, pressed): 79 | global events 80 | events.append(MouseEvent(time.time(), 0 if button == Button.left else 81 | 1 if button == Button.right else 2, x, y, pressed=pressed)) 82 | 83 | 84 | 85 | def mouse_scroll(x, y, dx, dy): 86 | global events 87 | if events[-1].get_event_type() != EventType.MOUSE_WHEEL: 88 | events.append(ScrollEvent(time.time(), dx, dy)) 89 | else: 90 | events[-1].displace(x, y) 91 | 92 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 93 | 94 | window_event = default_window_event 95 | 96 | windows = [] 97 | 98 | delay = 0 99 | 100 | def run_events(x, y, time): 101 | global window_event, events, default_window_event, delay, update_window 102 | if len(events) == 0: 103 | print("[Replay] Test finished!") 104 | return False 105 | 106 | event = events[0] 107 | 108 | while event.should_perform_event(time - delay): 109 | if isinstance(event, WindowChangeEvent): 110 | window_event = event 111 | print("[Replay] Changed expected window! (%s)" % window_event.wm_name) 112 | update_window = True 113 | 114 | if not window_event == default_window_event: 115 | if isinstance(event, MouseEvent): 116 | dx = x-window_event.position[0] 117 | dy = y-window_event.position[1] 118 | event.displace(dx, dy) 119 | elif isinstance(event, ScrollEvent): 120 | dx = x-window_event.position[0] 121 | dy = y-window_event.position[1] 122 | event.displace(0, 0, dx, dy) 123 | 124 | event.perform() 125 | events.pop(0) 126 | event = events[0] 127 | return True 128 | 129 | 130 | def gen_window_model(window_event, proc_events, clusterer=Clusterer()): 131 | global default_window_event 132 | if window_event == default_window_event.wm_name: 133 | return None 134 | 135 | assignments = {} 136 | clustered_events = {} 137 | 138 | for et in proc_events[window_event]: 139 | clusterer.clear_data() 140 | 141 | if et is EventType.NONE: 142 | continue 143 | 144 | try: 145 | for e in proc_events[window_event][et]: 146 | f = e.get_features() 147 | if len(f) == 0: 148 | break 149 | clusterer.append_data(f) 150 | if clusterer.shape[1] == 0: 151 | continue 152 | centroids, assigns = clusterer.cluster(clusterer.recommend_clusters(), 10) 153 | 154 | clustered_events[str(et)] = centroids 155 | 156 | for i in range(len(proc_events[window_event][et])): 157 | assignments[proc_events[window_event][et][i]] = assigns[i] 158 | 159 | except NotImplementedError as e: 160 | print(e) 161 | pass 162 | 163 | ngram = Ngram("") 164 | 165 | clustered_windowed_events = windowed_events[window_event][:] 166 | 167 | for i in range(len(clustered_windowed_events)): 168 | we = clustered_windowed_events[i] 169 | name = str(we.event_type) 170 | 171 | id = we.get_identifier() 172 | if not id is None: 173 | name += "[" + id + "]" 174 | 175 | if we in assignments: 176 | assignment = "{" + str(assignments[we]) + "}" 177 | if "{cluster}" in name: 178 | name = name.replace("{cluster}", assignment) 179 | else: 180 | name += "[" + assignment + "]" 181 | 182 | clustered_windowed_events[i] = name 183 | 184 | sequence = " ".join(clustered_windowed_events).replace("EventType.NONE", ngram.delimiter) 185 | ngram.construct(sequence, 5) 186 | 187 | ngram.calculate_probabilities() 188 | 189 | window_model = WindowModel(ngram, clustered_events) 190 | 191 | return window_model 192 | 193 | 194 | if __name__ == '__main__': 195 | # Collect events until released 196 | wd = os.getcwd() 197 | 198 | if len(sys.argv) < 2: 199 | print("Must have filename argument!", file=sys.stderr) 200 | sys.exit(1) 201 | input_dirs = sys.argv[1].split(":") 202 | output_dir = input_dirs[0] 203 | 204 | if len(sys.argv) > 2: 205 | output_dir = sys.argv[2] 206 | 207 | event_constructor = EventConstructor() 208 | 209 | for f in input_dirs: 210 | file = f + "/" + "events.evnt" 211 | 212 | 213 | 214 | with open(file, "r") as f: 215 | for line in f: 216 | events.append(event_constructor.construct(line)) 217 | events.append(Event(events[-1].timestamp+1, EventType.NONE)) 218 | 219 | events = sorted(events, key=lambda x : x.timestamp) 220 | 221 | proc_events = {} 222 | 223 | windowed_events = {} 224 | 225 | for e in events: 226 | 227 | key = window_event.filename_string() 228 | 229 | if isinstance(e, WindowChangeEvent): 230 | window_event = e 231 | windows.append(e) 232 | 233 | if not key in proc_events: 234 | proc_events[key] = {} 235 | 236 | if not e.event_type in proc_events[key]: 237 | proc_events[key][e.event_type] = [] 238 | 239 | if not None in proc_events: 240 | proc_events[None] = {} 241 | 242 | if not e.event_type in proc_events[None]: 243 | proc_events[None][e.event_type] = [] 244 | 245 | 246 | if normalise and not window_event == default_window_event: 247 | e.change_window(window_event) 248 | 249 | proc_events[key][e.event_type].append(e) 250 | proc_events[None][e.event_type].append(e) 251 | 252 | if not isinstance(e, WindowChangeEvent) or window_event.filename_string() != key: 253 | 254 | if not None in windowed_events: 255 | windowed_events[None] = [] 256 | 257 | windowed_events[None].append(e) 258 | 259 | if not key in windowed_events: 260 | windowed_events[key] = [] 261 | 262 | windowed_events[key].append(e) 263 | 264 | user_model = UserModel() 265 | 266 | for window_event in proc_events: 267 | 268 | 269 | window_model = gen_window_model(window_event, proc_events) 270 | 271 | if window_model is None: 272 | continue 273 | 274 | user_model.add_window_model(window_event, window_model) 275 | 276 | for i in range(100): 277 | print("Gen:", window_model.next()) 278 | 279 | default_model = gen_window_model(None, proc_events) 280 | 281 | user_model.set_default_model(default_model) 282 | 283 | for i in range(100): 284 | print("Gen:", default_model.next()) 285 | 286 | if not os.path.isdir(output_dir): 287 | os.makedirs(output_dir, exist_ok=True) 288 | try: 289 | os.mkdir(output_dir) 290 | except OSError as e: 291 | #folder exists 292 | pass 293 | 294 | with open(output_dir + "/user_model.mdl", "bw+") as f: 295 | pickle.dump(user_model, f) 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/manual_tester.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import config as cfg 4 | from yolo import Yolo 5 | import cv2 6 | import numpy as np 7 | import tensorflow as tf 8 | import sys 9 | import gc 10 | import math 11 | import random 12 | import os 13 | import pyautogui 14 | from operator import itemgetter 15 | import time 16 | import subprocess 17 | import Xlib 18 | from pynput import keyboard 19 | import signal 20 | import timeit 21 | from pynput.mouse import Listener, Button 22 | from test_helper import get_focused_window, screenshot, perform_interaction, is_running 23 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent 24 | from threading import Thread 25 | 26 | sub_window = False 27 | 28 | running = True 29 | 30 | debug = False 31 | 32 | quit_counter = 3 33 | 34 | working_dir = "" 35 | aut_command = "" 36 | process_id = -1 37 | 38 | def on_release(key): 39 | global running, quit_counter, shift_modifier, ctrl_modifier, alt_modifier, altgr_modifier, fn_modifier, cmd_modifier 40 | 41 | if hasattr(key, "name"): 42 | name = key.name 43 | elif hasattr(key, "char"): 44 | name = key.char 45 | else: 46 | name = str(key) 47 | 48 | if key == keyboard.Key.shift: 49 | shift_modifier = 0 50 | if key == keyboard.Key.ctrl: 51 | ctrl_modifier = 0 52 | if key == keyboard.Key.alt: 53 | alt_modifier = 0 54 | if key == keyboard.Key.alt_gr: 55 | altgr_modifier = 0 56 | if key == keyboard.Key.cmd: 57 | cmd_modifier = 0 58 | 59 | add_event(KeyEvent(time.time(), name, pressed=False, shift=shift_modifier, ctrl=ctrl_modifier, alt=alt_modifier, 60 | altgr=altgr_modifier, fn=fn_modifier, cmd=cmd_modifier)) 61 | 62 | if key == keyboard.Key.esc: 63 | quit_counter -= 1 64 | if quit_counter == 0: 65 | running = False 66 | print("[Manual] Killing tester.") 67 | 68 | wd = "" 69 | 70 | def on_press(key): 71 | global shift_modifier, ctrl_modifier, alt_modifier, altgr_modifier, fn_modifier, cmd_modifier 72 | name = "" 73 | 74 | if hasattr(key, "name"): 75 | name = key.name 76 | elif hasattr(key, "char"): 77 | name = key.char 78 | else: 79 | name = str(key) 80 | 81 | if key == keyboard.Key.shift: 82 | shift_modifier = 1 83 | if key == keyboard.Key.ctrl: 84 | ctrl_modifier = 1 85 | if key == keyboard.Key.alt: 86 | alt_modifier = 1 87 | if key == keyboard.Key.alt_gr: 88 | altgr_modifier = 1 89 | if key == keyboard.Key.cmd: 90 | cmd_modifier = 1 91 | 92 | add_event(KeyEvent(time.time(), name, pressed=True, shift=shift_modifier, ctrl=ctrl_modifier, alt=alt_modifier, 93 | altgr=altgr_modifier, fn=fn_modifier, cmd=cmd_modifier)) 94 | 95 | def kill_old_process(): 96 | global process_id 97 | 98 | if process_id != -1: 99 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 100 | process_id = -1 101 | 102 | def start_aut(): 103 | global working_dir, aut_command, process_id, wd 104 | 105 | if aut_command != "": 106 | 107 | print("[Manual] Starting AUT") 108 | 109 | kill_old_process() 110 | 111 | os.chdir(working_dir) 112 | 113 | with open('stdout.log', "a+") as outfile: 114 | with open('stderr.log', "a+") as errfile: 115 | outfile.write("\n" + str(time.time()) + "\n") 116 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 117 | 118 | time.sleep(10) 119 | else: 120 | print("[Manual] Could not find AUT to start!") 121 | 122 | os.chdir(wd) 123 | 124 | mouse_pos = (0, 0) 125 | 126 | events = [] 127 | 128 | shift_modifier=0 129 | ctrl_modifier=0 130 | alt_modifier=0 131 | altgr_modifier=0 132 | fn_modifier=0 133 | cmd_modifier=0 134 | out_folder = "" 135 | 136 | w_name, w_class, app_x, app_y, app_w, app_h = "", "", 0, 0, 0, 0 137 | 138 | def capture_screen(): 139 | global app_x, app_y, app_w, app_h, exec_time 140 | 141 | time.sleep(1) 142 | 143 | img_folder = out_folder + "/images" 144 | 145 | img_out = img_folder + "/"+ window_event.filename_string() 146 | 147 | counter = int(exec_time/5) 148 | 149 | if not (os.path.isfile(img_out + str(counter) + ".png")): 150 | image = screenshot() 151 | 152 | if not image is None: 153 | 154 | if not cfg.fullscreen: 155 | image = image[app_y:app_y+app_h, app_x:app_x+app_w] 156 | 157 | image = np.array(image) 158 | 159 | 160 | if not os.path.isdir(img_folder): 161 | os.makedirs(img_folder, exist_ok=True) 162 | try: 163 | os.mkdir(img_folder) 164 | except OSError as e: 165 | #folder exists 166 | pass 167 | 168 | if not (os.path.isfile(img_out + str(counter) + ".png")): 169 | cv2.imwrite(img_out + str(counter) + ".png", image) 170 | 171 | def add_event(event): 172 | global events 173 | 174 | t = Thread(target=capture_screen) 175 | t.start() 176 | 177 | events.append(event) 178 | 179 | def mouse_move(x, y): 180 | global mouse_pos 181 | mouse_pos = (x, y) 182 | 183 | 184 | def mouse_click(x, y, button, pressed): 185 | global events 186 | add_event( 187 | MouseEvent(time.time(), 0 if button == Button.left else 188 | 1 if button == Button.right else 2, x, y, pressed=pressed) 189 | ) 190 | 191 | 192 | 193 | def mouse_scroll(x, y, dx, dy): 194 | global events 195 | if events[-1].get_event_type() != EventType.MOUSE_WHEEL or dy * events[-1].velocity_y < 0 \ 196 | or dx * events[-1].velocity_x < 0: 197 | add_event(ScrollEvent(time.time(), dx, dy, x, y)) 198 | else: 199 | events[-1].displace(x, y, 0, 0) 200 | 201 | window_event = WindowChangeEvent(0, "", "", (0, 0), (0, 0)) 202 | exec_time = 0 203 | if __name__ == '__main__': 204 | # Collect events until released 205 | wd = os.getcwd() 206 | with Listener(on_move=mouse_move, 207 | on_click=mouse_click, 208 | on_scroll=mouse_scroll) as listener: 209 | with keyboard.Listener(on_release=on_release, 210 | on_press=on_press) as klistener: 211 | if len(sys.argv) > 1: 212 | cfg.window_name = sys.argv[1] 213 | 214 | if len(sys.argv) > 3: 215 | working_dir = sys.argv[2] 216 | aut_command = sys.argv[3] 217 | 218 | start_aut() 219 | 220 | if len(sys.argv) > 5: 221 | cfg.test_time = int(sys.argv[5]) 222 | cfg.use_iterations = False 223 | print("[Manual] Using time limit: " + str(cfg.test_time)) 224 | 225 | out_folder = sys.argv[4] 226 | 227 | output_file = out_folder + "/events.evnt" 228 | 229 | print("[Manual] Starting") 230 | 231 | start_time = time.time() 232 | 233 | runtime = cfg.test_time 234 | 235 | actions = 0 236 | 237 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 238 | 239 | with open(csv_file, "w+") as p_f: 240 | p_f.write("time,actions,technique,iteration_time,window_name\n") 241 | 242 | while is_running(start_time, runtime, actions, running): 243 | 244 | iteration_time = time.time() 245 | 246 | exec_time = time.time() - start_time 247 | 248 | #os.system('wmctrl -c "firefox"') 249 | 250 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 251 | 252 | no_focus = 0 253 | 254 | while app_w == 0: 255 | 256 | time.sleep(1) 257 | 258 | if no_focus >= 20: 259 | kill_old_process() 260 | 261 | start_aut() 262 | 263 | w_name, w_class, app_x, app_y, app_w, app_h = get_focused_window(cfg.window_name) 264 | if not is_running(start_time, runtime, actions, running): 265 | print("[Manual] Couldn't find application window!") 266 | break 267 | 268 | no_focus += 1 269 | 270 | if w_name != window_event.wm_name or "(" + w_class[0] + "," + w_class[1] + ")" != window_event.wm_class: 271 | window_event = WindowChangeEvent(time.time(), w_name, w_class, (app_x, app_y), 272 | (app_w, app_h)) 273 | add_event(window_event) 274 | 275 | if not is_running(start_time, runtime, actions, running): 276 | break 277 | 278 | with open(csv_file, "a+") as p_f: 279 | p_f.write(str(exec_time) + "," + str(actions) + ",manual," + str(iteration_time) + "," + cfg.window_name + "\n") 280 | kill_old_process() 281 | 282 | time.sleep(5) 283 | print("[Manual] Finished testing!") 284 | 285 | os.chdir(wd) 286 | 287 | if not os.path.exists(os.path.dirname(output_file)): 288 | try: 289 | os.makedirs(os.path.dirname(output_file)) 290 | except OSError as exc: 291 | pass 292 | 293 | with open(output_file, "w+") as f: 294 | for event in events: 295 | f.write(event.hashcode() + "\n") 296 | 297 | 298 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/swing_test.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import sys 6 | import gc 7 | import math 8 | import random 9 | import os 10 | import pyautogui 11 | from operator import itemgetter 12 | import time 13 | import subprocess 14 | import Xlib 15 | import time 16 | from pynput import keyboard 17 | import data_loader 18 | import signal 19 | import timeit 20 | import signal 21 | from test_helper import get_window_size, screenshot, perform_interaction, is_running 22 | 23 | 24 | running = True 25 | 26 | debug = False 27 | 28 | quit_counter = 3 29 | 30 | working_dir = "" 31 | aut_command = "" 32 | process_id = -1 33 | 34 | pyautogui.FAILSAFE = False # disables the fail-safe 35 | 36 | def on_release(key): 37 | global running, quit_counter 38 | if key == keyboard.Key.f1: 39 | quit_counter -= 1 40 | if quit_counter == 0: 41 | running = False 42 | print("[Swing Test] Killing tester.") 43 | 44 | sub_window = False 45 | 46 | error_count = -1 47 | 48 | def kill_old_process(): 49 | global process_id 50 | 51 | if process_id != -1: 52 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 53 | process_id = -1 54 | 55 | 56 | def handler(signum, frame): 57 | print('Signal handler called with signal', signum) 58 | 59 | kill_old_process() 60 | 61 | raise TimeoutError("Could not retrieve bounding boxes!") 62 | 63 | signal.signal(signal.SIGALRM,handler) 64 | 65 | def start_aut(): 66 | global working_dir, aut_command, process_id 67 | 68 | if aut_command != "": 69 | 70 | print("[Swing Test] Starting AUT") 71 | 72 | kill_old_process() 73 | 74 | os.chdir(working_dir) 75 | 76 | with open('stdout.log', "a+") as outfile: 77 | with open('stderr.log', "a+") as errfile: 78 | outfile.write("\n" + str(time.time()) + "\n") 79 | cmds = aut_command.split() 80 | print(cmds) 81 | process_id = subprocess.Popen(aut_command.split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=errfile, preexec_fn=os.setsid, shell=False, 82 | bufsize=1, 83 | universal_newlines=True) 84 | time.sleep(10) 85 | else: 86 | print("[Swing Test] Could not find AUT to start!") 87 | 88 | 89 | def generate_input_string(): 90 | if random.random() < 0.5: 91 | return "Hello World!" 92 | else: 93 | return str(random.randint(-10000, 10000)) 94 | 95 | def convert_coords(x, y, w, h, aspect): 96 | if aspect > 1: # width is bigger than height 97 | h = h * aspect 98 | y = 0.5 + ((y - 0.5)*aspect) 99 | elif aspect < 1: 100 | w = w / aspect 101 | x = 0.5 + ((x - 0.5)/aspect) 102 | 103 | return x, y, w, h 104 | 105 | def select_random_box(proc_boxes): 106 | best_box = random.sample(proc_boxes.tolist(), 1)[0] 107 | 108 | return best_box 109 | 110 | def prepare_screenshot(raw_image): 111 | st = time.time() 112 | raw_image = data_loader.pad_image(raw_image) 113 | et = time.time() 114 | if debug: 115 | print("[Swing Test] PADDING:", et - st) 116 | image = cv2.resize(raw_image, (cfg.width, cfg.height)) 117 | 118 | images = np.reshape(image, [1, cfg.width, cfg.height, 1]) 119 | 120 | imgs = (images/127.5)-1 121 | 122 | return imgs 123 | 124 | def convert_boxes(boxes): 125 | return yolo.convert_net_to_bb(boxes, filter_top=False)[0] 126 | 127 | if __name__ == '__main__': 128 | # Collect events until released 129 | with keyboard.Listener(on_release=on_release) as listener: 130 | if len(sys.argv) > 1: 131 | cfg.window_name = sys.argv[1] 132 | 133 | event = 0 134 | 135 | yolo = Yolo() 136 | 137 | states = [] 138 | 139 | runtime = round(time.time()) 140 | 141 | output_dir = cfg.output_dir + "/" + str(runtime) 142 | 143 | try: 144 | if not os.path.exists(output_dir): 145 | os.makedirs(output_dir) 146 | except: 147 | pass 148 | 149 | try: 150 | if not os.path.exists(output_dir + "/images"): 151 | os.makedirs(output_dir + "/images") 152 | except: 153 | pass 154 | 155 | test_file = output_dir + "/test.txt" 156 | 157 | with open(test_file, "w+") as t_f: 158 | t_f.write("") 159 | 160 | 161 | if len(sys.argv) > 3: 162 | working_dir = sys.argv[2] 163 | aut_command = sys.argv[3] 164 | 165 | start_aut() 166 | 167 | yolo.set_training(False) 168 | 169 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 170 | 171 | start_time = time.time() 172 | 173 | last_image = None 174 | 175 | boxes = [] 176 | 177 | interactions = [] 178 | 179 | runtime = cfg.test_time #5 mins 180 | 181 | last_u_input = "" 182 | 183 | actions = 0 184 | 185 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 186 | 187 | with open(csv_file, "w+") as p_f: 188 | p_f.write("time,actions,technique,iteration_time,window_name\n") 189 | 190 | while is_running(start_time, runtime, actions, running): 191 | 192 | signal.alarm(60) 193 | 194 | iteration_time = time.time() 195 | 196 | time.sleep(1) 197 | 198 | exec_time = time.time() - start_time 199 | 200 | os.system('killall "firefox"') 201 | 202 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 203 | 204 | counter = 0 205 | 206 | while app_w == 0: 207 | 208 | start_aut() 209 | counter += 1 210 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 211 | if counter >= 3: 212 | print("[Swing Test] Couldn't find application window!") 213 | break 214 | 215 | if not is_running(start_time, runtime, actions, running): 216 | break 217 | 218 | image = screenshot()[app_y:app_y+app_h, app_x:app_x+app_w] 219 | 220 | image = np.array(image) 221 | 222 | # raw_image = image[app_y:app_y+app_h, app_x:app_x+an.write(b"bounds")pp_w] 223 | 224 | ih, iw = image.shape[:2] 225 | 226 | aspect = iw/ih 227 | 228 | imgs = prepare_screenshot(image) 229 | 230 | gen_boxes = True 231 | 232 | for l in states: 233 | diff = np.sum(np.square(image-l[0]))/image.size 234 | if diff < 2: 235 | gen_boxes = False 236 | proc_boxes = l[1] 237 | 238 | last_image = image 239 | 240 | if True: #gen_boxes or len(proc_boxes) < 3: 241 | 242 | #print("New state found!", len(states), "states found total.") 243 | 244 | st = time.time() 245 | 246 | process_id.stdin.write("bounds\n") 247 | 248 | line = "first line" 249 | 250 | p_boxes = [] 251 | 252 | while len(line) > 1: 253 | line = process_id.stdout.readline() 254 | 255 | if "AUT Not Supported!" in line: 256 | print(line) 257 | kill_old_process() 258 | sys.exit(-1) 259 | 260 | l_arr = line.split(",") 261 | 262 | if len(l_arr) > 4: 263 | error_count = 0 264 | p_boxes.append([yolo.names.index(l_arr[0]), 265 | (float(l_arr[1]) - app_x)/app_w, 266 | (float(l_arr[2]) - app_y)/app_h, 267 | float(l_arr[3]) / app_w, 268 | float(l_arr[4]) / app_h]) 269 | 270 | #p_boxes = np.array(p_boxes) 271 | 272 | if len(p_boxes) == 0: 273 | error_count += 1 274 | 275 | if error_count > 5: 276 | kill_old_process() 277 | print("5 errors in a row. Is application supported?") 278 | sys.exit(1) 279 | 280 | continue 281 | 282 | proc_boxes = np.array(p_boxes) 283 | 284 | for box_num in range(1): 285 | 286 | input_string = generate_input_string() 287 | 288 | if (process_id.poll() != None): 289 | sys.exit(-1) 290 | 291 | best_box = select_random_box(proc_boxes) 292 | 293 | rand_num = random.random() 294 | 295 | #np.delete(proc_boxes, best_box) 296 | 297 | height, width = image.shape[:2] 298 | 299 | current_box = best_box 300 | 301 | perform_interaction(best_box, app_x, app_y, app_w, app_h, input_string) 302 | 303 | actions += 1 304 | 305 | end_iteration_time = time.time() 306 | if debug: 307 | print("[Swing Test] Iteration Time:", end_iteration_time - iteration_time) 308 | 309 | # write test info 310 | with open(csv_file, "a") as p_f: 311 | p_f.write(str(exec_time) + "," + str(actions) + ",swing," + str(iteration_time) + "," + cfg.window_name + "\n") 312 | 313 | kill_old_process() 314 | 315 | time.sleep(20) 316 | print("[Swing Test] Finished testing!") 317 | 318 | 319 | -------------------------------------------------------------------------------- /scripts/process_results_users.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | library(ggthemes) 7 | library(xtable) 8 | library(effsize) 9 | library(viridis) 10 | 11 | setwd('/home/thomas/experiments/test_experiments') 12 | 13 | dir = "output" 14 | random_name = "detection" 15 | detection_name = "user-models" 16 | truth_name = "truth" 17 | #truth_name = "random" 18 | 19 | p_random_name = "Pred" 20 | p_detection_name = "WindowM" 21 | p_truth_name = "API" 22 | 23 | out_name = "issta-user-comparison" 24 | 25 | config = 3 26 | 27 | if (config == 1){ 28 | random_name = "random" 29 | detection_name = "user-models" 30 | truth_name = "user-random" 31 | 32 | p_random_name = "ISSTA-Random" 33 | p_detection_name = "WindowM" 34 | p_truth_name = "User Random" 35 | 36 | out_name = "issta-random-comparison" 37 | } 38 | 39 | if (config == 2){ 40 | random_name = "user-random" 41 | detection_name = "user-models" 42 | truth_name = "user-models-oneout" 43 | 44 | p_random_name = "User Random" 45 | p_detection_name = "WindowM" 46 | p_truth_name = "AppM" 47 | 48 | out_name = "user-comparison" 49 | } 50 | 51 | 52 | if (config == 3){ 53 | random_name = "user-models-app" 54 | detection_name = "user-models" 55 | truth_name = "user-random" 56 | 57 | p_random_name = "Application" 58 | p_detection_name = "Window" 59 | p_truth_name = "Random Events" 60 | 61 | out_name = "window-app-comparison" 62 | } 63 | 64 | 65 | load_file <- function(file){ 66 | filename = paste(getwd(), dir, file, sep="/") 67 | cat(paste("\r", filename)) 68 | row = readr::read_csv(filename) 69 | info=strsplit(file, "/") 70 | app = info[[1]][3] 71 | if (app == "Java_FireLands"){ 72 | app = "Java_FabledLands" 73 | } 74 | row = mutate(row, ITERATION=info[[1]][2]) 75 | row = mutate(row, APPLICATION=app) 76 | row = mutate(row, FILE=file) 77 | row = mutate(row, TECHNIQUE=info[[1]][1]) 78 | return(row) 79 | } 80 | test_file = "test.txt" 81 | 82 | images_dir = "images" 83 | labels_dir = "labels" 84 | 85 | 86 | # ------------------------ 87 | 88 | load_data <- function(){ 89 | if (file.exists("user_data.RData")){ 90 | load("user_data.RData") 91 | } else { 92 | pattern="coverage.csv" 93 | temp = list.files(path = dir, pattern=pattern, recursive = TRUE) 94 | r_data = bind_rows(lapply(temp, load_file)) 95 | #data = mutate(data, BRANCH_COVERAGE=BRANCH_COVERED/(BRANCH_MISSED+BRANCH_COVERED+1)) 96 | save(r_data, file="user_data.RData") 97 | } 98 | return(r_data) 99 | } 100 | 101 | 102 | process_data <- function(data){ 103 | data$TECHNIQUE = factor(data$TECHNIQUE, levels=c(random_name, detection_name, truth_name)) 104 | 105 | data = data %>% 106 | mutate(BRANCH_COVERED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_COVERED, BRANCH_COVERED), 107 | BRANCH_MISSED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_MISSED, BRANCH_MISSED)) 108 | 109 | #data$APPLICATION = substr(data$APPLICATION, 0, 8) 110 | 111 | #data[(data$BRANCH_COVERED + data$BRANCH_MISSED)==0,]$BRANCH_COVERED = 1 112 | 113 | #data$BRANCH_COVERAGE=data$BRANCH_COVERED/(data$BRANCH_COVERED+data$BRANCH_MISSED) 114 | 115 | p_data = data %>% 116 | group_by(ITERATION, APPLICATION, TECHNIQUE) %>% 117 | summarise(BRANCHES_COVERED_TOTAL=sum(BRANCH_COVERED), 118 | BRANCHES_MISSED_TOTAL=sum(BRANCH_MISSED), 119 | BRANCHES_TOTAL=sum(BRANCH_COVERED+BRANCH_MISSED), 120 | LINES_TOTAL=sum(LINE_COVERED+LINE_MISSED), 121 | BRANCH_COVERAGE=sum(BRANCH_COVERED)) 122 | 123 | p_data = p_data[complete.cases(p_data),] 124 | apps = unique(p_data$APPLICATION) 125 | 126 | for (a in apps){ 127 | if (nrow(p_data %>% filter(APPLICATION == a, TECHNIQUE==detection_name)) == 0){ 128 | p_data = p_data %>% filter(APPLICATION != a) 129 | } else { 130 | mb = max((p_data %>% filter(APPLICATION == a))$BRANCHES_TOTAL) 131 | p_data = p_data %>% mutate(BRANCH_COVERAGE=if_else(APPLICATION == a, BRANCH_COVERAGE/mb, as.double(BRANCH_COVERAGE)), 132 | BRANCHES_TOTAL=if_else(APPLICATION == a, mb, BRANCHES_TOTAL)) 133 | } 134 | } 135 | 136 | p_data = p_data %>% filter(BRANCH_COVERAGE!=0) 137 | 138 | p_data = p_data %>% mutate(Technique = TECHNIQUE) 139 | p_data = p_data %>% mutate(Application = APPLICATION) 140 | 141 | p_data_r = p_data %>% filter(TECHNIQUE==random_name) %>% mutate(Technique = p_random_name) 142 | p_data_d = p_data %>% filter(TECHNIQUE==detection_name) %>% mutate(Technique = p_detection_name) 143 | p_data_t = p_data %>% filter(TECHNIQUE==truth_name) %>% mutate(Technique = p_truth_name) 144 | 145 | p_data = rbind(p_data_r, p_data_d, p_data_t) 146 | 147 | p_data$Technique = factor(p_data$Technique, c(p_detection_name, p_random_name, p_truth_name)) 148 | 149 | #p_data = p_data %>% mutate(BRANCH_COVERAGE=BRANCHES_COVERED_TOTAL/(BRANCHES_COVERED_TOTAL+BRANCHES_MISSED_TOTAL)) 150 | 151 | return(p_data) 152 | } 153 | 154 | get_table_row <- function(){ 155 | names = c('Application', '\\textbf{Win}dowM Cov.', '\\textbf{Pred}iction Cov.', 'P$_v$(Pred)', '\\^A$_{12}$(Pred)', '\\textbf{API} Cov.', 'P$_v$(Win, API)', '\\^A$_{12}$(Win, API)') 156 | if (config == 1){ 157 | names = c('Application', '\\textbf{Win}dowM Cov.', '\\textbf{Rand}om Cov.', 'P$_v$(Win, Rand)', '\\^A$_{12}$(Win, Rand)', '\\textbf{U-R}and Cov.', 'P$_v$(Win, U-R)', '\\^A$_{12}$(Win, U-R)') 158 | } else if (config == 2){ 159 | names = c('Application', '\\textbf{Win}dowM Cov.', '\\textbf{U-R}and Cov.', 'P$_v$(Win, U-R)', '\\^A$_{12}$(Win, U-R)', '\\textbf{App}M Cov.', 'P$_v$(Win, App)', '\\^A$_{12}$(Win, App)') 160 | } 161 | d = data.frame(matrix(ncol = length(names), nrow = 0), stringsAsFactors=FALSE) 162 | colnames(d) <- names 163 | 164 | 165 | return(d) 166 | } 167 | 168 | r3dp <- function(x){ 169 | if (is.numeric(x)){ 170 | if (is.nan(x)){ 171 | return("1.000") 172 | } 173 | if(x < 0.001){ 174 | return("$<$0.001") 175 | } 176 | return(format(round(x, 3), nsmall=3)) 177 | } else { 178 | return(gsub("_", "-", x)) 179 | } 180 | } 181 | 182 | bold <- function(x) {paste('{\\textbf{',r3dp(x),'}}', sep ='')} 183 | 184 | get_table <- function(data){ 185 | apps = unique(data$APPLICATION) 186 | 187 | table_data = get_table_row() 188 | 189 | a_db = 0 190 | a_rb = 0 191 | a_drp= 0 192 | a_dra= 0 193 | a_tb= 0 194 | a_dtp= 0 195 | a_dta= 0 196 | 197 | a12_random = c() 198 | a12_truth = c() 199 | i = 0 200 | 201 | for (a in apps){ 202 | i = i + 1 203 | print(a) 204 | 205 | style = r3dp 206 | 207 | rand = data %>% filter(APPLICATION == a, TECHNIQUE == random_name) 208 | det = data %>% filter(APPLICATION == a, TECHNIQUE == detection_name) 209 | tru = data %>% filter(APPLICATION == a, TECHNIQUE == truth_name) 210 | 211 | pv = wilcox.test(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 212 | pvt = wilcox.test(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 213 | style_t = r3dp 214 | 215 | if (!is.nan(pv) && pv < 0.05){ 216 | style = bold 217 | } 218 | 219 | if (!is.nan(pvt) && pvt < 0.05){ 220 | style_t = bold 221 | } 222 | 223 | a_rb = a_rb + mean(rand$BRANCH_COVERAGE) 224 | a_db = a_db + mean(det$BRANCH_COVERAGE) 225 | a_tb = a_tb + mean(tru$BRANCH_COVERAGE) 226 | if (is.nan(pv)){ 227 | a_drp = a_drp + 1 228 | } else { 229 | a_drp = a_drp + pv 230 | } 231 | 232 | if (is.nan(pvt)){ 233 | a_dtp = a_dtp + 1 234 | } else { 235 | a_dtp = a_dtp + pvt 236 | } 237 | a_dra = a_dra + VD.A(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate 238 | a_dta = a_dta + VD.A(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate 239 | 240 | a12_random <- c(a12_random, a_dra) 241 | a12_truth <- c(a12_truth, a_dta) 242 | 243 | 244 | row = get_table_row() 245 | row[1,] = list(style(a), r3dp(mean(det$BRANCH_COVERAGE)), style(mean(rand$BRANCH_COVERAGE)), 246 | style(pv), style(VD.A(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate), style_t(mean(tru$BRANCH_COVERAGE)), style_t(pvt), style_t(VD.A(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$estimate)) 247 | 248 | table_data = rbind(table_data, row) 249 | } 250 | 251 | row = get_table_row() 252 | row[1,] = list("Mean", r3dp(a_db / length(apps)), r3dp(a_rb / length(apps)), 253 | r3dp(a_drp / length(apps)), r3dp(a_dra / length(apps)), r3dp(a_tb / length(apps)), 254 | r3dp(a_dtp / length(apps)), r3dp(a_dta / length(apps))) 255 | 256 | table_data = rbind(table_data, row) 257 | 258 | table_data = table_data %>% 259 | mutate_each(funs(if(is.double(.)) as.double(.) else .)) 260 | 261 | return(xtable(table_data, caption="Branch Coverage of Random against Object Detection. \\textbf{Bold} indicates significance. ")) 262 | } 263 | 264 | box_plots <- function(data, file){ 265 | # colscale <- c("Random"="#8D8B97", p_detection_name="#B3BEAD", 266 | # p_truth_name="#FFE6CB") 267 | # fillscale <- scale_fill_manual(name="Technique", values=colscale) 268 | 269 | p = data %>% ggplot(aes(x=Technique, y=BRANCH_COVERAGE, fill=Technique)) + 270 | geom_boxplot() + 271 | xlab("Application") + 272 | ylab("Branch Coverage") + 273 | #fillscale + 274 | #scale_y_log10() + 275 | theme_bw() + 276 | theme(axis.text.x = element_text(angle = 30, hjust = 1), legend.position = "none") + 277 | facet_wrap(~APPLICATION, scales = "free", ncol=5) + 278 | scale_fill_viridis(discrete=TRUE, option="viridis", begin=0.5) 279 | 280 | 281 | ggsave(file, p, width = 8, height=5) 282 | 283 | return(p) 284 | } 285 | 286 | print("Loading data...") 287 | r_data = load_data() 288 | print("Processing data...") 289 | data = process_data(r_data) 290 | 291 | p = box_plots(data, paste("chapter4-", out_name, ".pdf", sep="")) 292 | 293 | #data_rq4 = data %>% filter(grepl("detection", TECHNIQUE) | grepl("truth", TECHNIQUE)) 294 | 295 | #p = box_plots(data_rq4, "rq4.pdf") 296 | 297 | print(p) 298 | 299 | table = get_table(data) 300 | 301 | latex = print(table, sanitize.text.function = function(x) x, include.rownames=FALSE) 302 | 303 | write(latex, file=paste("chapter4-", out_name, ".tex", sep="")) 304 | 305 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/swing_plot.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | import cv2 4 | import numpy as np 5 | import sys 6 | import gc 7 | import math 8 | import random 9 | import os 10 | import pyautogui 11 | from operator import itemgetter 12 | import time 13 | import subprocess 14 | import Xlib 15 | import time 16 | from pynput import keyboard 17 | import data_loader 18 | import signal 19 | import timeit 20 | import signal 21 | from test_helper import get_window_size, screenshot, perform_interaction 22 | 23 | 24 | running = True 25 | 26 | debug = False 27 | 28 | quit_counter = 3 29 | 30 | working_dir = "" 31 | aut_command = "" 32 | process_id = -1 33 | 34 | def on_release(key): 35 | global running, quit_counter 36 | if key == keyboard.Key.f1: 37 | quit_counter -= 1 38 | if quit_counter == 0: 39 | running = False 40 | print("[Swing Test] Killing tester.") 41 | 42 | sub_window = False 43 | 44 | error_count = -1 45 | 46 | def kill_old_process(): 47 | global process_id 48 | 49 | if process_id != -1: 50 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 51 | process_id = -1 52 | 53 | 54 | def handler(signum, frame): 55 | print('Signal handler called with signal', signum) 56 | 57 | kill_old_process() 58 | 59 | raise TimeoutError("Could not retrieve bounding boxes!") 60 | 61 | signal.signal(signal.SIGALRM,handler) 62 | 63 | def start_aut(): 64 | global working_dir, aut_command, process_id 65 | 66 | if aut_command != "": 67 | 68 | print("[Swing Test] Starting AUT") 69 | 70 | kill_old_process() 71 | 72 | os.chdir(working_dir) 73 | 74 | with open('stdout.log', "a+") as outfile: 75 | with open('stderr.log', "a+") as errfile: 76 | outfile.write("\n" + str(time.time()) + "\n") 77 | cmds = aut_command.split() 78 | print(cmds) 79 | process_id = subprocess.Popen(aut_command.split(), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=errfile, preexec_fn=os.setsid, shell=False, 80 | bufsize=1, 81 | universal_newlines=True) 82 | time.sleep(10) 83 | else: 84 | print("[Swing Test] Could not find AUT to start!") 85 | 86 | 87 | def generate_input_string(): 88 | if random.random() < 0.5: 89 | return "Hello World!" 90 | else: 91 | return str(random.randint(-10000, 10000)) 92 | 93 | def convert_coords(x, y, w, h, aspect): 94 | if aspect > 1: # width is bigger than height 95 | h = h * aspect 96 | y = 0.5 + ((y - 0.5)*aspect) 97 | elif aspect < 1: 98 | w = w / aspect 99 | x = 0.5 + ((x - 0.5)/aspect) 100 | 101 | return x, y, w, h 102 | 103 | def select_random_box(proc_boxes): 104 | best_box = random.sample(proc_boxes.tolist(), 1)[0] 105 | 106 | return best_box 107 | 108 | def prepare_screenshot(raw_image): 109 | st = time.time() 110 | raw_image = data_loader.pad_image(raw_image) 111 | et = time.time() 112 | if debug: 113 | print("[Swing Test] PADDING:", et - st) 114 | image = cv2.resize(raw_image, (cfg.width, cfg.height)) 115 | 116 | images = np.reshape(image, [1, cfg.width, cfg.height, 1]) 117 | 118 | imgs = (images/127.5)-1 119 | 120 | return imgs 121 | 122 | def convert_boxes(boxes): 123 | return yolo.convert_net_to_bb(boxes, filter_top=False)[0] 124 | 125 | if __name__ == '__main__': 126 | # Collect events until released 127 | with keyboard.Listener(on_release=on_release) as listener: 128 | if len(sys.argv) > 1: 129 | cfg.window_name = sys.argv[1] 130 | 131 | event = 0 132 | 133 | yolo = Yolo() 134 | 135 | states = [] 136 | 137 | runtime = round(time.time()) 138 | 139 | output_dir = cfg.output_dir + "/" + str(runtime) 140 | 141 | if not os.path.exists(output_dir): 142 | os.makedirs(output_dir) 143 | 144 | if not os.path.exists(output_dir + "/images"): 145 | os.makedirs(output_dir + "/images") 146 | 147 | test_file = output_dir + "/test.txt" 148 | 149 | with open(test_file, "w+") as t_f: 150 | t_f.write("") 151 | 152 | 153 | if len(sys.argv) > 3: 154 | working_dir = sys.argv[2] 155 | aut_command = sys.argv[3] 156 | 157 | start_aut() 158 | 159 | yolo.set_training(False) 160 | 161 | anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) 162 | 163 | start_time = time.time() 164 | 165 | last_image = None 166 | 167 | boxes = [] 168 | 169 | interactions = [] 170 | 171 | runtime = cfg.test_time #5 mins 172 | 173 | last_u_input = "" 174 | 175 | actions = 0 176 | 177 | csv_file = cfg.log_dir + "/" + str(start_time) + "-test.csv" 178 | 179 | with open(csv_file, "w+") as p_f: 180 | p_f.write("time,actions,technique,iteration_time,window_name\n") 181 | 182 | while ((time.time() - start_time < runtime and not cfg.use_iterations) or 183 | (actions < cfg.test_iterations and cfg.use_iterations)) and running: 184 | 185 | iteration_time = time.time() 186 | 187 | time.sleep(1) 188 | 189 | exec_time = time.time() - start_time 190 | 191 | os.system('killall "firefox"') 192 | 193 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 194 | 195 | signal.alarm(60) 196 | 197 | counter = 0 198 | 199 | while app_w == 0: 200 | 201 | start_aut() 202 | counter += 1 203 | app_x, app_y, app_w, app_h = get_window_size(cfg.window_name) 204 | if counter >= 3: 205 | print("[Swing Test] Couldn't find application window!") 206 | break 207 | 208 | if time.time() - start_time > runtime: 209 | break 210 | 211 | image = screenshot()[app_y:app_y+app_h, app_x:app_x+app_w] 212 | 213 | image = np.array(image) 214 | 215 | # raw_image = image[app_y:app_y+app_h, app_x:app_x+an.write(b"bounds")pp_w] 216 | 217 | ih, iw = image.shape[:2] 218 | 219 | aspect = iw/ih 220 | 221 | imgs = prepare_screenshot(image) 222 | 223 | gen_boxes = True 224 | 225 | for l in states: 226 | diff = np.sum(np.square(image-l[0]))/image.size 227 | if diff < 2: 228 | gen_boxes = False 229 | proc_boxes = l[1] 230 | 231 | last_image = image 232 | 233 | if True: #gen_boxes or len(proc_boxes) < 3: 234 | 235 | #print("New state found!", len(states), "states found total.") 236 | 237 | st = time.time() 238 | 239 | process_id.stdin.write("bounds\n") 240 | 241 | line = "first line" 242 | 243 | p_boxes = [] 244 | 245 | while len(line) > 1: 246 | line = process_id.stdout.readline() 247 | 248 | print(line) 249 | 250 | if "AUT Not Supported!" in line: 251 | print(line) 252 | kill_old_process() 253 | sys.exit(-1) 254 | 255 | l_arr = line.split(",") 256 | 257 | if len(l_arr) > 4: 258 | error_count = 0 259 | p_boxes.append([yolo.names.index(l_arr[0]), 260 | (float(l_arr[1]) - app_x)/app_w, 261 | (float(l_arr[2]) - app_y)/app_h, 262 | float(l_arr[3]) / app_w, 263 | float(l_arr[4]) / app_h]) 264 | 265 | #p_boxes = np.array(p_boxes) 266 | 267 | print(p_boxes) 268 | 269 | if len(p_boxes) == 0: 270 | error_count += 1 271 | 272 | if error_count > 5: 273 | kill_old_process() 274 | print("5 errors in a row. Is application supported?") 275 | sys.exit(1) 276 | 277 | continue 278 | 279 | proc_boxes = np.array(p_boxes) 280 | 281 | height, width = image.shape 282 | 283 | for box in range(len(proc_boxes)): 284 | lbl = proc_boxes[box] 285 | 286 | col = 0 287 | 288 | # lbl[0] = lbl[0] / cfg.grid_shape[0] 289 | # lbl[1] = lbl[1] / cfg.grid_shape[1] 290 | # lbl[2] = lbl[2] / cfg.grid_shape[0] 291 | # lbl[3] = lbl[3] / cfg.grid_shape[1] 292 | x1, y1 = (int(width * (lbl[1] - lbl[3]/2)), 293 | int(height * (lbl[2] - lbl[4]/2))) 294 | x2, y2 = (int(width * (lbl[1] + lbl[3]/2)), 295 | int(height * (lbl[2] + lbl[4]/2))) 296 | cv2.rectangle(image, 297 | (x1, y1), 298 | (x2, y2), 299 | 127, 3, 4) 300 | 301 | cv2.rectangle(image, 302 | (int((x1+x2)/2-1), int((y1+y2)/2-1)), 303 | (int((x1+x2)/2+1), int((y1+y2)/2+1)), 304 | 127, 3, 4) 305 | 306 | cv2.imshow('image',image) 307 | cv2.waitKey(0) 308 | cv2.destroyAllWindows() 309 | 310 | for box_num in range(1): 311 | 312 | input_string = generate_input_string() 313 | 314 | if (process_id.poll() != None): 315 | sys.exit(-1) 316 | 317 | best_box = select_random_box(proc_boxes) 318 | 319 | rand_num = random.random() 320 | 321 | #np.delete(proc_boxes, best_box) 322 | 323 | height, width = image.shape[:2] 324 | 325 | current_box = best_box 326 | 327 | perform_interaction(best_box, app_x, app_y, app_w, app_h, input_string) 328 | 329 | actions += 1 330 | 331 | end_iteration_time = time.time() 332 | if debug: 333 | print("[Swing Test] Iteration Time:", end_iteration_time - iteration_time) 334 | 335 | # write test info 336 | with open(csv_file, "a") as p_f: 337 | p_f.write(str(exec_time) + "," + str(actions) + ",detection," + str(iteration_time) + "," + cfg.window_name + "\n") 338 | 339 | kill_old_process() 340 | 341 | time.sleep(20) 342 | print("[Swing Test] Finished testing!") 343 | 344 | 345 | -------------------------------------------------------------------------------- /scripts/process_results_users_min.R: -------------------------------------------------------------------------------- 1 | library(dplyr) 2 | library(ggplot2) 3 | library(tidyr) 4 | library(readr) 5 | library(RColorBrewer) 6 | library(ggthemes) 7 | library(xtable) 8 | library(effsize) 9 | library(viridis) 10 | 11 | setwd('/home/thomas/experiments/test_experiments') 12 | 13 | dir = "output" 14 | random_name = "detection" 15 | detection_name = "user-models" 16 | truth_name = "truth" 17 | #truth_name = "random" 18 | 19 | p_random_name = "PREDICTION" 20 | p_detection_name = "WINMODEL" 21 | p_truth_name = "API" 22 | table_label = "chapter4:table:issta-comparison" 23 | table_caption = "Branch Coverage of User-Learned Models, Object Detection and API." 24 | 25 | out_name = "issta-user-comparison" 26 | 27 | config = 3 28 | 29 | limited_config = "user-random" 30 | 31 | if (config == 1){ 32 | random_name = "user-random" 33 | detection_name = "user-models" 34 | truth_name = "random" 35 | 36 | p_random_name = "EVENTRAND" 37 | p_detection_name = "WINMODEL" 38 | p_truth_name = "CLICKRAND" 39 | 40 | out_name = "issta-random-comparison" 41 | table_label = "chapter4:table:random-comparison" 42 | } 43 | 44 | if (config == 2){ 45 | random_name = "user-models-random" 46 | detection_name = "user-models" 47 | truth_name = "user-models-oneout" 48 | 49 | p_random_name = "WINMODEL+RAND" 50 | p_detection_name = "WINMODEL+APP" 51 | p_truth_name = "WINMODEL-AUT" 52 | out_name = "user-window-app-comparison" 53 | table_label = "chapter4:table:window-comparison" 54 | } 55 | 56 | if (config == 3){ 57 | random_name = "user-models-app" 58 | detection_name = "user-models" 59 | truth_name = "user-models-oneout" 60 | 61 | p_random_name = "APPMODEL" 62 | p_detection_name = "WINMODEL" 63 | p_truth_name = "WINMODEL-AUT" 64 | out_name = "user-window-app-oneout-comparison" 65 | table_label = "chapter4:table:window-app-comparison" 66 | } 67 | 68 | table_caption = paste("Branch Coverage of techniques ", p_detection_name, ", ", p_random_name, ", and ", p_truth_name, ".", sep="") 69 | 70 | table_caption = paste(table_caption, "\\textbf{Bold} indicates significance. * Indicates Effect Size (*Small, **Medium and ***Large).") 71 | 72 | load_file <- function(file){ 73 | filename = paste(getwd(), dir, file, sep="/") 74 | cat(paste("\r", filename)) 75 | row = readr::read_csv(filename) 76 | info=strsplit(file, "/") 77 | app = info[[1]][3] 78 | if (app == "Java_FireLands"){ 79 | app = "Java_FabledLands" 80 | } 81 | row = mutate(row, ITERATION=info[[1]][2]) 82 | row = mutate(row, APPLICATION=app) 83 | row = mutate(row, FILE=file) 84 | row = mutate(row, TECHNIQUE=info[[1]][1]) 85 | return(row) 86 | } 87 | test_file = "test.txt" 88 | 89 | images_dir = "images" 90 | labels_dir = "labels" 91 | 92 | 93 | # ------------------------ 94 | 95 | load_data <- function(){ 96 | if (file.exists("user_data.RData")){ 97 | load("user_data.RData") 98 | } else { 99 | pattern="coverage.csv" 100 | temp = list.files(path = dir, pattern=pattern, recursive = TRUE) 101 | r_data = bind_rows(lapply(temp, load_file)) 102 | #data = mutate(data, BRANCH_COVERAGE=BRANCH_COVERED/(BRANCH_MISSED+BRANCH_COVERED+1)) 103 | save(r_data, file="user_data.RData") 104 | } 105 | return(r_data) 106 | } 107 | 108 | 109 | process_data <- function(data){ 110 | data = data %>% 111 | mutate(BRANCH_COVERED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_COVERED, BRANCH_COVERED), 112 | BRANCH_MISSED=if_else(BRANCH_COVERED+BRANCH_MISSED==0, METHOD_MISSED, BRANCH_MISSED)) 113 | 114 | #data$APPLICATION = substr(data$APPLICATION, 0, 8) 115 | 116 | #data[(data$BRANCH_COVERED + data$BRANCH_MISSED)==0,]$BRANCH_COVERED = 1 117 | 118 | #data$BRANCH_COVERAGE=data$BRANCH_COVERED/(data$BRANCH_COVERED+data$BRANCH_MISSED) 119 | 120 | p_data = data %>% 121 | group_by(ITERATION, APPLICATION, TECHNIQUE) %>% 122 | summarise(BRANCHES_COVERED_TOTAL=sum(BRANCH_COVERED), 123 | BRANCHES_MISSED_TOTAL=sum(BRANCH_MISSED), 124 | BRANCHES_TOTAL=sum(BRANCH_COVERED+BRANCH_MISSED), 125 | LINES_TOTAL=sum(LINE_COVERED+LINE_MISSED), 126 | BRANCH_COVERAGE=sum(BRANCH_COVERED)) 127 | 128 | apps = unique(p_data$APPLICATION) 129 | 130 | for (a in apps){ 131 | if (nrow(p_data %>% filter(APPLICATION == a, TECHNIQUE==limited_config)) == 0){ 132 | p_data = p_data %>% filter(APPLICATION != a) 133 | } else { 134 | mb = max((p_data %>% filter(APPLICATION == a, TECHNIQUE=="truth"))$BRANCHES_TOTAL) 135 | p_data = p_data %>% mutate(BRANCH_COVERAGE=if_else(APPLICATION == a, BRANCH_COVERAGE/mb, as.double(BRANCH_COVERAGE)), 136 | BRANCHES_TOTAL=if_else(APPLICATION == a, mb, BRANCHES_TOTAL)) 137 | } 138 | } 139 | 140 | p_data$TECHNIQUE = factor(p_data$TECHNIQUE, levels=c(random_name, detection_name, truth_name)) 141 | p_data = p_data[complete.cases(p_data),] 142 | 143 | p_data = p_data %>% filter(BRANCH_COVERAGE!=0) 144 | 145 | p_data = p_data %>% mutate(Technique = TECHNIQUE) 146 | p_data = p_data %>% mutate(Application = APPLICATION) 147 | 148 | p_data_r = p_data %>% filter(TECHNIQUE==random_name) %>% mutate(Technique = p_random_name) 149 | p_data_d = p_data %>% filter(TECHNIQUE==detection_name) %>% mutate(Technique = p_detection_name) 150 | p_data_t = p_data %>% filter(TECHNIQUE==truth_name) %>% mutate(Technique = p_truth_name) 151 | 152 | p_data = rbind(p_data_r, p_data_d, p_data_t) 153 | 154 | p_data$Technique = factor(p_data$Technique, c(p_detection_name, p_random_name, p_truth_name)) 155 | 156 | # p_data = p_data %>% mutate(BRANCH_COVERAGE=BRANCHES_COVERED_TOTAL/(BRANCHES_COVERED_TOTAL+BRANCHES_MISSED_TOTAL)) 157 | 158 | return(p_data) 159 | } 160 | 161 | get_table_row <- function(){ 162 | # names = c('Application', 'Window Model', 'Prediction', 'Window Model', 'API') 163 | # if (config == 1){ 164 | # names = c('Application', 'Window Model', 'ISSTA Random', 'Window Model', 'Random') 165 | # } else if (config == 2){ 166 | # names = c('Application', 'Window Model', 'Random', 'Window Model', 'App Model') 167 | # } else if (config == 3){ 168 | # names = c('Application', 'Window Model', 'Random', 'Window Model', 'Window/Random') 169 | # } 170 | names = c('Application', p_detection_name, p_random_name, p_detection_name, p_truth_name) 171 | 172 | d = data.frame(matrix(ncol = length(names), nrow = 0), stringsAsFactors=FALSE) 173 | colnames(d) <- names 174 | 175 | 176 | return(d) 177 | } 178 | 179 | r3dp <- function(x, effsize){ 180 | if (is.numeric(x)){ 181 | if (is.nan(x)){ 182 | return("1.000") 183 | } 184 | if(x < 0.001){ 185 | return("$<$0.001") 186 | } 187 | return(format(round(x, 3), nsmall=3)) 188 | } else { 189 | return(gsub("_", "-", x)) 190 | } 191 | } 192 | 193 | bold <- function(x, effsize) { 194 | 195 | if (effsize < 0.5){ 196 | return(r3dp(x)) 197 | } 198 | 199 | asize = "" 200 | if (effsize > 0.56){ 201 | asize = "*" 202 | } 203 | 204 | if (effsize > 0.64){ 205 | asize = "**" 206 | } 207 | 208 | if (effsize > 0.71){ 209 | asize = "***" 210 | } 211 | 212 | paste('{\\textbf{', asize, r3dp(x),'}}', sep ='') 213 | } 214 | 215 | get_table <- function(data){ 216 | apps = unique(data$APPLICATION) 217 | 218 | table_data = get_table_row() 219 | 220 | a_db = 0 221 | a_rb = 0 222 | a_drp= 0 223 | a_dra= 0 224 | a_tb= 0 225 | a_dtp= 0 226 | a_dta= 0 227 | 228 | a12_random = c() 229 | a12_truth = c() 230 | i = 0 231 | 232 | for (a in apps){ 233 | i = i + 1 234 | print(a) 235 | 236 | style = r3dp 237 | 238 | rand = data %>% filter(APPLICATION == a, TECHNIQUE == random_name) 239 | det = data %>% filter(APPLICATION == a, TECHNIQUE == detection_name) 240 | tru = data %>% filter(APPLICATION == a, TECHNIQUE == truth_name) 241 | 242 | pv = wilcox.test(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 243 | pvt = wilcox.test(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE)$p.value 244 | style_t = r3dp 245 | 246 | style_d = r3dp 247 | style_dt = r3dp 248 | 249 | if (!is.nan(pv) && pv < 0.05){ 250 | style = bold 251 | style_d = bold 252 | } 253 | 254 | if (!is.nan(pvt) && pvt < 0.05){ 255 | style_t = bold 256 | style_dt = bold 257 | } 258 | 259 | a_rb = a_rb + mean(rand$BRANCH_COVERAGE) 260 | a_db = a_db + mean(det$BRANCH_COVERAGE) 261 | a_tb = a_tb + mean(tru$BRANCH_COVERAGE) 262 | 263 | a12_random <- VD.A(rand$BRANCH_COVERAGE, det$BRANCH_COVERAGE) 264 | a12_truth <- VD.A(tru$BRANCH_COVERAGE, det$BRANCH_COVERAGE) 265 | 266 | 267 | row = get_table_row() 268 | row[1,] = list(r3dp(a), style_d(median(det$BRANCH_COVERAGE), 1-a12_random$estimate), 269 | style(median(rand$BRANCH_COVERAGE), a12_random$estimate), 270 | style_dt(median(det$BRANCH_COVERAGE), 1-a12_truth$estimate), 271 | style_t(median(tru$BRANCH_COVERAGE), a12_truth$estimate)) 272 | 273 | table_data = rbind(table_data, row) 274 | } 275 | 276 | row = get_table_row() 277 | row[1,] = list("Mean", r3dp(a_db / length(apps)), r3dp(a_rb / length(apps)), r3dp(a_db / length(apps)), r3dp(a_tb / length(apps))) 278 | 279 | table_data = rbind(table_data, row) 280 | 281 | # table_data = table_data %>% 282 | # mutate_each(funs(if(is.double(.)) as.double(.) else .)) 283 | 284 | return(xtable(table_data, caption=table_caption, 285 | label=table_label, align='ll|rr|rr')) 286 | } 287 | 288 | box_plots <- function(data, file){ 289 | # colscale <- c("Random"="#8D8B97", p_detection_name="#B3BEAD", 290 | # p_truth_name="#FFE6CB") 291 | # fillscale <- scale_fill_manual(name="Technique", values=colscale) 292 | 293 | p = data %>% ggplot(aes(x=Technique, y=BRANCH_COVERAGE, fill=Technique)) + 294 | geom_boxplot() + 295 | xlab("Application") + 296 | ylab("Branch Coverage") + 297 | #fillscale + 298 | #scale_y_log10() + 299 | theme_bw() + 300 | theme(axis.text.x = element_text(angle = 30, hjust = 1), legend.position = "none") + 301 | facet_wrap(~APPLICATION, scales = "free", ncol=5) + 302 | scale_fill_viridis(discrete=TRUE, option="viridis", begin=0.5) 303 | 304 | 305 | ggsave(file, p, width = 8, height=5) 306 | 307 | return(p) 308 | } 309 | 310 | print("Loading data...") 311 | r_data = load_data() 312 | print("Processing data...") 313 | data = process_data(r_data) 314 | 315 | p = box_plots(data, paste("chapter4-", out_name, ".pdf", sep="")) 316 | 317 | #data_rq4 = data %>% filter(grepl("detection", TECHNIQUE) | grepl("truth", TECHNIQUE)) 318 | 319 | #p = box_plots(data_rq4, "rq4.pdf") 320 | 321 | print(p) 322 | 323 | table = get_table(data) 324 | 325 | hlines <- c(-1, 0, nrow(table)-1, nrow(table)) 326 | 327 | latex = print(table, sanitize.text.function = function(x) x, include.rownames=FALSE, 328 | scalebox = 0.7, booktabs = TRUE, hline.after = hlines) 329 | 330 | write(latex, file=paste("chapter4-", out_name, ".tex", sep="")) 331 | 332 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/model_info.py: -------------------------------------------------------------------------------- 1 | import config as cfg 2 | from yolo import Yolo 3 | from cv2 import imread, resize 4 | import numpy as np 5 | import tensorflow as tf 6 | import sys 7 | import gc 8 | import math 9 | import random 10 | import os 11 | import pickle 12 | import re 13 | import cv2 14 | 15 | tf.logging.set_verbosity(tf.logging.INFO) 16 | 17 | totals = [] 18 | yolo = Yolo() 19 | 20 | def normalise_point(point, val): 21 | i = point*val 22 | return i, math.floor(i) 23 | 24 | def normalise_label(label): 25 | px, cx = normalise_point(max(0, min(1, label[0])), cfg.grid_shape[0]) 26 | py, cy = normalise_point(max(0, min(1, label[1])), cfg.grid_shape[1]) 27 | return [ 28 | px, 29 | py, 30 | max(0, min(cfg.grid_shape[0], label[2]*cfg.grid_shape[0])), 31 | max(0, min(cfg.grid_shape[1], label[3]*cfg.grid_shape[1])), 32 | label[4] 33 | ], (cx, cy) 34 | 35 | def load_files(raw_files): 36 | raw_files = [f.replace("/data/acp15tdw", "/home/thomas/experiments") for f in raw_files] 37 | label_files = [f.replace("/images/", "/labels/") for f in raw_files] 38 | label_files = [f.replace(".png", ".txt") for f in label_files] 39 | 40 | pickle_files = [f.replace("/images/", "/pickle/") for f in raw_files] 41 | pickle_files = [f.replace(".png", ".pickle") for f in pickle_files] 42 | 43 | areas = [] 44 | images = [] 45 | labels = [] 46 | object_detection = [] 47 | 48 | for i in range(len(raw_files)): 49 | f = raw_files[i] 50 | f_l = label_files[i] 51 | d = imread(f, 0) 52 | 53 | if not os.path.isfile(f_l) or not os.path.isfile(f) or d is None: 54 | continue 55 | 56 | image = np.int16(d) 57 | 58 | height, width = image.shape 59 | 60 | areas.append(height*width) 61 | 62 | images.append(image) 63 | 64 | # read in format [c, x, y, width, height] 65 | # store in format [c], [x, y, width, height] 66 | with open(f_l, "r") as l: 67 | obj_detect = [[0 for i in 68 | range(cfg.grid_shape[0])]for i in 69 | range(cfg.grid_shape[1])] 70 | imglabs = [[[0 for i in 71 | range(5)]for i in 72 | range(cfg.grid_shape[1])] for i in 73 | range(cfg.grid_shape[0])] 74 | 75 | for line in l: 76 | elements = line.split(" ") 77 | #print(elements[1:3]) 78 | normalised_label, centre = normalise_label([float(elements[1]), float(elements[2]), 79 | float(elements[3]), float(elements[4]), 1]) 80 | x = max(0, min(int(centre[0]), cfg.grid_shape[0]-1)) 81 | y = max(0, min(int(centre[1]), cfg.grid_shape[1]-1)) 82 | imglabs[y][x] = normalised_label 83 | obj_detect[y][x] = int(elements[0]) 84 | #obj_detect[y][x][int(elements[0])] = 1 85 | 86 | object_detection.append(obj_detect) 87 | labels.append(imglabs) 88 | 89 | return images, labels, object_detection, areas 90 | 91 | def proc_subset(imgs, labels, classes, dataset, areas): 92 | density_f = np.array([[[0 for i in 93 | range(len(yolo.names)+1)]for i in 94 | range(cfg.grid_shape[1])] for i in 95 | range(cfg.grid_shape[0])]) 96 | 97 | pixels_f = np.array([0 for i in 98 | range(30)]) 99 | quantities_f = [] 100 | 101 | lim = imgs.shape[0] 102 | 103 | 104 | for i in range(lim): 105 | 106 | widget_q = 0 107 | 108 | class_count = np.array([0 for i in 109 | range(len(yolo.names))]) 110 | 111 | displacement = 255/30 112 | 113 | widget_area = 0 114 | image_area = areas[i] 115 | 116 | img = imgs[i] 117 | 118 | for j in range(30): 119 | ub = (j+1)*displacement 120 | lb = j*displacement 121 | quantity = np.sum((img < ub).astype(np.int32) * 122 | (img > lb).astype(np.int32)).astype(np.float32) 123 | pixels_f[j] += quantity 124 | 125 | 126 | for x in range(cfg.grid_shape[0]): 127 | for y in range(cfg.grid_shape[1]): 128 | if (labels[i,x,y,4] == 1): 129 | c = classes[i,x,y] 130 | density_f[y,x,0] = density_f[x,y,0] + 1 131 | density_f[y,x,c+1] = density_f[x,y,c+1] + 1 132 | class_count[c] += 1 133 | 134 | widget_q += 1 135 | 136 | with open("label_dims.csv", "a") as file: 137 | area = str(labels[i,x,y,2]/cfg.grid_shape[0] * labels[i,x,y,3]/cfg.grid_shape[1]) 138 | file.write(str(i) + "," + yolo.names[c] + ",width," + str(labels[i,x,y,2]/cfg.grid_shape[0]) + "," + dataset + "\n") 139 | file.write(str(i) + "," + yolo.names[c] + ",height," + str(labels[i,x,y,3]/cfg.grid_shape[1]) + "," + dataset + "\n") 140 | file.write(str(i) + "," + yolo.names[c] + ",area," + area + "," + dataset + "\n") 141 | widget_area += image_area*labels[i,x,y,2]/cfg.grid_shape[0]*labels[i,x,y,3]/cfg.grid_shape[0] 142 | 143 | quantities_f.append(widget_q) 144 | 145 | with open("white_space.csv", "a") as file: 146 | file.write(str(i) + "," + str(image_area) + "," + str(np.sum(class_count)) + "," + str(widget_area) + "," + dataset + "\n") 147 | 148 | for c in range(len(class_count)): 149 | with open("class_count.csv", "a") as file: 150 | file.write(str(i) + "," + yolo.names[c] + "," + str(class_count[c]/np.sum(class_count)) + "," + dataset + "\n") 151 | 152 | return density_f, pixels_f, quantities_f 153 | 154 | def print_info(imgs, labels, classes, dataset, areas): 155 | 156 | return proc_subset(imgs, labels, classes, dataset, areas) 157 | 158 | def run_dataset(files, dataset): 159 | 160 | density = np.array([[[0 for i in 161 | range(len(yolo.names)+1)]for i in 162 | range(cfg.grid_shape[1])] for i in 163 | range(cfg.grid_shape[0])]) 164 | 165 | pixels = np.array([0 for i in 166 | range(30)]) 167 | quantities = [] 168 | 169 | max_run = 500 170 | 171 | iterations = int(math.ceil(len(files) / max_run)) 172 | last_prog = 0 173 | 174 | print(dataset, "set ---") 175 | 176 | for i in range(iterations): 177 | 178 | if i / iterations > last_prog: 179 | print(last_prog*100, "%") 180 | last_prog += 0.1 181 | 182 | 183 | f_s = files[i*max_run:min((i+1)*max_run, len(files))] 184 | 185 | v_imgs, v_labels, v_obj_detection, areas = load_files( 186 | f_s) 187 | 188 | v_imgs = np.array(v_imgs) 189 | 190 | v_labels = np.array(v_labels) 191 | 192 | v_obj_detection = np.array(v_obj_detection) 193 | 194 | 195 | 196 | density_p, pixels_p, quantities_p = print_info(v_imgs, v_labels, v_obj_detection, dataset, areas) 197 | 198 | density += density_p 199 | pixels += pixels_p 200 | quantities += quantities_p 201 | 202 | del(v_imgs, v_labels, v_obj_detection) 203 | 204 | print("img_hist:", pixels) 205 | 206 | for j in range(30): 207 | with open("img_hist.csv", "a") as file: 208 | file.write(str(j) + "," + str(pixels[j]) + "," + dataset + "\n") 209 | 210 | for x in range(cfg.grid_shape[0]): 211 | for y in range(cfg.grid_shape[1]): 212 | for c in range(len(yolo.names)): 213 | with open("label_heat.csv", "a") as file: 214 | dens = (density[x, y, c] + 1) / (np.amax(density[..., c]) + 1) 215 | 216 | if (density[x, y, c] == 0): 217 | dens = 0 218 | 219 | cl = "total" 220 | if (c > 0): 221 | cl = yolo.names[c-1] 222 | file.write(str(x) + "," + str(y) + "," + str(dens) + "," + cl + "," + dataset + "\n") 223 | 224 | 225 | 226 | areas = np.array(areas) 227 | 228 | med_area = np.median(areas) 229 | 230 | lower_quat_area = np.quantile(areas, 0.25) 231 | upper_quat_area = np.quantile(areas, 0.75) 232 | 233 | quantities = np.array(quantities) 234 | 235 | med_quantities = np.median(quantities) 236 | 237 | lower_quat_quantities = np.quantile(quantities, 0.25) 238 | upper_quat_quantities = np.quantile(quantities, 0.75) 239 | 240 | print("AREAS:", "BIG:", upper_quat_area, "MED:", med_area, "SMALL:", lower_quat_area) 241 | print("QUANTITIES:", "BIG:", upper_quat_quantities, "MED:", med_quantities, "SMALL:", lower_quat_quantities) 242 | 243 | if __name__ == '__main__': 244 | training_file = cfg.data_dir + "/../backup/data/train.txt" 245 | 246 | valid_images = [] 247 | 248 | real_images = [] 249 | 250 | pattern = re.compile(".*\/([0-9]+).*") 251 | 252 | with open(training_file, "r") as tfile: 253 | for l in tfile: 254 | 255 | file_num = int(pattern.findall(l)[-1]) 256 | 257 | if file_num <= 243: 258 | real_images.append(l.strip()) 259 | 260 | 261 | 262 | valid_file = cfg.data_dir + "/../backup/data/validate.txt" 263 | 264 | with open(valid_file, "r") as tfile: 265 | for l in tfile: 266 | file_num = int(pattern.findall(l)[-1]) 267 | 268 | if file_num <= 243: 269 | real_images.append(l.strip()) 270 | 271 | 272 | valid_file = cfg.data_dir + "/../backup/data/test.txt" 273 | 274 | with open(valid_file, "r") as tfile: 275 | for l in tfile: 276 | file_num = int(pattern.findall(l)[-1]) 277 | 278 | if file_num <= 243: 279 | real_images.append(l.strip()) 280 | 281 | real_images = [f.replace("/data/acp15tdw", "/data/acp15tdw/backup") for f in real_images] 282 | 283 | real_images_manual = [os.path.join(dp, f) for dp, dn, fn in os.walk(os.path.expanduser(cfg.data_dir + "/../real/data/images")) for f in fn] 284 | 285 | for ri in real_images_manual: 286 | real_images.append(ri) 287 | 288 | 289 | with open("img_hist.csv", "w") as file: 290 | file.write("pixel_value,quantity,dataset" + "\n") 291 | 292 | with open("label_heat.csv", "w") as file: 293 | file.write("x,y,density,class,dataset" + "\n") 294 | 295 | with open("class_count.csv", "w") as file: 296 | file.write("img,class,count,dataset" + "\n") 297 | 298 | with open("label_dims.csv", "w") as file: 299 | file.write("img,class,dimension,value,dataset" + "\n") 300 | 301 | with open("white_space.csv", "w+") as file: 302 | file.write("img,area,widget_count,widget_area,dataset\n") 303 | 304 | valid_file = cfg.data_dir + "/train-10000.txt" 305 | 306 | with open(valid_file, "r") as tfile: 307 | for l in tfile: 308 | file_num = int(pattern.findall(l)[-1]) 309 | valid_images.append(l.strip()) 310 | 311 | valid_images = [f.replace("/home/thomas/work/GuiImages/public", "/data/acp15tdw/data") for f in valid_images] 312 | 313 | 314 | run_dataset(valid_images, "synthetic") 315 | 316 | 317 | run_dataset(real_images, "real") 318 | 319 | gc.collect() 320 | -------------------------------------------------------------------------------- /gui-tester/src/main/python/gui_interaction/batch_data_processor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo python3 2 | 3 | import sys 4 | import os 5 | import time 6 | import subprocess 7 | from pynput import keyboard 8 | import signal 9 | from pynput.mouse import Listener, Button 10 | from event import MouseEvent, KeyEvent, ScrollEvent, EventType, WindowChangeEvent, EventConstructor, Event 11 | from clusterer import Clusterer 12 | from user_model import WindowModel, UserModel 13 | from ngram import Ngram 14 | import pickle 15 | import cv2 16 | 17 | 18 | apps = [ 19 | # ("00", "bellmanzadeh"), 20 | # ("02", "JabRef"), 21 | # ("03", "Java_FireLands"), 22 | # ("05", "ordrumbox"), 23 | # ("11", "Dietetics"), 24 | # ("12", "Minesweeper"), 25 | # ("13", "SQuiz"), 26 | # ("14", "BlackJack"), 27 | # ("15", "UPM"), 28 | ("16", "Simple_Calculator")] 29 | 30 | sub_window = False 31 | 32 | normalise = True 33 | 34 | running = True 35 | 36 | debug = False 37 | 38 | quit_counter = 3 39 | 40 | working_dir = "" 41 | aut_command = "" 42 | process_id = -1 43 | 44 | def on_release(key): 45 | global running, quit_counter 46 | 47 | if key == keyboard.Key.f1: 48 | quit_counter -= 1 49 | if quit_counter == 0: 50 | running = False 51 | print("[Replay] Killing tester.") 52 | 53 | def kill_old_process(): 54 | global process_id 55 | 56 | if process_id != -1: 57 | os.killpg(os.getpgid(process_id.pid), signal.SIGTERM) 58 | process_id = -1 59 | 60 | def start_aut(): 61 | global working_dir, aut_command, process_id, delay 62 | 63 | if aut_command != "": 64 | 65 | print("[Replay] Starting AUT") 66 | 67 | kill_old_process() 68 | 69 | os.chdir(working_dir) 70 | 71 | with open('stdout.log', "a+") as outfile: 72 | with open('stderr.log', "a+") as errfile: 73 | outfile.write("\n" + str(time.time()) + "\n") 74 | process_id = subprocess.Popen(aut_command.split(), stdout=outfile, stderr=errfile, preexec_fn=os.setsid) 75 | 76 | time.sleep(10) 77 | delay += 10 78 | else: 79 | print("[Replay] Could not find AUT to start!") 80 | 81 | mouse_pos = (0, 0) 82 | 83 | events = [] 84 | 85 | update_window = True 86 | 87 | def mouse_move(x, y): 88 | global mouse_pos 89 | mouse_pos = (x, y) 90 | 91 | 92 | def mouse_click(x, y, button, pressed): 93 | global events 94 | events.append(MouseEvent(time.time(), 0 if button == Button.left else 95 | 1 if button == Button.right else 2, x, y, pressed=pressed)) 96 | 97 | 98 | 99 | def mouse_scroll(x, y, dx, dy): 100 | global events 101 | if events[-1].get_event_type() != EventType.MOUSE_WHEEL: 102 | events.append(ScrollEvent(time.time(), dx, dy)) 103 | else: 104 | events[-1].displace(x, y) 105 | 106 | default_window_event = WindowChangeEvent(0, "nowindow", "('nowindow', 'nowindow')", (0,0), (0,0)) 107 | 108 | window_event = default_window_event 109 | 110 | windows = [] 111 | 112 | delay = 0 113 | 114 | def run_events(x, y, time): 115 | global window_event, events, default_window_event, delay, update_window 116 | if len(events) == 0: 117 | print("[Replay] Test finished!") 118 | return False 119 | 120 | event = events[0] 121 | 122 | while event.should_perform_event(time - delay): 123 | if isinstance(event, WindowChangeEvent): 124 | window_event = event 125 | print("[Replay] Changed expected window! (%s)" % window_event.wm_name) 126 | update_window = True 127 | 128 | if not window_event == default_window_event: 129 | if isinstance(event, MouseEvent): 130 | dx = x-window_event.position[0] 131 | dy = y-window_event.position[1] 132 | event.displace(dx, dy) 133 | elif isinstance(event, ScrollEvent): 134 | dx = x-window_event.position[0] 135 | dy = y-window_event.position[1] 136 | event.displace(0, 0, dx, dy) 137 | 138 | event.perform() 139 | events.pop(0) 140 | event = events[0] 141 | return True 142 | 143 | 144 | def gen_window_model(window_event, proc_events, windowed_events, clusterer=Clusterer()): 145 | global default_window_event 146 | if window_event == default_window_event.wm_name: 147 | return None 148 | 149 | assignments = {} 150 | clustered_events = {} 151 | 152 | for et in proc_events[window_event]: 153 | clusterer.clear_data() 154 | 155 | if et is EventType.NONE: 156 | continue 157 | 158 | try: 159 | for e in proc_events[window_event][et]: 160 | f = e.get_features() 161 | if len(f) == 0: 162 | break 163 | clusterer.append_data(f) 164 | if clusterer.shape[1] == 0: 165 | continue 166 | centroids, assigns = clusterer.cluster(clusterer.recommend_clusters(), 2000) 167 | 168 | clustered_events[str(et)] = centroids 169 | 170 | for i in range(len(proc_events[window_event][et])): 171 | assignments[proc_events[window_event][et][i]] = assigns[i] 172 | 173 | except NotImplementedError as e: 174 | print(e) 175 | pass 176 | 177 | ngram = Ngram("") 178 | 179 | clustered_windowed_events = windowed_events[window_event][:] 180 | 181 | for i in range(len(clustered_windowed_events)): 182 | we = clustered_windowed_events[i] 183 | name = str(we.event_type) 184 | 185 | id = we.get_identifier() 186 | if not id is None: 187 | name += "[" + id + "]" 188 | 189 | if we in assignments: 190 | assignment = "{" + str(assignments[we]) + "}" 191 | if "{cluster}" in name: 192 | name = name.replace("{cluster}", assignment) 193 | else: 194 | name += "[" + assignment + "]" 195 | 196 | clustered_windowed_events[i] = name 197 | 198 | sequence = " ".join(clustered_windowed_events).replace("EventType.NONE", ngram.delimiter) 199 | ngram.construct(sequence, 5) 200 | 201 | ngram.calculate_probabilities() 202 | 203 | window_model = WindowModel(ngram, clustered_events) 204 | 205 | return window_model 206 | 207 | def find(name, path): 208 | for root, dirs, files in os.walk(path): 209 | for f in files: 210 | if name in f: 211 | return os.path.join(root, f) 212 | 213 | 214 | def process_inputs(input_dirs, output_dir): 215 | event_constructor = EventConstructor() 216 | events = [] 217 | window_event = default_window_event 218 | 219 | for f in input_dirs: 220 | file = f + "/" + "events.evnt" 221 | 222 | if not os.path.isfile(file): 223 | print("Cannot find file: " + file) 224 | continue 225 | 226 | 227 | 228 | with open(file, "r") as f: 229 | for line in f: 230 | events.append(event_constructor.construct(line)) 231 | events.append(Event(events[-1].timestamp+1, EventType.NONE)) 232 | 233 | events = sorted(events, key=lambda x : x.timestamp) 234 | 235 | proc_events = {} 236 | 237 | windowed_events = {} 238 | 239 | for e in events: 240 | 241 | key = window_event.filename_string() 242 | 243 | if isinstance(e, WindowChangeEvent): 244 | window_event = e 245 | windows.append(e) 246 | 247 | if not key in proc_events: 248 | proc_events[key] = {} 249 | 250 | if not e.event_type in proc_events[key]: 251 | proc_events[key][e.event_type] = [] 252 | 253 | if not None in proc_events: 254 | proc_events[None] = {} 255 | 256 | if not e.event_type in proc_events[None]: 257 | proc_events[None][e.event_type] = [] 258 | 259 | 260 | if normalise and not window_event == default_window_event: 261 | e.change_window(window_event) 262 | 263 | proc_events[key][e.event_type].append(e) 264 | proc_events[None][e.event_type].append(e) 265 | 266 | if not isinstance(e, WindowChangeEvent) or window_event.filename_string() != key: 267 | 268 | if not None in windowed_events: 269 | windowed_events[None] = [] 270 | 271 | windowed_events[None].append(e) 272 | 273 | if not key in windowed_events: 274 | windowed_events[key] = [] 275 | 276 | windowed_events[key].append(e) 277 | 278 | user_model = UserModel() 279 | 280 | for window_event in proc_events: 281 | 282 | window_model = gen_window_model(window_event, proc_events, windowed_events) 283 | 284 | if window_model is None: 285 | continue 286 | 287 | user_model.add_window_model(window_event, window_model) 288 | 289 | if not window_event is None: 290 | 291 | for input_file_num in range(len(input_dirs)): 292 | 293 | screenshot = find(window_event, input_dirs[input_file_num] + "/images/") 294 | 295 | cols = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (0, 255, 255), (255, 0, 255), (0, 0, 0)] 296 | 297 | if not screenshot is None and os.path.isfile(screenshot): 298 | img = cv2.imread(screenshot) 299 | if not img is None: 300 | ci = 0 301 | h, w = img.shape[:2] 302 | for c in window_model.clusters: 303 | if c[-4:] == "DOWN": 304 | for cl in window_model.clusters[c]: 305 | print(cl) 306 | cv2.rectangle(img, 307 | (int(cl[0]*w)-2, int(cl[1]*h)-2), 308 | (int(cl[0]*w)+2, int(cl[1]*h)+2), 309 | cols[ci], -1, 8) 310 | ci += 1 311 | imgout = output_dir + "/imgs/" 312 | 313 | if not os.path.isdir(imgout): 314 | os.makedirs(imgout, exist_ok=True) 315 | try: 316 | os.mkdir(imgout) 317 | except OSError as e: 318 | #folder exists 319 | pass 320 | 321 | 322 | cv2.imwrite(imgout + window_event + ".png", img) 323 | 324 | continue 325 | 326 | default_model = gen_window_model(None, proc_events, windowed_events) 327 | 328 | user_model.set_default_model(default_model) 329 | 330 | for i in range(100): 331 | print("Gen:", default_model.next()) 332 | 333 | if not os.path.isdir(output_dir): 334 | os.makedirs(output_dir, exist_ok=True) 335 | try: 336 | os.mkdir(output_dir) 337 | except OSError as e: 338 | #folder exists 339 | pass 340 | 341 | with open(output_dir + "/user_model.mdl", "bw+") as f: 342 | pickle.dump(user_model, f) 343 | 344 | 345 | 346 | if __name__ == '__main__': 347 | # Collect events until released 348 | 349 | wd = os.getcwd() 350 | 351 | for application in apps: 352 | window_event = default_window_event 353 | 354 | if len(sys.argv) < 2: 355 | print("Must have filename argument!", file=sys.stderr) 356 | sys.exit(1) 357 | input_dir = sys.argv[1] 358 | 359 | input_dirs = "" 360 | 361 | for i in range(1, 11): 362 | p_dir = "/participant-" + str(i) 363 | input_dirs += input_dir + p_dir + "/tasks/" + application[1] + ";" 364 | input_dirs += input_dir + p_dir + "/warmup/" + application[1] + ";" 365 | 366 | input_dirs = input_dirs[:-1] 367 | input_dirs = input_dirs.split(";") 368 | 369 | output_dir = input_dirs[0] 370 | 371 | if len(sys.argv) > 2: 372 | output_dir = sys.argv[2] + "/" + application[1] 373 | 374 | process_inputs(input_dirs, output_dir) 375 | 376 | 377 | # One out training 378 | for application in apps: 379 | window_event = default_window_event 380 | 381 | if len(sys.argv) < 2: 382 | print("Must have filename argument!", file=sys.stderr) 383 | sys.exit(1) 384 | input_dir = sys.argv[1] 385 | 386 | input_dirs = "" 387 | 388 | for ap in apps: 389 | if ap != application: 390 | for i in range(1, 11): 391 | p_dir = "/participant-" + str(i) 392 | input_dirs += input_dir + p_dir + "/tasks/" + ap[1] + ";" 393 | input_dirs += input_dir + p_dir + "/warmup/" + ap[1] + ";" 394 | 395 | input_dirs = input_dirs[:-1] 396 | input_dirs = input_dirs.split(";") 397 | 398 | output_dir = input_dirs[0] 399 | 400 | if len(sys.argv) > 2: 401 | output_dir = sys.argv[2] + "/" + application[1] + "-one-out" 402 | 403 | process_inputs(input_dirs, output_dir) 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | --------------------------------------------------------------------------------