├── image ├── 1462.png ├── 3553.png └── 17059.png ├── .gitignore ├── .idea ├── vcs.xml ├── misc.xml ├── modules.xml └── rico_json_processing.iml ├── log_parsing └── log_parse.py ├── draw_log_parse.py ├── compress.py ├── main.py ├── compare_org_draw.py ├── delete.py ├── settings.py ├── README.md ├── draw_image.py └── element.py /image/1462.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jin1ming/rico-json-processing/HEAD/image/1462.png -------------------------------------------------------------------------------- /image/3553.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jin1ming/rico-json-processing/HEAD/image/3553.png -------------------------------------------------------------------------------- /image/17059.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jin1ming/rico-json-processing/HEAD/image/17059.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | json/ 2 | comparison/ 3 | original_picture/ 4 | layout/ 5 | compressed_comp/ 6 | __pycache__/ 7 | .idea/ 8 | 9 | *.pyc 10 | *.txt 11 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /log_parsing/log_parse.py: -------------------------------------------------------------------------------- 1 | file = open("log.txt") 2 | pout = open("perr.txt", mode="w") 3 | mout = open("merr.txt", mode="w") 4 | 5 | for line in file: 6 | if line[0:2] != 'Do': 7 | if line[0] == 'M': 8 | mout.write(line) 9 | elif line[0] == 'P': 10 | pout.write(line) 11 | # print(line.split(": ")[1].rstrip().split(".")[0] + ".jpg", end=' ') -------------------------------------------------------------------------------- /draw_log_parse.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | log = open("./draw_log.txt", "r") 4 | delete_log = open("./del_log.txt", "w") 5 | 6 | for line in log: 7 | if len(line.split(":")) > 2 and (line.split(":")[1] == "WebView" or line.split(":")[1] == "google.android.gms.ads"): 8 | print(line) 9 | delete_log.write(line) 10 | os.system("./delete.py " + line.split(":")[0].split(".")[0]) 11 | -------------------------------------------------------------------------------- /.idea/rico_json_processing.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | -------------------------------------------------------------------------------- /compress.py: -------------------------------------------------------------------------------- 1 | from multiprocessing.dummy import Pool 2 | from PIL import Image 3 | from settings import * 4 | import os 5 | 6 | 7 | def work(file_name): 8 | try: 9 | origin = Image.open(os.path.join(os.path.curdir, 'comparison', file_name + '_comp.png')) 10 | except FileNotFoundError: 11 | print("File not found: " + file_name + "_comp.png") 12 | return 13 | 14 | compressed_screen_size = (int(2880 / compress_ratio), int(2560 // compress_ratio)) 15 | 16 | new = origin.resize(compressed_screen_size, Image.ANTIALIAS) 17 | new.save(os.path.join(os.path.curdir, 'compressed_comp', file_name + '_' + str(compress_ratio) + 'compressed.png'), 'PNG') 18 | print(os.path.join(os.path.curdir, 'compressed_comp', file_name + '_' + str(compress_ratio) + 'compressed.png')) 19 | 20 | 21 | try: 22 | compress_ratio = float(input("Please input compression ratio: ")) 23 | except: 24 | print("Wrong input. Default value(2.0) will be used.") 25 | compress_ratio = 2.0 26 | 27 | threads = Pool(40) 28 | threads.map(work, FILE_LIST) 29 | threads.close() 30 | threads.join() 31 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import json, os, csv 2 | from settings import * 3 | from element import element 4 | from multiprocessing.dummy import Pool 5 | 6 | app_info_iter = csv.reader(APP_METADATA) 7 | app_info = {i[0]: i[0:3] for i in app_info_iter} 8 | 9 | 10 | # Chunks that actually works 11 | def work(FILE_NAME): 12 | print("Do: " + os.path.join(os.path.curdir, 'json', 'raw', FILE_NAME)) 13 | FILE = open(os.path.join(os.path.curdir, 'json', 'raw', FILE_NAME)) 14 | 15 | data = json.load(FILE) 16 | parsed_data = {} 17 | 18 | try: 19 | parsed_data['app'] = {'package_name': data['activity_name'].split('/')[0], 20 | 'store_name': app_info[data['activity_name'].split('/')[0]][1], 21 | 'category': app_info[data['activity_name'].split('/')[0]][2]} 22 | except: 23 | print("Metadata error: " + FILE_NAME + " : " + data['activity_name'].split('/')[0]) 24 | parsed_data['activity_name'] = data['activity_name'].split('/')[1].split('.')[-1] 25 | parsed_data['keyboard'] = data['is_keyboard_deployed'] 26 | try: 27 | parsed_data['hierarchy'] = element(data['activity']['root']).to_dict() 28 | except: 29 | print("Parsing error: " + FILE_NAME + " : " + data['activity_name'].split('/')[0]) 30 | 31 | output = open(os.path.join(os.path.curdir, 'json', 'refined', FILE_NAME), mode='w') 32 | json.dump(parsed_data, output, indent=2, sort_keys=True) 33 | print("Done: " + os.path.join(os.path.curdir, 'json', 'refined', FILE_NAME)) 34 | 35 | # Simple codes that initiate multi-threads. 36 | threads = Pool(40) 37 | threads.map(work, FILE_LIST) 38 | threads.close() 39 | threads.join() 40 | -------------------------------------------------------------------------------- /compare_org_draw.py: -------------------------------------------------------------------------------- 1 | from settings import * 2 | from PIL import Image 3 | import os 4 | from multiprocessing.dummy import Pool 5 | 6 | def work(FILE_NAME): 7 | # for FILE_NAME in FILE_LIST: 8 | canvas = Image.new("RGBA", (2880, 2560), 'white') 9 | try: 10 | origin = Image.open(os.path.join(os.path.curdir, 'original_picture', FILE_NAME.split('.')[0] + '.jpg')) 11 | except: 12 | err = open('comp_err_log.txt', 'a') 13 | err.write(FILE_NAME + ": Error occurred while opening original pic.\n") 14 | print(FILE_NAME + ": Error occurred while opening original pic.") 15 | err.close() 16 | origin = Image.new("RGBA", (1440, 2560), "black") 17 | pass 18 | try: 19 | layout = Image.open(os.path.join(os.path.curdir, 'layout', FILE_NAME + '_out.png')) 20 | except: 21 | err = open('comp_err_log.txt', 'a') 22 | err.write(FILE_NAME + ": Error occurred while opening layout pic.\n") 23 | err.close() 24 | print(FILE_NAME + ": Error occurred while opening layout pic.") 25 | layout = Image.new("RGBA", (1440, 2560), "black") 26 | pass 27 | # origin = Image.open('original_picture/' + FILE_NAME.split('.')[0] + '.jpg') 28 | # layout = Image.open('layout/' + FILE_NAME + '_out.png') 29 | 30 | origin = origin.resize((1440, 2560), Image.ANTIALIAS) 31 | 32 | canvas.paste(origin, (0, 0)) 33 | canvas.paste(layout, (1440, 0)) 34 | 35 | canvas.save(os.path.join(os.path.curdir, 'comparison', FILE_NAME + '_comp.png'), 'PNG') 36 | print(os.path.join(os.path.curdir, 'comparison', FILE_NAME + '_comp.png')) 37 | # canvas.save('./comparison/' + FILE_NAME + '_comp.png', 'PNG') 38 | # print('./comparison/' + FILE_NAME + '_comp.png') 39 | 40 | 41 | threads = Pool(40) 42 | threads.map(work, FILE_LIST) 43 | threads.close() 44 | threads.join() 45 | -------------------------------------------------------------------------------- /delete.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import json, sys, os 3 | 4 | FILES = sys.argv[1:] 5 | 6 | for FILE_NUM in FILES: 7 | try: 8 | FILE_JSON = json.load(open(os.path.join(os.path.curdir, 'json', 'refined', FILE_NUM + '.json'))) 9 | except FileNotFoundError: 10 | print("File not found: %s" % os.path.join(os.path.curdir, 'json', 'refined', FILE_NUM + '.json')) 11 | continue 12 | 13 | print("File you want to delete is:") 14 | print(" - %s" % os.path.join(os.path.curdir, 'json', 'refined', FILE_NUM + '.json')) 15 | print(" - %s" % os.path.join(os.path.curdir, 'json', 'raw', FILE_NUM + '.json')) 16 | print(" - %s" % os.path.join(os.path.curdir, 'layout', FILE_NUM + '.json_out.png')) 17 | print(" - %s" % os.path.join(os.path.curdir, 'comparison', FILE_NUM + '.json_comp.png')) 18 | print("App info: ") 19 | print(" - Package name: %s" % FILE_JSON['app']['package_name']) 20 | print(" - Store name : %s" % FILE_JSON['app']['store_name']) 21 | print(" - Category : %s" % FILE_JSON['app']['category']) 22 | print("\nPlease check the information before you delete.\nDo you really want to delete this info? (Y/N) ") 23 | 24 | while True: 25 | n = input() 26 | if n == 'Y' or n == 'y': 27 | try: 28 | os.remove(os.path.join(os.path.curdir, 'json', 'refined', FILE_NUM + '.json')) 29 | except: 30 | pass 31 | try: 32 | os.remove(os.path.join(os.path.curdir, 'json', 'raw', FILE_NUM + '.json')) 33 | except: 34 | pass 35 | try: 36 | os.remove(os.path.join(os.path.curdir, 'layout', FILE_NUM + '.json_out.png')) 37 | except: 38 | pass 39 | try: 40 | os.remove(os.path.join(os.path.curdir, 'comparison', FILE_NUM + '.json_comp.png')) 41 | except: 42 | pass 43 | print("Delete complete.") 44 | break 45 | elif n == 'N' or n == 'n': 46 | print("Deletion aborted.") 47 | break 48 | else: 49 | print("Please answer with Y/N. ") 50 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Information about json files 4 | FILE_LIST = [name for name in os.listdir(os.path.join(os.path.curdir, 'json', 'raw'))] 5 | # FILE_LIST = ['1001.json'] 6 | 7 | # Information about layout images 8 | SCREEN_SIZE = (1440, 2560) # Do NOT change 9 | ELEMENT_COLOR = {'EditText': 'orange', 10 | #'edit': 'orange', 11 | 'CheckedTextView': 'lime', 12 | 'Text': 'blue', 13 | 'text': 'blue', 14 | 'TextView': 'blue', 15 | 'RadioButton': 'lime', 16 | 'Button': 'green', 17 | #'button': 'green', 18 | 'Image': 'red', 19 | 'image': 'red', 20 | 'icon': 'red', 21 | 'CheckBox': 'pink', 22 | 'checkbox': 'pink', 23 | 'Checkbox': 'pink', 24 | 'checkBox': 'pink', 25 | 'WebView': 'purple', 26 | 'AdView': 'brown', 27 | 'Banner': 'brown', 28 | 'NumberPicker': 'gray', 29 | 'SpeedometerGauge': 'red', 30 | 'Gauge': 'red', 31 | 'ArcProgress': 'red', 32 | 'SimpleDraweeView': 'red', 33 | 'MonthView': 'yellow', 34 | 'AccessibleDateAnimator': 'yellow', 35 | 'MapView': 'skyblue', 36 | 'VideoView': 'navy', 37 | 'Chart': 'red', 38 | 'SeekBar': 'gold', 39 | 'PictureView': 'red', 40 | 'Picture': 'red', 41 | 'ActionMenuItemView': 'green', 42 | 'ImageSurfaceView': 'red', 43 | 'ImageView': 'red', 44 | 'SurfaceView': 'red', 45 | 'Graph': 'red', 46 | 'RadialPicker': 'ivory', 47 | 'time_picker': 'ivory', 48 | 'TimePicker': 'ivory', 49 | 'ProgressBar': 'magenta', 50 | } 51 | 52 | ELEMENT_EXCEPT = [#'feeditem', 53 | #'edit_profile', 54 | #'credit', 55 | 'background_holder', 56 | 'background_image', 57 | ] 58 | 59 | FULL_NAME_COLOR = {'google.maps.api.android': 'skyblue', # Google Maps API 60 | 'google.android.gms.ads': 'brown', # Google Ads 61 | 'mopub.mobileads': 'brown', # Mopub Ads 62 | 'maps.D': 'skyblue', # Google Maps 63 | 'maps.ad.ay$a': 'skyblue', # Google Maps 64 | 'drumpadmachine.ui.Pad': 'red', # Game image element 65 | } 66 | 67 | 68 | # Others 69 | APP_METADATA = open(os.path.join(os.path.curdir, 'metadata', 'app_details.csv')) 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JSON Processing of RICO Dataset 2 | 3 | ## **This is an amendment to the original version.** 4 | ## **这是一个对原来代码的修正版本** 5 | 6 | 如果有疑问，欢迎提issue交流 7 | 8 | --- 9 | 10 | 11 | These scripts are used to process JSON layouts of [RICO dataset](http://interactionmining.org/rico).
12 | The result of processed layouts has been used in [a paper in CHI '20](https://dl.acm.org/doi/abs/10.1145/3313831.3376327). 13 | 14 | ## Scripts 15 | 16 | We used python 3.6.2 to execute the scripts below. 17 | 18 | ### `python main.py` 19 | 20 | This script extracts required metadata from the raw RICO json files, as a raw file contains too many information about the interface.
21 | RICO json files should be stored in `json/raw/` folder, and the results will be stored in `json/refined` folder. 22 | 23 | ### `python draw_image.py` 24 | 25 | Using the refined json files produced by `main.py`, this script draws bounding boxes of layout components.
26 | You may find the mapping between colors and component categories in the `settings.py` file.
27 | The results will be stored in a `layout/{filename}_out.png` format. 28 | 29 | ### `python compare_org_draw.py` 30 | 31 | This script draws the comparison image of a UI capture image and its layout image.
32 | The purpose of this comparison is because **a few RICO images do not match to their json hierarchy information**, so we had to produce the comparsion images and manually check if all the images are correct. 33 | 34 | This script requires the original UI images in `original_picutre/` folder as well as the layout images produced by `draw_image.py`, and the results will be stored in a `comparison/{filename}_comp.png` format. 35 | 36 | ### `python compress.py` 37 | 38 | This script compresses the drawn layout images produced by `compare_org_draw.py` script.
39 | The purpose of compression is because the dimension of a raw layout image is 2880 x 2560, which is a bit large.
40 | The results will be stored in a `compressed_comp/{filename}_{compress_ratio}compressed.png` format. 41 | 42 | Here are a few examples of the produced outputs after `compress.py`, compressed by 4.0: 43 | 1462 | 3553 | 17059 44 | ---- | ---- | ---- 45 | ![Layout of Image No. 1462](image/1462.png) | ![Layout of Image No. 3553](image/3553.png) | ![Layout of Image No. 17059](image/17059.png) 46 | 47 | ### `delete.py {image_numbers}` 48 | 49 | This script receives a list of RICO image numbers and deletes the data of them produced by the scripts above. 50 | 51 | ## Citation 52 | 53 | If you found these scripts helpful, please consider citing our paper. 54 | ```bibtex 55 | @inproceedings{lee2020guicomp, 56 | author = {Lee, Chunggi and Kim, Sanghoon and Han, Dongyun and Yang, Hongjun and Park, Young-Woo and Kwon, Bum Chul and Ko, Sungahn}, 57 | title = {GUIComp: A GUI Design Assistant with Real-Time, Multi-Faceted Feedback}, 58 | year = {2020}, 59 | publisher = {Association for Computing Machinery}, 60 | address = {New York, NY, USA}, 61 | url = {https://doi.org/10.1145/3313831.3376327}, 62 | doi = {10.1145/3313831.3376327}, 63 | booktitle = {Proceedings of the 2020 CHI Conference on Human Factors in Computing Systems}, 64 | pages = {1–13}, 65 | series = {CHI '20} 66 | } 67 | ``` 68 | -------------------------------------------------------------------------------- /draw_image.py: -------------------------------------------------------------------------------- 1 | import json, os 2 | from settings import * 3 | from PIL import Image 4 | from multiprocessing.dummy import Pool 5 | 6 | def draw(hier, img, FILE_NAME): 7 | # 8 | # Some non-standard layout/views, they don't have standard name. 9 | # Therefore, we have to guess the label using their name and resource-id. 10 | # 11 | checked = False 12 | for element_name in ELEMENT_COLOR.keys(): 13 | if hier['name'].count(element_name) > 0: 14 | except_occur = False 15 | for element_except in ELEMENT_EXCEPT: 16 | if hier['resource_id'] is not None and hier['resource_id'].count(element_except) > 0: 17 | except_occur = True 18 | break 19 | if not except_occur: 20 | hier['name'] = element_name 21 | checked = True 22 | break 23 | 24 | if not checked and hier['resource_id'] is not None: 25 | for element_name in ELEMENT_COLOR.keys(): 26 | if hier['resource_id'].count(element_name) > 0: 27 | except_occur = False 28 | for element_except in ELEMENT_EXCEPT: 29 | if hier['resource_id'].count(element_except) > 0: 30 | except_occur = True 31 | break 32 | if not except_occur: 33 | hier['name'] = element_name 34 | break 35 | 36 | # 37 | # Some non-standard layout/views which can't be handled by name/resource-id, 38 | # they should be processed using full_name(class information). 39 | # 40 | full_name_checked = False 41 | for full_name in FULL_NAME_COLOR.keys(): 42 | if hier['full_name'].count(full_name) > 0 and hier['visible']: 43 | full_name_checked = True 44 | b = hier['bounds'] 45 | try: 46 | block = Image.new('RGBA', (b[2] - b[0], b[3] - b[1]), FULL_NAME_COLOR[full_name]) 47 | img.paste(block, (b[0], b[1])) 48 | if ((b[2] - b[0]) * (b[3] - b[1]))/(SCREEN_SIZE[0] * SCREEN_SIZE[1]) > 0.8: 49 | print(FILE_NAME + ":" + full_name + ":Too large:" + str(((b[2] - b[0]) * (b[3] - b[1]))/(SCREEN_SIZE[0] * SCREEN_SIZE[1]))) 50 | except ValueError: 51 | pass 52 | 53 | # Coloring 54 | if not full_name_checked and hier['name'] in ELEMENT_COLOR.keys() and hier['visible']: 55 | b = hier['bounds'] 56 | try: 57 | block = Image.new('RGBA', (b[2] - b[0], b[3] - b[1]), ELEMENT_COLOR[hier['name']]) 58 | img.paste(block, (b[0], b[1])) 59 | if ((b[2] - b[0]) * (b[3] - b[1])) / (SCREEN_SIZE[0] * SCREEN_SIZE[1]) > 0.8: 60 | print(FILE_NAME + ":" + hier['name'] + ":Too large:" + str( 61 | ((b[2] - b[0]) * (b[3] - b[1])) / (SCREEN_SIZE[0] * SCREEN_SIZE[1]))) 62 | except ValueError: 63 | pass 64 | 65 | # Recursion 66 | if hier['children'] is not None: 67 | for child in hier['children']: 68 | draw(child, img, FILE_NAME) 69 | 70 | 71 | def work(FILE_NAME): 72 | with open(os.path.join(os.path.curdir, 'json', 'refined', FILE_NAME), mode='r') as file: 73 | data = json.load(file) 74 | 75 | img = Image.new('RGBA', SCREEN_SIZE, 'white') 76 | try: 77 | draw(data['hierarchy'], img, FILE_NAME) 78 | img.save(os.path.join(os.path.curdir, 'layout', FILE_NAME + '_out.png'), 'PNG') 79 | print(os.path.join(os.path.curdir, 'layout', FILE_NAME + '_out.png')) 80 | except: 81 | print("ERROR: " + os.path.join(os.path.curdir, 'layout', FILE_NAME + '_out.png')) 82 | pass 83 | 84 | 85 | # Multi-thread parts 86 | threads = Pool(40) 87 | threads.map(work, FILE_LIST) 88 | threads.close() 89 | threads.join() 90 | -------------------------------------------------------------------------------- /element.py: -------------------------------------------------------------------------------- 1 | class element(object): 2 | def __init__(self, hier): 3 | if hier.__class__ != dict: 4 | self.visible = False 5 | return 6 | self.visible = hier['visible-to-user'] 7 | self.clickable = hier['clickable'] 8 | self.bounds = hier['bounds'] 9 | self.name = hier['class'].split('.')[-1] 10 | self.full_name = hier['class'] 11 | self.scrollable = {'horizontal': hier['scrollable-horizontal'], 12 | 'vertical': hier['scrollable-vertical']} 13 | self.resource_id = hier['resource-id'].split('/')[-1] if 'resource-id' in hier else None 14 | self.text = hier['text'] if 'text' in hier else None 15 | self.children = [element(i) for i in hier['children']] if 'children' in hier else list() 16 | 17 | # Exception postprocess 18 | # - if children is invisible, delete it 19 | wait_del = list() 20 | if self.children is not None: 21 | for child in self.children: 22 | if not child.visible: 23 | wait_del.append(child) 24 | 25 | for it in wait_del: 26 | self.children.remove(it) 27 | 28 | # Exception postprocess 29 | # - if DrawerLayout has 2+ layout/views (which means, drawer has opened), 30 | # delete original layout/view (which priors to the drawer view) 31 | if self.name.count("DrawerLayout") and len(self.children) > 1: 32 | self.children.remove(self.children[0]) 33 | 34 | # Exception postprocess 35 | # - if SlidingMenu has 2+ layout/views (which means, slide has opened), 36 | # delete original layout/view (which priors to the slide view) 37 | if self.name == "SlidingMenu" and len(self.children) > 1: 38 | self.children.remove(self.children[1]) 39 | 40 | # Exception postprocess 41 | # - if element's width/height is under 0 (which is confusing), 42 | # delete the element 43 | if self.children is not None: 44 | for child in self.children: 45 | if (child.bounds[2] - child.bounds[0]) < 0 or (child.bounds[3] - child.bounds[1]) < 0: 46 | self.children.remove(child) 47 | 48 | # Exception postprocess 49 | # - if FanView has 2+ layout/views (which means, fan has opened), 50 | # delete original layout/view (which priors to the slide view) 51 | if self.name == "FanView" and len(self.children[0].children) > 1: 52 | self.children[0].children = [self.children[0].children[0]] 53 | 54 | def __str__(self): 55 | out = '{' \ 56 | + ("visible: %s, " % self.visible) \ 57 | + ("clickable: %s, " % self.clickable) \ 58 | + ("bounds: %s, " % self.bounds) \ 59 | + ("name: '%s', " % self.name) \ 60 | + ("full_name: '%s', " % self.full_name) \ 61 | + ("scrollable: %s, " % self.scrollable) \ 62 | + ("resource_id: %s, " % ("'" + self.resource_id + "'" if self.resource_id is not None else "None")) \ 63 | + ("text: %s" % ("'" + self.text + "'" if self.text is not None else "None")) \ 64 | + ("children: %s" % self.children) \ 65 | + '}' 66 | 67 | return out 68 | 69 | def __repr__(self): 70 | return self.__str__() 71 | 72 | def to_dict(self): 73 | out = {'visible': self.visible, 74 | 'clickable': self.clickable, 75 | 'bounds': self.bounds, 76 | 'name': self.name, 77 | 'full_name': self.full_name, 78 | 'scrollable': self.scrollable, 79 | 'resource_id': self.resource_id, 80 | 'text': self.text, 81 | 'children': [child.to_dict() for child in self.children] if self.children is not None else None} 82 | 83 | return out 84 | --------------------------------------------------------------------------------