├── .gitignore
├── .gitattributes
├── photo.png
├── thresh.png
├── photo_out.png
├── requirements.txt
├── app.py
├── data.json
├── README.md
├── main.py
├── GUI.py
└── flowchart_recognition.py


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | *.pyc
3 | venv/
4 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 | 


--------------------------------------------------------------------------------
/photo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SnoopyDevelops/hand_drawn_flowchart_recognition/HEAD/photo.png


--------------------------------------------------------------------------------
/thresh.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SnoopyDevelops/hand_drawn_flowchart_recognition/HEAD/thresh.png


--------------------------------------------------------------------------------
/photo_out.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SnoopyDevelops/hand_drawn_flowchart_recognition/HEAD/photo_out.png


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask~=2.3.2
2 | opencv-python~=4.7.0.72
3 | imutils~=0.5.4
4 | pytesseract~=0.3.10
5 | tabulate~=0.9.0
6 | requests~=2.31.0
7 | 


--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask
2 | 
3 | UPLOAD_FOLDER = 'uploads'
4 | 
5 | app = Flask(__name__)
6 | app.secret_key = "secret key"
7 | app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
8 | app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
9 | 


--------------------------------------------------------------------------------
/data.json:
--------------------------------------------------------------------------------
1 | {"Node": [{"Id": 0, "Name": "DATABASE\n\f", "Position": "Inside", "Shape": "square", "Line": ""}, {"Id": 2, "Name": "LORD BALANCER\n\f", "Position": "Inside", "Shape": "square", "Line": ""}, {"Id": 4, "Name": "SERVER\n\f", "Position": "Inside", "Shape": "square", "Line": ""}]}


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # HandWritten Flowchart Recognition with Flask Backend Service
 2 |  
 3 |  Sample Input Image<br/>
 4 | ![Sample Input Image](photo.png)
 5 | 
 6 | - command line: python flowchart_recognition.py "image file name" [parameters]<br/>
 7 | If image file name were not given, it'll be asking image file name
 8 | 
 9 | - GUI: python GUI.py
10 | 
11 | - flask backend service: python main.py
12 | 
13 | Result
14 | 
15 | | Id | Name | Position | Shape | Line |
16 | | - | - | - | - | - |
17 | | 0 | DATABASE| Inside | square |  |
18 | | 2 | LORD BALANCER | Inside | square | DATABASE |
19 | | 4 | LMN | Inside | square | LORD BALANCER|
20 | 
21 | * Position: inside / outside
22 | * Shape: triangle / square / circle
23 | * Line: connected nodes to go
24 | 
25 | parameters:
26 | 
27 | - padding: the distance range between inside text and boundingRect of shapes
28 | - offset: used to decide direction(AC or BD) of curved lines
29 | - arrow: length of arrow
30 | 
31 | Output Image for Sample Input Image<br/>
32 | ![Output Image](photo_out.png)
33 | 


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | 
 3 | from flask import request, jsonify
 4 | from werkzeug.utils import secure_filename
 5 | 
 6 | from app import app
 7 | from flowchart_recognition import flowchart
 8 | 
 9 | ALLOWED_EXTENSIONS = {'jpg', 'png'}
10 | 
11 | 
12 | def allowed_file(filename):
13 |     return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
14 | 
15 | 
16 | @app.route('/', methods=['POST'])
17 | def upload_file():
18 |     # check if the post request has the file part
19 |     if 'file' not in request.files:
20 |         resp = jsonify({'message': 'No file part in the request', 'status': 400})
21 |         resp.status_code = 400
22 |         return resp
23 | 
24 |     file = request.files['file']
25 |     if file.filename == '':
26 |         resp = jsonify({'message': 'No file selected for uploading', 'status': 400})
27 |         resp.status_code = 400
28 |         return resp
29 | 
30 |     if file and allowed_file(file.filename):
31 |         filename = secure_filename(file.filename)
32 |         path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
33 |         # path = secure_filename(file.filename)
34 |         file.save(path)
35 | 
36 |         if '-p' in request.values.keys():
37 |             padding = int(request.values['-p'])
38 |         else:
39 |             padding = 25
40 | 
41 |         if '-o' in request.values.keys():
42 |             offset = int(request.values['-o'])
43 |         else:
44 |             offset = 10
45 | 
46 |         if '-a' in request.values.keys():
47 |             arrow = int(request.values['-a'])
48 |         else:
49 |             arrow = 30
50 | 
51 |         nodes = flowchart(
52 |             filename=path, padding=padding, offset=offset, arrow=arrow, gui=False
53 |         )
54 | 
55 |         os.remove(path)
56 |         resp = jsonify({'message': 'File successfully uploaded', 'data': nodes, 'status': 200})
57 |         resp.status_code = 201
58 |         return resp
59 | 
60 |     else:
61 |         resp = jsonify({'message': 'Allowed file types are jpg and png', 'status': 400})
62 |         resp.status_code = 400
63 |         return resp
64 | 
65 | 
66 | if __name__ == "__main__":
67 |     app.run(host="0.0.0.0", port=80)
68 | 


--------------------------------------------------------------------------------
/GUI.py:
--------------------------------------------------------------------------------
 1 | from tkinter import Tk, Label, Button, filedialog, Menu, Toplevel
 2 | 
 3 | import cv2
 4 | 
 5 | from flowchart_recognition import flowchart
 6 | 
 7 | 
 8 | def alert_popup(title, message):
 9 |     """Generate a pop-up window for special messages."""
10 |     root = Tk()
11 |     root.title(title)
12 |     w, h = 400, 200  # popup window width and height
13 |     sw, sh = root.winfo_screenwidth(), root.winfo_screenheight()
14 |     x, y = (sw - w) / 2, (sh - h) / 2
15 |     root.geometry('%dx%d+%d+%d' % (w, h, x, y))
16 |     m = message + '\n'
17 |     w = Label(root, text=m, width=50, height=10)
18 |     w.pack()
19 |     b = Button(root, text="OK", command=root.destroy, width=5)
20 |     b.pack()
21 | 
22 | 
23 | def database():
24 |     # pytesseract.tesseract_cmd = 'C:\\Program Files\\Tesseract-OCR\\tesseract.exe'
25 |     try:
26 |         filename = root.filename
27 |     except:
28 |         alert_popup('Error message!!!', 'Select a file first')
29 |         return
30 |     if filename == '':
31 |         alert_popup('Error message!!!', 'Select a file first')
32 |         return
33 | 
34 |     popup = Toplevel()
35 |     Label(popup, text="Processing..").grid(row=0, column=0)
36 | 
37 |     popup.pack_slaves()
38 |     popup.update()
39 |     flowchart(filename)
40 |     popup.destroy()
41 | 
42 |     root.filename = ''
43 |     alert_popup('Completed', 'Your task has been completed..')
44 | 
45 | 
46 | def OpenFile():
47 |     root.filename = filedialog.askopenfilename(
48 |         title="Select file",
49 |         filetypes=(("jpeg files", "*.jpg"), ("png files", "*.png"), ("all files", "*.*"))
50 |     )
51 |     cv2.namedWindow("output", cv2.WINDOW_NORMAL)  # Create window with freedom of dimensions
52 |     cv2.imshow("output", cv2.resize(cv2.imread(root.filename), (2000, 750)))  # Show image
53 |     cv2.waitKey(0)
54 | 
55 | 
56 | def About():
57 |     popup = Toplevel()
58 |     Label(popup, text="Recognition of Hand Drawn Flowcharts").grid(row=10, column=1)
59 | 
60 | 
61 | def show():
62 |     root.filename = filedialog.askopenfilename(
63 |         title="Select file",
64 |         filetypes=(("png files", "*.png"), ("jpeg files", "*.jpg"), ("all files", "*.*"))
65 |     )
66 | 
67 | 
68 | if __name__ == "__main__":
69 |     root = Tk()
70 |     root.geometry('500x400')
71 |     root.title("hand_drawn_flowchart_recognition")
72 | 
73 |     menu = Menu(root)
74 |     root.config(menu=menu)
75 | 
76 |     filemenu = Menu(menu)
77 |     menu.add_cascade(label="File", menu=filemenu)
78 | 
79 |     filemenu.add_command(label="Open...", command=OpenFile)
80 |     filemenu.add_separator()
81 | 
82 |     filemenu.add_command(label="Exit", command=root.destroy)
83 | 
84 |     help_menu = Menu(menu)
85 |     menu.add_cascade(label="Help", menu=help_menu)
86 |     help_menu.add_command(label="About...")
87 | 
88 |     label_0 = Label(root, text="Hand-Drawn Flowchart \n Recognition", width=20, font=("bold", 20))
89 |     label_0.place(x=100, y=50)
90 | 
91 |     Button(root, text='Select Image', width=30, bg='blue', fg='white', command=show).place(x=150, y=150)
92 |     Button(root, text='Start', width=30, bg='brown', fg='white', command=database).place(x=150, y=250)
93 | 
94 |     root.mainloop()
95 | 


--------------------------------------------------------------------------------
/flowchart_recognition.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from argparse import ArgumentParser
  3 | from json import dump
  4 | from math import sqrt
  5 | from operator import itemgetter
  6 | from sys import argv
  7 | 
  8 | import cv2
  9 | import numpy as np
 10 | from imutils import resize
 11 | from numpy.linalg import norm
 12 | from pytesseract import pytesseract
 13 | from tabulate import tabulate
 14 | 
 15 | 
 16 | def contour_circumstance(contour):
 17 |     temp, circumstance = contour[0], 0
 18 |     for point in contour[1:]:
 19 |         circumstance += sqrt((point[0][0] - temp[0][0]) ** 2 + (point[0][1] - temp[0][1]) ** 2)
 20 |         temp = point
 21 |     return circumstance
 22 | 
 23 | 
 24 | def distance(A, B, P):
 25 |     """ segment line AB, point P, where each one is an array([x, y]) """
 26 |     if np.arccos(np.dot((P - A) / norm(P - A), (B - A) / norm(B - A))) > np.pi / 2:
 27 |         return norm(P - A)
 28 |     if np.arccos(np.dot((P - B) / norm(P - B), (A - B) / norm(A - B))) > np.pi / 2:
 29 |         return norm(P - B)
 30 |     return norm(np.cross(A - B, A - P)) / norm(B - A)
 31 | 
 32 | 
 33 | def flowchart(filename, padding=25, offset=10, arrow=30, gui=True):
 34 |     img = cv2.imread(filename, 0)
 35 | 
 36 |     thresh = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 111, 48)
 37 |     cv2.imwrite('thresh.png', thresh)
 38 |     img = cv2.imread('thresh.png')
 39 | 
 40 |     image = img.copy()
 41 |     blur = cv2.GaussianBlur(image, (5, 5), 0)
 42 | 
 43 |     # resize the image
 44 |     resized = resize(blur, width=2 * image.shape[1])
 45 |     ratio = 0.5
 46 | 
 47 |     # convert the resized image to grayscale
 48 |     gray = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)
 49 | 
 50 |     # histogram
 51 |     clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(4, 4))
 52 |     cl = clahe.apply(gray)
 53 | 
 54 |     denoised_cl = cv2.fastNlMeansDenoising(cl, None, 270, 7, 21)
 55 | 
 56 |     # Otsu's thresholding
 57 |     blur = cv2.GaussianBlur(denoised_cl, (25, 25), 0)
 58 | 
 59 |     _, thresh = cv2.threshold(blur, 10, 100, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
 60 | 
 61 |     # detect edges
 62 |     edge = cv2.Canny(thresh, 0, 255, apertureSize=3)
 63 | 
 64 |     # opening image for less noise
 65 |     blur = cv2.GaussianBlur(edge, (9, 9), 0)
 66 |     kernel = np.ones((3, 3), np.uint8)
 67 |     opening = cv2.morphologyEx(blur, cv2.MORPH_OPEN, kernel)
 68 | 
 69 |     # find contours in the thresholded image and initialize the shape detector
 70 |     contours, _ = cv2.findContours(opening.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 71 | 
 72 |     # construct the list of bounding boxes
 73 |     boundingBoxes = [cv2.boundingRect(contour) for contour in contours]
 74 |     contours, boundingBoxes = zip(*sorted(zip(contours, boundingBoxes), key=lambda b: b[1][1], reverse=False))
 75 | 
 76 |     # contours = contours[0] if imutils.is_cv2() else contours[1]
 77 | 
 78 |     nodes, index, outside_texts, shapes, arrow_lines = {}, 1, {}, [], {}
 79 |     # loop over the contours
 80 |     for idx, contour in enumerate(contours):
 81 |         # compute the center of the contour, then detect the name of the shape using only the contour
 82 |         if len(contour) >= 5:
 83 |             area = cv2.contourArea(contour)
 84 |             _, (MA, ma), angle = cv2.fitEllipse(contour)
 85 |             perimeter = cv2.arcLength(contour, True)
 86 |             approx = cv2.approxPolyDP(contour, 0.04 * perimeter, True)
 87 | 
 88 |             # (x,y) be the top-left coordinate of the rectangle and (w,h) be its width and height.
 89 |             x, y, w, h = cv2.boundingRect(approx)
 90 |             x = int(x * ratio)
 91 |             y = int(y * ratio)
 92 |             w = int(w * ratio)
 93 |             h = int(h * ratio)
 94 | 
 95 |             if (int(ma) < int(0.5 * resized.shape[0]) & int(MA) < int(0.5 * resized.shape[1])) or \
 96 |                     area > 0.001 * (image.shape[0] * image.shape[1]):
 97 |                 M = cv2.moments(contour)
 98 |                 cx = int((M["m10"] / (M["m00"])) * ratio)
 99 |                 cy = int((M["m01"] / (M["m00"])) * ratio)
100 | 
101 |                 circumstance = contour_circumstance(contour)
102 | 
103 |                 if area / circumstance < 30:
104 |                     shape = 'arrow'
105 |                 else:
106 |                     perimeter = cv2.arcLength(contour, True)
107 |                     approx = cv2.approxPolyDP(contour, 0.04 * perimeter, True)
108 | 
109 |                     # if 3 vertices, triangle
110 |                     if len(approx) == 3:
111 |                         shape = "triangle"
112 | 
113 |                     # if 4 vertices, square
114 |                     elif len(approx) == 4:
115 |                         shape = "square"
116 | 
117 |                     # otherwise, we assume the shape is a circle
118 |                     else:
119 |                         shape = "circle"
120 | 
121 |                 # multiply the contour (x, y)-coordinates by the resize ratio,
122 |                 # then draw the contours and the name of the  shape on the image
123 |                 contour = contour.astype("float")
124 |                 contour *= ratio
125 |                 contour = contour.astype("int")
126 | 
127 |                 if shape in ['square', 'circle', 'triangle']:
128 |                     shapes.append(idx)
129 | 
130 |                     cropped = img.copy()
131 |                     cv2.drawContours(cropped, [contour], -1, (255, 255, 255), padding)
132 |                     cropped = cropped[y:y + h, x:x + w]
133 |                     text = pytesseract.image_to_string(cropped)
134 |                     # print(text)
135 |                     if text == '':
136 |                         position = 'Outside'
137 |                     else:
138 |                         position = 'Inside'
139 | 
140 |                     nodes[idx] = {
141 |                         'Id': idx,
142 |                         'Name': text,
143 |                         'Position': position,
144 |                         'Shape': shape,
145 |                         'Line': []
146 |                     }
147 |                     index += 1
148 |                 else:
149 |                     margin = 10
150 |                     if y <= margin or x <= margin or y + h >= img.shape[1] - margin or x + w >= img.shape[0] - margin:
151 |                         continue
152 |                     text_offset = 5
153 |                     cropped = img[y - text_offset:y + h + text_offset, x - text_offset:x + w + text_offset]
154 |                     text = pytesseract.image_to_string(cropped)
155 |                     if text != '':
156 |                         shape = 'text'
157 |                         outside_texts[idx] = text
158 |                     else:
159 |                         # print(idx)
160 |                         A, B, C, D = (x, y), (x, y + h), (x + w, y + h), (x + w, y)
161 |                         start_point, end_point = None, None
162 |                         for point in contour:
163 |                             if point[0][0] < x + offset and point[0][1] < y + offset:
164 |                                 start_point = A
165 |                                 # print('A')
166 |                                 break
167 |                             elif point[0][0] < x + offset and point[0][1] > y + h - offset:
168 |                                 start_point = B
169 |                                 # print('B')
170 |                                 break
171 |                         if start_point is None:
172 |                             start_point = A
173 |                             # print(start_point)
174 | 
175 |                         for point in contour:
176 |                             if point[0][0] > x + w - offset and point[0][1] > y + h - offset:
177 |                                 end_point = C
178 |                                 # print('C')
179 |                                 break
180 |                             elif point[0][0] > x + w - offset and point[0][1] < y + offset:
181 |                                 end_point = D
182 |                                 # print('D')
183 |                                 break
184 |                         if end_point is None:
185 |                             end_point = C
186 |                             # print(end_point)
187 | 
188 |                         start_contour_count, end_contour_count = 0, 0
189 |                         for point in contour:
190 |                             if (point[0][0] - start_point[0]) ** 2 + (point[0][1] - start_point[1]) ** 2 < arrow ** 2:
191 |                                 start_contour_count += 1
192 |                             elif (point[0][0] - end_point[0]) ** 2 + (point[0][1] - end_point[1]) ** 2 < arrow ** 2:
193 |                                 end_contour_count += 1
194 | 
195 |                         # print(idx, start_point, end_point)
196 |                         if start_contour_count > end_contour_count:
197 |                             # print(start_contour_count, end_contour_count)
198 |                             reverse = True
199 |                         else:
200 |                             reverse = False
201 |                         # print(idx, reverse)
202 |                         arrow_lines[idx] = (np.array(start_point), np.array(end_point), reverse)
203 | 
204 |                 # print(idx, shape, area/circumstance)
205 |                 cv2.drawContours(image, [contour], -1, (0, 255, 0), 2)
206 |                 cv2.putText(
207 |                     img=image, text=' '.join([str(idx), shape]), org=(cx + 10, cy - 20),
208 |                     fontFace=cv2.FONT_HERSHEY_COMPLEX_SMALL, fontScale=1, color=(255, 0, 0), thickness=2
209 |                 )
210 |                 cv2.putText(
211 |                     img=image, text=text, org=(cx + 10, cy + 40),
212 |                     fontFace=cv2.FONT_HERSHEY_COMPLEX_SMALL, fontScale=1, color=(255, 0, 0), thickness=2
213 |                 )
214 | 
215 |         else:
216 |             cv2.drawContours(image, contours, -1, (0, 0, 0), -1)
217 | 
218 |     l = len(contours)
219 |     for i in nodes.keys():
220 |         if nodes[i]['Name'] == '':
221 |             for j in range(1, max(i + 1, l - i)):
222 |                 mn, mx = min(i + j, l - 1), max(0, i - j)
223 | 
224 |                 if mn in outside_texts.keys():
225 |                     nodes[i]['Name'] = outside_texts[mn]
226 |                     break
227 |                 elif mx in outside_texts.keys():
228 |                     nodes[i]['Name'] = outside_texts[mx]
229 |                     break
230 | 
231 |     for arrow_line in arrow_lines.keys():
232 |         start_point, end_point, reverse = arrow_lines[arrow_line]
233 |         # print(arrow_line, start_point, end_point, reverse)
234 |         start_point *= 2
235 |         end_point *= 2
236 | 
237 |         start_distances, end_distances = {}, {}
238 |         for i in range(l):
239 |             if i in shapes:
240 |                 x, y, w, h = boundingBoxes[i]
241 |                 A, B, C, D = np.array((x, y)), np.array((x, y + h)), np.array((x + w, y + h)), np.array((x + w, y))
242 | 
243 |                 start_distance_bc = distance(B, C, start_point)
244 |                 start_distance_cd = distance(C, D, start_point)
245 |                 start_distances[i] = min(start_distance_bc, start_distance_cd)
246 | 
247 |                 end_distance_ab = distance(A, B, end_point)
248 |                 end_distance_da = distance(D, A, end_point)
249 |                 end_distances[i] = min(end_distance_ab, end_distance_da)
250 | 
251 |         if len(start_distances) > 0:
252 |             start_node, _ = sorted(start_distances.items(), key=itemgetter(1))[0]
253 |             end_node, _ = sorted(end_distances.items(), key=itemgetter(1))[0]
254 |             # print(arrow_line, start_node, end_node)
255 | 
256 |             if reverse:
257 |                 nodes[end_node]['Line'].append(nodes[start_node]['Name'])
258 |             else:
259 |                 nodes[start_node]['Line'].append(nodes[end_node]['Name'])
260 | 
261 |     for node in nodes.keys():
262 |         nodes[node]['Line'] = ', '.join(nodes[node]['Line'])
263 |     nodes = list(nodes.values())
264 |     print(tabulate(nodes, headers='keys'))
265 | 
266 |     filename = filename.replace('.jpg', '_out.jpg')
267 |     filename = filename.replace('.png', '_out.png')
268 |     cv2.imwrite(filename, image)
269 |     with open('data.json', 'w') as file:
270 |         dump({'Node': nodes}, file)
271 | 
272 |     if gui:
273 |         cv2.imshow("Image", image)
274 |         cv2.waitKey(0)
275 | 
276 |     try:
277 |         os.remove("text.png")
278 |     except:
279 |         pass
280 | 
281 |     return nodes
282 | 
283 | 
284 | if __name__ == '__main__':
285 |     if len(argv) > 1:
286 |         ap = ArgumentParser()
287 | 
288 |         ap.add_argument('-f', '--filename', required=True)
289 |         ap.add_argument('-p', '--padding', type=int, default=25, required=False)
290 |         ap.add_argument('-o', '--offset', type=int, default=10, required=False)
291 |         ap.add_argument('-a', '--arrow', type=int, default=30, required=False)
292 | 
293 |         args = ap.parse_args()
294 | 
295 |         flowchart(**vars(args))
296 |     else:
297 |         input_file = input('image file: ')
298 |         flowchart(filename=input_file)
299 | 


--------------------------------------------------------------------------------