├── Dependencies_list.txt ├── app.py ├── collect-data.py ├── image_processing.py ├── length.py ├── preprocessing.py └── train.py /Dependencies_list.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.4.0 2 | alabaster==0.7.12 3 | asn1crypto==0.24.0 4 | astor==0.7.1 5 | astroid==2.0.4 6 | Babel==2.6.0 7 | backcall==0.1.0 8 | bleach==3.0.2 9 | certifi==2018.10.15 10 | cffi==1.11.5 11 | chardet==3.0.4 12 | cloudpickle==0.6.1 13 | cryptography==2.3.1 14 | cryptography-vectors==2.3.1 15 | cycler==0.10.0 16 | decorator==4.3.0 17 | defusedxml==0.5.0 18 | docutils==0.14 19 | entrypoints==0.2.3 20 | gast==0.2.0 21 | grpcio==1.14.1 22 | h5py==2.8.0 23 | html5lib==1.0.1 24 | hunspell==0.5.5 25 | idna==2.7 26 | imagesize==1.1.0 27 | ipykernel==5.1.0 28 | ipython==7.1.1 29 | ipython-genutils==0.2.0 30 | ipywidgets==7.4.2 31 | isort==4.3.4 32 | jedi==0.13.1 33 | jeepney==0.4 34 | Jinja2==2.10 35 | jsonschema==2.6.0 36 | jupyter==1.0.0 37 | jupyter-client==5.2.3 38 | jupyter-console==6.0.0 39 | jupyter-core==4.4.0 40 | Keras==2.2.4 41 | Keras-Applications==1.0.6 42 | Keras-Preprocessing==1.0.5 43 | keras-resnet==0.1.0 44 | keyring==16.0.0 45 | kiwisolver==1.0.1 46 | lazy-object-proxy==1.3.1 47 | llvmlite==0.25.0 48 | lxml==4.2.5 49 | Markdown==2.6.11 50 | MarkupSafe==1.1.0 51 | mkl-fft==1.0.6 52 | mkl-random==1.0.1 53 | nbconvert==5.4.0 54 | nbformat==4.4.0 55 | notebook==5.7.0 56 | numba==0.40.0 57 | numexpr==2.6.8 58 | numpy==1.15.4 59 | numpydoc==0.8.0 60 | olefile==0.46 61 | opencv-python==3.4.3.18 62 | packaging==18.0 63 | pandas==0.23.4 64 | pandas-datareader==0.7.0 65 | pandocfilters==1.4.2 66 | parso==0.3.1 67 | pexpect==4.6.0 68 | pickleshare==0.7.5 69 | prometheus-client==0.4.2 70 | prompt-toolkit==2.0.7 71 | protobuf==3.6.1 72 | psutil==5.4.8 73 | ptyprocess==0.6.0 74 | pycodestyle==2.4.0 75 | pycparser==2.19 76 | pyflakes==2.0.0 77 | Pygments==2.2.0 78 | pylint==2.1.1 79 | pyOpenSSL==18.0.0 80 | pyparsing==2.3.0 81 | PySocks==1.6.8 82 | python-dateutil==2.7.5 83 | pytz==2018.7 84 | PyYAML==3.13 85 | pyzmq==17.1.2 86 | QtAwesome==0.5.2 87 | qtconsole==4.4.2 88 | QtPy==1.5.2 89 | requests==2.19.1 90 | requests-file==1.4.3 91 | requests-ftp==0.3.1 92 | rope==0.11.0 93 | scikit-learn==0.20.0 94 | scipy==1.1.0 95 | SecretStorage==3.1.0 96 | Send2Trash==1.5.0 97 | simplegeneric==0.8.1 98 | six==1.11.0 99 | snowballstemmer==1.2.1 100 | Sphinx==1.8.1 101 | sphinxcontrib-websupport==1.1.0 102 | spyder==3.3.1 103 | spyder-kernels==0.2.6 104 | tensorboard==1.9.0 105 | tensorflow==1.9.0 106 | tensorflow-gpu==1.10.0 107 | termcolor==1.1.0 108 | terminado==0.8.1 109 | testpath==0.4.2 110 | tornado==5.1.1 111 | traitlets==4.3.2 112 | typed-ast==1.1.0 113 | urllib3==1.23 114 | wcwidth==0.1.7 115 | webencodings==0.5.1 116 | Werkzeug==0.14.1 117 | widgetsnbextension==3.4.2 118 | wrapt==1.10.11 119 | matplotlib==3.0.1 120 | mistune==0.8.4 121 | Pillow==5.2.0 122 | mccabe==0.6.1 123 | matplotlib==3.0.1 124 | mistune==0.8.4 125 | Pillow==5.2.0 126 | mccabe==0.6.1 127 | pyenchant==2.0.0 -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from tkinter.constants import COMMAND 2 | from PIL import Image, ImageTk 3 | import tkinter as tk 4 | import cv2 5 | from keras.models import model_from_json 6 | import operator 7 | from string import ascii_uppercase 8 | import pyttsx3 9 | 10 | 11 | class Application: 12 | def __init__(self): 13 | 14 | # Setup GUI 15 | self.root = tk.Tk() 16 | self.root.title("Indian Sign Language Recognition") 17 | self.root.config(background="#000") 18 | self.root.protocol("WM_DELETE_WINDOW", self.destructor) 19 | 20 | self.directory = "model/" 21 | 22 | self.vs = cv2.VideoCapture(cv2.CAP_DSHOW) 23 | # self.vs.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 24 | # self.vs.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 25 | 26 | self.current_image = None 27 | self.current_image2 = None 28 | 29 | self.json_file = open(self.directory + "model_az.json", "r") 30 | self.model_json = self.json_file.read() 31 | self.json_file.close() 32 | self.loaded_model = model_from_json(self.model_json) 33 | self.loaded_model.load_weights(self.directory + "model_az.h5") 34 | 35 | self.ct = {} 36 | self.ct["blank"] = 0 37 | self.blank_flag = 0 38 | for i in ascii_uppercase: 39 | self.ct[i] = 0 40 | print("Loaded model from disk") 41 | 42 | def talk(): 43 | engine = pyttsx3.init() 44 | engine.say(self.str) 45 | engine.runAndWait() 46 | 47 | def clearWord(): 48 | self.word = "" 49 | 50 | def clearSentence(): 51 | self.str = "" 52 | 53 | self.root.geometry("900x1100") 54 | self.panel = tk.Label(self.root) 55 | self.panel.place(x=135, y=10, width=640, height=480) 56 | self.panel2 = tk.Label(self.root) 57 | self.panel2.place(x=950, y=20, width=310, height=310) 58 | 59 | self.panel3 = tk.Label(self.root) 60 | self.panel3.place(x=420, y=560) 61 | 62 | self.T1 = tk.Label(self.root) 63 | self.T1.place(x=300, y=560) 64 | self.T1.config(text="Character :", font=("Raleway", 10, "bold")) 65 | 66 | self.panel4 = tk.Label(self.root) 67 | self.panel4.place(x=420, y=600) 68 | 69 | self.T2 = tk.Label(self.root) 70 | self.T2.place(x=300, y=600) 71 | self.T2.config(text="Word :", font=("Raleway", 10, "bold")) 72 | 73 | self.panel5 = tk.Label(self.root) 74 | self.panel5.place(x=420, y=640) 75 | 76 | self.T3 = tk.Label(self.root) 77 | self.T3.place(x=300, y=640) 78 | self.T3.config(text="Sentence :", font=("Raleway", 10, "bold")) 79 | 80 | my_button = tk.Button(self.root, text="speak", command=talk) 81 | my_button.place(x=620, y=640) 82 | 83 | clear_button = tk.Button(self.root, text="Clear", command=clearWord) 84 | clear_button.place(x=720, y=600) 85 | 86 | clear_button1 = tk.Button(self.root, text="Clear", command=clearSentence) 87 | clear_button1.place(x=720, y=640) 88 | 89 | self.str = "" 90 | self.word = "" 91 | self.current_symbol = "Empty" 92 | self.photo = "Empty" 93 | self.video_loop() 94 | 95 | def video_loop(self): 96 | ok, frame = self.vs.read() 97 | 98 | if ok: 99 | cv2image = cv2.flip(frame, 1) 100 | x1 = int(0.5 * frame.shape[1]) 101 | y1 = 10 102 | x2 = frame.shape[1] - 10 103 | y2 = int(0.5 * frame.shape[1]) 104 | 105 | cv2.rectangle(cv2image, (x1, y1 + 1), (x2, y2 - 1), (255, 0, 0), 1) 106 | 107 | cv2image = cv2.cvtColor(cv2image, cv2.COLOR_BGR2RGBA) 108 | 109 | self.current_image = Image.fromarray(cv2image) 110 | imgtk = ImageTk.PhotoImage(image=self.current_image) 111 | self.panel.imgtk = imgtk 112 | self.panel.config(image=imgtk) 113 | 114 | cv2image = cv2image[y1 : x1 + 1, y2 : x2 - 1] 115 | 116 | cv2image = cv2.resize(cv2image, (128, 128)) 117 | 118 | gray = cv2.cvtColor(cv2image, cv2.COLOR_BGR2GRAY) 119 | blur = cv2.GaussianBlur(gray, (5, 5), 0) 120 | 121 | ret, res = cv2.threshold( 122 | blur, 90, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU 123 | ) 124 | 125 | self.predict(res) 126 | 127 | self.current_image2 = Image.fromarray(res) 128 | imgtk = ImageTk.PhotoImage(image=self.current_image2) 129 | self.panel2.imgtk = imgtk 130 | self.panel2.config(image=imgtk) 131 | self.panel3.config(text=self.current_symbol, font=("Raleway", 10)) 132 | self.panel4.config(text=self.word, font=("Raleway", 10)) 133 | self.panel5.config(text=self.str, font=("Raleway", 10)) 134 | 135 | self.root.after(17, self.video_loop) 136 | 137 | def predict(self, test_image): 138 | test_image = cv2.resize(test_image, (128, 128)) 139 | result = self.loaded_model.predict(test_image.reshape(1, 128, 128, 1)) 140 | 141 | prediction = {} 142 | prediction["blank"] = result[0][0] 143 | index = 1 144 | 145 | for i in ascii_uppercase: 146 | prediction[i] = result[0][index] 147 | index += 1 148 | 149 | # LAYER 1 150 | prediction = sorted( 151 | prediction.items(), key=operator.itemgetter(1), reverse=True 152 | ) 153 | self.current_symbol = prediction[0][0] 154 | 155 | if self.current_symbol == "blank": 156 | for i in ascii_uppercase: 157 | self.ct[i] = 0 158 | self.ct[self.current_symbol] += 1 159 | if self.ct[self.current_symbol] > 20: 160 | for i in ascii_uppercase: 161 | if i == self.current_symbol: 162 | continue 163 | tmp = self.ct[self.current_symbol] - self.ct[i] 164 | if tmp < 0: 165 | tmp *= -1 166 | if tmp <= 20: 167 | self.ct["blank"] = 0 168 | for i in ascii_uppercase: 169 | self.ct[i] = 0 170 | return 171 | self.ct["blank"] = 0 172 | for i in ascii_uppercase: 173 | self.ct[i] = 0 174 | if self.current_symbol == "blank": 175 | if self.blank_flag == 0: 176 | self.blank_flag = 1 177 | if len(self.str) > 0: 178 | self.str += " " 179 | self.str += self.word 180 | self.word = "" 181 | else: 182 | if len(self.str) > 16: 183 | self.str = "" 184 | self.blank_flag = 0 185 | self.word += self.current_symbol 186 | 187 | def destructor(self): 188 | print("Closing Application...") 189 | self.root.destroy() 190 | self.vs.release() 191 | cv2.destroyAllWindows() 192 | 193 | def destructor1(self): 194 | print("Closing Application...") 195 | self.root1.destroy() 196 | 197 | 198 | print("Starting Application...") 199 | pba = Application() 200 | pba.root.mainloop() 201 | -------------------------------------------------------------------------------- /collect-data.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import os 4 | import string 5 | 6 | 7 | # Create the directory structure 8 | if not os.path.exists("myData"): 9 | os.makedirs("myData") 10 | if not os.path.exists("myData/train"): 11 | os.makedirs("myData/train") 12 | if not os.path.exists("myData/test"): 13 | os.makedirs("myData/test") 14 | 15 | for i in range(1): 16 | if not os.path.exists("myData/train/" + str(i)): 17 | os.makedirs("myData/train/" + str(i)) 18 | if not os.path.exists("myData/test/" + str(i)): 19 | os.makedirs("myData/test/" + str(i)) 20 | 21 | # If you want to make only specific character directory folder 22 | # string.ascii_uppercase = ['C', 'I', 'L', 'O', 'U', 'V'] 23 | 24 | for i in string.ascii_uppercase: 25 | if not os.path.exists("myData/train/" + i): 26 | os.makedirs("myData/train/" + i) 27 | if not os.path.exists("myData/test/" + i): 28 | os.makedirs("myData/test/" + i) 29 | 30 | 31 | # Train or test 32 | mode = "train" 33 | directory = "myData/" + mode + "/" 34 | minValue = 90 35 | 36 | cap = cv2.VideoCapture(cv2.CAP_DSHOW) 37 | cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 38 | cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 39 | interrupt = -1 40 | 41 | while True: 42 | _, frame = cap.read() 43 | # Simulating mirror image 44 | frame = cv2.flip(frame, 1) 45 | 46 | # Getting count of existing images 47 | count = { 48 | "0": len(os.listdir(directory + "/0")), 49 | # "1": len(os.listdir(directory + "/1")), 50 | "a": len(os.listdir(directory + "/A")), 51 | "b": len(os.listdir(directory + "/B")), 52 | "c": len(os.listdir(directory + "/C")), 53 | "d": len(os.listdir(directory + "/D")), 54 | "e": len(os.listdir(directory + "/E")), 55 | "f": len(os.listdir(directory + "/F")), 56 | "g": len(os.listdir(directory + "/G")), 57 | "h": len(os.listdir(directory + "/H")), 58 | "i": len(os.listdir(directory + "/I")), 59 | "j": len(os.listdir(directory + "/J")), 60 | "k": len(os.listdir(directory + "/K")), 61 | "l": len(os.listdir(directory + "/L")), 62 | "m": len(os.listdir(directory + "/M")), 63 | "n": len(os.listdir(directory + "/N")), 64 | "o": len(os.listdir(directory + "/O")), 65 | "p": len(os.listdir(directory + "/P")), 66 | "q": len(os.listdir(directory + "/Q")), 67 | "r": len(os.listdir(directory + "/R")), 68 | "s": len(os.listdir(directory + "/S")), 69 | "t": len(os.listdir(directory + "/T")), 70 | "u": len(os.listdir(directory + "/U")), 71 | "v": len(os.listdir(directory + "/V")), 72 | "w": len(os.listdir(directory + "/W")), 73 | "x": len(os.listdir(directory + "/X")), 74 | "y": len(os.listdir(directory + "/Y")), 75 | "z": len(os.listdir(directory + "/Z")), 76 | } 77 | 78 | # Printing the count in each set to the screen 79 | cv2.putText( 80 | frame, 81 | "0 : " + str(count["0"]), 82 | (10, 80), 83 | cv2.FONT_HERSHEY_PLAIN, 84 | 1, 85 | (0, 255, 255), 86 | 1, 87 | ) 88 | # cv2.putText( 89 | # frame, 90 | # "1 : " + str(count["1"]), 91 | # (10, 90), 92 | # cv2.FONT_HERSHEY_PLAIN, 93 | # 1, 94 | # (0, 255, 255), 95 | # 1, 96 | # ) 97 | cv2.putText( 98 | frame, 99 | "a : " + str(count["a"]), 100 | (10, 100), 101 | cv2.FONT_HERSHEY_PLAIN, 102 | 1, 103 | (0, 255, 255), 104 | 1, 105 | ) 106 | cv2.putText( 107 | frame, 108 | "b : " + str(count["b"]), 109 | (10, 110), 110 | cv2.FONT_HERSHEY_PLAIN, 111 | 1, 112 | (0, 255, 255), 113 | 1, 114 | ) 115 | cv2.putText( 116 | frame, 117 | "c : " + str(count["c"]), 118 | (10, 120), 119 | cv2.FONT_HERSHEY_PLAIN, 120 | 1, 121 | (0, 255, 255), 122 | 1, 123 | ) 124 | cv2.putText( 125 | frame, 126 | "d : " + str(count["d"]), 127 | (10, 130), 128 | cv2.FONT_HERSHEY_PLAIN, 129 | 1, 130 | (0, 255, 255), 131 | 1, 132 | ) 133 | cv2.putText( 134 | frame, 135 | "e : " + str(count["e"]), 136 | (10, 140), 137 | cv2.FONT_HERSHEY_PLAIN, 138 | 1, 139 | (0, 255, 255), 140 | 1, 141 | ) 142 | cv2.putText( 143 | frame, 144 | "f : " + str(count["f"]), 145 | (10, 150), 146 | cv2.FONT_HERSHEY_PLAIN, 147 | 1, 148 | (0, 255, 255), 149 | 1, 150 | ) 151 | cv2.putText( 152 | frame, 153 | "g : " + str(count["g"]), 154 | (10, 160), 155 | cv2.FONT_HERSHEY_PLAIN, 156 | 1, 157 | (0, 255, 255), 158 | 1, 159 | ) 160 | cv2.putText( 161 | frame, 162 | "h : " + str(count["h"]), 163 | (10, 170), 164 | cv2.FONT_HERSHEY_PLAIN, 165 | 1, 166 | (0, 255, 255), 167 | 1, 168 | ) 169 | cv2.putText( 170 | frame, 171 | "i : " + str(count["i"]), 172 | (10, 180), 173 | cv2.FONT_HERSHEY_PLAIN, 174 | 1, 175 | (0, 255, 255), 176 | 1, 177 | ) 178 | cv2.putText( 179 | frame, 180 | "j : " + str(count["j"]), 181 | (10, 190), 182 | cv2.FONT_HERSHEY_PLAIN, 183 | 1, 184 | (0, 255, 255), 185 | 1, 186 | ) 187 | cv2.putText( 188 | frame, 189 | "k : " + str(count["k"]), 190 | (10, 200), 191 | cv2.FONT_HERSHEY_PLAIN, 192 | 1, 193 | (0, 255, 255), 194 | 1, 195 | ) 196 | cv2.putText( 197 | frame, 198 | "l : " + str(count["l"]), 199 | (10, 210), 200 | cv2.FONT_HERSHEY_PLAIN, 201 | 1, 202 | (0, 255, 255), 203 | 1, 204 | ) 205 | cv2.putText( 206 | frame, 207 | "m : " + str(count["m"]), 208 | (10, 220), 209 | cv2.FONT_HERSHEY_PLAIN, 210 | 1, 211 | (0, 255, 255), 212 | 1, 213 | ) 214 | cv2.putText( 215 | frame, 216 | "n : " + str(count["n"]), 217 | (10, 230), 218 | cv2.FONT_HERSHEY_PLAIN, 219 | 1, 220 | (0, 255, 255), 221 | 1, 222 | ) 223 | cv2.putText( 224 | frame, 225 | "o : " + str(count["o"]), 226 | (10, 240), 227 | cv2.FONT_HERSHEY_PLAIN, 228 | 1, 229 | (0, 255, 255), 230 | 1, 231 | ) 232 | cv2.putText( 233 | frame, 234 | "p : " + str(count["p"]), 235 | (10, 250), 236 | cv2.FONT_HERSHEY_PLAIN, 237 | 1, 238 | (0, 255, 255), 239 | 1, 240 | ) 241 | cv2.putText( 242 | frame, 243 | "q : " + str(count["q"]), 244 | (10, 260), 245 | cv2.FONT_HERSHEY_PLAIN, 246 | 1, 247 | (0, 255, 255), 248 | 1, 249 | ) 250 | cv2.putText( 251 | frame, 252 | "r : " + str(count["r"]), 253 | (10, 270), 254 | cv2.FONT_HERSHEY_PLAIN, 255 | 1, 256 | (0, 255, 255), 257 | 1, 258 | ) 259 | cv2.putText( 260 | frame, 261 | "s : " + str(count["s"]), 262 | (10, 280), 263 | cv2.FONT_HERSHEY_PLAIN, 264 | 1, 265 | (0, 255, 255), 266 | 1, 267 | ) 268 | cv2.putText( 269 | frame, 270 | "t : " + str(count["t"]), 271 | (10, 290), 272 | cv2.FONT_HERSHEY_PLAIN, 273 | 1, 274 | (0, 255, 255), 275 | 1, 276 | ) 277 | cv2.putText( 278 | frame, 279 | "u : " + str(count["u"]), 280 | (10, 300), 281 | cv2.FONT_HERSHEY_PLAIN, 282 | 1, 283 | (0, 255, 255), 284 | 1, 285 | ) 286 | cv2.putText( 287 | frame, 288 | "v : " + str(count["v"]), 289 | (10, 310), 290 | cv2.FONT_HERSHEY_PLAIN, 291 | 1, 292 | (0, 255, 255), 293 | 1, 294 | ) 295 | cv2.putText( 296 | frame, 297 | "w : " + str(count["w"]), 298 | (10, 320), 299 | cv2.FONT_HERSHEY_PLAIN, 300 | 1, 301 | (0, 255, 255), 302 | 1, 303 | ) 304 | cv2.putText( 305 | frame, 306 | "x : " + str(count["x"]), 307 | (10, 330), 308 | cv2.FONT_HERSHEY_PLAIN, 309 | 1, 310 | (0, 255, 255), 311 | 1, 312 | ) 313 | cv2.putText( 314 | frame, 315 | "y : " + str(count["y"]), 316 | (10, 340), 317 | cv2.FONT_HERSHEY_PLAIN, 318 | 1, 319 | (0, 255, 255), 320 | 1, 321 | ) 322 | cv2.putText( 323 | frame, 324 | "z : " + str(count["z"]), 325 | (10, 350), 326 | cv2.FONT_HERSHEY_PLAIN, 327 | 1, 328 | (0, 255, 255), 329 | 1, 330 | ) 331 | 332 | # Coordinates of the ROI 333 | x1 = int(0.5 * frame.shape[1]) 334 | y1 = 10 335 | x2 = frame.shape[1] - 10 336 | y2 = int(0.5 * frame.shape[1]) 337 | 338 | # Drawing the ROI 339 | # The increment/decrement by 1 is to compensate for the bounding box 340 | cv2.rectangle(frame, (x1, y1 + 4), (x2, y2 + 2), (255, 0, 0), 1) 341 | # Extracting the ROI 342 | roi = frame[y1 : x1 + 1, y2 : x2 - 1] 343 | # roi = frame[10:410, 220:520] 344 | roi = cv2.resize(roi, (128, 128)) 345 | 346 | cv2.imshow("Frame", frame) 347 | gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) 348 | 349 | blur = cv2.GaussianBlur(gray, (5, 5), 0) 350 | 351 | # th3 = cv2.adaptiveThreshold( 352 | # blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 2) 353 | 354 | ret, test_image = cv2.threshold( 355 | blur, minValue, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU 356 | ) 357 | 358 | test_image = cv2.resize(test_image, (310, 310)) 359 | cv2.imshow("test", test_image) 360 | 361 | interrupt = cv2.waitKey(10) 362 | if interrupt & 0xFF == 27: # esc key 363 | break 364 | if interrupt & 0xFF == ord("0"): 365 | cv2.imwrite(directory + "0/" + "0" + str(count["0"]) + ".jpg", roi) 366 | # if interrupt & 0xFF == ord("1"): 367 | # cv2.imwrite(directory + "1/" + "1" + str(count["1"]) + ".jpg", roi) 368 | if interrupt & 0xFF == ord("a"): 369 | cv2.imwrite(directory + "A/" + "A" + str(count["a"]) + ".jpg", roi) 370 | if interrupt & 0xFF == ord("b"): 371 | cv2.imwrite(directory + "B/" + "B" + str(count["b"]) + ".jpg", roi) 372 | if interrupt & 0xFF == ord("c"): 373 | cv2.imwrite(directory + "C/" + "C" + str(count["c"]) + ".jpg", roi) 374 | if interrupt & 0xFF == ord("d"): 375 | cv2.imwrite(directory + "D/" + "D" + str(count["d"]) + ".jpg", roi) 376 | if interrupt & 0xFF == ord("e"): 377 | cv2.imwrite(directory + "E/" + "E" + str(count["e"]) + ".jpg", roi) 378 | if interrupt & 0xFF == ord("f"): 379 | cv2.imwrite(directory + "F/" + "F" + str(count["f"]) + ".jpg", roi) 380 | if interrupt & 0xFF == ord("g"): 381 | cv2.imwrite(directory + "G/" + "G" + str(count["g"]) + ".jpg", roi) 382 | if interrupt & 0xFF == ord("h"): 383 | cv2.imwrite(directory + "H/" + "H" + str(count["h"]) + ".jpg", roi) 384 | if interrupt & 0xFF == ord("i"): 385 | cv2.imwrite(directory + "I/" + "I" + str(count["i"]) + ".jpg", roi) 386 | if interrupt & 0xFF == ord("j"): 387 | cv2.imwrite(directory + "J/" + "J" + str(count["j"]) + ".jpg", roi) 388 | if interrupt & 0xFF == ord("k"): 389 | cv2.imwrite(directory + "K/" + "K" + str(count["k"]) + ".jpg", roi) 390 | if interrupt & 0xFF == ord("l"): 391 | cv2.imwrite(directory + "L/" + "L" + str(count["l"]) + ".jpg", roi) 392 | if interrupt & 0xFF == ord("m"): 393 | cv2.imwrite(directory + "M/" + "M" + str(count["m"]) + ".jpg", roi) 394 | if interrupt & 0xFF == ord("n"): 395 | cv2.imwrite(directory + "N/" + "N" + str(count["n"]) + ".jpg", roi) 396 | if interrupt & 0xFF == ord("o"): 397 | cv2.imwrite(directory + "O/" + "O" + str(count["o"]) + ".jpg", roi) 398 | if interrupt & 0xFF == ord("p"): 399 | cv2.imwrite(directory + "P/" + "P" + str(count["p"]) + ".jpg", roi) 400 | if interrupt & 0xFF == ord("q"): 401 | cv2.imwrite(directory + "Q/" + "Q" + str(count["q"]) + ".jpg", roi) 402 | if interrupt & 0xFF == ord("r"): 403 | cv2.imwrite(directory + "R/" + "R" + str(count["r"]) + ".jpg", roi) 404 | if interrupt & 0xFF == ord("s"): 405 | cv2.imwrite(directory + "S/" + "S" + str(count["s"]) + ".jpg", roi) 406 | if interrupt & 0xFF == ord("t"): 407 | cv2.imwrite(directory + "T/" + "T" + str(count["t"]) + ".jpg", roi) 408 | if interrupt & 0xFF == ord("u"): 409 | cv2.imwrite(directory + "U/" + "U" + str(count["u"]) + ".jpg", roi) 410 | if interrupt & 0xFF == ord("v"): 411 | cv2.imwrite(directory + "V/" + "V" + str(count["v"]) + ".jpg", roi) 412 | if interrupt & 0xFF == ord("w"): 413 | cv2.imwrite(directory + "W/" + "W" + str(count["w"]) + ".jpg", roi) 414 | if interrupt & 0xFF == ord("x"): 415 | cv2.imwrite(directory + "X/" + "X" + str(count["x"]) + ".jpg", roi) 416 | if interrupt & 0xFF == ord("y"): 417 | cv2.imwrite(directory + "Y/" + "Y" + str(count["y"]) + ".jpg", roi) 418 | if interrupt & 0xFF == ord("z"): 419 | cv2.imwrite(directory + "Z/" + "Z" + str(count["z"]) + ".jpg", roi) 420 | 421 | cap.release() 422 | cv2.destroyAllWindows() 423 | -------------------------------------------------------------------------------- /image_processing.py: -------------------------------------------------------------------------------- 1 | # STEP 2 (PART1) : CONVERTING RGB IMAGES TO BLACK AND WHITE IMAGES 2 | 3 | import numpy as np 4 | import cv2 5 | 6 | minValue = 70 7 | 8 | 9 | def func(path): 10 | frame = cv2.imread(path) 11 | gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 12 | blur = cv2.GaussianBlur(gray, (5, 5), 2) 13 | 14 | _, res = cv2.threshold(blur, minValue, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) 15 | return res 16 | -------------------------------------------------------------------------------- /length.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | 4 | path = "myProcessData" 5 | train_path = "myProcessdata/train" 6 | test_path = "myProcessdata/test" 7 | 8 | for (dirpath, dirnames, filenames) in os.walk(train_path): 9 | for dirname in dirnames: 10 | print(dirname) 11 | 12 | for (direcpath, direcnames, files) in os.walk(train_path + "/" + dirname): 13 | num = len(files) 14 | print("train : ", num) 15 | 16 | for (direcpath, direcnames, files) in os.walk(test_path + "/" + dirname): 17 | num = len(files) 18 | num += num 19 | print("test : ", num) 20 | -------------------------------------------------------------------------------- /preprocessing.py: -------------------------------------------------------------------------------- 1 | # STEP 2 (PART2): IMAGE PREPROCESSING 2 | # TURNING IMAGE WITH A LOT OF FEATURES INTO A SIMPLE ONE TO FEED IT TO THE NN SO THAT IT WILL REQUIRE LESS COMPUTATIONAL POWER. 3 | 4 | import numpy as np 5 | import cv2 6 | import os 7 | 8 | from image_processing import func 9 | 10 | if not os.path.exists("myProcessData"): 11 | os.makedirs("myProcessData") 12 | if not os.path.exists("myProcessData/train"): 13 | os.makedirs("myProcessData/train") 14 | if not os.path.exists("myProcessData/test"): 15 | os.makedirs("myProcessData/test") 16 | path = "myData/train" 17 | output = "myProcessData" 18 | a = ["label"] 19 | 20 | for i in range(128 * 128): 21 | a.append("pixel" + str(i)) 22 | # outputLine = a.tolist() 23 | 24 | label = 0 25 | var = 0 26 | c1 = 0 27 | c2 = 0 28 | 29 | for (dirpath, dirnames, filenames) in os.walk(path): 30 | for dirname in dirnames: 31 | print(dirname) 32 | for (direcpath, direcnames, files) in os.walk(path + "/" + dirname): 33 | if not os.path.exists(output + "/train/" + dirname): 34 | os.makedirs(output + "/train/" + dirname) 35 | if not os.path.exists(output + "/test/" + dirname): 36 | os.makedirs(output + "/test/" + dirname) 37 | num = 0.70 * len(files) 38 | i = 0 39 | for file in files: 40 | var += 1 41 | actual_path = path + "/" + dirname + "/" + file 42 | output_path = output + "/" + "train/" + dirname + "/" + file 43 | output_path_test = output + "/" + "test/" + dirname + "/" + file 44 | img = cv2.imread(actual_path, 0) 45 | bw_image = func(actual_path) 46 | if i < num: 47 | c1 += 1 48 | cv2.imwrite(output_path, bw_image) 49 | else: 50 | c2 += 1 51 | cv2.imwrite(output_path_test, bw_image) 52 | 53 | i = i + 1 54 | 55 | label = label + 1 56 | 57 | print(var) 58 | print(c1) 59 | print(c2) 60 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | # Importing the Keras libraries and packages 2 | from keras.preprocessing.image import ImageDataGenerator 3 | from keras.models import Sequential 4 | from keras.layers import Convolution2D 5 | from keras.layers import MaxPooling2D 6 | from keras.layers import Flatten 7 | from keras.layers import Dense, Dropout 8 | import os 9 | 10 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 11 | sz = 128 12 | # Step 1 - Building the CNN 13 | 14 | # Initializing the CNN 15 | classifier = Sequential() 16 | 17 | # First convolution layer and pooling 18 | classifier.add(Convolution2D(64, (3, 3), input_shape=(sz, sz, 1), activation="relu")) 19 | classifier.add(MaxPooling2D(pool_size=(2, 2))) 20 | 21 | # Second convolution layer and pooling 22 | classifier.add(Convolution2D(64, (3, 3), activation="relu")) 23 | classifier.add(MaxPooling2D(pool_size=(2, 2))) 24 | 25 | # third convolution layer and pooling 26 | classifier.add(Convolution2D(64, (3, 3), activation="relu")) 27 | classifier.add(MaxPooling2D(pool_size=(2, 2))) 28 | 29 | # If needed..... 30 | # fourth convolution layer and pooling 31 | # classifier.add(Convolution2D(32, (3, 3), activation='relu')) 32 | # classifier.add(MaxPooling2D(pool_size=(2, 2))) 33 | 34 | # Flattening the layers 35 | classifier.add(Flatten()) 36 | 37 | # Adding a fully connected layer 38 | 39 | # classifier.add(Dense(units=128, activation='relu')) 40 | # classifier.add(Dropout(0.5)) 41 | 42 | classifier.add(Dense(units=128, activation="relu")) 43 | classifier.add(Dropout(0.5)) 44 | 45 | classifier.add(Dense(units=27, activation="relu")) 46 | classifier.add(Dropout(0.5)) 47 | 48 | # softmax for more than 2 outputs neuron 49 | classifier.add(Dense(units=27, activation="softmax")) 50 | 51 | # Compiling the CNN 52 | classifier.compile( 53 | optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] 54 | ) # categorical_crossentropy for more than 2 55 | 56 | 57 | # Step 2 - Preparing the train/test data and training the model 58 | classifier.summary() 59 | 60 | train_datagen = ImageDataGenerator( 61 | rescale=1.0 / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True 62 | ) 63 | 64 | test_datagen = ImageDataGenerator(rescale=1.0 / 255) 65 | 66 | training_set = train_datagen.flow_from_directory( 67 | "myProcessdata/train", 68 | target_size=(sz, sz), 69 | batch_size=5, 70 | color_mode="grayscale", 71 | class_mode="categorical", 72 | ) 73 | 74 | test_set = test_datagen.flow_from_directory( 75 | "myProcessData/test", 76 | target_size=(sz, sz), 77 | batch_size=5, 78 | color_mode="grayscale", 79 | class_mode="categorical", 80 | ) 81 | classifier.fit_generator( 82 | training_set, 83 | steps_per_epoch=945, # No of images in training set 84 | epochs=5, 85 | validation_data=test_set, 86 | validation_steps=405, 87 | ) # No of images in test set 88 | 89 | 90 | # Saving the model 91 | model_json = classifier.to_json() 92 | with open("model_az.json", "w") as json_file: 93 | json_file.write(model_json) 94 | print("Model Saved") 95 | classifier.save_weights("model_az.h5") 96 | print("Model saved") 97 | --------------------------------------------------------------------------------