├── .gitignore ├── CODE_OF_CONDUCT.md ├── GUI.jpg ├── Gesture_Controller.py ├── Gesture_Controller_Gloved.py ├── README.md ├── app.py ├── aura.py ├── eyen.py ├── haarcascade_frontalface_default.xml ├── kc.wav ├── main.py ├── mn.png ├── model.yml ├── samvkn.py └── tempCodeRunnerFile.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | Final/__pycache__/Gesture_Controller.cpython-38.pyc 3 | Final/__pycache__/Gesture_Controller.cpython-38.pyc 4 | src/.vs/slnx.sqlite 5 | src/.vs/slnx.sqlite 6 | *.json 7 | *.sqlite 8 | *.sqlite 9 | *.pyc 10 | *.json 11 | .vs/Repo/v16/.suo 12 | .vs/Gesture_Controller/v16/.suo 13 | *.sqlite 14 | *.vs 15 | .vs/slnx.sqlite 16 | *.gif 17 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /GUI.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RAGHURAM5276/VIRTUAL-MOUSE-USING-HAND-GESTURE-RECOGNITION/04f21b436fd236c1ef73e5582d55314a057a36ec/GUI.jpg -------------------------------------------------------------------------------- /Gesture_Controller.py: -------------------------------------------------------------------------------- 1 | # Imports 2 | 3 | import cv2 4 | import mediapipe as mp 5 | import pyautogui 6 | import math 7 | from enum import IntEnum 8 | from ctypes import cast, POINTER 9 | from comtypes import CLSCTX_ALL 10 | from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume 11 | from google.protobuf.json_format import MessageToDict 12 | import screen_brightness_control as sbcontrol 13 | 14 | pyautogui.FAILSAFE = False 15 | mp_drawing = mp.solutions.drawing_utils 16 | mp_hands = mp.solutions.hands 17 | 18 | # Gesture Encodings 19 | 20 | def gest_control(): 21 | class Gest(IntEnum): 22 | FIST = 0 23 | PINKY = 1 24 | RING = 2 25 | MID = 4 26 | LAST3 = 7 27 | INDEX = 8 28 | FIRST2 = 12 29 | LAST4 = 15 30 | THUMB = 16 31 | PALM = 31 32 | 33 | # Extra Mappings 34 | V_GEST = 33 35 | TWO_FINGER_CLOSED = 34 36 | PINCH_MAJOR = 35 37 | PINCH_MINOR = 36 38 | 39 | # Multi-handedness Labels 40 | class HLabel(IntEnum): 41 | MINOR = 0 42 | MAJOR = 1 43 | 44 | # Convert Mediapipe Landmarks to recognizable Gestures 45 | class HandRecog: 46 | def __init__(self, hand_label): 47 | self.finger = 0 48 | self.ori_gesture = Gest.PALM 49 | self.prev_gesture = Gest.PALM 50 | self.frame_count = 0 51 | self.hand_result = None 52 | self.hand_label = hand_label 53 | 54 | def update_hand_result(self, hand_result): 55 | self.hand_result = hand_result 56 | 57 | def get_signed_dist(self, point): 58 | sign = -1 59 | if self.hand_result.landmark[point[0]].y < self.hand_result.landmark[point[1]].y: 60 | sign = 1 61 | dist = (self.hand_result.landmark[point[0]].x - self.hand_result.landmark[point[1]].x)**2 62 | dist += (self.hand_result.landmark[point[0]].y - self.hand_result.landmark[point[1]].y)**2 63 | dist = math.sqrt(dist) 64 | return dist*sign 65 | 66 | def get_dist(self, point): 67 | dist = (self.hand_result.landmark[point[0]].x - self.hand_result.landmark[point[1]].x)**2 68 | dist += (self.hand_result.landmark[point[0]].y - self.hand_result.landmark[point[1]].y)**2 69 | dist = math.sqrt(dist) 70 | return dist 71 | 72 | def get_dz(self,point): 73 | return abs(self.hand_result.landmark[point[0]].z - self.hand_result.landmark[point[1]].z) 74 | 75 | # Function to find Gesture Encoding using current finger_state. 76 | # Finger_state: 1 if finger is open, else 0 77 | def set_finger_state(self): 78 | if self.hand_result == None: 79 | return 80 | 81 | points = [[8,5,0],[12,9,0],[16,13,0],[20,17,0]] 82 | self.finger = 0 83 | self.finger = self.finger | 0 #thumb 84 | for idx,point in enumerate(points): 85 | dist = self.get_signed_dist(point[:2]) 86 | dist2 = self.get_signed_dist(point[1:]) 87 | 88 | try: 89 | ratio = round(dist/dist2,1) 90 | except: 91 | ratio = round(dist1/0.01,1) 92 | 93 | self.finger = self.finger << 1 94 | if ratio > 0.5 : 95 | self.finger = self.finger | 1 96 | 97 | 98 | # Handling Fluctations due to noise 99 | def get_gesture(self): 100 | if self.hand_result == None: 101 | return Gest.PALM 102 | 103 | current_gesture = Gest.PALM 104 | if self.finger in [Gest.LAST3,Gest.LAST4] and self.get_dist([8,4]) < 0.05: 105 | if self.hand_label == HLabel.MINOR : 106 | current_gesture = Gest.PINCH_MINOR 107 | else: 108 | current_gesture = Gest.PINCH_MAJOR 109 | 110 | elif Gest.FIRST2 == self.finger : 111 | point = [[8,12],[5,9]] 112 | dist1 = self.get_dist(point[0]) 113 | dist2 = self.get_dist(point[1]) 114 | ratio = dist1/dist2 115 | if ratio > 1.7: 116 | current_gesture = Gest.V_GEST 117 | else: 118 | if self.get_dz([8,12]) < 0.1: 119 | current_gesture = Gest.TWO_FINGER_CLOSED 120 | else: 121 | current_gesture = Gest.MID 122 | 123 | else: 124 | current_gesture = self.finger 125 | 126 | if current_gesture == self.prev_gesture: 127 | self.frame_count += 1 128 | else: 129 | self.frame_count = 0 130 | 131 | self.prev_gesture = current_gesture 132 | 133 | if self.frame_count > 4 : 134 | self.ori_gesture = current_gesture 135 | return self.ori_gesture 136 | 137 | # Executes commands according to detected gestures 138 | class Controller: 139 | tx_old = 0 140 | ty_old = 0 141 | trial = True 142 | flag = False 143 | grabflag = False 144 | pinchmajorflag = False 145 | pinchminorflag = False 146 | pinchstartxcoord = None 147 | pinchstartycoord = None 148 | pinchdirectionflag = None 149 | prevpinchlv = 0 150 | pinchlv = 0 151 | framecount = 0 152 | prev_hand = None 153 | pinch_threshold = 0.3 154 | 155 | def getpinchylv(hand_result): 156 | dist = round((Controller.pinchstartycoord - hand_result.landmark[8].y)*10,1) 157 | return dist 158 | 159 | def getpinchxlv(hand_result): 160 | dist = round((hand_result.landmark[8].x - Controller.pinchstartxcoord)*10,1) 161 | return dist 162 | 163 | def changesystembrightness(): 164 | currentBrightnessLv = sbcontrol.get_brightness(display=0)[-1]/100.0 165 | currentBrightnessLv += Controller.pinchlv/50.0 166 | if currentBrightnessLv > 1.0: 167 | currentBrightnessLv = 1.0 168 | elif currentBrightnessLv < 0.0: 169 | currentBrightnessLv = 0.0 170 | sbcontrol.fade_brightness(int(100*currentBrightnessLv) , start = sbcontrol.get_brightness(display=0)) 171 | 172 | def changesystemvolume(): 173 | devices = AudioUtilities.GetSpeakers() 174 | interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None) 175 | volume = cast(interface, POINTER(IAudioEndpointVolume)) 176 | currentVolumeLv = volume.GetMasterVolumeLevelScalar() 177 | currentVolumeLv += Controller.pinchlv/50.0 178 | if currentVolumeLv > 1.0: 179 | currentVolumeLv = 1.0 180 | elif currentVolumeLv < 0.0: 181 | currentVolumeLv = 0.0 182 | volume.SetMasterVolumeLevelScalar(currentVolumeLv, None) 183 | 184 | def scrollVertical(): 185 | pyautogui.scroll(120 if Controller.pinchlv>0.0 else -120) 186 | 187 | 188 | def scrollHorizontal(): 189 | pyautogui.keyDown('shift') 190 | pyautogui.keyDown('ctrl') 191 | pyautogui.scroll(-120 if Controller.pinchlv>0.0 else 120) 192 | pyautogui.keyUp('ctrl') 193 | pyautogui.keyUp('shift') 194 | 195 | # Locate Hand to get Cursor Position 196 | # Stabilize cursor by Dampening 197 | def get_position(hand_result): 198 | point = 9 199 | position = [hand_result.landmark[point].x ,hand_result.landmark[point].y] 200 | sx,sy = pyautogui.size() 201 | x_old,y_old = pyautogui.position() 202 | x = int(position[0]*sx) 203 | y = int(position[1]*sy) 204 | if Controller.prev_hand is None: 205 | Controller.prev_hand = x,y 206 | delta_x = x - Controller.prev_hand[0] 207 | delta_y = y - Controller.prev_hand[1] 208 | 209 | distsq = delta_x**2 + delta_y**2 210 | ratio = 1 211 | Controller.prev_hand = [x,y] 212 | 213 | if distsq <= 25: 214 | ratio = 0 215 | elif distsq <= 900: 216 | ratio = 0.07 * (distsq ** (1/2)) 217 | else: 218 | ratio = 2.1 219 | x , y = x_old + delta_x*ratio , y_old + delta_y*ratio 220 | return (x,y) 221 | 222 | def pinch_control_init(hand_result): 223 | Controller.pinchstartxcoord = hand_result.landmark[8].x 224 | Controller.pinchstartycoord = hand_result.landmark[8].y 225 | Controller.pinchlv = 0 226 | Controller.prevpinchlv = 0 227 | Controller.framecount = 0 228 | 229 | # Hold final position for 5 frames to change status 230 | def pinch_control(hand_result, controlHorizontal, controlVertical): 231 | if Controller.framecount == 5: 232 | Controller.framecount = 0 233 | Controller.pinchlv = Controller.prevpinchlv 234 | 235 | if Controller.pinchdirectionflag == True: 236 | controlHorizontal() #x 237 | 238 | elif Controller.pinchdirectionflag == False: 239 | controlVertical() #y 240 | 241 | lvx = Controller.getpinchxlv(hand_result) 242 | lvy = Controller.getpinchylv(hand_result) 243 | 244 | if abs(lvy) > abs(lvx) and abs(lvy) > Controller.pinch_threshold: 245 | Controller.pinchdirectionflag = False 246 | if abs(Controller.prevpinchlv - lvy) < Controller.pinch_threshold: 247 | Controller.framecount += 1 248 | else: 249 | Controller.prevpinchlv = lvy 250 | Controller.framecount = 0 251 | 252 | elif abs(lvx) > Controller.pinch_threshold: 253 | Controller.pinchdirectionflag = True 254 | if abs(Controller.prevpinchlv - lvx) < Controller.pinch_threshold: 255 | Controller.framecount += 1 256 | else: 257 | Controller.prevpinchlv = lvx 258 | Controller.framecount = 0 259 | 260 | def handle_controls(gesture, hand_result): 261 | x,y = None,None 262 | if gesture != Gest.PALM : 263 | x,y = Controller.get_position(hand_result) 264 | 265 | # flag reset 266 | if gesture != Gest.FIST and Controller.grabflag: 267 | Controller.grabflag = False 268 | pyautogui.mouseUp(button = "left") 269 | 270 | if gesture != Gest.PINCH_MAJOR and Controller.pinchmajorflag: 271 | Controller.pinchmajorflag = False 272 | 273 | if gesture != Gest.PINCH_MINOR and Controller.pinchminorflag: 274 | Controller.pinchminorflag = False 275 | 276 | # implementation 277 | if gesture == Gest.V_GEST: 278 | Controller.flag = True 279 | pyautogui.moveTo(x, y, duration = 0.1) 280 | 281 | elif gesture == Gest.FIST: 282 | if not Controller.grabflag : 283 | Controller.grabflag = True 284 | pyautogui.mouseDown(button = "left") 285 | pyautogui.moveTo(x, y, duration = 0.1) 286 | 287 | elif gesture == Gest.MID and Controller.flag: 288 | pyautogui.click() 289 | Controller.flag = False 290 | 291 | elif gesture == Gest.INDEX and Controller.flag: 292 | pyautogui.click(button='right') 293 | Controller.flag = False 294 | 295 | elif gesture == Gest.TWO_FINGER_CLOSED and Controller.flag: 296 | pyautogui.doubleClick() 297 | Controller.flag = False 298 | 299 | elif gesture == Gest.PINCH_MINOR: 300 | if Controller.pinchminorflag == False: 301 | Controller.pinch_control_init(hand_result) 302 | Controller.pinchminorflag = True 303 | Controller.pinch_control(hand_result,Controller.scrollHorizontal, Controller.scrollVertical) 304 | 305 | elif gesture == Gest.PINCH_MAJOR: 306 | if Controller.pinchmajorflag == False: 307 | Controller.pinch_control_init(hand_result) 308 | Controller.pinchmajorflag = True 309 | Controller.pinch_control(hand_result,Controller.changesystembrightness, Controller.changesystemvolume) 310 | 311 | class GestureController: 312 | gc_mode = 0 313 | cap = None 314 | CAM_HEIGHT = None 315 | CAM_WIDTH = None 316 | hr_major = None # Right Hand by default 317 | hr_minor = None # Left hand by default 318 | dom_hand = True 319 | 320 | def __init__(self): 321 | GestureController.gc_mode = 1 322 | GestureController.cap = cv2.VideoCapture(0) 323 | GestureController.CAM_HEIGHT = GestureController.cap.get(cv2.CAP_PROP_FRAME_HEIGHT) 324 | GestureController.CAM_WIDTH = GestureController.cap.get(cv2.CAP_PROP_FRAME_WIDTH) 325 | 326 | def classify_hands(results): 327 | left , right = None,None 328 | try: 329 | handedness_dict = MessageToDict(results.multi_handedness[0]) 330 | if handedness_dict['classification'][0]['label'] == 'Right': 331 | right = results.multi_hand_landmarks[0] 332 | else : 333 | left = results.multi_hand_landmarks[0] 334 | except: 335 | pass 336 | 337 | try: 338 | handedness_dict = MessageToDict(results.multi_handedness[1]) 339 | if handedness_dict['classification'][0]['label'] == 'Right': 340 | right = results.multi_hand_landmarks[1] 341 | else : 342 | left = results.multi_hand_landmarks[1] 343 | except: 344 | pass 345 | 346 | if GestureController.dom_hand == True: 347 | GestureController.hr_major = right 348 | GestureController.hr_minor = left 349 | else : 350 | GestureController.hr_major = left 351 | GestureController.hr_minor = right 352 | 353 | def start(self): 354 | 355 | handmajor = HandRecog(HLabel.MAJOR) 356 | handminor = HandRecog(HLabel.MINOR) 357 | 358 | with mp_hands.Hands(max_num_hands = 2,min_detection_confidence=0.5, min_tracking_confidence=0.5) as hands: 359 | while GestureController.cap.isOpened() and GestureController.gc_mode: 360 | success, image = GestureController.cap.read() 361 | 362 | if not success: 363 | print("Ignoring empty camera frame.") 364 | continue 365 | 366 | image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB) 367 | image.flags.writeable = False 368 | results = hands.process(image) 369 | 370 | image.flags.writeable = True 371 | image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) 372 | 373 | if results.multi_hand_landmarks: 374 | GestureController.classify_hands(results) 375 | handmajor.update_hand_result(GestureController.hr_major) 376 | handminor.update_hand_result(GestureController.hr_minor) 377 | 378 | handmajor.set_finger_state() 379 | handminor.set_finger_state() 380 | gest_name = handminor.get_gesture() 381 | 382 | if gest_name == Gest.PINCH_MINOR: 383 | Controller.handle_controls(gest_name, handminor.hand_result) 384 | else: 385 | gest_name = handmajor.get_gesture() 386 | Controller.handle_controls(gest_name, handmajor.hand_result) 387 | 388 | for hand_landmarks in results.multi_hand_landmarks: 389 | mp_drawing.draw_landmarks(image, hand_landmarks, mp_hands.HAND_CONNECTIONS) 390 | else: 391 | Controller.prev_hand = None 392 | cv2.imshow('Gesture Controller', image) 393 | if cv2.waitKey(5) & 0xFF == 13: 394 | break 395 | GestureController.cap.release() 396 | cv2.destroyAllWindows() 397 | 398 | # uncomment to run directly 399 | gc1 = GestureController() 400 | gc1.start() 401 | 402 | #gest_control() -------------------------------------------------------------------------------- /Gesture_Controller_Gloved.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import cv2.aruco as aruco 4 | import os 5 | import glob 6 | import math 7 | import pyautogui 8 | import time 9 | 10 | class Marker: 11 | def __init__(self, dict_type = aruco.DICT_4X4_50, thresh_constant = 1): 12 | self.aruco_dict = aruco.Dictionary_get(dict_type) 13 | self.parameters = aruco.DetectorParameters_create() 14 | self.parameters.adaptiveThreshConstant = thresh_constant 15 | self.corners = None # corners of Marker 16 | self.marker_x2y = 1 # width:height ratio 17 | self.mtx, self.dist = Marker.calibrate() 18 | 19 | def calibrate(): 20 | criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001) 21 | objp = np.zeros((6*7,3), np.float32) 22 | objp[:,:2] = np.mgrid[0:7,0:6].T.reshape(-1,2) 23 | objpoints = [] # 3d point in real world space 24 | imgpoints = [] # 2d points in image plane. 25 | path = os.path.dirname(os.path.abspath(__file__)) 26 | p1 = path + r'\calib_images\checkerboard\*.jpg' 27 | images = glob.glob(p1) 28 | for fname in images: 29 | img = cv2.imread(fname) 30 | gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) 31 | ret, corners = cv2.findChessboardCorners(gray, (7,6),None) 32 | if ret == True: 33 | objpoints.append(objp) 34 | corners2 = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria) 35 | imgpoints.append(corners2) 36 | img = cv2.drawChessboardCorners(img, (7,6), corners2,ret) 37 | 38 | ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1],None,None) 39 | 40 | #mtx = [[534.34144579,0.0,339.15527836],[0.0,534.68425882,233.84359493],[0.0,0.0,1.0]] 41 | #dist = [[-2.88320983e-01, 5.41079685e-02, 1.73501622e-03, -2.61333895e-04, 2.04110465e-01]] 42 | return mtx, dist 43 | 44 | def detect(self, frame): 45 | gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 46 | self.corners, ids, rejectedImgPoints = aruco.detectMarkers(gray_frame, self.aruco_dict, parameters = self.parameters) 47 | if np.all(ids != None): 48 | rvec, tvec ,_ = aruco.estimatePoseSingleMarkers(self.corners, 0.05, self.mtx, self.dist) 49 | else: 50 | self.corners = None 51 | 52 | def is_detected(self): 53 | if self.corners: 54 | return True 55 | return False 56 | 57 | def draw_marker(self, frame): 58 | aruco.drawDetectedMarkers(frame, self.corners) 59 | 60 | 61 | 62 | def ecu_dis(p1, p2): 63 | dist = np.sqrt((p1[0]-p2[0])**2 + (p1[1]-p2[1])**2) 64 | return dist 65 | 66 | def find_HSV(samples): 67 | try: 68 | color = np.uint8([ samples ]) 69 | except: 70 | color = np.uint8([ [[105,105,50]] ]) 71 | hsv_color = cv2.cvtColor(color,cv2.COLOR_RGB2HSV) 72 | #print( hsv_color ) 73 | return hsv_color 74 | 75 | def draw_box(frame, points, color=(0,255,127)): 76 | if points: 77 | frame = cv2.line(frame, points[0], points[1], color, thickness=2, lineType=8) #top 78 | frame = cv2.line(frame, points[1], points[2], color, thickness=2, lineType=8) #right 79 | frame = cv2.line(frame, points[2], points[3], color, thickness=2, lineType=8) #bottom 80 | frame = cv2.line(frame, points[3], points[0], color, thickness=2, lineType=8) #left 81 | 82 | def in_cam(val, type_): 83 | if type_ == 'x': 84 | if val<0: 85 | return 0 86 | if val>GestureController.cam_width: 87 | return GestureController.cam_width 88 | elif type_ == 'y': 89 | if val<0: 90 | return 0 91 | if val>GestureController.cam_height: 92 | return GestureController.cam_height 93 | return val 94 | 95 | 96 | class ROI: 97 | def __init__(self, roi_alpha1=1.5, roi_alpha2=1.5, roi_beta=2.5, hsv_alpha = 0.3, hsv_beta = 0.5, hsv_lift_up = 0.3): 98 | self.roi_alpha1 = roi_alpha1 99 | self.roi_alpha2 = roi_alpha2 100 | self.roi_beta = roi_beta 101 | self.roi_corners = None 102 | 103 | self.hsv_alpha = hsv_alpha 104 | self.hsv_beta = hsv_beta 105 | self.hsv_lift_up = hsv_lift_up 106 | self.hsv_corners = None 107 | 108 | self.marker_top = None 109 | self.glove_hsv = None 110 | 111 | def findROI(self, frame, marker): 112 | rec_coor = marker.corners[0][0] 113 | c1 = (int(rec_coor[0][0]),int(rec_coor[0][1])) 114 | c2 = (int(rec_coor[1][0]),int(rec_coor[1][1])) 115 | c3 = (int(rec_coor[2][0]),int(rec_coor[2][1])) 116 | c4 = (int(rec_coor[3][0]),int(rec_coor[3][1])) 117 | 118 | try: 119 | marker.marker_x2y = np.sqrt((c1[0]-c2[0])**2 + (c1[1]-c2[1])**2) / np.sqrt((c3[0]-c2[0])**2 + (c3[1]-c2[1])**2) 120 | except: 121 | marker.marker_x2y = 999.0 122 | 123 | #mid-point of top line of Marker 124 | cx = (c1[0] + c2[0])/2 125 | cy = (c1[1] + c2[1])/2 126 | 127 | self.marker_top = [cx, cy] 128 | 129 | l = np.absolute(ecu_dis(c1,c4)) 130 | 131 | try: 132 | slope_12 = (c1[1]-c2[1])/(c1[0]-c2[0]) 133 | except: 134 | slope_12 = (c1[1]-c2[1])*999.0 + 0.1 135 | 136 | try: 137 | slope_14 = -1 / slope_12 138 | except: 139 | slope_14 = -999.0 140 | 141 | if slope_14 < 0: 142 | sign = 1 143 | else: 144 | sign = -1 145 | 146 | bot_rx = int(cx + self.roi_alpha2 * l * np.sqrt(1/(1+slope_12**2))) 147 | bot_ry = int(cy + self.roi_alpha2 * slope_12 * l * np.sqrt(1/(1+slope_12**2))) 148 | 149 | bot_lx = int(cx - self.roi_alpha1 * l * np.sqrt(1/(1+slope_12**2))) 150 | bot_ly = int(cy - self.roi_alpha1 * slope_12 * l * np.sqrt(1/(1+slope_12**2))) 151 | 152 | top_lx = int(bot_lx + sign * self.roi_beta * l * np.sqrt(1/(1+slope_14**2))) 153 | top_ly = int(bot_ly + sign * self.roi_beta * slope_14 * l * np.sqrt(1/(1+slope_14**2))) 154 | 155 | top_rx = int(bot_rx + sign * self.roi_beta * l * np.sqrt(1/(1+slope_14**2))) 156 | top_ry = int(bot_ry + sign * self.roi_beta * slope_14 * l * np.sqrt(1/(1+slope_14**2))) 157 | 158 | bot_lx = in_cam(bot_lx, 'x') 159 | bot_ly = in_cam(bot_ly, 'y') 160 | 161 | bot_rx = in_cam(bot_rx, 'x') 162 | bot_ry = in_cam(bot_ry, 'y') 163 | 164 | top_lx = in_cam(top_lx, 'x') 165 | top_ly = in_cam(top_ly, 'y') 166 | 167 | top_rx = in_cam(top_rx, 'x') 168 | top_ry = in_cam(top_ry, 'y') 169 | 170 | self.roi_corners = [(bot_lx,bot_ly), (bot_rx,bot_ry), (top_rx,top_ry), (top_lx,top_ly)] 171 | 172 | 173 | def find_glove_hsv(self, frame, marker): 174 | rec_coor = marker.corners[0][0] 175 | c1 = (int(rec_coor[0][0]),int(rec_coor[0][1])) 176 | c2 = (int(rec_coor[1][0]),int(rec_coor[1][1])) 177 | c3 = (int(rec_coor[2][0]),int(rec_coor[2][1])) 178 | c4 = (int(rec_coor[3][0]),int(rec_coor[3][1])) 179 | 180 | l = np.absolute(ecu_dis(c1,c4)) 181 | 182 | try: 183 | slope_12 = (c1[1]-c2[1])/(c1[0]-c2[0]) 184 | except: 185 | slope_12 = (c1[1]-c2[1])*999.0 + 0.1 186 | try: 187 | slope_14 = -1 / slope_12 188 | except: 189 | slope_14 = -999.0 190 | 191 | if slope_14 < 0: 192 | sign = 1 193 | else: 194 | sign = -1 195 | 196 | bot_rx = int(self.marker_top[0] + self.hsv_alpha * l * np.sqrt(1/(1+slope_12**2))) 197 | bot_ry = int(self.marker_top[1] - self.hsv_lift_up*l + self.hsv_alpha * slope_12 * l * np.sqrt(1/(1+slope_12**2))) 198 | 199 | bot_lx = int(self.marker_top[0] - self.hsv_alpha * l * np.sqrt(1/(1+slope_12**2))) 200 | bot_ly = int(self.marker_top[1] - self.hsv_lift_up*l - self.hsv_alpha * slope_12 * l * np.sqrt(1/(1+slope_12**2))) 201 | 202 | top_lx = int(bot_lx + sign * self.hsv_beta * l * np.sqrt(1/(1+slope_14**2))) 203 | top_ly = int(bot_ly + sign * self.hsv_beta * slope_14 * l * np.sqrt(1/(1+slope_14**2))) 204 | 205 | top_rx = int(bot_rx + sign * self.hsv_beta * l * np.sqrt(1/(1+slope_14**2))) 206 | top_ry = int(bot_ry + sign * self.hsv_beta * slope_14 * l * np.sqrt(1/(1+slope_14**2))) 207 | 208 | region = frame[top_ry:bot_ry , top_lx:bot_rx] 209 | b, g, r = np.mean(region, axis=(0, 1)) 210 | 211 | self.hsv_glove = find_HSV([[r,g,b]]) 212 | self.hsv_corners = [(bot_lx,bot_ly), (bot_rx,bot_ry), (top_rx,top_ry), (top_lx,top_ly)] 213 | 214 | 215 | def cropROI(self, frame): 216 | pts = np.array(self.roi_corners) 217 | 218 | ## (1) Crop the bounding rect 219 | rect = cv2.boundingRect(pts) 220 | x,y,w,h = rect 221 | croped = frame[y:y+h, x:x+w].copy() 222 | 223 | ## (2) make mask 224 | pts = pts - pts.min(axis=0) 225 | 226 | mask = np.zeros(croped.shape[:2], np.uint8) 227 | cv2.drawContours(mask, [pts], -1, (255, 255, 255), -1, cv2.LINE_AA) 228 | 229 | ## (3) do bit-op 230 | dst = cv2.bitwise_and(croped, croped, mask=mask) 231 | 232 | ## (4) add the white background 233 | bg = np.ones_like(croped, np.uint8)*255 234 | cv2.bitwise_not(bg,bg, mask=mask) 235 | 236 | kernelOpen = np.ones((3,3),np.uint8) 237 | kernelClose = np.ones((5,5),np.uint8) 238 | 239 | hsv = cv2.cvtColor(dst, cv2.COLOR_BGR2HSV) 240 | 241 | lower_range = np.array([self.hsv_glove[0][0][0]//1-5,50,50]) 242 | upper_range = np.array([self.hsv_glove[0][0][0]//1+5,255,255]) 243 | 244 | mask = cv2.inRange(hsv, lower_range, upper_range) 245 | #mask = cv2.dilate(mask,kernelOpen,iterations = 1) 246 | Opening =cv2.morphologyEx(mask,cv2.MORPH_OPEN,kernelOpen) 247 | Closing =cv2.morphologyEx(Opening,cv2.MORPH_CLOSE,kernelClose) 248 | FinalMask = Closing 249 | 250 | return FinalMask 251 | 252 | 253 | class Glove: 254 | 255 | def __init__(self): 256 | self.fingers = 0 257 | self.arearatio = 0 258 | self.gesture = 0 259 | 260 | def find_fingers(self, FinalMask): 261 | conts,h=cv2.findContours(FinalMask,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE) 262 | hull = [cv2.convexHull(c) for c in conts] 263 | 264 | try: 265 | cnt = max(conts, key = lambda x: cv2.contourArea(x)) 266 | #approx the contour a little 267 | epsilon = 0.0005*cv2.arcLength(cnt,True) 268 | approx= cv2.approxPolyDP(cnt,epsilon,True) 269 | #make convex hull around hand 270 | hull = cv2.convexHull(cnt) 271 | #define area of hull and area of hand 272 | areahull = cv2.contourArea(hull) 273 | areacnt = cv2.contourArea(cnt) 274 | #find the percentage of area not covered by hand in convex hull 275 | self.arearatio=((areahull-areacnt)/areacnt)*100 276 | #find the defects in convex hull with respect to hand 277 | hull = cv2.convexHull(approx, returnPoints=False) 278 | defects = cv2.convexityDefects(approx, hull) 279 | except: 280 | print("No Contours found in FinalMask") 281 | 282 | # l = no. of defects 283 | l=0 284 | try: 285 | #code for finding no. of defects due to fingers 286 | for i in range(defects.shape[0]): 287 | s,e,f,d = defects[i,0] 288 | start = tuple(approx[s][0]) 289 | end = tuple(approx[e][0]) 290 | far = tuple(approx[f][0]) 291 | 292 | # find length of all sides of triangle 293 | a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2) 294 | b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2) 295 | c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2) 296 | s = (a+b+c)/2 297 | ar = math.sqrt(s*(s-a)*(s-b)*(s-c)) 298 | 299 | #distance between point and convex hull 300 | d=(2*ar)/a 301 | 302 | # apply cosine rule here 303 | angle = math.acos((b**2 + c**2 - a**2)/(2*b*c)) * 57 304 | 305 | # ignore angles > 90 and ignore points very close to convex hull(they generally come due to noise) 306 | if angle <= 90 and d>30: 307 | l += 1 308 | #cv2.circle(frame, far, 3, [255,255,255], -1) 309 | 310 | #draw lines around hand 311 | cv2.line(FinalMask,start, end, [255,255,255], 2) 312 | 313 | l+=1 314 | except: 315 | l = 0 316 | print("No Defects found in mask") 317 | 318 | self.fingers = l 319 | 320 | def find_gesture(self, frame): 321 | font = cv2.FONT_HERSHEY_SIMPLEX 322 | self.gesture = 0 323 | if self.fingers==1: 324 | #cv2.putText(frame, str(int(arearatio)), (10,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 325 | if self.arearatio<15: 326 | cv2.putText(frame,'0',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 327 | self.gesture = 0 328 | elif self.arearatio<25: 329 | cv2.putText(frame,'2 fingers',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 330 | self.gesture = 2 331 | else: 332 | cv2.putText(frame,'1 finger',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 333 | self.gesture = 1 334 | 335 | elif self.fingers==2: 336 | cv2.putText(frame,'2',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 337 | self.gesture = 3 338 | ''' 339 | elif self.fingers==3: 340 | #cv2.putText(frame,'3',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 341 | 342 | elif self.fingers==4: 343 | #cv2.putText(frame,'4',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 344 | 345 | elif self.fingers==5: 346 | #cv2.putText(frame,'5',(0,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 347 | 348 | else : 349 | # cv2.putText(frame,'reposition',(10,50), font, 2, (0,0,255), 3, cv2.LINE_AA) 350 | ''' 351 | 352 | class Tracker: 353 | def __init__(self): 354 | self.tracker_started = False 355 | self.tracker = None 356 | self.start_time = 0.0 357 | self.now_time = 0.0 358 | self.tracker_bbox = None 359 | 360 | def corners_to_tracker(self, corners): 361 | csrt_minX = int( min( [corners[0][0][0][0], corners[0][0][1][0], corners[0][0][2][0], corners[0][0][3][0]] )) 362 | csrt_maxX = int( max( [corners[0][0][0][0], corners[0][0][1][0], corners[0][0][2][0], corners[0][0][3][0]] )) 363 | csrt_minY = int( min( [corners[0][0][0][1], corners[0][0][1][1], corners[0][0][2][1], corners[0][0][3][1]] )) 364 | csrt_maxY = int( max( [corners[0][0][0][1], corners[0][0][1][1], corners[0][0][2][1], corners[0][0][3][1]] )) 365 | self.tracker_bbox = [csrt_minX, csrt_minY, csrt_maxX-csrt_minX, csrt_maxY-csrt_minY] 366 | 367 | def tracker_to_corner(self, final_bbox): 368 | if self.tracker_bbox == None: 369 | return None 370 | final_bbox = [[[1,2],[3,4],[5,6],[7,8]]] 371 | final_bbox[0][0] = [self.tracker_bbox[0],self.tracker_bbox[1]] 372 | final_bbox[0][1] = [self.tracker_bbox[0]+ self.tracker_bbox[2],self.tracker_bbox[1]] 373 | final_bbox[0][2] = [self.tracker_bbox[0]+ self.tracker_bbox[2],self.tracker_bbox[1] + self.tracker_bbox[3]] 374 | final_bbox[0][3] = [self.tracker_bbox[0],self.tracker_bbox[1] +self.tracker_bbox[3]] 375 | return [np.array(final_bbox, dtype = 'f')] 376 | 377 | def CSRT_tracker(self, frame): 378 | if self.tracker_bbox == None and self.tracker_started == False: 379 | return 380 | 381 | if self.tracker_started == False: 382 | if self.tracker == None: 383 | self.tracker = cv2.TrackerCSRT_create() 384 | 385 | if self.tracker_bbox != None: 386 | try: 387 | self.start_time = time.time() 388 | ok = self.tracker.init(frame, self.tracker_bbox) 389 | self.tracker_started = True 390 | except: 391 | print("tracker.init failed") 392 | try: 393 | ok, self.tracker_bbox = self.tracker.update(frame) 394 | except: 395 | ok = None 396 | print("tracker.update failed") 397 | self.now_time = time.time() 398 | 399 | if self.now_time-self.start_time >= 2.0 : 400 | #cv2.putText(frame, "Please posture your hand correctly", (10,50), cv2.FONT_HERSHEY_SIMPLEX, 1,(0,0,255),1) 401 | cv2.putText(frame,'Posture your hand correctly',(10,10), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0,0,255), 1, cv2.LINE_AA) 402 | #print("tracking timeout") 403 | self.tracker_started = False 404 | self.tracker_bbox = None 405 | return 406 | 407 | if ok: 408 | # Tracking success 409 | p1 = (int(self.tracker_bbox[0]), int(self.tracker_bbox[1])) 410 | p2 = (int(self.tracker_bbox[0] + self.tracker_bbox[2]), int(self.tracker_bbox[1] + self.tracker_bbox[3])) 411 | cv2.rectangle(frame, p1, p2, (80, 255, 255), 2, 1) 412 | else : 413 | # Tracking failure 414 | self.tracker_started = False 415 | cv2.putText(frame, "Tracking failure detected", (100,80), cv2.FONT_HERSHEY_SIMPLEX, 0.75,(0,0,255),2) 416 | print("Tracking failure detected") 417 | #reintiallize code to tackle tracking failure 418 | 419 | 420 | 421 | 422 | 423 | 424 | class Mouse: 425 | def __init__(self): 426 | self.tx_old = 0 427 | self.ty_old = 0 428 | self.trial = True 429 | self.flag = 0 430 | 431 | def move_mouse(self,frame,position,gesture): 432 | 433 | (sx,sy)=pyautogui.size() 434 | (camx,camy) = (frame.shape[:2][0],frame.shape[:2][1]) 435 | (mx_old,my_old) = pyautogui.position() 436 | 437 | 438 | Damping = 2 # Hyperparameter we will have to adjust 439 | tx = position[0] 440 | ty = position[1] 441 | if self.trial: 442 | self.trial, self.tx_old, self.ty_old = False, tx, ty 443 | 444 | delta_tx = tx - self.tx_old 445 | delta_ty = ty - self.ty_old 446 | self.tx_old,self.ty_old = tx,ty 447 | 448 | if (gesture == 3): 449 | self.flag = 0 450 | mx = mx_old + (delta_tx*sx) // (camx*Damping) 451 | my = my_old + (delta_ty*sy) // (camy*Damping) 452 | pyautogui.moveTo(mx,my, duration = 0.1) 453 | 454 | elif(gesture == 0): 455 | if self.flag == 0: 456 | pyautogui.doubleClick() 457 | self.flag = 1 458 | elif(gesture == 1): 459 | print('1 Finger Open') 460 | 461 | 462 | 463 | 464 | 465 | class GestureController: 466 | gc_mode = 0 467 | pyautogui.FAILSAFE = False 468 | f_start_time = 0 469 | f_now_time = 0 470 | 471 | cam_width = 0 472 | cam_height = 0 473 | 474 | aru_marker = Marker() 475 | hand_roi = ROI(2.5, 2.5, 6, 0.45, 0.6, 0.4) 476 | glove = Glove() 477 | csrt_track = Tracker() 478 | mouse = Mouse() 479 | 480 | def __init__(self): 481 | GestureController.cap = cv2.VideoCapture(0) 482 | if GestureController.cap.isOpened(): 483 | GestureController.cam_width = int( GestureController.cap.get(cv2.CAP_PROP_FRAME_WIDTH) ) 484 | GestureController.cam_height = int( GestureController.cap.get(cv2.CAP_PROP_FRAME_HEIGHT) ) 485 | else: 486 | print("CANNOT OPEN CAMERA") 487 | 488 | GestureController.gc_mode = 1 489 | GestureController.f_start_time = time.time() 490 | GestureController.f_now_time = time.time() 491 | 492 | def start(self): 493 | while (True): 494 | #mode checking 495 | if not GestureController.gc_mode: 496 | print('Exiting Gesture Controller') 497 | break 498 | #fps control 499 | fps = 30.0 500 | GestureController.f_start_time = time.time() 501 | while (GestureController.f_now_time-GestureController.f_start_time <= 1.0/fps): 502 | GestureController.f_now_time = time.time() 503 | 504 | #read camera 505 | ret, frame = GestureController.cap.read() 506 | frame = cv2.flip(frame, 1) 507 | 508 | #detect Marker, find ROI, find glove HSV, get FinalMask on glove 509 | GestureController.aru_marker.detect(frame) 510 | if GestureController.aru_marker.is_detected(): 511 | GestureController.csrt_track.corners_to_tracker(GestureController.aru_marker.corners) 512 | GestureController.csrt_track.CSRT_tracker(frame) 513 | 514 | else: 515 | GestureController.csrt_track.tracker_bbox = None 516 | GestureController.csrt_track.CSRT_tracker(frame) 517 | GestureController.aru_marker.corners = GestureController.csrt_track.tracker_to_corner(GestureController.aru_marker.corners) 518 | 519 | if GestureController.aru_marker.is_detected(): 520 | GestureController.hand_roi.findROI(frame, GestureController.aru_marker) 521 | GestureController.hand_roi.find_glove_hsv(frame, GestureController.aru_marker) 522 | FinalMask = GestureController.hand_roi.cropROI(frame) 523 | GestureController.glove.find_fingers(FinalMask) 524 | GestureController.glove.find_gesture(frame) 525 | GestureController.mouse.move_mouse(frame,GestureController.hand_roi.marker_top,GestureController.glove.gesture) 526 | 527 | #draw call 528 | if GestureController.aru_marker.is_detected(): 529 | GestureController.aru_marker.draw_marker(frame) 530 | draw_box(frame, GestureController.hand_roi.roi_corners, (255,0,0)) 531 | draw_box(frame, GestureController.hand_roi.hsv_corners, (0,0,250)) 532 | cv2.imshow('FinalMask',FinalMask) 533 | 534 | #display frame 535 | cv2.imshow('frame',frame) 536 | if cv2.waitKey(1) & 0xFF == ord('q'): 537 | break 538 | 539 | # When everything done, release the capture 540 | GestureController.cap.release() 541 | cv2.destroyAllWindows() 542 | 543 | 544 | 545 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Virtual-Mouse-using-hand-Gesture-Recognition 2 | 3 | 4 | The gesture-controlled virtual mouse is an innovative technology that allows people to interact with computers without the need for physical contact. The system employs modern machine learning and computer vision techniques to detect hand gestures, voice commands, and eye movements using internal and external cameras. The system consists of two modules: the first one involves hand detection using MediaPipe's technology, while the second employs a glove with a single color. Users can control the virtual keyboard and mouse by moving their fingers in the air, which is made possible through computer vision technology and artificial intelligence. The system uses modules such as Hand Tracking, CVzone Hand Detector, and the Controller imported from the Pynput keyboard to recognize hand gestures and control input and output processes.The system functions as a virtual keyboard and mouse without the need for any external devices, wires, or connections. It employs a Convolutional Neural Network(CNN)-like model implemented by MediaPipe running on top of pybind11. The system enables users to work in the air and access keyboard keys by maneuvering fingers, making it an ideal tool for people who are physically challenged or have mobility issues. The webcam is the only piece of hardware required in this system, which is used to record images, recognize hand gestures, eye movements, and receive voice instructions using the Pyttsx3 module. The suggested system can improve the quality of life for people with disabilities or mobility issues, as it allows them to interact with computers without the need for physical contact or external devices. Overall, the gesture-controlled virtual mouse is a remarkable advancement in computer technology that has the potential to revolutionize the way people interact with computers. 5 | 6 | 7 | Note: Use Python version: 3.8.5 8 | -Install Anaconda Distribution 9 | # Install Necessary libraries using pip: 10 | 11 | -Install PyAudio 12 | -Install pywin32 13 | -pyttsx3==2.71 14 | -SpeechRecognition==3.8.1 15 | -pynput==1.7.3 16 | -pyautogui==0.9.53 17 | -wikipedia==1.4.0 18 | -opencv-python==4.5.3.56 19 | -mediapipe==0.8.6.2 20 | -comtypes==1.1.11 21 | -pycaw==20181226 22 | -screen-brightness-control==0.9.0 23 | -eel==0.14.0 24 | 25 | ## Features 26 | 27 | - VoiceBot 28 | - Current Date and Time 29 | - Google Search 30 | - Find Location 31 | - Sleep/Wake-up 32 | 33 | - KeyBoard 34 | - Eye Movements 35 | - Gesture Recognition: 36 | - Neutral Gesture 37 | - Move Cursor 38 | - Left Click 39 | - Right Click 40 | - Double Click 41 | - Scrolling 42 | - Drag and Drop 43 | - Multiple Item Selection 44 | - Volume Control 45 | ## Procedure 46 | Step 1: 47 | ```bash 48 | conda create --name gest python=3.8.5 49 | ``` 50 | Step 2: 51 | ```bash 52 | conda activate gest 53 | ``` 54 | Step 3: 55 | ```bash 56 | pip install -r requirements.txt 57 | ``` 58 | Step 4: 59 | ```bash 60 | conda install PyAudio 61 | 62 | conda install pywin32 63 | ``` 64 | Step 5: 65 | ```bash 66 | cd to the GitHub Repo till src folder 67 | ``` 68 | Command may look like: cd C:\Users\.....\Gesture-Controlled-Virtual-Mouse\src 69 | 70 | Step 6: 71 | For running GUI: 72 | ```bash 73 | python main.py 74 | ``` 75 | Or to run only Gesture Recognition without the voice assisstant: 76 | 77 | Uncomment last 2 lines of Code in the file Gesture_Controller.py 78 | 79 | 80 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import eel 2 | import os 3 | from queue import Queue 4 | 5 | class ChatBot: 6 | 7 | started = False 8 | userinputQueue = Queue() 9 | 10 | def isUserInput(): 11 | return not ChatBot.userinputQueue.empty() 12 | 13 | def popUserInput(): 14 | return ChatBot.userinputQueue.get() 15 | 16 | def close_callback(route, websockets): 17 | # if not websockets: 18 | # print('Bye!') 19 | exit() 20 | 21 | @eel.expose 22 | def getUserInput(msg): 23 | ChatBot.userinputQueue.put(msg) 24 | print(msg) 25 | 26 | def close(): 27 | ChatBot.started = False 28 | 29 | def addUserMsg(msg): 30 | eel.addUserMsg(msg) 31 | 32 | def addAppMsg(msg): 33 | eel.addAppMsg(msg) 34 | 35 | def start(): 36 | path = os.path.dirname(os.path.abspath(__file__)) 37 | eel.init(path + r'\web', allowed_extensions=['.js', '.html']) 38 | try: 39 | eel.start('index.html', mode='chrome', 40 | host='localhost', 41 | port=27005, 42 | block=False, 43 | size=(350, 480), 44 | position=(10,100), 45 | disable_cache=True, 46 | close_callback=ChatBot.close_callback) 47 | ChatBot.started = True 48 | while ChatBot.started: 49 | try: 50 | eel.sleep(10.0) 51 | except: 52 | #main thread exited 53 | break 54 | 55 | except: 56 | pass 57 | -------------------------------------------------------------------------------- /aura.py: -------------------------------------------------------------------------------- 1 | import pyttsx3 2 | import speech_recognition as sr 3 | from datetime import date 4 | import time 5 | import webbrowser 6 | import datetime 7 | from pynput.keyboard import Key, Controller 8 | import os 9 | from os import listdir 10 | from os.path import isfile, join 11 | import app 12 | from threading import Thread 13 | 14 | # Initialize text-to-speech 15 | engine = pyttsx3.init('sapi5') 16 | engine.setProperty('voice', engine.getProperty('voices')[0].id) 17 | 18 | # Initialize recognizer 19 | r = sr.Recognizer() 20 | 21 | # Globals 22 | today = date.today() 23 | keyboard = Controller() 24 | file_exp_status = False 25 | files = [] 26 | path = '' 27 | is_awake = True # Bot status 28 | 29 | def aura_chat(): 30 | def reply(audio): 31 | """Respond via text-to-speech and chatbot interface.""" 32 | app.ChatBot.addAppMsg(audio) 33 | print(audio) 34 | engine.say(audio) 35 | engine.runAndWait() 36 | 37 | def wish(): 38 | """Greet the user based on the current time.""" 39 | hour = datetime.datetime.now().hour 40 | if hour < 12: 41 | reply("Good Morning!") 42 | elif hour < 18: 43 | reply("Good Afternoon!") 44 | else: 45 | reply("Good Evening!") 46 | reply("I am Aura. How may I help you?") 47 | 48 | # Configure microphone and audio input 49 | def record_audio(): 50 | """Capture audio and convert it to text.""" 51 | with sr.Microphone() as source: 52 | print("Listening...") 53 | r.energy_threshold = 300 54 | r.pause_threshold = 0.8 55 | try: 56 | audio = r.listen(source, timeout=5, phrase_time_limit=5) 57 | voice_data = r.recognize_google(audio) 58 | print(f"Recognized: {voice_data}") 59 | return voice_data.lower() 60 | except sr.RequestError: 61 | reply("Sorry, my service is down. Please check your internet connection.") 62 | except sr.UnknownValueError: 63 | reply("I didn't catch that. Could you please repeat?") 64 | return "" 65 | 66 | # Command processing 67 | def respond(voice_data): 68 | global file_exp_status, files, is_awake, path 69 | 70 | if 'wake up' in voice_data and not is_awake: 71 | is_awake = True 72 | wish() 73 | return 74 | 75 | if not is_awake: 76 | return 77 | 78 | if 'hello' in voice_data: 79 | wish() 80 | elif 'what is your name' in voice_data: 81 | reply('My name is Aura!') 82 | elif 'date' in voice_data: 83 | reply(today.strftime("%B %d, %Y")) 84 | elif 'time' in voice_data: 85 | reply(datetime.datetime.now().strftime("%H:%M:%S")) 86 | elif 'search' in voice_data: 87 | query = voice_data.split('search', 1)[-1].strip() 88 | reply(f'Searching for {query}') 89 | url = f'https://google.com/search?q={query}' 90 | try: 91 | webbrowser.open(url) 92 | reply("Here's what I found.") 93 | except: 94 | reply("Please check your internet connection.") 95 | elif 'location' in voice_data: 96 | reply('Which place are you looking for?') 97 | location = record_audio() 98 | if location: 99 | reply(f"Locating {location}...") 100 | url = f'https://google.nl/maps/place/{location}/&' 101 | try: 102 | webbrowser.open(url) 103 | reply("Here's the location.") 104 | except: 105 | reply("Please check your internet connection.") 106 | elif 'bye' in voice_data: 107 | reply("Goodbye! Have a great day.") 108 | is_awake = False 109 | elif 'exit' in voice_data: 110 | reply("Exiting. Goodbye!") 111 | sys.exit() 112 | else: 113 | reply("I am not programmed to do that yet!") 114 | 115 | # Main loop 116 | t1 = Thread(target=app.ChatBot.start) 117 | t1.start() 118 | 119 | while not app.ChatBot.started: 120 | time.sleep(0.5) 121 | 122 | wish() 123 | while True: 124 | voice_data = app.ChatBot.popUserInput() if app.ChatBot.isUserInput() else record_audio() 125 | try: 126 | respond(voice_data) 127 | except Exception as e: 128 | print(f"Error: {e}") 129 | break 130 | 131 | if __name__ == "__main__": 132 | aura_chat() 133 | -------------------------------------------------------------------------------- /eyen.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import pyautogui 4 | import time 5 | 6 | # Initialize webcam and mediapipe face mesh 7 | cam = cv2.VideoCapture(0) 8 | face_mesh = mp.solutions.face_mesh.FaceMesh(refine_landmarks=True) 9 | screen_w, screen_h = pyautogui.size() 10 | 11 | def eye_move(): 12 | blink_threshold = 0.2 # Threshold for blink detection 13 | blink_cooldown = 1 # Time in seconds between blinks for clicks 14 | last_blink_time = time.time() 15 | 16 | while True: 17 | # Read frame from webcam 18 | ret, frame = cam.read() 19 | if not ret: 20 | break 21 | 22 | # Flip and process frame 23 | frame = cv2.flip(frame, 1) 24 | rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 25 | output = face_mesh.process(rgb_frame) 26 | landmark_points = output.multi_face_landmarks 27 | frame_h, frame_w, _ = frame.shape 28 | 29 | if landmark_points: 30 | landmarks = landmark_points[0].landmark 31 | 32 | # Cursor movement based on eye landmarks 33 | cursor_point = landmarks[474] # Pupil landmark 34 | screen_x = int(cursor_point.x * screen_w) 35 | screen_y = int(cursor_point.y * screen_h) 36 | pyautogui.moveTo(screen_x, screen_y) 37 | 38 | # Eye blink detection 39 | left_eye = [landmarks[145], landmarks[159]] # Top and bottom of left eye 40 | vertical_distance = abs(left_eye[0].y - left_eye[1].y) 41 | if vertical_distance < blink_threshold: 42 | current_time = time.time() 43 | if current_time - last_blink_time > blink_cooldown: 44 | pyautogui.click() 45 | last_blink_time = current_time 46 | 47 | # Draw landmarks for debugging 48 | for point in left_eye: 49 | x = int(point.x * frame_w) 50 | y = int(point.y * frame_h) 51 | cv2.circle(frame, (x, y), 3, (0, 255, 255), -1) 52 | cv2.circle(frame, (int(cursor_point.x * frame_w), int(cursor_point.y * frame_h)), 5, (255, 0, 0), -1) 53 | 54 | # Display the frame 55 | cv2.imshow('Eye Controlled Mouse', frame) 56 | 57 | # Exit on pressing 'Q' 58 | if cv2.waitKey(1) & 0xFF == ord('q'): 59 | break 60 | 61 | # Release resources 62 | cam.release() 63 | cv2.destroyAllWindows() 64 | 65 | # Run the program 66 | eye_move() 67 | -------------------------------------------------------------------------------- /kc.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RAGHURAM5276/VIRTUAL-MOUSE-USING-HAND-GESTURE-RECOGNITION/04f21b436fd236c1ef73e5582d55314a057a36ec/kc.wav -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import tkinter as tk 2 | import tkinter.font as font 3 | from Gesture_Controller import gest_control 4 | from eye import eye_move 5 | from samvk import vk_keyboard 6 | from PIL import Image, ImageTk 7 | from aura import aura_chat 8 | 9 | # Constants for dimensions 10 | WINDOW_WIDTH = 1080 11 | WINDOW_HEIGHT = 700 12 | ICON_SIZE = (200, 200) 13 | BUTTON_IMAGE_SIZE = (50, 50) 14 | 15 | # Initialize main window 16 | window = tk.Tk() 17 | window.title("Gesture Controlled Virtual Mouse and Keyboard") 18 | window.iconphoto(False, tk.PhotoImage(file='mn.png')) 19 | window.geometry(f'{WINDOW_WIDTH}x{WINDOW_HEIGHT}') 20 | window.configure(bg="white") 21 | 22 | # Define fonts 23 | title_font = font.Font(size=25, weight='bold', family='Helvetica') 24 | button_font = font.Font(size=20) 25 | 26 | # Title Label 27 | label_title = tk.Label(window, text="Gesture Controlled Virtual Mouse and Keyboard", font=title_font, bg="white") 28 | label_title.grid(row=0, column=0, columnspan=4, pady=(20, 10)) 29 | 30 | # Load and resize images 31 | def load_image(path, size): 32 | try: 33 | image = Image.open(path) 34 | image = image.resize(size, Image.LANCZOS) 35 | return ImageTk.PhotoImage(image) 36 | except Exception as e: 37 | print(f"Error loading image {path}: {e}") 38 | return None 39 | 40 | # Center Icon 41 | center_icon = load_image('icons/man.jpeg', ICON_SIZE) 42 | label_icon = tk.Label(window, image=center_icon, bg="white") 43 | label_icon.grid(row=1, column=1, columnspan=2, pady=(10, 20)) 44 | 45 | # Button images 46 | btn1_image = load_image('icons/bot.png', BUTTON_IMAGE_SIZE) 47 | btn2_image = load_image('icons/keyboard.png', BUTTON_IMAGE_SIZE) 48 | btn3_image = load_image('icons/eye.jpeg', BUTTON_IMAGE_SIZE) 49 | btn4_image = load_image('icons/hand.png', BUTTON_IMAGE_SIZE) 50 | 51 | # Buttons 52 | btn1 = tk.Button(window, text='VoiceBot', fg='green', command=aura_chat, image=btn1_image, compound='top', font=button_font) 53 | btn1.grid(row=2, column=0, padx=20, pady=10) 54 | 55 | btn2 = tk.Button(window, text='Keyboard', fg='red', command=vk_keyboard, image=btn2_image, compound='top', font=button_font) 56 | btn2.grid(row=2, column=3, padx=20, pady=10) 57 | 58 | btn3 = tk.Button(window, text='Eye', fg='blue', command=eye_move, image=btn3_image, compound='top', font=button_font) 59 | btn3.grid(row=3, column=0, padx=20, pady=10) 60 | 61 | btn4 = tk.Button(window, text='Gesture', fg='orange', command=gest_control, image=btn4_image, compound='top', font=button_font) 62 | btn4.grid(row=3, column=3, padx=20, pady=10) 63 | 64 | # Exit Button (Centered below other buttons) 65 | btn5 = tk.Button(window, text='Exit', fg='red', command=window.quit, font=button_font) 66 | btn5.grid(row=4, column=1, columnspan=2, pady=20) 67 | 68 | window.mainloop() 69 | -------------------------------------------------------------------------------- /mn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RAGHURAM5276/VIRTUAL-MOUSE-USING-HAND-GESTURE-RECOGNITION/04f21b436fd236c1ef73e5582d55314a057a36ec/mn.png -------------------------------------------------------------------------------- /samvkn.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | from time import sleep 5 | import math 6 | from pynput.keyboard import Controller 7 | 8 | mpHands = mp.solutions.hands 9 | hands = mpHands.Hands(static_image_mode=False, max_num_hands=1, min_detection_confidence=0.5, min_tracking_confidence=0.5) 10 | mpdraw = mp.solutions.drawing_utils 11 | 12 | keyboard = Controller() 13 | 14 | cap = cv2.VideoCapture(0) 15 | cap.set(2, 150) 16 | 17 | text = "" 18 | tx = "" 19 | 20 | # Define Button class 21 | class Button(): 22 | def __init__(self, pos, text, size=[70, 70]): 23 | self.pos = pos 24 | self.size = size 25 | self.text = text 26 | 27 | keys = [["Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "CL"], 28 | ["A", "S", "D", "F", "G", "H", "J", "K", "L", ";", "SP"], 29 | ["Z", "X", "C", "V", "B", "N", "M", ",", ".", "/", "APR"]] 30 | keys1 = [["q", "w", "e", "r", "t", "y", "u", "i", "o", "p", "CL"], 31 | ["a", "s", "d", "f", "g", "h", "j", "k", "l", ";", "SP"], 32 | ["z", "x", "c", "v", "b", "n", "m", ",", ".", "/", "APR"]] 33 | 34 | # Function to draw the keyboard 35 | def vk_keyboard(frame, buttonList): 36 | for button in buttonList: 37 | x, y = button.pos 38 | w, h = button.size 39 | cv2.rectangle(frame, button.pos, (x + w, y + h), (96, 96, 96), cv2.FILLED) 40 | cv2.putText(frame, button.text, (x + 10, y + 40), 41 | cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 255), 2) 42 | return frame 43 | 44 | buttonList = [] 45 | buttonList1 = [] 46 | 47 | # Initialize buttons for the keyboard 48 | for i in range(len(keys)): 49 | for j, key in enumerate(keys[i]): 50 | buttonList.append(Button([80 * j + 10, 80 * i + 10], key)) 51 | for i in range(len(keys1)): 52 | for j, key in enumerate(keys1[i]): 53 | buttonList1.append(Button([80 * j + 10, 80 * i + 10], key)) 54 | 55 | # Function to calculate distance 56 | def calculate_distance(x1, y1, x2, y2): 57 | return math.sqrt((x2 - x1)**2 + (y2 - y1)**2) 58 | 59 | # Coefficients for distance measurement 60 | x = [300, 245, 200, 170, 145, 130, 112, 103, 93, 87, 80, 75, 70, 67, 62, 59, 57] 61 | y = [20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100] 62 | coff = np.polyfit(x, y, 2) 63 | 64 | # Main loop 65 | app = 0 66 | delay = 0 67 | while True: 68 | success, frame = cap.read() 69 | frame = cv2.resize(frame, (1000, 580)) 70 | frame = cv2.flip(frame, 1) 71 | img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 72 | results = hands.process(img) 73 | lanmark = [] 74 | 75 | if app == 0: 76 | frame = vk_keyboard(frame, buttonList) 77 | buttonListToUse = buttonList 78 | r = "up" 79 | else: 80 | frame = vk_keyboard(frame, buttonList1) 81 | buttonListToUse = buttonList1 82 | r = "down" 83 | 84 | if results.multi_hand_landmarks: 85 | for hn in results.multi_hand_landmarks: 86 | for id, lm in enumerate(hn.landmark): 87 | hl, wl, cl = frame.shape 88 | cx, cy = int(lm.x * wl), int(lm.y * hl) 89 | lanmark.append([id, cx, cy]) 90 | 91 | if lanmark != 0: 92 | try: 93 | x5, y5 = lanmark[5][1], lanmark[5][2] 94 | x17, y17 = lanmark[17][1], lanmark[17][2] 95 | dis = calculate_distance(x5, y5, x17, y17) 96 | A, B, C = coff 97 | distanceCM = A * dis ** 2 + B * dis + C 98 | if 20 < distanceCM < 50: 99 | x, y = lanmark[8][1], lanmark[8][2] 100 | x2, y2 = lanmark[6][1], lanmark[6][2] 101 | x3, y3 = lanmark[12][1], lanmark[12][2] 102 | cv2.circle(frame, (x, y), 20, (255, 0, 255), cv2.FILLED) 103 | cv2.circle(frame, (x3, y3), 20, (255, 0, 255), cv2.FILLED) 104 | 105 | if y2 > y: 106 | for button in buttonListToUse: 107 | xb, yb = button.pos 108 | wb, hb = button.size 109 | 110 | if (xb < x < xb + wb) and (yb < y < yb + hb): 111 | cv2.rectangle(frame, (xb - 5, yb - 5), (xb + wb + 5, yb + hb + 5), (160, 160, 160), cv2.FILLED) 112 | cv2.putText(frame, button.text, (xb + 20, yb + 65), cv2.FONT_HERSHEY_PLAIN, 4, (255, 255, 255), 4) 113 | dis = calculate_distance(x, y, x3, y3) 114 | if dis < 50 and delay == 0: 115 | k = button.text 116 | cv2.rectangle(frame, (xb - 5, yb - 5), (xb + wb + 5, yb + hb + 5), (255, 255, 255), cv2.FILLED) 117 | cv2.putText(frame, k, (xb + 20, yb + 65), cv2.FONT_HERSHEY_PLAIN, 4, (0, 0, 0), 4) 118 | 119 | if k == "SP": 120 | tx = ' ' 121 | text += tx 122 | keyboard.press(tx) 123 | 124 | elif k == "CL": 125 | tx = text[:-1] 126 | text = "" 127 | text += tx 128 | keyboard.press('\b') 129 | 130 | elif k == "APR" and r == "up": 131 | app = 1 132 | 133 | elif k == "APR" and r == "down": 134 | app = 0 135 | 136 | else: 137 | text += k 138 | keyboard.press(k) 139 | delay = 1 140 | except: 141 | pass 142 | 143 | if delay != 0: 144 | delay += 1 145 | if delay > 10: 146 | delay = 0 147 | 148 | cv2.rectangle(frame, (20, 250), (850, 400), (255, 255, 255), cv2.FILLED) 149 | cv2.putText(frame, text, (30, 300), cv2.FONT_HERSHEY_PLAIN, 3, (0, 0, 0), 3) 150 | cv2.imshow('virtual keyboard', frame) 151 | 152 | if cv2.waitKey(1) & 0xff == ord('q'): 153 | break 154 | -------------------------------------------------------------------------------- /tempCodeRunnerFile.py: -------------------------------------------------------------------------------- 1 | 2 | gest_name = handmajor.get_gesture() 3 | Controller.handle_controls(gest_name, handmajor.hand_result) 4 | 5 | for hand_landmarks in results.multi_hand_landmarks: 6 | mp_drawing.draw_landmarks(image, hand_landmarks, mp_hands.HAND_CONNECTIONS) 7 | else: 8 | Controller.prev_hand = None 9 | cv2.imshow('Gesture Controller', imag --------------------------------------------------------------------------------