├── GestureAPI.pyc ├── screenshots ├── 1.jpg ├── 2.jpg ├── 3.jpg ├── 4.jpg ├── 5.jpg ├── 6.jpg ├── 7.jpg ├── 8.jpg └── 9.jpg ├── docs └── Documentation.pdf ├── LICENSE ├── GestureAPI.py ├── README.md └── HandRecognition.py /GestureAPI.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/GestureAPI.pyc -------------------------------------------------------------------------------- /screenshots/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/1.jpg -------------------------------------------------------------------------------- /screenshots/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/2.jpg -------------------------------------------------------------------------------- /screenshots/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/3.jpg -------------------------------------------------------------------------------- /screenshots/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/4.jpg -------------------------------------------------------------------------------- /screenshots/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/5.jpg -------------------------------------------------------------------------------- /screenshots/6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/6.jpg -------------------------------------------------------------------------------- /screenshots/7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/7.jpg -------------------------------------------------------------------------------- /screenshots/8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/8.jpg -------------------------------------------------------------------------------- /screenshots/9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/screenshots/9.jpg -------------------------------------------------------------------------------- /docs/Documentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahaveerverma/hand-gesture-recognition-opencv/HEAD/docs/Documentation.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Mahaveer Verma 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /GestureAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import math 4 | import numpy as np 5 | 6 | class Gesture(object): 7 | def __init__(self,name): 8 | self.name=name 9 | def getName(self): 10 | return self.name 11 | def set_palm(self,hand_center,hand_radius): 12 | self.hand_center=hand_center 13 | self.hand_radius=hand_radius 14 | def set_finger_pos(self,finger_pos): 15 | self.finger_pos=finger_pos 16 | self.finger_count=len(finger_pos) 17 | def calc_angles(self): 18 | self.angle=np.zeros(self.finger_count,dtype=int) 19 | for i in range(self.finger_count): 20 | y = self.finger_pos[i][1] 21 | x = self.finger_pos[i][0] 22 | self.angle[i]=abs(math.atan2((self.hand_center[1]-y),(x-self.hand_center[0]))*180/math.pi) 23 | 24 | def DefineGestures(): 25 | dict={} 26 | # 1. BEGIN ------------------------------------# 27 | V=Gesture("V") 28 | V.set_palm((475,225),45) 29 | V.set_finger_pos([(490,90),(415,105)]) 30 | V.calc_angles() 31 | dict[V.getName()]=V 32 | # 1. END --------------------------------------# 33 | # 2. BEGIN ------------------------------------# 34 | L_right=Gesture("L_right") 35 | L_right.set_palm((475,225),50) 36 | L_right.set_finger_pos([(450,62),(345,200)]) 37 | L_right.calc_angles() 38 | dict[L_right.getName()]=L_right 39 | # 2. END --------------------------------------# 40 | # 3. BEGIN ------------------------------------# 41 | Index_Pointing=Gesture("Index_Pointing") 42 | Index_Pointing.set_palm((480,230),43) 43 | Index_Pointing.set_finger_pos([(475,102)]) 44 | Index_Pointing.calc_angles() 45 | dict[Index_Pointing.getName()]=Index_Pointing 46 | # 3. END --------------------------------------# 47 | return dict 48 | 49 | def CompareGestures(src1,src2): 50 | if(src1.finger_count==src2.finger_count): 51 | if(src1.finger_count==1): 52 | angle_diff=src1.angle[0]-src2.angle[0] 53 | if(angle_diff>20): 54 | result=0 55 | else: 56 | len1 = np.sqrt((src1.finger_pos[0][0]- src1.hand_center[0])**2 + (src1.finger_pos[0][1] - src1.hand_center[1])**2) 57 | len2 = np.sqrt((src2.finger_pos[0][0]- src2.hand_center[0])**2 + (src2.finger_pos[0][1] - src2.hand_center[1])**2) 58 | length_diff=len1/len2 59 | radius_diff=src1.hand_radius/src2.hand_radius 60 | length_score=abs(length_diff-radius_diff) 61 | if(length_score<0.09): 62 | result=src2.getName() 63 | else: 64 | result=0 65 | else: 66 | angle_diff=[] 67 | for i in range(src1.finger_count): 68 | angle_diff.append(src1.angle[i]-src2.angle[i]) 69 | angle_score=max(angle_diff)-min(angle_diff) 70 | if(angle_score<15): 71 | length_diff=[] 72 | for i in range(src1.finger_count): 73 | len1 = np.sqrt((src1.finger_pos[i][0]- src1.hand_center[0])**2 + (src1.finger_pos[i][1] - src1.hand_center[1])**2) 74 | len2 = np.sqrt((src2.finger_pos[i][0]- src2.hand_center[0])**2 + (src2.finger_pos[i][1] - src2.hand_center[1])**2) 75 | length_diff.append(len1/len2) 76 | length_score=max(length_diff)-min(length_diff) 77 | if(length_score<0.06): 78 | result=src2.getName() 79 | else: 80 | result=0 81 | else: 82 | result=0 83 | else: 84 | result=0 85 | return result 86 | 87 | def DecideGesture(src,GestureDictionary): 88 | result_list=[] 89 | for k in GestureDictionary.keys(): 90 | src2='"'+k+'"' 91 | result=CompareGestures(src,GestureDictionary[k]) 92 | if(result!=0): 93 | return result 94 | return "NONE" 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HAND GESTURE RECOGNITION 2 | 3 | ### INTRODUCTION 4 | This project implements a hand recognition and hand gesture recognition system using OpenCV on Python 2.7. A histogram based approach is used to separate out a hand from the background image. Background cancellation techniques are used to obtain optimum results. The detected hand is then processed and modelled by finding contours and convex hull to recognize finger and palm positions and dimensions. Finally, a gesture object is created from the recognized pattern which is compared to a defined gesture dictionary. 5 | 6 | **Platform:** Python 2.7 7 | 8 | **Libraries:** OpenCV 2.4.8, Numpy 9 | 10 | **Hardware Requirements:** Camera/Webcam 11 | 12 | ### USAGE 13 | 14 | Run HandRecognition.py to begin the program. 15 | 16 | **Note for Windows users:** 17 | Remove this line from all .py files: '#!/usr/bin/python' or else you might get some error. 18 | 19 | You will find a window that shows your camera feed. Notice a rectangular frame on the right side of the window. That's the frame where all the detection and recognition works. 20 | 21 | To begin, keep your hand and body outside the frame, so as to capture just the background environment, and press 'b'. This will capture the background and create a model of it. This model will be used to remove background from every frame captured once the program setup is complete. 22 | 23 | Now, you have to capture your hand histogram. Place your hand over the 9 small boxes in the frame so as to capture the maximum range of shades of your hand. Don't let any shadow or air gap show on the boxed areas for best results. Press 'c' to capture the hand and generate a histogram. 24 | 25 | The setup is now complete. Now you will see, by keeping your hand inside the rectangular frame, it gets detected and you will notice a circle inside your palm area, with lines projecting out from it towards your fingers. Try moving your hands, hiding a few fingers or giving it one of the sample gestures implemented in the program. 26 | 27 | The sample gestures implemented are described with screenshots in the documentation. 28 | 29 | They are: 30 | 31 | 1. "V" with your index and middle finger 32 | 33 | 2. A flipped "L" with thumb and index finger 34 | 35 | 3. Pointing with your index finger in vertical position 36 | 37 | **Note:** Press 'q' at any time to stop the program or 'r' to restart the program. 38 | 39 | ### HOW DOES IT WORK? 40 | 41 | Read full documentation for detailed explanation about implementation in "docs" folder. 42 | 43 | During setup, first a background model is generated when the user presses 'b'. Then, a histogram is generated when the user provides his hand as a sample by pressing 'c'. When the setup is completed, the program goes into an infinite while loop which does as follows. 44 | 45 | Camera input frame is saved to a numpy array. A mask is generated based on background model and applied on the frame. This removes background from the captured frame. Now the frame containing only the foreground is converted to HSV color space, followed by histogram comparison (generating back projection). This leaves us with the detected hand. Morphology and smoothening is applied to get a proper hand shape out of the frame. A threshold converts this into a binary image. 46 | 47 | Next, we find contours of the binary image obtained, look for the largest contour and find its convex hull. 48 | 49 | Using points from the largest contour we determine center of the palm by finding the largest circle inscribed inside the contour and then the dimension of palm. Using the center of palm as reference, we eliminate all points from the convex hull which do not seem to be part of hand. Also, nearby convex hull points are eliminated so that we are left with exactly only those many points as the number of fingers stretched out. 50 | 51 | Using the positions of fingers and palm dimensions, we model our hand. 52 | 53 | Then we compare the model with a dictionary of Gestures defined in GestureAPI.py to determine presence of gestures. 54 | 55 | **Full explanation with screenshots is provided in /docs/Documentation.pdf** 56 | 57 | For any queries, contact: mahaveer.verma1@gmail.com -------------------------------------------------------------------------------- /HandRecognition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # ********************************************** 4 | # * Hand Gesture Recognition Implementation v1.0 5 | # * 2 July 2016 6 | # * Mahaveer Verma 7 | # ********************************************** 8 | 9 | import cv2 10 | import numpy as np 11 | import math 12 | from GestureAPI import * 13 | 14 | # Variables & parameters 15 | hsv_thresh_lower=150 16 | gaussian_ksize=11 17 | gaussian_sigma=0 18 | morph_elem_size=13 19 | median_ksize=3 20 | capture_box_count=9 21 | capture_box_dim=20 22 | capture_box_sep_x=8 23 | capture_box_sep_y=18 24 | capture_pos_x=500 25 | capture_pos_y=150 26 | cap_region_x_begin=0.5 # start point/total width 27 | cap_region_y_end=0.8 # start point/total width 28 | finger_thresh_l=2.0 29 | finger_thresh_u=3.8 30 | radius_thresh=0.04 # factor of width of full frame 31 | first_iteration=True 32 | finger_ct_history=[0,0] 33 | 34 | # ------------------------ Function declarations ------------------------ # 35 | 36 | # 1. Hand capture histogram 37 | def hand_capture(frame_in,box_x,box_y): 38 | hsv = cv2.cvtColor(frame_in, cv2.COLOR_BGR2HSV) 39 | ROI = np.zeros([capture_box_dim*capture_box_count,capture_box_dim,3], dtype=hsv.dtype) 40 | for i in xrange(capture_box_count): 41 | ROI[i*capture_box_dim:i*capture_box_dim+capture_box_dim,0:capture_box_dim] = hsv[box_y[i]:box_y[i]+capture_box_dim,box_x[i]:box_x[i]+capture_box_dim] 42 | hand_hist = cv2.calcHist([ROI],[0, 1], None, [180, 256], [0, 180, 0, 256]) 43 | cv2.normalize(hand_hist,hand_hist, 0, 255, cv2.NORM_MINMAX) 44 | return hand_hist 45 | 46 | # 2. Filters and threshold 47 | def hand_threshold(frame_in,hand_hist): 48 | frame_in=cv2.medianBlur(frame_in,3) 49 | hsv=cv2.cvtColor(frame_in,cv2.COLOR_BGR2HSV) 50 | hsv[0:int(cap_region_y_end*hsv.shape[0]),0:int(cap_region_x_begin*hsv.shape[1])]=0 # Right half screen only 51 | hsv[int(cap_region_y_end*hsv.shape[0]):hsv.shape[0],0:hsv.shape[1]]=0 52 | back_projection = cv2.calcBackProject([hsv], [0,1],hand_hist, [00,180,0,256], 1) 53 | disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (morph_elem_size,morph_elem_size)) 54 | cv2.filter2D(back_projection, -1, disc, back_projection) 55 | back_projection=cv2.GaussianBlur(back_projection,(gaussian_ksize,gaussian_ksize), gaussian_sigma) 56 | back_projection=cv2.medianBlur(back_projection,median_ksize) 57 | ret, thresh = cv2.threshold(back_projection, hsv_thresh_lower, 255, 0) 58 | 59 | return thresh 60 | 61 | # 3. Find hand contour 62 | def hand_contour_find(contours): 63 | max_area=0 64 | largest_contour=-1 65 | for i in range(len(contours)): 66 | cont=contours[i] 67 | area=cv2.contourArea(cont) 68 | if(area>max_area): 69 | max_area=area 70 | largest_contour=i 71 | if(largest_contour==-1): 72 | return False,0 73 | else: 74 | h_contour=contours[largest_contour] 75 | return True,h_contour 76 | 77 | # 4. Detect & mark fingers 78 | def mark_fingers(frame_in,hull,pt,radius): 79 | global first_iteration 80 | global finger_ct_history 81 | finger=[(hull[0][0][0],hull[0][0][1])] 82 | j=0 83 | 84 | cx = pt[0] 85 | cy = pt[1] 86 | 87 | for i in range(len(hull)): 88 | dist = np.sqrt((hull[-i][0][0] - hull[-i+1][0][0])**2 + (hull[-i][0][1] - hull[-i+1][0][1])**2) 89 | if (dist>18): 90 | if(j==0): 91 | finger=[(hull[-i][0][0],hull[-i][0][1])] 92 | else: 93 | finger.append((hull[-i][0][0],hull[-i][0][1])) 94 | j=j+1 95 | 96 | temp_len=len(finger) 97 | i=0 98 | while(ifinger_thresh_u*radius or finger[i][1]>cy+radius): 101 | finger.remove((finger[i][0],finger[i][1])) 102 | temp_len=temp_len-1 103 | else: 104 | i=i+1 105 | 106 | temp_len=len(finger) 107 | if(temp_len>5): 108 | for i in range(1,temp_len+1-5): 109 | finger.remove((finger[temp_len-i][0],finger[temp_len-i][1])) 110 | 111 | palm=[(cx,cy),radius] 112 | 113 | if(first_iteration): 114 | finger_ct_history[0]=finger_ct_history[1]=len(finger) 115 | first_iteration=False 116 | else: 117 | finger_ct_history[0]=0.34*(finger_ct_history[0]+finger_ct_history[1]+len(finger)) 118 | 119 | if((finger_ct_history[0]-int(finger_ct_history[0]))>0.8): 120 | finger_count=int(finger_ct_history[0])+1 121 | else: 122 | finger_count=int(finger_ct_history[0]) 123 | 124 | finger_ct_history[1]=len(finger) 125 | 126 | count_text="FINGERS:"+str(finger_count) 127 | cv2.putText(frame_in,count_text,(int(0.62*frame_in.shape[1]),int(0.88*frame_in.shape[0])),cv2.FONT_HERSHEY_DUPLEX,1,(0,255,255),1,8) 128 | 129 | for k in range(len(finger)): 130 | cv2.circle(frame_in,finger[k],10,255,2) 131 | cv2.line(frame_in,finger[k],(cx,cy),255,2) 132 | return frame_in,finger,palm 133 | 134 | # 5. Mark hand center circle 135 | 136 | def mark_hand_center(frame_in,cont): 137 | max_d=0 138 | pt=(0,0) 139 | x,y,w,h = cv2.boundingRect(cont) 140 | for ind_y in xrange(int(y+0.3*h),int(y+0.8*h)): #around 0.25 to 0.6 region of height (Faster calculation with ok results) 141 | for ind_x in xrange(int(x+0.3*w),int(x+0.6*w)): #around 0.3 to 0.6 region of width (Faster calculation with ok results) 142 | dist= cv2.pointPolygonTest(cont,(ind_x,ind_y),True) 143 | if(dist>max_d): 144 | max_d=dist 145 | pt=(ind_x,ind_y) 146 | if(max_d>radius_thresh*frame_in.shape[1]): 147 | thresh_score=True 148 | cv2.circle(frame_in,pt,int(max_d),(255,0,0),2) 149 | else: 150 | thresh_score=False 151 | return frame_in,pt,max_d,thresh_score 152 | 153 | # 6. Find and display gesture 154 | 155 | def find_gesture(frame_in,finger,palm): 156 | frame_gesture.set_palm(palm[0],palm[1]) 157 | frame_gesture.set_finger_pos(finger) 158 | frame_gesture.calc_angles() 159 | gesture_found=DecideGesture(frame_gesture,GestureDictionary) 160 | gesture_text="GESTURE:"+str(gesture_found) 161 | cv2.putText(frame_in,gesture_text,(int(0.56*frame_in.shape[1]),int(0.97*frame_in.shape[0])),cv2.FONT_HERSHEY_DUPLEX,1,(0,255,255),1,8) 162 | return frame_in,gesture_found 163 | 164 | # 7. Remove bg from image 165 | 166 | def remove_bg(frame): 167 | fg_mask=bg_model.apply(frame) 168 | kernel = np.ones((3,3),np.uint8) 169 | fg_mask=cv2.erode(fg_mask,kernel,iterations = 1) 170 | frame=cv2.bitwise_and(frame,frame,mask=fg_mask) 171 | return frame 172 | 173 | # ------------------------ BEGIN ------------------------ # 174 | 175 | # Camera 176 | camera = cv2.VideoCapture(0) 177 | capture_done=0 178 | bg_captured=0 179 | GestureDictionary=DefineGestures() 180 | frame_gesture=Gesture("frame_gesture") 181 | 182 | while(1): 183 | # Capture frame from camera 184 | ret, frame = camera.read() 185 | frame=cv2.bilateralFilter(frame,5,50,100) 186 | # Operations on the frame 187 | frame=cv2.flip(frame,1) 188 | cv2.rectangle(frame,(int(cap_region_x_begin*frame.shape[1]),0),(frame.shape[1],int(cap_region_y_end*frame.shape[0])),(255,0,0),1) 189 | frame_original=np.copy(frame) 190 | if(bg_captured): 191 | fg_frame=remove_bg(frame) 192 | 193 | if (not (capture_done and bg_captured)): 194 | if(not bg_captured): 195 | cv2.putText(frame,"Remove hand from the frame and press 'b' to capture background",(int(0.05*frame.shape[1]),int(0.97*frame.shape[0])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,255,255),1,8) 196 | else: 197 | cv2.putText(frame,"Place hand inside boxes and press 'c' to capture hand histogram",(int(0.08*frame.shape[1]),int(0.97*frame.shape[0])),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,255,255),1,8) 198 | first_iteration=True 199 | finger_ct_history=[0,0] 200 | box_pos_x=np.array([capture_pos_x,capture_pos_x+capture_box_dim+capture_box_sep_x,capture_pos_x+2*capture_box_dim+2*capture_box_sep_x,capture_pos_x,capture_pos_x+capture_box_dim+capture_box_sep_x,capture_pos_x+2*capture_box_dim+2*capture_box_sep_x,capture_pos_x,capture_pos_x+capture_box_dim+capture_box_sep_x,capture_pos_x+2*capture_box_dim+2*capture_box_sep_x],dtype=int) 201 | box_pos_y=np.array([capture_pos_y,capture_pos_y,capture_pos_y,capture_pos_y+capture_box_dim+capture_box_sep_y,capture_pos_y+capture_box_dim+capture_box_sep_y,capture_pos_y+capture_box_dim+capture_box_sep_y,capture_pos_y+2*capture_box_dim+2*capture_box_sep_y,capture_pos_y+2*capture_box_dim+2*capture_box_sep_y,capture_pos_y+2*capture_box_dim+2*capture_box_sep_y],dtype=int) 202 | for i in range(capture_box_count): 203 | cv2.rectangle(frame,(box_pos_x[i],box_pos_y[i]),(box_pos_x[i]+capture_box_dim,box_pos_y[i]+capture_box_dim),(255,0,0),1) 204 | else: 205 | frame=hand_threshold(fg_frame,hand_histogram) 206 | contour_frame=np.copy(frame) 207 | contours,hierarchy=cv2.findContours(contour_frame,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 208 | found,hand_contour=hand_contour_find(contours) 209 | if(found): 210 | hand_convex_hull=cv2.convexHull(hand_contour) 211 | frame,hand_center,hand_radius,hand_size_score=mark_hand_center(frame_original,hand_contour) 212 | if(hand_size_score): 213 | frame,finger,palm=mark_fingers(frame,hand_convex_hull,hand_center,hand_radius) 214 | frame,gesture_found=find_gesture(frame,finger,palm) 215 | else: 216 | frame=frame_original 217 | 218 | # Display frame in a window 219 | cv2.imshow('Hand Gesture Recognition v1.0',frame) 220 | interrupt=cv2.waitKey(10) 221 | 222 | # Quit by pressing 'q' 223 | if interrupt & 0xFF == ord('q'): 224 | break 225 | # Capture hand by pressing 'c' 226 | elif interrupt & 0xFF == ord('c'): 227 | if(bg_captured): 228 | capture_done=1 229 | hand_histogram=hand_capture(frame_original,box_pos_x,box_pos_y) 230 | # Capture background by pressing 'b' 231 | elif interrupt & 0xFF == ord('b'): 232 | bg_model = cv2.BackgroundSubtractorMOG2(0,10) 233 | bg_captured=1 234 | # Reset captured hand by pressing 'r' 235 | elif interrupt & 0xFF == ord('r'): 236 | capture_done=0 237 | bg_captured=0 238 | 239 | # Release camera & end program 240 | camera.release() 241 | cv2.destroyAllWindows() 242 | --------------------------------------------------------------------------------