├── GestureRecognitionSourceFiles.zip ├── Gesture recognitionIACV2FinalReport.pdf ├── README.md ├── merge_files.py ├── LICENSE ├── rnn.py ├── pred.py └── src.py /GestureRecognitionSourceFiles.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/niharikabalachandra/Gesture-Recognition-Using-RNN/HEAD/GestureRecognitionSourceFiles.zip -------------------------------------------------------------------------------- /Gesture recognitionIACV2FinalReport.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/niharikabalachandra/Gesture-Recognition-Using-RNN/HEAD/Gesture recognitionIACV2FinalReport.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gesture-Recognition-Using-RNN 2 | This project investigates gesture recognition techniques using Leap Motion Controller to collect 3 dimensional hand features such as coordinates, position of the hand, etc. We then use this raw data to train a Long Short Term Memory Recurrent Neural Network that is able to recognize basic hand gestures 3 | -------------------------------------------------------------------------------- /merge_files.py: -------------------------------------------------------------------------------- 1 | import csv 2 | fout=open("./one_file.csv","a") 3 | # first file: 4 | count=0 5 | with open('./CSV/0.csv', 'rb') as f: 6 | reader = csv.reader(f, delimiter=',') 7 | for row in reader: 8 | if row[1]==str(0.0) and row [2]==str(0.0) and row[3]==str(0.0): 9 | count+=1 10 | else: 11 | pass 12 | #Eliminate the samples which had erroneous input recorded 13 | if count==0: 14 | for line in open("./CSV/0.csv"): 15 | fout.write(line) 16 | 17 | # Rest of the files: 18 | for num in range(1,503): 19 | count=0 20 | with open('./CSV/'+str(num)+'.csv', 'rb') as f: 21 | reader = csv.reader(f, delimiter=',') 22 | for row in reader: 23 | if row[1]==str(0.0) and row [2]==str(0.0) and row[3]==str(0.0): 24 | count+=1 25 | else: 26 | pass 27 | if count==0: 28 | f = open("./CSV/"+str(num)+".csv") 29 | for line in f: 30 | fout.write(line) 31 | f.close() 32 | fout.close() 33 | 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Niharika Balachandra 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /rnn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from keras.models import Sequential 4 | from keras.layers import Dense 5 | from keras.layers import LSTM 6 | from keras.models import load_model 7 | from keras.utils import np_utils 8 | from imblearn.over_sampling import SMOTE 9 | import time 10 | 11 | # Import dataset 12 | dataset = pd.read_csv('~/LeapDeveloperKit_2.3.1+31549_linux/LeapSDK/src/one_file.csv',header=None) 13 | 14 | #Split to Data samples and labels 15 | x = dataset.iloc[:, 1:11].values 16 | y = dataset.iloc[:, 11].values 17 | 18 | #Number of time series to work on is 60 19 | t=60 20 | n_samples=len(dataset)/t 21 | 22 | 23 | #Reshape input samples to Nx60x11 24 | x = np.reshape(x, (x.shape[0]/t, t, x.shape[1])) 25 | #Set labels to these N samples 26 | indices=[ind for ind in range(len(y)) if ind%t==0] 27 | y=y[indices] 28 | 29 | # One-hot encode the classes 30 | from sklearn.preprocessing import LabelEncoder 31 | label_encoder=LabelEncoder() 32 | y_new=label_encoder.fit_transform(y) 33 | y_one_hot=np_utils.to_categorical(y_new) 34 | y=y_one_hot 35 | 36 | #Split data to training and testing data 37 | from sklearn.model_selection import train_test_split 38 | X_train, X_test, y_train, y_test = train_test_split(x, y, test_size = 0.2) 39 | 40 | #Construct the model 41 | n_neurons=7 42 | start_time=time.time() 43 | model = Sequential() 44 | model.add(LSTM(n_neurons, input_shape=(X_train.shape[1],X_train.shape[2]))) 45 | model.add(Dense(output_dim=2,init = 'uniform', activation='sigmoid')) 46 | model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy']) 47 | 48 | #Train the model 49 | model.fit(X_train, y_train, epochs=100, batch_size=5) 50 | 51 | #Evaluation of the Training time 52 | print "%s Minutes of Execution" %str((time.time()-start_time)/60) 53 | 54 | #Save the model for prediction 55 | model.save('model_test.h5') 56 | print "Model Saved" 57 | 58 | # Evaluate the model on CV Data 59 | scores = model.evaluate(X_test, y_test, verbose=0) 60 | print "%s: %.2f%%" % (model.metrics_names[1], scores[1]*100) 61 | -------------------------------------------------------------------------------- /pred.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from keras.models import load_model 4 | from keras.utils import np_utils 5 | import time 6 | import sys 7 | sys.path.insert(0, "../lib") 8 | sys.path.insert(0, "../lib/x64") 9 | import Leap 10 | import tensorflow as tf 11 | import os 12 | 13 | leftHand=[] 14 | rightHand=[] 15 | previous_time=0 16 | 17 | #Load the model 18 | model=load_model('model_test.h5') 19 | print "Model Loaded" 20 | 21 | gesture=["Swipe Right","Swipe Left"] 22 | audio=["aplay right.wav","aplay left.wav"] 23 | graph=tf.get_default_graph() 24 | 25 | class SimpleListener(Leap.Listener): 26 | 27 | def on_connect(self, controller): 28 | print "Connected" 29 | 30 | def on_frame(self, controller): 31 | global model, graph,previous_time, gesture, audio 32 | frame=controller.frame() 33 | hand=frame.hands 34 | lFlag=False 35 | rFlag=False 36 | 37 | if not hand.is_empty: 38 | for h in hand: 39 | if h.is_left: 40 | leftHand=h 41 | lFlag=True 42 | elif h.is_right: 43 | rightHand=h 44 | rFlag=True 45 | if lFlag and rFlag: 46 | pass 47 | 48 | elif lFlag: 49 | 50 | print "Hand Detected" 51 | time.sleep(1.5) 52 | print "Record now" 53 | time.sleep(0.5) 54 | frame=controller.frame() 55 | lefthand=frame.hands[0] 56 | current_time=frame.timestamp 57 | time_delta=current_time-previous_time 58 | data_left=[[] for _ in range(12)] 59 | if time_delta>4e6 and time_delta<1e9: 60 | for i in range(59,-1,-1): 61 | data_left[0].append(0) 62 | data_left[1].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[0],4)) 63 | data_left[2].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[1],4)) 64 | data_left[3].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[2],4)) 65 | data_left[4].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[0],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[0],4)) 66 | data_left[5].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[1],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[1],4)) 67 | data_left[6].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[2],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[2],4)) 68 | data_left[7].append(round(controller.frame(i).hand(lefthand.id).palm_normal.roll,4)) 69 | data_left[8].append(round(controller.frame(i).hand(lefthand.id).direction.yaw,4)) 70 | data_left[9].append(round(controller.frame(i).hand(lefthand.id).direction.pitch,4)) 71 | data_left[10].append(int(rFlag)) 72 | data_left[11].append(1) 73 | 74 | df_left=pd.DataFrame(data=data_left) 75 | df_left=df_left.transpose() 76 | 77 | x=df_left.iloc[:,1:11].values 78 | with graph.as_default(): 79 | y_pred=model.predict(np.reshape(x,(1,x.shape[0],x.shape[1]))) 80 | pred=np.argmax(y_pred) 81 | # Eliminate the False Postives 82 | if (y_pred[0][1] or y_pred[0][0])>=0.3: 83 | print gesture[pred] 84 | os.system(audio[pred]) 85 | else: 86 | print "try again" 87 | else: 88 | print "Try Again" 89 | 90 | time.sleep(3) 91 | print "Ready" 92 | time.sleep(1) 93 | previous_time=current_time 94 | 95 | 96 | 97 | elif rFlag: 98 | 99 | 100 | print "Hand Detected" 101 | time.sleep(1.5) 102 | print "Record now" 103 | time.sleep(0.5) 104 | 105 | frame=controller.frame() 106 | righthand=frame.hands[0] 107 | current_time=frame.timestamp 108 | time_delta=current_time-previous_time 109 | data_right=[[] for _ in range(12)] 110 | if time_delta>5e6 and time_delta<1e9: 111 | for i in range(59,-1,-1): 112 | data_right[0].append(0) 113 | data_right[1].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[0],4)) 114 | data_right[2].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[1],4)) 115 | data_right[3].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[2],4)) 116 | data_right[4].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[0],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[0],4)) 117 | data_right[5].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[1],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[1],4)) 118 | data_right[6].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[2],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[2],4)) 119 | data_right[7].append(round(controller.frame(i).hand(righthand.id).palm_normal.roll,4)) 120 | data_right[8].append(round(controller.frame(i).hand(righthand.id).direction.yaw,4)) 121 | data_right[9].append(round(controller.frame(i).hand(righthand.id).direction.pitch,4)) 122 | data_right[10].append(int(rFlag)) 123 | data_right[11].append(0) 124 | df_right=pd.DataFrame(data=data_right) 125 | df_right=df_right.transpose() 126 | 127 | x = df_right.iloc[:, 1:11].values 128 | with graph.as_default(): 129 | y_pred=model.predict(np.reshape(x,(1,x.shape[0],x.shape[1]))) 130 | pred=np.argmax(y_pred) 131 | # Eliminate the False Postives 132 | if (y_pred[0][0] or y_pred[0][1])>=0.3: 133 | print gesture[pred] 134 | os.system(audio[pred]) 135 | else: 136 | print "try again" 137 | else: 138 | print "Try Again" 139 | 140 | time.sleep(3) 141 | print "Ready" 142 | time.sleep(1) 143 | previous_time=current_time 144 | 145 | else: 146 | pass 147 | 148 | 149 | 150 | def main(): 151 | listener = SimpleListener() 152 | controller = Leap.Controller() 153 | controller.add_listener(listener) 154 | 155 | print "Hit Enter to quit!" 156 | try: 157 | sys.stdin.readline() 158 | except KeyboardInterrupt: 159 | pass 160 | finally: 161 | controller.remove_listener(listener) 162 | 163 | if __name__ == "__main__": 164 | main() 165 | -------------------------------------------------------------------------------- /src.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | sys.path.insert(0, "../lib") 3 | sys.path.insert(0, "../lib/x64") 4 | import Leap 5 | import time 6 | import pandas as pd 7 | import numpy as np 8 | from PIL import Image 9 | 10 | leftHand=[] 11 | rightHand=[] 12 | count=502 13 | left_count=238 14 | right_count=259 15 | count_free=0 16 | previous_time=0 17 | frame_count=0 18 | class SimpleListener(Leap.Listener): 19 | 20 | def on_connect(self, controller): 21 | print "Connected" 22 | 23 | 24 | 25 | def on_frame(self, controller): 26 | global count,left_count,right_count, count_free,previous_time,frame_count 27 | 28 | frame=controller.frame() 29 | hand=frame.hands 30 | lFlag=False 31 | rFlag=False 32 | 33 | if not hand.is_empty: 34 | for h in hand: 35 | if h.is_left: 36 | leftHand=h 37 | lFlag=True 38 | elif h.is_right: 39 | rightHand=h 40 | rFlag=True 41 | if lFlag and rFlag: 42 | pass 43 | 44 | elif lFlag: 45 | 46 | 47 | # Recording Left Gesture 48 | 49 | if (leftHand.palm_normal.roll* Leap.RAD_TO_DEG<=35 and leftHand.palm_normal.roll* Leap.RAD_TO_DEG>0) and len(leftHand.fingers.extended())==1: 50 | print "Hand Detected for Left" 51 | time.sleep(1.5) 52 | print "Record now" 53 | time.sleep(0.5) 54 | frame=controller.frame() 55 | lefthand=frame.hands[0] 56 | current_time=frame.timestamp 57 | time_delta=current_time-previous_time 58 | 59 | data_left=[[] for _ in range(12)] 60 | # Record only if frame rate during gesture is low 61 | if time_delta>5e6 and time_delta<1e9: 62 | for i in range(59,-1,-1): 63 | data_left[0].append(count) 64 | data_left[1].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[0],4)) 65 | data_left[2].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[1],4)) 66 | data_left[3].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[2],4)) 67 | data_left[4].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[0],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[0],4)) 68 | data_left[5].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[1],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[1],4)) 69 | data_left[6].append(round(controller.frame(i).hand(lefthand.id).stabilized_palm_position[2],4)-round(controller.frame(59).hand(lefthand.id).stabilized_palm_position[2],4)) 70 | data_left[7].append(round(controller.frame(i).hand(lefthand.id).palm_normal.roll,4)) 71 | data_left[8].append(round(controller.frame(i).hand(lefthand.id).direction.yaw,4)) 72 | data_left[9].append(round(controller.frame(i).hand(lefthand.id).direction.pitch,4)) 73 | data_left[10].append(int(rFlag)) 74 | data_left[11].append(1) 75 | 76 | df_left=pd.DataFrame(data=data_left) 77 | df_left=df_left.transpose() 78 | df_left.to_csv("./CSV/%s.csv"%(str(count)),sep=',',index=False, header=False) 79 | count+=1 80 | left_count+=1 81 | print "Swipe Left Gesture Recorded %s" %(str(left_count)) 82 | else: 83 | print "Try Again" 84 | time.sleep(3) 85 | print "Ready" 86 | time.sleep(1) 87 | previous_time=current_time 88 | 89 | 90 | # Recording Right Gesture 91 | 92 | if ((rightHand.palm_normal.roll* Leap.RAD_TO_DEG>=-35 and rightHand.palm_normal.roll* Leap.RAD_TO_DEG<0) and (len(rightHand.fingers.extended())==1)): 93 | print "Hand Detected for Swipe Right" 94 | time.sleep(1.5) 95 | print "Record now" 96 | time.sleep(0.5) 97 | frame=controller.frame() 98 | righthand=frame.hands[0] 99 | current_time=frame.timestamp 100 | time_delta=current_time-previous_time 101 | data_right=[[] for _ in range(12)] 102 | # Record only if frame rate during gesture is low 103 | if time_delta>5e6 and time_delta<1e9: 104 | for i in range(59,-1,-1): 105 | data_right[0].append(count) 106 | data_right[1].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[0],4)) 107 | data_right[2].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[1],4)) 108 | data_right[3].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[2],4)) 109 | data_right[4].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[0],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[0],4)) 110 | data_right[5].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[1],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[1],4)) 111 | data_right[6].append(round(controller.frame(i).hand(righthand.id).stabilized_palm_position[2],4)-round(controller.frame(59).hand(righthand.id).stabilized_palm_position[2],4)) 112 | data_right[7].append(round(controller.frame(i).hand(righthand.id).palm_normal.roll,4)) 113 | data_right[8].append(round(controller.frame(i).hand(righthand.id).direction.yaw,4)) 114 | data_right[9].append(round(controller.frame(i).hand(righthand.id).direction.pitch,4)) 115 | data_right[10].append(int(rFlag)) 116 | data_right[11].append(0) 117 | 118 | df_right=pd.DataFrame(data=data_right) 119 | df_right=df_right.transpose() 120 | df_right.to_csv("./CSV/%s.csv"%(str(count)),sep=',',index=False, header=False) 121 | count+=1 122 | right_count+=1 123 | print "Swipe Right Gesture Recorded %s" %(str(right_count)) 124 | else: 125 | print "Try Again" 126 | time.sleep(3) 127 | print "Ready" 128 | time.sleep(1) 129 | previous_time=current_time 130 | 131 | 132 | else: 133 | pass 134 | 135 | 136 | 137 | def main(): 138 | listener = SimpleListener() 139 | controller = Leap.Controller() 140 | controller.add_listener(listener) 141 | # Run infinitely until Enter is pressed 142 | print "Hit Enter to quit!" 143 | try: 144 | sys.stdin.readline() 145 | except KeyboardInterrupt: 146 | pass 147 | finally: 148 | controller.remove_listener(listener) 149 | 150 | if __name__ == "__main__": 151 | main() 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | --------------------------------------------------------------------------------