├── 0.npy ├── CollectData.py ├── NeuralNetwork.py ├── README.md ├── Run.py ├── RunNoMarker.py └── action.h5 /0.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evarghese563/Sign-Language-Detection-Using-LSTM/ff3df538a2b54025bbfe7ee9b62df07f52c9fa9f/0.npy -------------------------------------------------------------------------------- /CollectData.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import os 4 | from matplotlib import pyplot as plt 5 | import time 6 | import mediapipe as mp 7 | 8 | mp_holistic = mp.solutions.holistic 9 | mp_drawing = mp.solutions.drawing_utils 10 | 11 | def mediapipe_detection(image,model): 12 | image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB) 13 | image.flags.writeable = False 14 | results = model.process(image) 15 | image.flags.writeable = True 16 | image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR) 17 | return image,results 18 | 19 | def draw_styled_landmarks(image,results): 20 | mp_drawing.draw_landmarks(image,results.face_landmarks, mp_holistic.FACEMESH_TESSELATION, 21 | mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1), 22 | mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1) 23 | ) 24 | 25 | mp_drawing.draw_landmarks(image,results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, 26 | mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4), 27 | mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2) 28 | ) 29 | 30 | mp_drawing.draw_landmarks(image,results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 31 | mp_drawing.DrawingSpec(color=(121,22,76), thickness=1, circle_radius=4), 32 | mp_drawing.DrawingSpec(color=(121,44,250), thickness=1, circle_radius=2) 33 | ) 34 | 35 | mp_drawing.draw_landmarks(image,results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 36 | mp_drawing.DrawingSpec(color=(245,117,66), thickness=1, circle_radius=1), 37 | mp_drawing.DrawingSpec(color=(245,66,230), thickness=1, circle_radius=1) 38 | ) 39 | 40 | def extract_keypoints(results): 41 | pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4) 42 | lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3) 43 | rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3) 44 | face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3) 45 | return np.concatenate([pose,face,lh,rh]) 46 | 47 | 48 | DATA_PATH = os.path.join('MP_Data') 49 | #Actions 50 | actions = np.array(['hello','thanks','iloveyou']) 51 | #30 videos worth of data 52 | no_sequences = 30 53 | #30 frames 54 | sequence_length = 30 55 | 56 | for action in actions: 57 | for sequence in range(no_sequences): 58 | try: 59 | os.makedirs(os.path.join(DATA_PATH,action,str(sequence))) 60 | except: 61 | pass 62 | 63 | cap = cv2.VideoCapture(0) 64 | 65 | #Mediapipe Model 66 | with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic: 67 | 68 | #Loop through Actions 69 | for action in actions: 70 | #Loop through Videos 71 | for sequence in range(no_sequences): 72 | #Loop through video length aka sequence length 73 | for frame_num in range(sequence_length): 74 | 75 | 76 | #Read Feed 77 | ret, frame = cap.read() 78 | 79 | #Make detections 80 | image,results = mediapipe_detection(frame,holistic) 81 | 82 | #Draw Styled Landmarks 83 | draw_styled_landmarks(image,results) 84 | 85 | #Wait Logic 86 | if frame_num==0: 87 | cv2.putText(image,'Starting Collection',(120,200), 88 | cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,0),4,cv2.LINE_AA) 89 | cv2.putText(image,'Collecting frames for {} Video Number {}'.format(action,sequence),(15,12), 90 | cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),4,cv2.LINE_AA) 91 | cv2.waitKey(2000) 92 | 93 | else: 94 | cv2.putText(image,'Collecting frames for {} Video Number {}'.format(action,sequence),(15,12), 95 | cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),4,cv2.LINE_AA) 96 | 97 | #NEW Export keypoints 98 | keypoints = extract_keypoints(results) 99 | npy_path = os.path.join(DATA_PATH,action,str(sequence), str(frame_num)) 100 | np.save(npy_path,keypoints) 101 | 102 | #Show to Screen 103 | cv2.imshow('OpenCV feed', image) 104 | 105 | #Breaking the Feed 106 | if cv2.waitKey(10) & 0xFF == ord('q'): 107 | break 108 | 109 | cap.release() 110 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /NeuralNetwork.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sklearn.model_selection import train_test_split 3 | from tensorflow.keras.utils import to_categorical 4 | import numpy as np 5 | from tensorflow.keras.models import Sequential 6 | from tensorflow.keras.layers import LSTM,Dense 7 | from tensorflow.keras.callbacks import TensorBoard 8 | 9 | 10 | 11 | DATA_PATH = os.path.join('MP_Data') 12 | #Actions 13 | actions = np.array(['hello','thanks','iloveyou']) 14 | #30 videos worth of data 15 | no_sequences = 30 16 | #30 frames 17 | sequence_length = 30 18 | label_map = {label:num for num, label in enumerate(actions)} 19 | 20 | sequences, labels = [], [] 21 | for action in actions: 22 | for sequence in range(no_sequences): 23 | window = [] 24 | for frame_num in range(sequence_length): 25 | res = np.load(os.path.join(DATA_PATH,action, str(sequence),"{}.npy".format(frame_num))) 26 | window.append(res) 27 | sequences.append(window) 28 | labels.append(label_map[action]) 29 | 30 | X = np.array(sequences) 31 | y = to_categorical(labels).astype(int) 32 | 33 | x_train, x_test, y_train, y_test = train_test_split(X,y,test_size = 0.05) 34 | 35 | log_dir = os.path.join('Logs') 36 | tb_callback = TensorBoard(log_dir = log_dir) 37 | 38 | model = Sequential() 39 | model.add(LSTM(64,return_sequences=True, activation='relu', input_shape=(30,1662))) 40 | model.add(LSTM(128,return_sequences=True, activation = 'relu')) 41 | model.add(LSTM(64, return_sequences = False,activation='relu')) 42 | model.add(Dense(64,activation='relu')) 43 | model.add(Dense(32,activation = 'relu')) 44 | model.add(Dense(actions.shape[0],activation='softmax')) 45 | 46 | res = [.2,0.7,.01] 47 | print(actions[np.argmax(res)]) 48 | 49 | model.compile(optimizer = 'Adam',loss='categorical_crossentropy',metrics=['categorical_accuracy']) 50 | 51 | model.fit(x_train,y_train,epochs = 2000, callbacks=[tb_callback]) 52 | model.summary() 53 | 54 | res = model.predict(x_test) 55 | print(actions[np.argmax(res[1])]) 56 | print(actions[np.argmax(y_test[1])]) 57 | model.save('action.h5') -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sign-Language-Recognition-Using-LSTM 2 | 3 | ## General 4 | 5 | After the start of the pandemic the use of E-Learning has increased drastically. By 2026 the E-learning industry is seat to reach at least $400 billion. With the rise of this new fashion of education no child should be left behind. Children with the disability of hearing loss are held at a disadvantage due to the reason that they have a hard time hearing the lectures that are present on the screen. Other factors would be a mute child who would not be able to speak in according to the response of the lecturer. In order for these children to cope with education the American Sign Language was created not only to help them with education but to provide ease for their daily lives. To help these children with their education we proposed a model that will help the student make ASL gestures to the camera and have it read and provide feedback on what language was read. For this we used OpenCV with Mediapipe Holistic to identify the key markers of the poser with all the values to be collected and then be trained on the Long Short Term Memory Architecture. 6 | 7 | ## MediaPipe Holistic 8 | The initial step of the program is to identify the Keypoints of the user. MP Holistic is an API created by Mediapipe which identifies the keypoints on a body by identifying the holistic key points 9 | 10 | `mp_holistic = mp.solutions.holistic 11 | mp_drawing = mp.solutions.drawing_utils` 12 | 13 | These two api calls, are calls the Holistic API uses to identify the keypoints holistic keypoints and draw it 14 | 15 | 16 | 17 | The function 18 | `draw_landmarks(image, results)` 19 | 20 | Draws landmarks of the identified keypoints on the user 21 | 22 | ![Unstyled](https://github.com/evarghese563/Images/blob/main/Sign/unstyled.gif) 23 | 24 | 25 | However wth a few extra lines of code 26 | `draw_styled_landmarks(image,results)` 27 | 28 | The keypoints can be color coded for the viewer to identify the different parts of the human 29 | 30 | ![Styled](https://github.com/evarghese563/Images/blob/main/Sign/styled.gif) 31 | 32 | 33 | ## Collecting the Data 34 | After the mediapipe has been defined the keypoints must be collected and stored to be trained on later. 35 | 36 | `def extract_keypoints(results)` 37 | 38 | This takes in the extracted keypoints from the video and concatenated the values into a numpy array. 39 | 40 | 41 | The collected data is abeled as 'Hello', 'Thanks', and 'I Love You'. In order to collect the data the user poses for the three different action in for 30 sequences for 30 frames. For a total of 90 collectd sequences 42 | 43 | The identified keypoints are stored in the folder called MP_Data, with each sequence in its respective action folder in the .npy format 44 | 45 | 46 | ## Training the Long Short Term Memory 47 | To train the Long Short Term Memory the model selected was a Sequential model trained on 3 layers and 2 dense layers. Since it is catagorical data the model was compiled as: 48 | 49 | `model.compile(optimizer = 'Adam',loss='categorical_crossentropy',metrics=['categorical_accuracy'])` 50 | 51 | The model was then Trained on 2000 epochs. 52 | 53 | ## Testing in Real time 54 | These are the testing results in real time from the project after training the data on the LSTM Architecture using MP Holistic 55 | 56 | ### Hello 57 | 58 | ![Hello](https://github.com/evarghese563/Images/blob/main/Sign/hello.gif) 59 | 60 | ### Thanks 61 | 62 | ![Thanks](https://github.com/evarghese563/Images/blob/main/Sign/thanks.gif) 63 | 64 | ### I Love You 65 | 66 | ![ILY](https://github.com/evarghese563/Images/blob/main/Sign/ily.gif) 67 | 68 | ### Extras 69 | 70 | #### 1. Color Coded Probability Viewer 71 | 72 | - This is a probability bar which shows the gesture that is being identified the most in the test. The bar rises with the gesture it recognizes most. The color for each bar is as follows: 73 | 74 | - Blue/Hello 75 | - Green/Thanks 76 | - Orange/I Love You 77 | 78 | 79 | ## Run 80 | ### Run on Old Weights 81 | 83 | To run on the previous versions weights the user has to simply run 'Run.py' or ' RunNoMarker.py'. Make sure either one of the files in the same folder with actions.h5 84 | 85 | ### Run on New Weights 86 | To run on a new dataset: 87 | 88 | 1. CollectData.py 89 | 90 | - When the camera opens pose in the Sign Language poses for 30 sequences in 'Hello', 'Thanks', and 'I Love You'. 91 | 92 | 2. NeuralNetwork.py 93 | 94 | - To train the collected data on the Long Short Term Memory Architecture hit run 95 | 96 | 3. Run.py or RunNoMarker.py 97 | 98 | - Test the model in real time 99 | - Run RunNoMarker.py if the user would like to run the model without any of the viewing markers the user can run this file to access the application without any of the landmarks 100 | 101 | -------------------------------------------------------------------------------- /Run.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import os 4 | from matplotlib import pyplot as plt 5 | import mediapipe as mp 6 | from tensorflow.keras.models import Sequential 7 | from tensorflow.keras.layers import LSTM,Dense 8 | from tensorflow.keras.callbacks import TensorBoard 9 | 10 | 11 | mp_holistic = mp.solutions.holistic 12 | mp_drawing = mp.solutions.drawing_utils 13 | 14 | def mediapipe_detection(image,model): 15 | image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB) 16 | image.flags.writeable = False 17 | results = model.process(image) 18 | image.flags.writeable = True 19 | image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR) 20 | return image,results 21 | 22 | def draw_styled_landmarks(image,results): 23 | mp_drawing.draw_landmarks(image,results.face_landmarks, mp_holistic.FACEMESH_TESSELATION, 24 | mp_drawing.DrawingSpec(color=(80,110,10), thickness=1, circle_radius=1), 25 | mp_drawing.DrawingSpec(color=(80,256,121), thickness=1, circle_radius=1) 26 | ) 27 | 28 | mp_drawing.draw_landmarks(image,results.pose_landmarks, mp_holistic.POSE_CONNECTIONS, 29 | mp_drawing.DrawingSpec(color=(80,22,10), thickness=2, circle_radius=4), 30 | mp_drawing.DrawingSpec(color=(80,44,121), thickness=2, circle_radius=2) 31 | ) 32 | 33 | mp_drawing.draw_landmarks(image,results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 34 | mp_drawing.DrawingSpec(color=(121,22,76), thickness=1, circle_radius=4), 35 | mp_drawing.DrawingSpec(color=(121,44,250), thickness=1, circle_radius=2) 36 | ) 37 | 38 | mp_drawing.draw_landmarks(image,results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS, 39 | mp_drawing.DrawingSpec(color=(245,117,66), thickness=1, circle_radius=1), 40 | mp_drawing.DrawingSpec(color=(245,66,230), thickness=1, circle_radius=1) 41 | ) 42 | 43 | def extract_keypoints(results): 44 | pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4) 45 | lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3) 46 | rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3) 47 | face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3) 48 | return np.concatenate([pose,face,lh,rh]) 49 | 50 | 51 | colors = [(245,117,16),(117,245,16),(16,117,245)] 52 | def prob_viz(res,actions,input_frame,colors): 53 | output_frame = input_frame.copy() 54 | for num,prob in enumerate(res): 55 | cv2.rectangle(output_frame, (0,60+num*40), (int(prob*100), 90+num*40),colors[num], -1) 56 | cv2.putText(output_frame,actions[num],(0,85+num*40), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2,cv2.LINE_AA) 57 | return output_frame 58 | 59 | 60 | DATA_PATH = os.path.join('MP_Data') 61 | #Actions 62 | actions = np.array(['hello','thanks','iloveyou']) 63 | #30 videos worth of data 64 | no_sequences = 30 65 | #30 frames 66 | sequence_length = 30 67 | 68 | for action in actions: 69 | for sequence in range(no_sequences): 70 | try: 71 | os.makedirs(os.path.join(DATA_PATH,action,str(sequence))) 72 | except: 73 | pass 74 | 75 | label_map = {label:num for num, label in enumerate(actions)} 76 | sequences, labels = [], [] 77 | for action in actions: 78 | for sequence in range(no_sequences): 79 | window = [] 80 | for frame_num in range(sequence_length): 81 | res = np.load(os.path.join(DATA_PATH,action, str(sequence),"{}.npy".format(frame_num))) 82 | window.append(res) 83 | sequences.append(window) 84 | labels.append(label_map[action]) 85 | 86 | log_dir = os.path.join('Logs') 87 | tb_callback = TensorBoard(log_dir = log_dir) 88 | 89 | model = Sequential() 90 | model.add(LSTM(64,return_sequences=True, activation='relu', input_shape=(30,1662))) 91 | model.add(LSTM(128,return_sequences=True, activation = 'relu')) 92 | model.add(LSTM(64, return_sequences = False,activation='relu')) 93 | model.add(Dense(64,activation='relu')) 94 | model.add(Dense(32,activation = 'relu')) 95 | model.add(Dense(actions.shape[0],activation='softmax')) 96 | 97 | res = [.2,0.7,.01] 98 | 99 | actions[np.argmax(res)] 100 | model.compile(optimizer = 'Adam',loss='categorical_crossentropy',metrics=['categorical_accuracy']) 101 | actions[np.argmax(res[1])] 102 | 103 | model.load_weights('action.h5') 104 | 105 | #New Detection Variables 106 | sequence = [] 107 | sentence = [] 108 | threshold = .4 109 | 110 | cap = cv2.VideoCapture(0) 111 | #Mediapipe Model 112 | with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic: 113 | while cap.isOpened(): 114 | 115 | #Read Feed 116 | ret, frame = cap.read() 117 | 118 | #Make detections 119 | image,results = mediapipe_detection(frame,holistic) 120 | 121 | #Draw Styled Landmarks 122 | draw_styled_landmarks(image,results) 123 | 124 | #Prediciton Logic 125 | keypoints = extract_keypoints(results) 126 | sequence.insert(0,keypoints) 127 | sequence = sequence[:30] 128 | 129 | if len(sequence) == 30: 130 | res = model.predict(np.expand_dims(sequence,axis=0))[0] 131 | 132 | #Visualization 133 | if res[np.argmax(res)] > threshold: 134 | if len(sentence) > 0: 135 | if actions[np.argmax(res)] != sentence[-1]: 136 | sentence.append(actions[np.argmax(res)]) 137 | else: 138 | sentence.append(actions[np.argmax(res)]) 139 | 140 | if len(sentence)>5: 141 | sentence = sentence[-5:] 142 | 143 | 144 | #Viz probability 145 | image = prob_viz(res,actions,image,colors) 146 | 147 | 148 | cv2.rectangle(image,(0,0),(640,40),(245,117,16),-1) 149 | cv2.putText(image, ' '.join(sentence),(3,30), 150 | cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2,cv2.LINE_AA) 151 | 152 | #Show to Screen 153 | cv2.imshow('OpenCV feed', image) 154 | 155 | #Breaking the Feed 156 | if cv2.waitKey(10) & 0xFF == ord('q'): 157 | break 158 | 159 | cap.release() 160 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /RunNoMarker.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import os 4 | from matplotlib import pyplot as plt 5 | import mediapipe as mp 6 | from tensorflow.keras.models import Sequential 7 | from tensorflow.keras.layers import LSTM,Dense 8 | from tensorflow.keras.callbacks import TensorBoard 9 | 10 | 11 | mp_holistic = mp.solutions.holistic 12 | mp_drawing = mp.solutions.drawing_utils 13 | 14 | def mediapipe_detection(image,model): 15 | image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB) 16 | image.flags.writeable = False 17 | results = model.process(image) 18 | image.flags.writeable = True 19 | image = cv2.cvtColor(image,cv2.COLOR_RGB2BGR) 20 | return image,results 21 | 22 | 23 | def extract_keypoints(results): 24 | pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4) 25 | lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3) 26 | rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3) 27 | face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3) 28 | return np.concatenate([pose,face,lh,rh]) 29 | 30 | 31 | colors = [(245,117,16),(117,245,16),(16,117,245)] 32 | def prob_viz(res,actions,input_frame,colors): 33 | output_frame = input_frame.copy() 34 | for num,prob in enumerate(res): 35 | cv2.rectangle(output_frame, (0,60+num*40), (int(prob*100), 90+num*40),colors[num], -1) 36 | cv2.putText(output_frame,actions[num],(0,85+num*40), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2,cv2.LINE_AA) 37 | return output_frame 38 | 39 | 40 | DATA_PATH = os.path.join('MP_Data') 41 | #Actions 42 | actions = np.array(['hello','thanks','iloveyou']) 43 | #30 videos worth of data 44 | no_sequences = 30 45 | #30 frames 46 | sequence_length = 30 47 | 48 | for action in actions: 49 | for sequence in range(no_sequences): 50 | try: 51 | os.makedirs(os.path.join(DATA_PATH,action,str(sequence))) 52 | except: 53 | pass 54 | 55 | label_map = {label:num for num, label in enumerate(actions)} 56 | sequences, labels = [], [] 57 | for action in actions: 58 | for sequence in range(no_sequences): 59 | window = [] 60 | for frame_num in range(sequence_length): 61 | res = np.load(os.path.join(DATA_PATH,action, str(sequence),"{}.npy".format(frame_num))) 62 | window.append(res) 63 | sequences.append(window) 64 | labels.append(label_map[action]) 65 | 66 | log_dir = os.path.join('Logs') 67 | tb_callback = TensorBoard(log_dir = log_dir) 68 | 69 | model = Sequential() 70 | model.add(LSTM(64,return_sequences=True, activation='relu', input_shape=(30,1662))) 71 | model.add(LSTM(128,return_sequences=True, activation = 'relu')) 72 | model.add(LSTM(64, return_sequences = False,activation='relu')) 73 | model.add(Dense(64,activation='relu')) 74 | model.add(Dense(32,activation = 'relu')) 75 | model.add(Dense(actions.shape[0],activation='softmax')) 76 | 77 | res = [.2,0.7,.01] 78 | 79 | actions[np.argmax(res)] 80 | model.compile(optimizer = 'Adam',loss='categorical_crossentropy',metrics=['categorical_accuracy']) 81 | actions[np.argmax(res[1])] 82 | 83 | model.load_weights('action.h5') 84 | 85 | #New Detection Variables 86 | sequence = [] 87 | sentence = [] 88 | threshold = .4 89 | 90 | cap = cv2.VideoCapture(0) 91 | #Mediapipe Model 92 | with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic: 93 | while cap.isOpened(): 94 | 95 | #Read Feed 96 | ret, frame = cap.read() 97 | 98 | #Make detections 99 | image,results = mediapipe_detection(frame,holistic) 100 | 101 | #Prediciton Logic 102 | keypoints = extract_keypoints(results) 103 | sequence.insert(0,keypoints) 104 | sequence = sequence[:30] 105 | 106 | if len(sequence) == 30: 107 | res = model.predict(np.expand_dims(sequence,axis=0))[0] 108 | 109 | #Visualization 110 | if res[np.argmax(res)] > threshold: 111 | if len(sentence) > 0: 112 | if actions[np.argmax(res)] != sentence[-1]: 113 | sentence.append(actions[np.argmax(res)]) 114 | else: 115 | sentence.append(actions[np.argmax(res)]) 116 | 117 | if len(sentence)>5: 118 | sentence = sentence[-5:] 119 | 120 | 121 | #Viz probability 122 | image = prob_viz(res,actions,image,colors) 123 | 124 | 125 | cv2.rectangle(image,(0,0),(640,40),(245,117,16),-1) 126 | cv2.putText(image, ' '.join(sentence),(3,30), 127 | cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2,cv2.LINE_AA) 128 | 129 | #Show to Screen 130 | cv2.imshow('OpenCV feed', image) 131 | 132 | #Breaking the Feed 133 | if cv2.waitKey(10) & 0xFF == ord('q'): 134 | break 135 | 136 | cap.release() 137 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /action.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evarghese563/Sign-Language-Detection-Using-LSTM/ff3df538a2b54025bbfe7ee9b62df07f52c9fa9f/action.h5 --------------------------------------------------------------------------------