├── slr ├── model │ ├── __init__.py │ ├── slr_model.hdf5 │ ├── slr_model.tflite │ ├── label.csv │ ├── counter.json │ └── classifier.py ├── utils │ ├── __init__.py │ ├── cvfpscalc.py │ ├── args.py │ ├── draw_debug.py │ ├── logging.py │ ├── pre_process.py │ └── landmarks.py ├── __init__.py └── main.py ├── docs ├── SLR.png ├── result.png ├── demo-ss-a.png ├── demo-ss-s.png ├── flow-chart.png ├── hand-landmarks.png ├── hand_landmarks_o.jpg ├── hand_landmarks_o_r.png └── hand_landmarks_r.jpg ├── resources ├── result.png ├── background.png ├── background.psd ├── loggingpng.png ├── prediction.png ├── background-1.png ├── background_logging.png └── background_prediction.png ├── .gitignore ├── requirements.txt ├── app.py ├── SECURITY.md ├── LICENSE ├── train.py └── README.md /slr/model/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /slr/utils/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /slr/__init__.py: -------------------------------------------------------------------------------- 1 | from .main import main 2 | 3 | __version__ = "1.0.0" 4 | -------------------------------------------------------------------------------- /docs/SLR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/SLR.png -------------------------------------------------------------------------------- /docs/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/result.png -------------------------------------------------------------------------------- /docs/demo-ss-a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/demo-ss-a.png -------------------------------------------------------------------------------- /docs/demo-ss-s.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/demo-ss-s.png -------------------------------------------------------------------------------- /docs/flow-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/flow-chart.png -------------------------------------------------------------------------------- /resources/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/result.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | __pycache__ 3 | 4 | /.idea 5 | /data 6 | /static 7 | delete.* 8 | 9 | .env 10 | 11 | ss/ -------------------------------------------------------------------------------- /docs/hand-landmarks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/hand-landmarks.png -------------------------------------------------------------------------------- /resources/background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/background.png -------------------------------------------------------------------------------- /resources/background.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/background.psd -------------------------------------------------------------------------------- /resources/loggingpng.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/loggingpng.png -------------------------------------------------------------------------------- /resources/prediction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/prediction.png -------------------------------------------------------------------------------- /slr/model/slr_model.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/slr/model/slr_model.hdf5 -------------------------------------------------------------------------------- /docs/hand_landmarks_o.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/hand_landmarks_o.jpg -------------------------------------------------------------------------------- /docs/hand_landmarks_o_r.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/hand_landmarks_o_r.png -------------------------------------------------------------------------------- /docs/hand_landmarks_r.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/docs/hand_landmarks_r.jpg -------------------------------------------------------------------------------- /resources/background-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/background-1.png -------------------------------------------------------------------------------- /slr/model/slr_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/slr/model/slr_model.tflite -------------------------------------------------------------------------------- /resources/background_logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/background_logging.png -------------------------------------------------------------------------------- /resources/background_prediction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodingSamrat/Sign-Language-Recognition/HEAD/resources/background_prediction.png -------------------------------------------------------------------------------- /slr/model/label.csv: -------------------------------------------------------------------------------- 1 | A 2 | B 3 | C 4 | D 5 | E 6 | F 7 | G 8 | H 9 | I 10 | K 11 | L 12 | M 13 | N 14 | O 15 | P 16 | Q 17 | R 18 | S 19 | T 20 | U 21 | V 22 | W 23 | X 24 | Y -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.7.1 2 | mediapipe==0.10.0 3 | numpy==1.24.3 4 | opencv-python==4.6.0.66 5 | pandas==2.0.1 6 | Pillow==9.5.0 7 | python-dotenv==1.0.0 8 | scikit-learn==1.2.2 9 | seaborn==0.12.2 10 | tensorflow==2.12.0 11 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | """ 2 | app.py 3 | 4 | Description : This script is used to run and test the trained Sign Language Recognition system. 5 | It imports the main application from the `slr` module and executes it. 6 | 7 | Author : Sam 8 | Created : May 26, 2023 9 | """ 10 | 11 | from slr import main 12 | 13 | # Run the main function of the Sign Language Recognition system 14 | if __name__ == "__main__": 15 | main() 16 | -------------------------------------------------------------------------------- /slr/model/counter.json: -------------------------------------------------------------------------------- 1 | { 2 | "6": 1000, 3 | "4": 1000, 4 | "14": 1000, 5 | "16": 1000, 6 | "0": 1000, 7 | "17": 1000, 8 | "15": 1000, 9 | "8": 1000, 10 | "18": 1000, 11 | "7": 1000, 12 | "12": 1000, 13 | "19": 1000, 14 | "22": 1000, 15 | "9": 1000, 16 | "23": 1000, 17 | "20": 1000, 18 | "2": 1000, 19 | "1": 1000, 20 | "13": 1000, 21 | "10": 1000, 22 | "21": 1000, 23 | "5": 1000, 24 | "11": 1000, 25 | "3": 1000 26 | } -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Use this section to tell people about which versions of your project are 6 | currently being supported with security updates. 7 | 8 | | Version | Supported | 9 | | ------- | ------------------ | 10 | | 5.1.x | :white_check_mark: | 11 | | 5.0.x | :x: | 12 | | 4.0.x | :white_check_mark: | 13 | | < 4.0 | :x: | 14 | 15 | ## Reporting a Vulnerability 16 | 17 | Use this section to tell people how to report a vulnerability. 18 | 19 | Tell them where to go, how often they can expect to get an update on a 20 | reported vulnerability, what to expect if the vulnerability is accepted or 21 | declined, etc. 22 | -------------------------------------------------------------------------------- /slr/utils/cvfpscalc.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | import cv2 as cv 3 | 4 | 5 | class CvFpsCalc(object): 6 | def __init__(self, buffer_len=1): 7 | self._start_tick = cv.getTickCount() 8 | self._freq = 1000.0 / cv.getTickFrequency() 9 | self._difftimes = deque(maxlen=buffer_len) 10 | 11 | def get(self): 12 | current_tick = cv.getTickCount() 13 | different_time = (current_tick - self._start_tick) * self._freq 14 | self._start_tick = current_tick 15 | 16 | self._difftimes.append(different_time) 17 | 18 | fps = 1000.0 / (sum(self._difftimes) / len(self._difftimes)) 19 | fps_rounded = round(fps, 2) 20 | 21 | return fps_rounded 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Samrat Biswas & Akash Murmu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /slr/model/classifier.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | class KeyPointClassifier(object): 6 | def __init__( 7 | self, 8 | model_path='slr/model/slr_model.tflite', 9 | num_threads=1, 10 | ): 11 | #: Initializing tensor interpreter 12 | self.interpreter = tf.lite.Interpreter( 13 | model_path=model_path, 14 | num_threads=num_threads 15 | ) 16 | self.interpreter.allocate_tensors() 17 | 18 | #: Input Output details 19 | self.input_details = self.interpreter.get_input_details() 20 | self.output_details = self.interpreter.get_output_details() 21 | 22 | def __call__(self, landmark_list): 23 | 24 | input_details_tensor_index = self.input_details[0]['index'] 25 | 26 | #: Feeding landmarks to the tensor interpreter 27 | self.interpreter.set_tensor( 28 | input_details_tensor_index, 29 | np.array([landmark_list], dtype=np.float32) 30 | ) 31 | 32 | #: Invoking interpreter for prediction 33 | self.interpreter.invoke() 34 | 35 | #: Getting tensor index from output details 36 | output_details_tensor_index = self.output_details[0]['index'] 37 | # print(output_details_tensor_index) 38 | 39 | #: Getting all the prediction percentage 40 | result = self.interpreter.get_tensor(output_details_tensor_index) 41 | 42 | if max(np.squeeze(result)) > 0.5: 43 | #: Getting index of maximum accurate label 44 | result_index = np.argmax(np.squeeze(result)) 45 | 46 | return result_index 47 | else: 48 | return 25 49 | 50 | -------------------------------------------------------------------------------- /slr/utils/args.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | 4 | def get_args(): 5 | parser = argparse.ArgumentParser() 6 | 7 | #: Device 8 | parser.add_argument( 9 | "--device", 10 | type=int, 11 | default=0 12 | ) 13 | 14 | #: Image width 15 | parser.add_argument( 16 | "--width", 17 | type=int, 18 | default=640, 19 | help="width of image frame" 20 | ) 21 | 22 | #: Image height 23 | parser.add_argument( 24 | "--height", 25 | type=int, 26 | default=480, 27 | help="height of image frame" 28 | ) 29 | 30 | #: Static image mode 31 | parser.add_argument( 32 | '--use_static_image_mode', 33 | action='store_true' 34 | ) 35 | 36 | #: Number of hands 37 | parser.add_argument( 38 | "--max_num_hands", 39 | type=int, 40 | default=1, 41 | help="Number of hands to detect" 42 | ) 43 | 44 | #: Detection confidence 45 | parser.add_argument( 46 | "--min_detection_confidence", 47 | type=float, 48 | default=0.7, 49 | help="Minimum detection confidence" 50 | ) 51 | 52 | #: Tracking confidence 53 | parser.add_argument( 54 | "--min_tracking_confidence", 55 | type=float, 56 | default=0.5, 57 | help="Minimum tracking confidence" 58 | ) 59 | 60 | #: Bounding Rectangle 61 | parser.add_argument( 62 | "--use_brect", 63 | type=bool, 64 | default=True, 65 | help="Control if show bounding rectangle or not" 66 | ) 67 | 68 | #: Record mode 69 | parser.add_argument( 70 | "--mode", 71 | type=bool, 72 | default=0, 73 | help="Is it in record mode or in Normal mode" 74 | ) 75 | 76 | 77 | args = parser.parse_args() 78 | return args 79 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | """ 2 | train.py 3 | 4 | Description : This script trains a Sign Language Recognition model using hand keypoint data. 5 | It loads data from a CSV file, trains a neural network using TensorFlow/Keras, 6 | and saves both the HDF5 and TFLite versions of the trained model. 7 | 8 | Author : Sam 9 | Created : May 26, 2023 10 | """ 11 | 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn.model_selection import train_test_split 15 | import os 16 | 17 | # Set random seed for reproducibility 18 | RANDOM_SEED = 42 19 | np.random.seed(RANDOM_SEED) 20 | tf.random.set_seed(RANDOM_SEED) 21 | 22 | # Paths 23 | dataset_path = 'slr/model/keypoint.csv' # Make sure to place your dataset here 24 | model_h5_path = 'slr/model/slr_model.hdf5' 25 | model_tflite_path = 'slr/model/slr_model.tflite' 26 | 27 | # Number of classes (e.g., 24 alphabets excluding J & Z) 28 | NUM_CLASSES = 24 29 | 30 | # Load dataset 31 | print("[INFO] Loading dataset...") 32 | X = np.loadtxt(dataset_path, delimiter=',', dtype='float32', usecols=list(range(1, 43))) 33 | y = np.loadtxt(dataset_path, delimiter=',', dtype='int32', usecols=(0,)) 34 | 35 | # Split into training and test sets 36 | print("[INFO] Splitting dataset...") 37 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, random_state=RANDOM_SEED) 38 | 39 | # Define the model 40 | print("[INFO] Building model...") 41 | model = tf.keras.models.Sequential([ 42 | tf.keras.layers.Input((42,)), # 21 keypoints * 2 (x, y) 43 | tf.keras.layers.Dropout(0.2), 44 | tf.keras.layers.Dense(20, activation='relu'), 45 | tf.keras.layers.Dropout(0.4), 46 | tf.keras.layers.Dense(10, activation='relu'), 47 | tf.keras.layers.Dense(NUM_CLASSES, activation='softmax') 48 | ]) 49 | 50 | # Compile model 51 | model.compile(optimizer='adam', 52 | loss='sparse_categorical_crossentropy', 53 | metrics=['accuracy']) 54 | 55 | # Train the model 56 | print("[INFO] Training model...") 57 | model.fit(X_train, y_train, epochs=50, validation_data=(X_test, y_test)) 58 | 59 | # Save the model in HDF5 format 60 | print(f"[INFO] Saving model to {model_h5_path}...") 61 | model.save(model_h5_path) 62 | 63 | # Convert to TFLite 64 | print(f"[INFO] Converting model to TFLite and saving to {model_tflite_path}...") 65 | converter = tf.lite.TFLiteConverter.from_keras_model(model) 66 | tflite_model = converter.convert() 67 | with open(model_tflite_path, 'wb') as f: 68 | f.write(tflite_model) 69 | 70 | print("[INFO] Training complete.") 71 | -------------------------------------------------------------------------------- /slr/utils/draw_debug.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import numpy as np 3 | import cv2 as cv 4 | 5 | 6 | def draw_bounding_rect(image, use_brect, brect, outline_color=(255, 255, 255), pad=6): 7 | """ 8 | Draw bounding box draw the bounding box around the detected hand(s) 9 | 10 | :param use_brect: bool 11 | :param image: Debug image 12 | :param brect: B-Box coordinates 13 | :param outline_color: Color of the lines 14 | :return: Debug image 15 | :param pad: Padding 16 | """ 17 | 18 | if use_brect: 19 | #: Outer rectangle 20 | cv.rectangle( 21 | image, 22 | (brect[0] - pad, brect[1] - pad), 23 | (brect[2] + pad, brect[3] + pad), 24 | outline_color, 25 | 1 26 | ) 27 | 28 | return image 29 | 30 | 31 | def draw_hand_label(image, brect, handedness): 32 | """ 33 | Takes the following params and write information about 34 | detected hand gesture on the debug image. 35 | 36 | :param image: Debug image 37 | :param brect: B-Box coordinates 38 | :param handedness: Detected hand 39 | :return: Debug image 40 | """ 41 | 42 | cv.rectangle(image, (brect[0], brect[1]), (brect[2], brect[1] - 22), (0, 0, 0), -1) 43 | 44 | hand = handedness.classification[0].label[0:] 45 | # if hand_sign_text != "": 46 | # info_text = info_text + ':' + hand_sign_text 47 | 48 | cv.putText(image, hand, (brect[0] + 5, brect[1] - 4), cv.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1, cv.LINE_AA) 49 | 50 | return image 51 | 52 | 53 | def get_result_image(): 54 | image = np.ones([127, 299, 3], dtype=np.uint8) * 255 55 | return image 56 | 57 | 58 | def get_fps_log_image(): 59 | image = np.ones([30, 640, 3], dtype=np.uint8) * 255 60 | return image 61 | 62 | 63 | def show_result(image, handedness, hand_sign_text): 64 | """ 65 | :param hand_sign_text: Detected sign 66 | :param handedness: Detected hand 67 | :param image: Result to draw 68 | :return: Result image 69 | """ 70 | #: Detecting hand 71 | hand = handedness.classification[0].label[0:] 72 | 73 | #: Position of text 74 | position = (00, 00) 75 | 76 | #: Check if hand sign is empty or not 77 | #: Checking for right hand or left 78 | if hand_sign_text != "": 79 | if hand == "Right": 80 | position = (10, 80) 81 | cv.putText(image, hand_sign_text, position, cv.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 0), 6, cv.LINE_AA) 82 | 83 | elif hand == "Left": 84 | position = (10, 80) 85 | hand_sign_text = "Wrong Hand" 86 | cv.putText(image, hand_sign_text, position, cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 3, cv.LINE_AA) 87 | 88 | 89 | 90 | return image 91 | 92 | 93 | def show_fps_log(image, fps, log=""): 94 | """ 95 | 96 | :param image: Debug image 97 | :param fps: FPS 98 | :param log: log to print 99 | :return: Debug image 100 | """ 101 | # 116 / 674 102 | #: - 103 | #: FPS Functionality 104 | cv.putText(image, str(fps), (0, 22), cv.FONT_HERSHEY_SIMPLEX, 0.70, (0, 0, 0), 1, cv.LINE_AA) 105 | 106 | #: FPS Functionality 107 | if log != "": 108 | cv.putText(image, log, (90, 22), cv.FONT_HERSHEY_SIMPLEX, 0.60, (0, 0, 0), 1, cv.LINE_AA) 109 | 110 | return image 111 | 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /slr/utils/logging.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import cv2 as cv 3 | import json 4 | import numpy as np 5 | 6 | 7 | def log_keypoints(key, landmark_list, counter_obj, data_limit=1000): 8 | """ 9 | 10 | :param key: Keyboard key (latter) 11 | :param landmark_list: Preprocessed landmark list 12 | :param counter_obj: 13 | :param data_limit: How many row need for each sign 14 | :return: None 15 | """ 16 | counter_file = "slr/model/counter.json" 17 | 18 | _dict = {} 19 | csv_path = "slr/model/keypoint.csv" 20 | index = -1 21 | 22 | # counter_obj = _get_dict_form_list(csv_path) 23 | counter_obj = counter_obj 24 | 25 | #: Escaping 'J/j' 26 | if key == 106 or key == 74: 27 | return 28 | 29 | if 65 <= key <= 89 or 97 <= key <= 121: # A-Z / a-z 30 | 31 | #: Calculating index of letters 32 | if 65 <= key <= 90: # Capital letters 33 | index = key - 65 34 | 35 | #: Subtracting index by 1 after 'J' 36 | if key > 74: # J 37 | index -= 1 38 | 39 | elif 97 <= key <= 122: # Small letters 40 | index = key - 97 41 | 42 | #: Subtracting index by 1 after 'j' 43 | if key > 106: # j 44 | index -= 1 45 | 46 | #: Counting limit 47 | if str(index) in counter_obj.keys(): 48 | counter_obj[str(index)] += 1 49 | else: 50 | counter_obj[str(index)] = 1 51 | 52 | #: - 53 | #: Writing counter 54 | if counter_obj[str(index)] > data_limit: #: Limit of capturing image 55 | print(f"Dataset limit reached for {chr(key).upper()} [{counter_obj[str(index)]-1}/{data_limit}]") 56 | return 57 | else: 58 | with open(counter_file, "w") as cf: 59 | counter_obj_writable = json.dumps(counter_obj, indent=4) 60 | cf.write(counter_obj_writable) 61 | 62 | print(f"{chr(key).upper()} => {counter_obj[str(index)]}/{data_limit}") 63 | 64 | #: - 65 | #: Writing dataset 66 | with open(csv_path, 'a', newline="") as f: 67 | writer = csv.writer(f) 68 | writer.writerow([index, *landmark_list]) 69 | 70 | return 71 | 72 | 73 | def get_dict_form_list(file): 74 | _list = [] 75 | with open(file) as f: 76 | for row in f: 77 | _list.append(row.split(",")[0]) 78 | 79 | if len(_list)==0: 80 | return {} 81 | 82 | obj = {} 83 | set_list = set(_list) 84 | 85 | for i in set_list: 86 | obj[str(i)] = _list.count(i) 87 | 88 | return obj 89 | 90 | 91 | def _get_alphabet_index(key): 92 | """ 93 | 94 | :param key: Keyboard key (latter) 95 | :return: Index of alphabate 96 | """ 97 | cap_ascii_list = [ 98 | 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 99 | 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90 100 | ] 101 | sm_ascii_list = [ 102 | 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 103 | 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122 104 | ] 105 | 106 | index = 0 107 | return index 108 | 109 | 110 | def get_mode(key, _mode): 111 | """ 112 | :param key: Pressed key 113 | :param mode: Mode of program 114 | :return: mode 115 | """ 116 | mode = _mode 117 | if key == 48: 118 | mode = 0 119 | elif key == 49: 120 | mode = 1 121 | 122 | return mode 123 | -------------------------------------------------------------------------------- /slr/utils/pre_process.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import itertools 3 | from typing import List 4 | 5 | import numpy as np 6 | import cv2 as cv 7 | 8 | 9 | def calc_bounding_rect(image, landmarks) -> List: 10 | """ 11 | Takes Captures image and 3D coordinates of hand landmarks 12 | And Calculate the aria bounding box around the hand 13 | 14 | :param image: Captured Image 15 | :param landmarks: List of Hand's Landmarks 16 | :return: List of coordinate of bounding-box 17 | """ 18 | 19 | #: Getting image width & height 20 | image_width = image.shape[1] 21 | image_height = image.shape[0] 22 | 23 | #: Creating empty np array 24 | landmark_array = np.empty((0, 2), int) 25 | 26 | #: Iterating over all the landmarks 27 | for _, landmark in enumerate(landmarks.landmark): 28 | landmark_x = min(int(landmark.x * image_width), image_width -1) 29 | landmark_y = min(int(landmark.y * image_height), image_height -1) 30 | 31 | landmark_point = [np.array((landmark_x, landmark_y))] 32 | 33 | landmark_array = np.append(landmark_array, landmark_point, axis=0) 34 | 35 | x, y, w, h = cv.boundingRect(landmark_array) 36 | 37 | return [x, y, x+w, y+h] 38 | 39 | 40 | def calc_landmark_list(image, landmarks) -> List: 41 | """ 42 | Takes Captures image and 3D coordinates of hand landmarks 43 | And Calculate the Key-points of the hand landmarks 44 | 45 | :param image: Captured Image 46 | :param landmarks: Landmarks 47 | :return: List of Key-points 48 | """ 49 | 50 | #: Getting image width & height 51 | image_width = image.shape[1] 52 | image_height = image.shape[0] 53 | 54 | landmark_point = [] 55 | 56 | #: Keypoint 57 | for _, landmark in enumerate(landmarks.landmark): 58 | landmark_x = min(int(landmark.x * image_width), image_width - 1) 59 | landmark_y = min(int(landmark.y * image_height), image_height - 1) 60 | #: landmark_z = landmark.z 61 | 62 | landmark_point.append([landmark_x, landmark_y]) 63 | 64 | return landmark_point 65 | 66 | 67 | def pre_process_landmark(landmark_list) -> List: 68 | """ 69 | Pre Processing Landmark takes the list of calculated landmarks 70 | to calculate relative coordinates of all landmarks. 71 | 72 | For this it goes through some step: 73 | 74 | i. Making a deep-copy the list; 75 | ii. Set 0th landmark _(Wrist)_ as root (0, 0) 76 | iii. Then calculate other landmark's relative coordinate 77 | iv. Finally, normalized them all and return as list 78 | 79 | :param landmark_list: Calculated Landmarks 80 | :return: List of relative coordinates 81 | """ 82 | 83 | temp_landmark_list = copy.deepcopy(landmark_list) 84 | 85 | #: Convert to relative coordinates 86 | base_x, base_y = 0, 0 87 | for index, landmark_point in enumerate(temp_landmark_list): 88 | if index == 0: 89 | base_x, base_y = landmark_point[0], landmark_point[1] 90 | 91 | #: Overriding coordinates 92 | temp_landmark_list[index][0] = temp_landmark_list[index][0] - base_x 93 | temp_landmark_list[index][1] = temp_landmark_list[index][1] - base_y 94 | 95 | #: Convert into a one-dimensional list 96 | temp_landmark_list = list(itertools.chain.from_iterable(temp_landmark_list)) 97 | 98 | #: Normalization (-1 to 1) 99 | max_value = max(list(map(abs, temp_landmark_list))) 100 | 101 | def normalize_(n): 102 | return n / max_value 103 | 104 | temp_landmark_list = list(map(normalize_, temp_landmark_list)) 105 | 106 | return temp_landmark_list 107 | 108 | 109 | -------------------------------------------------------------------------------- /slr/main.py: -------------------------------------------------------------------------------- 1 | print("INFO: Initializing System") 2 | import copy 3 | import csv 4 | import os 5 | import datetime 6 | 7 | import pyautogui 8 | import cv2 as cv 9 | import mediapipe as mp 10 | from dotenv import load_dotenv 11 | 12 | from slr.model.classifier import KeyPointClassifier 13 | 14 | from slr.utils.args import get_args 15 | from slr.utils.cvfpscalc import CvFpsCalc 16 | from slr.utils.landmarks import draw_landmarks 17 | 18 | from slr.utils.draw_debug import get_result_image 19 | from slr.utils.draw_debug import get_fps_log_image 20 | from slr.utils.draw_debug import draw_bounding_rect 21 | from slr.utils.draw_debug import draw_hand_label 22 | from slr.utils.draw_debug import show_fps_log 23 | from slr.utils.draw_debug import show_result 24 | 25 | from slr.utils.pre_process import calc_bounding_rect 26 | from slr.utils.pre_process import calc_landmark_list 27 | from slr.utils.pre_process import pre_process_landmark 28 | 29 | from slr.utils.logging import log_keypoints 30 | from slr.utils.logging import get_dict_form_list 31 | from slr.utils.logging import get_mode 32 | 33 | 34 | 35 | def main(): 36 | #: - 37 | #: Getting all arguments 38 | load_dotenv() 39 | args = get_args() 40 | 41 | keypoint_file = "slr/model/keypoint.csv" 42 | counter_obj = get_dict_form_list(keypoint_file) 43 | 44 | #: cv Capture 45 | CAP_DEVICE = args.device 46 | CAP_WIDTH = args.width 47 | CAP_HEIGHT = args.height 48 | 49 | #: mp Hands 50 | # USE_STATIC_IMAGE_MODE = args.use_static_image_mode 51 | USE_STATIC_IMAGE_MODE = True 52 | MAX_NUM_HANDS = args.max_num_hands 53 | MIN_DETECTION_CONFIDENCE = args.min_detection_confidence 54 | MIN_TRACKING_CONFIDENCE = args.min_tracking_confidence 55 | 56 | #: Drawing Rectangle 57 | USE_BRECT = args.use_brect 58 | MODE = args.mode 59 | DEBUG = int(os.environ.get("DEBUG", "0")) == 1 60 | CAP_DEVICE = 0 61 | 62 | print("INFO: System initialization Successful") 63 | print("INFO: Opening Camera") 64 | 65 | #: - 66 | #: Capturing image 67 | cap = cv.VideoCapture(CAP_DEVICE) 68 | cap.set(cv.CAP_PROP_FRAME_WIDTH, CAP_WIDTH) 69 | cap.set(cv.CAP_PROP_FRAME_HEIGHT, CAP_HEIGHT) 70 | 71 | background_image = cv.imread("resources/background_prediction.png") 72 | 73 | #: Background Image 74 | background_image = cv.imread("resources/background.png") 75 | # result_image = cv.imread("resources/result.png") 76 | 77 | #: - 78 | #: Setup hands 79 | mp_hands = mp.solutions.hands 80 | hands = mp_hands.Hands( 81 | static_image_mode=USE_STATIC_IMAGE_MODE, 82 | max_num_hands=MAX_NUM_HANDS, 83 | min_detection_confidence=MIN_DETECTION_CONFIDENCE, 84 | min_tracking_confidence=MIN_TRACKING_CONFIDENCE 85 | ) 86 | 87 | #: - 88 | #: Load Model 89 | keypoint_classifier = KeyPointClassifier() 90 | 91 | #: Loading labels 92 | keypoint_labels_file = "slr/model/label.csv" 93 | with open(keypoint_labels_file, encoding="utf-8-sig") as f: 94 | key_points = csv.reader(f) 95 | keypoint_classifier_labels = [row[0] for row in key_points] 96 | 97 | #: - 98 | #: FPS Measurement 99 | cv_fps = CvFpsCalc(buffer_len=10) 100 | print("INFO: System is up & running") 101 | #: - 102 | #: Main Loop Start Here... 103 | while True: 104 | #: FPS of open cv frame or window 105 | fps = cv_fps.get() 106 | 107 | #: - 108 | #: Setup Quit key for program 109 | key = cv.waitKey(1) 110 | if key == 27: # ESC key 111 | print("INFO: Exiting...") 112 | break 113 | elif key == 57: # 9 114 | name = datetime.datetime.now().strftime("%m%d%Y-%H%M%S") 115 | myScreenshot = pyautogui.screenshot() 116 | myScreenshot.save(f'ss/{name}.png') 117 | 118 | #: - 119 | #: Camera capture 120 | success, image = cap.read() 121 | if not success: 122 | continue 123 | 124 | image = cv.resize(image, (CAP_WIDTH, CAP_HEIGHT)) 125 | 126 | #: Flip Image for mirror display 127 | image = cv.flip(image, 1) 128 | debug_image = copy.deepcopy(image) 129 | result_image = get_result_image() 130 | fps_log_image = get_fps_log_image() 131 | 132 | #: Converting to RBG from BGR 133 | image = cv.cvtColor(image, cv.COLOR_BGR2RGB) 134 | 135 | image.flags.writeable = False 136 | results = hands.process(image) #: Hand's landmarks 137 | image.flags.writeable = True 138 | 139 | #: - 140 | #: DEBUG - Showing Debug info 141 | if DEBUG: 142 | MODE = get_mode(key, MODE) 143 | fps_log_image = show_fps_log(fps_log_image, fps) 144 | 145 | #: - 146 | #: Start Detection 147 | if results.multi_hand_landmarks is not None: 148 | for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness): 149 | 150 | #: Calculate BoundingBox 151 | use_brect = True 152 | brect = calc_bounding_rect(debug_image, hand_landmarks) 153 | 154 | #: Landmark calculation 155 | landmark_list = calc_landmark_list(debug_image, hand_landmarks) 156 | 157 | #: Conversion to relative coordinates / normalized coordinates 158 | pre_processed_landmark_list = pre_process_landmark(landmark_list) 159 | 160 | #: - 161 | #: Checking if in Prediction Mode or in Logging Mode 162 | #: If Prediction Mode it will predict the hand gesture 163 | #: If in Logging Mode it will Log key-points or landmarks to the csv file 164 | 165 | if MODE == 0: #: Prediction Mode / Normal mode 166 | #: Hand sign classification 167 | hand_sign_id = keypoint_classifier(pre_processed_landmark_list) 168 | 169 | if hand_sign_id == 25: 170 | hand_sign_text = "" 171 | else: 172 | hand_sign_text = keypoint_classifier_labels[hand_sign_id] 173 | 174 | #: Showing Result 175 | result_image = show_result(result_image, handedness, hand_sign_text) 176 | 177 | elif MODE == 1: #: Logging Mode 178 | log_keypoints(key, pre_processed_landmark_list, counter_obj, data_limit=1000) 179 | 180 | #: - 181 | #: Drawing debug info 182 | debug_image = draw_bounding_rect(debug_image, use_brect, brect) 183 | debug_image = draw_landmarks(debug_image, landmark_list) 184 | debug_image = draw_hand_label(debug_image, brect, handedness) 185 | 186 | #: - 187 | #: Set main video footage on Background 188 | 189 | # if MODE == 0: #: Prediction Mode / Normal mode 190 | 191 | # #: Changing to Prediction Background and setting main video footage on Background 192 | # background_image = cv.imread("resources/background_prediction.png") 193 | # background_image[170:170 + 480, 50:50 + 640] = debug_image 194 | # background_image[240:240 + 127, 731:731 + 299] = result_image 195 | # background_image[678:678 + 30, 118:118 + 640] = fps_log_image 196 | 197 | # elif MODE == 1: #: Logging Mode 198 | 199 | # #: Changing to Logging Background and setting main video footage on Background 200 | # background_image = cv.imread("resources/background_logging.png") 201 | # background_image[170:170 + 480, 50:50 + 640] = debug_image 202 | # background_image[240:240 + 127, 731:731 + 299] = result_image 203 | # background_image[678:678 + 30, 118:118 + 640] = fps_log_image 204 | 205 | 206 | background_image[170:170 + 480, 50:50 + 640] = debug_image 207 | background_image[240:240 + 127, 731:731 + 299] = result_image 208 | background_image[678:678 + 30, 118:118 + 640] = fps_log_image 209 | 210 | # cv.imshow("Result", result_image) 211 | # cv.imshow("Main Frame", debug_image) 212 | 213 | cv.imshow("Sign Language Recognition", background_image) 214 | 215 | cap.release() 216 | cv.destroyAllWindows() 217 | 218 | print("INFO: Bye") 219 | 220 | 221 | if __name__ == "__main__": 222 | main() 223 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![MasterHead](docs/SLR.png) 2 | # Sign language Recognition 3 | 4 | This repository contains the source code and resources for a Sign Language Recognition System. The goal of this project is to develop a computer vision system that can recognize and interpret sign language gestures in real-time. 5 | 6 | Though the project name is `Sign Language Recognition`, it can be used for any hand gesture recognition. In addition, the system is fully customizable and can be trained to recognize any hand gesture. 7 | 8 | 19 | 20 | 21 |

22 | 23 | ## Introduction 24 | Sign language is a visual means of communication used by individuals with hearing impairments. This project aims to bridge the communication gap by developing an automated system that can understand and interpret sign language gestures. The system utilizes computer vision techniques and machine learning algorithms to recognize and translate these gestures into text or speech. 25 | 26 | The Sign Language Recognition System consists of several components: 27 | 28 | - **Data Collection:** A dataset of sign language gestures is collected, including various hand shapes. 29 | 30 | - **Preprocessing:** The collected data is preprocessed to enhance the quality, remove noise, and extract relevant features. 31 | 32 | - **Model Training:** Machine learning models, ie. convolutional neural networks (CNNs) is trained on the preprocessed data to learn the mapping between input gestures and their corresponding meanings. 33 | 34 | - **Real-time Recognition:** The trained model is deployed in a real-time environment, where it takes video input and performs gesture recognition on the fly. 35 | 36 | 37 |

38 | 39 | ## Installation 40 | To set up the Sign Language Recognition System on your local machine, follow these steps: 41 | 42 | 1. Clone the repository to your local machine. 43 | 44 | ``` bash 45 | git clone https://github.com/the-sam963/Sign-language-Recognition.git 46 | ``` 47 | 48 | 2. Install the required packages using the requirements.txt file. 49 | 50 | ``` bash 51 | # Windows 52 | cd Sign-language-Recognition 53 | virtualenv env 54 | .\env\Scripts\activate.ps1 55 | pip install -r requirements.txt 56 | 57 | # Linux/ macOS 58 | cd Sign-language-Recognition 59 | virtualenv env 60 | source env/bin/activate 61 | pip3 install -r requirements.txt 62 | ``` 63 | You are now ready to use the Sign Language Recognition System on your local machine. 64 | 65 | 66 |

67 | 68 | ## Usage 69 | To use the Sign Language Recognition System, follow these steps: 70 | 71 | 1. Ensure that the required dependencies and resources are properly installed and set up. 72 | 73 | 2. Train the model (if needed) 74 | ``` bash 75 | python train.py 76 | ``` 77 | 3. Run the main application 78 | ``` bash 79 | python app.py 80 | ``` 81 | 4. The application will open a video stream and start recognizing sign language gestures in real-time. 82 | 83 | 5. Interact with the system by making sign language gestures within the camera's view. The recognized gestures will be displayed on the screen 84 | 85 | 86 | Here is a demo of the Sign Language Recognition System in action: 87 | 88 | Coding 89 | Coding 90 | 91 | 92 |

93 | 94 | ## Customization 95 | The Sign Language Recognition System is fully customizable and can be trained to recognize any hand gesture. To customize the system, follow these steps: 96 | 97 | 1. Run the `app.py` file to open the application. 98 | 2. Press the `1` key to switch to the `Data Collection` mode. Press the `0` key if you want to switch to the `prediction` mode. 99 | 3. Make the hand gesture you want to add to the dataset. 100 | 4. Press the alphabate key to save the gesture to the dataset corresponding to the letter. 101 | 5. Add as many gestures as you want to the dataset. More data will result in better accuracy. 102 | 6. After adding all the gestures, press the `esc` key to quit program. Then run the `python train.py` to train the model on the new dataset. 103 | 104 | Enjoy your customized Sign Language Recognition System! 105 | 106 |

107 | 108 | ## System Overview 109 | In order to build such systems, data (keypoints of hands) must be obtained, then features involved in sign making must be extracted and finally combination of features must be analysed to describe the performed sign. 110 | 111 | ![System Overview](docs/flow-chart.png) 112 | 113 | 114 |

115 | 116 | ## Data Collection 117 | The first step in building a sign language recognition system is to collect a dataset of sign language gestures. The dataset is used to train the machine learning model to recognize and interpret these gestures. 118 | 119 | For this project we use our hand sign data. And the data is collected using the [MediaPipe](https://google.github.io/mediapipe/) library. The library provides a hand tracking solution that can detect and track 21 hand landmarks in real-time. The hand landmarks are the key points or 2D coordinates that can be used to determine the pose of the hand. 120 | 121 | ![Hand Landmarks](docs/hand-landmarks.png) 122 | 123 | The hand landmarks are used to extract features from the hand gestures. The features are then used to train a machine learning model to recognize and interpret these gestures. 124 | 125 | 126 |

127 | 128 | ## Preprocessing 129 | The collected data is preprocessed to enhance the quality, remove noise, and extract relevant features. The preprocessing steps include: 130 | 131 | 1. Getting the hand landmarks from the video stream. 132 | 133 | 2. Converting the hand landmarks relative to the `wrist` landmark's coordinate `(0, 0)`. This is done by subtracting the `wrist` landmark's coordinate from all the other landmarks. 134 | 135 | ![Hand Landmarks](docs/hand_landmarks_o_r.png) 136 | 137 | 138 | 3. Flatten the normalized hand landmarks to a 1 Dimensional list. 139 | 140 | ```python 141 | #: Convert into a one-dimensional list 142 | temp_landmark_list = list(itertools.chain.from_iterable(temp_landmark_list)) 143 | ``` 144 | 145 | 146 | 4. Normalizing the hand landmarks from `-1` to `1` value. 147 | 148 | ```python 149 | # Normalize the hand landmarks to a fixed size 150 | #: Normalization (-1 to 1) 151 | max_value = max(list(map(abs, temp_landmark_list))) 152 | 153 | def normalize_(n): 154 | return n / max_value 155 | 156 | landmark_list = list(map(normalize_, temp_landmark_list)) 157 | ``` 158 | 159 | 160 | 5. Adding the normalized hand landmarks to the dataset. 161 | 162 | ```python 163 | #: Writing dataset 164 | with open("slr/model/keypoint.csv", 'a', newline="") as f: 165 | writer = csv.writer(f) 166 | writer.writerow([index, *landmark_list]) 167 | ``` 168 | 169 | 170 |

171 | 172 | ## Model Training 173 | The preprocessed data is used to train a machine learning model to recognize and interpret sign language gestures. The model is trained using a convolutional neural network (CNN) architecture. The CNN is trained on the preprocessed data to learn the mapping between input gestures and their corresponding meanings. 174 | 175 | 1. Obtain or create a sign language dataset. 176 | 177 | 2. Split the dataset into training and validation sets. It is recommended to use a separate testing set for final evaluation. 178 | 179 | 3. Choose an appropriate machine learning algorithm or architecture, such as CNNs, RNNs, or their combinations, and implement it using a suitable framework like TensorFlow or PyTorch. For this project, we use a CNN architecture implemented using TensorFlow. 180 | 181 | 4. Train the model using the training dataset and tune hyperparameters to achieve optimal performance. 182 | 183 | 5. Validate the model using the validation dataset and make necessary adjustments to improve accuracy and generalization. 184 | 185 | 6. Test the final model using the testing dataset and evaluate its performance using appropriate metrics. 186 | 187 | 188 |

189 | 190 | ## Results 191 | The model was trained on a dataset of 24,000 hand gestures. The dataset was split into training and validation sets with a ratio of 80:20. The model was trained for 100 epochs with a batch size of 180. The training and validation accuracy and loss were recorded for each epoch. 192 | 193 | Our Proposed Model achieved an accuracy of `71.12%` on the validation set and `90.60%` on the testing set. The model was able to recognize and interpret sign language gestures in real-time with an accuracy of `71.12%`. 194 | 195 | 196 |

197 | 198 | ## Contributing 199 | Contributions to this project are welcome. If you encounter any issues or have suggestions for improvements, please open an issue or submit a pull request. Let's work together to make the Sign Language Recognition System even better! 200 | 201 | We appreciate your contributions, whether big or small, and we look forward to working together to enhance the Sign Language Recognition System. Let's make a positive impact on the lives of individuals with hearing impairments and promote inclusivity in communication. 202 | 203 | -------------------------------------------------------------------------------- /slr/utils/landmarks.py: -------------------------------------------------------------------------------- 1 | import cv2 as cv 2 | 3 | 4 | def draw_landmarks(image, landmark_point): 5 | 6 | #: Listing colours 7 | magenta = (255, 0, 255) 8 | cyan = (255, 255, 0) 9 | yellow = (0, 255, 255) 10 | neon_green = (20, 255, 57) 11 | neon_purple = (253, 18, 171) 12 | neon_blue = (255, 81, 31) 13 | neon_orange = (31, 91, 255) 14 | neon_red = (49, 49, 255) 15 | white = (255, 255, 255) 16 | black = (0, 0, 0) 17 | 18 | #:Setting default colours 19 | skeletal_color = white 20 | outline_color = black 21 | 22 | if len(landmark_point) > 0: 23 | 24 | #: Thumb [3, 4] 25 | skeletal_color = neon_green 26 | cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[3]), outline_color, 6) 27 | cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[3]), skeletal_color, 2) 28 | cv.line(image, tuple(landmark_point[3]), tuple(landmark_point[4]), outline_color, 6) 29 | cv.line(image, tuple(landmark_point[3]), tuple(landmark_point[4]), skeletal_color, 2) 30 | 31 | 32 | #: Index finger [6, 7, 8] 33 | skeletal_color = neon_purple 34 | cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[6]), outline_color, 6) 35 | cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[6]), skeletal_color, 2) 36 | cv.line(image, tuple(landmark_point[6]), tuple(landmark_point[7]), outline_color, 6) 37 | cv.line(image, tuple(landmark_point[6]), tuple(landmark_point[7]), skeletal_color, 2) 38 | cv.line(image, tuple(landmark_point[7]), tuple(landmark_point[8]), outline_color, 6) 39 | cv.line(image, tuple(landmark_point[7]), tuple(landmark_point[8]), skeletal_color, 2) 40 | 41 | 42 | #: Middle finger [10, 11, 12] 43 | skeletal_color = neon_red 44 | cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[10]), outline_color, 6) 45 | cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[10]), skeletal_color, 2) 46 | cv.line(image, tuple(landmark_point[10]), tuple(landmark_point[11]), outline_color, 6) 47 | cv.line(image, tuple(landmark_point[10]), tuple(landmark_point[11]), skeletal_color, 2) 48 | cv.line(image, tuple(landmark_point[11]), tuple(landmark_point[12]), outline_color, 6) 49 | cv.line(image, tuple(landmark_point[11]), tuple(landmark_point[12]), skeletal_color, 2) 50 | 51 | 52 | #: Ring finger [14, 15, 16] 53 | skeletal_color = neon_blue 54 | cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[14]), outline_color, 6) 55 | cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[14]), skeletal_color, 2) 56 | cv.line(image, tuple(landmark_point[14]), tuple(landmark_point[15]), outline_color, 6) 57 | cv.line(image, tuple(landmark_point[14]), tuple(landmark_point[15]), skeletal_color, 2) 58 | cv.line(image, tuple(landmark_point[15]), tuple(landmark_point[16]), outline_color, 6) 59 | cv.line(image, tuple(landmark_point[15]), tuple(landmark_point[16]), skeletal_color, 2) 60 | 61 | 62 | #: Little finger [18, 19, 20] 63 | skeletal_color = neon_orange 64 | cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[18]), outline_color, 6) 65 | cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[18]), skeletal_color, 2) 66 | cv.line(image, tuple(landmark_point[18]), tuple(landmark_point[19]), outline_color, 6) 67 | cv.line(image, tuple(landmark_point[18]), tuple(landmark_point[19]), skeletal_color, 2) 68 | cv.line(image, tuple(landmark_point[19]), tuple(landmark_point[20]), outline_color, 6) 69 | cv.line(image, tuple(landmark_point[19]), tuple(landmark_point[20]), skeletal_color, 2) 70 | 71 | 72 | #: Palm [1, 5, 9, 13, 17, 0] 73 | skeletal_color = yellow 74 | cv.line(image, tuple(landmark_point[0]), tuple(landmark_point[1]), outline_color, 6) 75 | cv.line(image, tuple(landmark_point[0]), tuple(landmark_point[1]), skeletal_color, 2) 76 | cv.line(image, tuple(landmark_point[1]), tuple(landmark_point[2]), outline_color, 6) 77 | cv.line(image, tuple(landmark_point[1]), tuple(landmark_point[2]), skeletal_color, 2) 78 | cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[5]), outline_color, 6) 79 | cv.line(image, tuple(landmark_point[2]), tuple(landmark_point[5]), skeletal_color, 2) 80 | cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[9]), outline_color, 6) 81 | cv.line(image, tuple(landmark_point[5]), tuple(landmark_point[9]), skeletal_color, 2) 82 | cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[13]), outline_color, 6) 83 | cv.line(image, tuple(landmark_point[9]), tuple(landmark_point[13]), skeletal_color, 2) 84 | cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[17]), outline_color, 6) 85 | cv.line(image, tuple(landmark_point[13]), tuple(landmark_point[17]), skeletal_color, 2) 86 | cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[0]), outline_color, 6) 87 | cv.line(image, tuple(landmark_point[17]), tuple(landmark_point[0]), skeletal_color, 2) 88 | 89 | #:Changed to default Skeletal colour 90 | skeletal_color = white 91 | 92 | #: Key Points 93 | for index, landmark in enumerate(landmark_point): 94 | 95 | #: wrist 1 96 | if index == 0: 97 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 98 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 99 | 100 | #: wrist 2 101 | if index == 1: 102 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 103 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 104 | 105 | #: thumb: root 106 | if index == 2: 107 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 108 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 109 | 110 | #: Thumb: 1st joint 111 | if index == 3: 112 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 113 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 114 | 115 | #: thumb: fingertip 116 | if index == 4: 117 | cv.circle(image, (landmark[0], landmark[1]), 8, skeletal_color, -1) 118 | cv.circle(image, (landmark[0], landmark[1]), 8, outline_color, 1) 119 | 120 | #: Index finger: root 121 | if index == 5: 122 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 123 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 124 | 125 | #: Index finger: 2nd joint 126 | if index == 6: 127 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 128 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 129 | 130 | #: Index finger: 1st joint 131 | if index == 7: 132 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 133 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 134 | 135 | #: index finger: fingertip 136 | if index == 8: 137 | cv.circle(image, (landmark[0], landmark[1]), 8, skeletal_color, -1) 138 | cv.circle(image, (landmark[0], landmark[1]), 8, outline_color, 1) 139 | 140 | #: Middle finger: root 141 | if index == 9: 142 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 143 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 144 | 145 | #: Middle finger: 2nd joint 146 | if index == 10: 147 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 148 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 149 | 150 | #: Middle finger: 1st joint 151 | if index == 11: 152 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 153 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 154 | 155 | #: middle finger: fingertip 156 | if index == 12: 157 | cv.circle(image, (landmark[0], landmark[1]), 8, skeletal_color, -1) 158 | cv.circle(image, (landmark[0], landmark[1]), 8, outline_color, 1) 159 | 160 | #: Ring finger: root 161 | if index == 13: 162 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 163 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 164 | 165 | #: Ring finger: 2nd joint 166 | if index == 14: 167 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 168 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 169 | 170 | #: Ring finger: 1st joint 171 | if index == 15: 172 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 173 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 174 | 175 | #: Ring finger: fingertip 176 | if index == 16: 177 | cv.circle(image, (landmark[0], landmark[1]), 8, skeletal_color, -1) 178 | cv.circle(image, (landmark[0], landmark[1]), 8, outline_color, 1) 179 | 180 | #: Little finger: root 181 | if index == 17: 182 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 183 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 184 | 185 | #: Little finger: 2nd joint 186 | if index == 18: 187 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 188 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 189 | 190 | #: Little finger: 1st joint 191 | if index == 19: 192 | cv.circle(image, (landmark[0], landmark[1]), 5, skeletal_color, -1) 193 | cv.circle(image, (landmark[0], landmark[1]), 5, outline_color, 1) 194 | 195 | #: Little finger: fingertip 196 | if index == 20: 197 | cv.circle(image, (landmark[0], landmark[1]), 8, skeletal_color, -1) 198 | cv.circle(image, (landmark[0], landmark[1]), 8, outline_color, 1) 199 | 200 | return image 201 | --------------------------------------------------------------------------------