├── .gitignore ├── README.md ├── ampere_law.py ├── code ├── 00_image.py ├── 01_video.py ├── 02_gesture.py ├── 03_game_rps.py ├── 04_hand_rom.py ├── 05_wrist_rom.py ├── 06_face_mask.py ├── 07_triangulate.py ├── 08_skeleton_3D.py ├── 09_objectron.py ├── utils_3d_reconstruct.py ├── utils_display.py ├── utils_joint_angle.py └── utils_mediapipe.py ├── data ├── canonical_face_model.obj ├── canonical_face_model.ply ├── gesture_train.csv ├── gesture_train_fy.csv ├── handrom_train.csv └── sample │ ├── hand.png │ ├── lower_limb1.png │ ├── lower_limb2.png │ ├── lower_limb3.png │ ├── lower_limb4.png │ ├── lower_limb5.png │ ├── lower_limb6.png │ ├── mona.png │ ├── object.png │ ├── upper_limb1.png │ ├── upper_limb2.png │ ├── upper_limb3.png │ ├── upper_limb4.png │ ├── upper_limb5.png │ ├── upper_limb6.png │ ├── wrist_extension.png │ ├── wrist_flexion.png │ ├── wrist_radial.png │ └── wrist_ulnar.png ├── doc ├── 00_image.gif ├── 02_gesture.gif ├── 03_game_rps.gif ├── 04_hand_rom.gif ├── 05_wrist_rom.gif ├── 06_face_mask.gif ├── 07_triangulate.gif ├── 08_skeleton_3D.gif ├── 09_objectron.gif └── rris_database.gif ├── dual.py ├── environment.yaml ├── fan.py ├── fy_filter.py ├── gather_dataset.py ├── java ├── README.md ├── elbow │ ├── codepen.css │ ├── codepen.html │ ├── codepen.js │ └── head.js ├── hand-fruit-ninja │ ├── codepen.css │ ├── codepen.html │ └── codepen.js ├── hand │ ├── codepen.css │ ├── codepen.html │ ├── codepen.js │ └── head.js └── pose-fruit-ninja │ ├── codepen.css │ ├── codepen.html │ └── codepen.js ├── result.gif ├── result_fan.gif ├── result_fy_filter.png └── single.py /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | __pycache__ 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rock Paper Scissors Machine 2 | 3 | Simplify [RRIS](https://github.com/ntu-rris)'s [code](https://github.com/ntu-rris/google-mediapipe) A.I. rock paper scissors machine using MediaPipe and KNN. 4 | 5 | ![](result.gif) 6 | 7 | ### Run 8 | 9 | ``` 10 | python dual.py 11 | ``` 12 | 13 | ## Fxck You Filter 14 | 15 | Mosaic the middle finger in video. 16 | 17 | ![](result_fy_filter.png) 18 | 19 | ``` 20 | python fy_filter.py 21 | ``` 22 | 23 | ## AI Fan 24 | 25 | Controlling the fan speed by AI gesture recognition 26 | BLDC Motor using [DynamiKontrol](https://dk.m47rix.com) Speed 27 | 28 | ![](result_fan.gif) 29 | 30 | ``` 31 | python fan.py 32 | ``` 33 | 34 | --- 35 | 36 | # [Google MediaPipe](https://github.com/google/mediapipe) for Pose Estimation 37 | 38 | [![](doc/rris_database.gif)](https://www.nature.com/articles/s41597-020-00627-7?sf237508323=1) 39 | 40 | [MediaPipe](https://opensource.google/projects/mediapipe) is a cross-platform framework for building multimodal applied machine learning pipelines including inference models and media processing functions. 41 | 42 | The main purpose of this repo is to: 43 | * Customize output of MediaPipe solutions 44 | * Customize visualization of 2D & 3D outputs 45 | * Demo some simple applications on Python (refer to [Demo Overview](#demo-overview)) 46 | * Demo some simple applications on JavaScript refer to [java](java/) folder 47 | 48 | 49 | ## Pose Estimation with Input Color Image 50 | Attractiveness of Google MediaPipe as compared to other SOTA (e.g. [FrankMocap](https://github.com/facebookresearch/frankmocap), [CMU OpenPose](https://github.com/CMU-Perceptual-Computing-Lab/openpose), [DeepPoseKit](https://github.com/jgraving/DeepPoseKit), [DeepLabCut](https://github.com/DeepLabCut/DeepLabCut), [MinimalHand](https://github.com/CalciferZh/minimal-hand)): 51 | * **Fast**: Runs at almost realtime rate on CPU and even mobile devices 52 | * **Open-source**: Codes are freely available at [github](https://github.com/google/mediapipe) (except that [details of network models are not released](https://github.com/google/mediapipe/issues/155)) 53 | * **User-friendly**: For python API just `pip install mediapipe` will work (but C++ API is much more troublesome to [build and use](https://google.github.io/mediapipe/getting_started/cpp)) 54 | * **Cross-platform**: Works across Android, iOS, desktop, [JavaScript](https://google.github.io/mediapipe/getting_started/javascript.html) and [web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html) (Note: this repo only focuses on using Python API for desktop usage) 55 | * **ML Solutions**: Apart from face, hand, body and object pose estimations, MediaPipe offers an array of machine learning applications refer to their [github](https://github.com/google/mediapipe) for more details 56 | 57 | ## Features 58 | Latest [MediaPipe Python API version 0.8.4.2](https://pypi.org/project/mediapipe/) (Released 11 May 2021) features: 59 | 60 | **Face Mesh** (468 **3D** face landmarks) 61 | 62 | * [**Blog**](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html) | [**Code**](https://google.github.io/mediapipe/solutions/face_mesh) | [**Paper**](https://arxiv.org/abs/1907.06724) | [**Video**](https://www.youtube.com/watch?v=JNSXC3E0-s4) | [**Model Card**](https://drive.google.com/file/d/1QvwWNfFoweGVjsXF3DXzcrCnz-mx-Lha/view) 63 | 64 | **Hands** (21 **3D** landmarks and able to support multiple hands) 65 | 66 | * [**Blog**](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html) | [**Code**](https://google.github.io/mediapipe/solutions/hands) | [**Paper**](https://arxiv.org/abs/2006.10214) | [**Video**](https://www.youtube.com/watch?v=I-UOrvxxXEk) | [**Model Card**](https://drive.google.com/file/d/1yiPfkhb4hSbXJZaSq9vDmhz24XVZmxpL/view) 67 | 68 | **Body Pose** (33 **3D** landmarks for whole body, **3 levels of model complexity (NEW)**) 69 | 70 | * [**Blog**](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html) | [**Code**](https://google.github.io/mediapipe/solutions/pose) | [**Paper**](https://arxiv.org/abs/2006.10204) | [**Video**](https://www.youtube.com/watch?v=YPpUOTRn5tA&feature=emb_logo) | [**Model Card**](https://drive.google.com/file/d/1zhYyUXhQrb_Gp0lKUFv1ADT3OCxGEQHS/view) 71 | 72 | **Holistic (Face + Hands + Body)** (A total of 543/535 landmarks: 468 face + 2 x 21 hands + 33/25 pose) 73 | 74 | * [**Blog**](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html) | [**Code**](https://google.github.io/mediapipe/solutions/holistic#smooth_landmarks) 75 | 76 | **Objectron (3D object detection and tracking)** (4 possible objects: Shoe / Chair / Camera / Cup) 77 | 78 | * [**Blog**](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html) | [**Code**](https://google.github.io/mediapipe/solutions/objectron) | [**Paper**](https://arxiv.org/abs/2003.03522) | [**Paper**](https://drive.google.com/file/d/1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8/view) | [**Model Card**](https://drive.google.com/file/d/1CMhN7Npdq0Dt2j0_z69mai2-m7oUTRKF/view) 79 | 80 | Note: The above videos are presented at [CVPR 2020 Fourth Workshop on Computer Vision for AR/VR](https://xr.cornell.edu/workshop/2020/papers), interested reader can refer to the link for other related works. 81 | 82 | 83 | ## Installation 84 | The simplest way to run our implementation is to use [anaconda](https://www.anaconda.com/). 85 | 86 | You can create an anaconda environment called `mp` with 87 | ``` 88 | conda env create -f environment.yaml 89 | conda activate mp 90 | ``` 91 | 92 | ## Demo Overview 93 | 94 | 95 | | Single Image | Video Input | Gesture Recognition | Rock Paper Scissor Game | 96 | | ------------ | ----------- | ------------------- | ----------------------- | 97 | | ![](doc/00_image.gif) | IMAGE ALT TEXT HERE | ![](doc/02_gesture.gif) | ![](doc/03_game_rps.gif) 98 | 99 | | Measure Hand ROM | Measure Wrist and Forearm ROM | Face Mask | Triangulate Points for 3D Pose | 100 | | ---------------- | ----------------------------- | --------- | ------------------------------ | 101 | | ![](doc/04_hand_rom.gif) | ![](doc/05_wrist_rom.gif)| ![](doc/06_face_mask.gif) | ![](doc/07_triangulate.gif) | 102 | 103 | | 3D Skeleton | 3D Object Detection | 104 | | ----------- | ------------------- | 105 | | ![](doc/08_skeleton_3D.gif) | ![](doc/09_objectron.gif)| 106 | 107 | 108 | 109 | ## Usage 110 | ### [0. Single Image](code/00_image.py) 111 | 112 | 4 different modes are available and sample images are located in [data/sample/](data/sample/) folder 113 | ``` 114 | python 00_image.py --mode face 115 | python 00_image.py --mode hand 116 | python 00_image.py --mode body 117 | python 00_image.py --mode holistic 118 | ``` 119 | Note: The sample images for subject with body marker are adapted from [An Asian-centric human movement database capturing activities of daily living](https://www.nature.com/articles/s41597-020-00627-7?sf237508323=1) and the image of Mona Lisa is adapted from [Wiki](https://upload.wikimedia.org/wikipedia/commons/e/ec/Mona_Lisa%2C_by_Leonardo_da_Vinci%2C_from_C2RMF_retouched.jpg) 120 | 121 | 122 | ### [1. Video Input](code/01_video.py) 123 | 124 | 4 different modes are available and video capture can be done online through [webcam](https://github.com/ntu-rris/google-mediapipe/blob/5e155130ba3477b84e873c57251c59f4206da3ee/code/01_video.py#L45) or offline from your own [.mp4 file](https://github.com/ntu-rris/google-mediapipe/blob/5e155130ba3477b84e873c57251c59f4206da3ee/code/01_video.py#L46) 125 | ``` 126 | python 01_video.py --mode face 127 | python 01_video.py --mode hand 128 | python 01_video.py --mode body 129 | python 01_video.py --mode holistic 130 | ``` 131 | 132 | Note: It takes around 10 to 30 FPS on CPU, depending on the mode selected. The [video](https://www.youtube.com/watch?v=rqFp-ZH5tpo) demonstrating supported mini-squats is adapted from [National Stroke Association](https://www.youtube.com/watch?v=WLjOoQUgWs4) 133 | 134 | 135 | ### [2. Gesture Recognition](code/02_gesture.py) 136 | 137 | 2 modes are available: Use evaluation mode to perform recognition of 11 gestures and use train mode to log your own training data 138 | ``` 139 | python 02_gesture.py --mode eval 140 | python 02_gesture.py --mode train 141 | ``` 142 | 143 | Note: A simple but effective K-nearest neighbor (KNN) algorithm is used as the classifier. For the hand gesture recognition demo, since 3D hand joints are available, we can compute flexion joint angles (feature vector) and use it to classify different hand poses. On the other hand, if 3D body joints are not yet reliable, the normalized pairwise distances between predifined lists of joints as described in [MediaPipe Pose Classification](https://google.github.io/mediapipe/solutions/pose_classification.html) could also be used as the feature vector for KNN. 144 | 145 | 146 | ### [3. Rock Paper Scissor Game](code/03_game_rps.py) 147 | 148 | Simple game of rock paper scissor requires a pair of hands facing the camera 149 | ``` 150 | python 03_game_rps.py 151 | ``` 152 | For another game of flappy bird refer to this [github](https://github.com/limgm/flappy-mediapipe) 153 | 154 | 155 | ### [4. Measure Hand Range of Motion](code/04_hand_rom.py) 156 | 157 | 2 modes are available: Use evaluation mode to perform hand ROM recognition and use train mode to log your own training data 158 | ``` 159 | python 04_hand_rom.py --mode eval 160 | python 04_hand_rom.py --mode train 161 | ``` 162 | 163 | ### [5. Measure Wrist and Forearm Range of Motion](code/05_wrist_rom.py) 164 | 165 | 3 modes are available and user has to input the side of the hand to be measured 166 | * 0: Wrist flexion/extension 167 | * 1: Wrist radial/ulnar deviation 168 | * 2: Forearm pronation/supination 169 | 170 | ``` 171 | python 05_wrist_rom.py --mode 0 --side right 172 | python 05_wrist_rom.py --mode 1 --side right 173 | python 05_wrist_rom.py --mode 2 --side right 174 | python 05_wrist_rom.py --mode 0 --side left 175 | python 05_wrist_rom.py --mode 1 --side left 176 | python 05_wrist_rom.py --mode 2 --side left 177 | ``` 178 | 179 | Note: For measuring forearm pronation/supination, the camera has to be placed at the same level as the hand such that palmar side of the hand is directly facing camera. For measuring wrist ROM, the camera has to be placed such that upper body of the subject is visible, refer to examples of wrist_XXX.png images in [data/sample/](data/sample/) folder. The wrist images are adapted from [Goni Wrist Flexion, Extension, Radial & Ulnar Deviation](https://www.youtube.com/watch?v=nIPaGkDh3dI) 180 | 181 | ### [6. Face Mask](code/06_face_mask.py) 182 | 183 | Overlay a 3D face mask on the detected face in image plane 184 | ``` 185 | python 06_face_mask.py 186 | ``` 187 | Note: The face image is adapted from [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html) 188 | 189 | 190 | ### [7. Triangulate Points](code/07_triangulate.py) 191 | 192 | Estimating 3D body pose from a single 2D image is an ill-posed problem and extremely challenging. 193 | One way to reconstruct 3D body pose is to make use of multiview setup and perform triangulation. 194 | For offline testing, use [CMU Panoptic Dataset](http://domedb.perception.cs.cmu.edu/171204_pose1.html), follow the instructions on [PanopticStudio Toolbox](https://github.com/CMU-Perceptual-Computing-Lab/panoptic-toolbox) to download a sample dataset 171204_pose1_sample into data/ folder 195 | ``` 196 | python 07_triangulate.py --mode body --use_panoptic_dataset 197 | ``` 198 | 199 | 205 | 206 | ### [8. 3D Skeleton](code/08_skeleton_3D.py) 207 | 208 | 3D pose estimation is available in full-body mode and this demo displays the estimated 3D skeleton of the hand and/or body. 3 different modes are available and video capture can be done online through [webcam](https://github.com/ntu-rris/google-mediapipe/blob/990a4a73c969450f6414940af5a832c7820f5c01/code/08_skeleton_3D.py#L24) or offline from your own [.mp4 file](https://github.com/ntu-rris/google-mediapipe/blob/990a4a73c969450f6414940af5a832c7820f5c01/code/08_skeleton_3D.py#L25) 209 | 210 | ``` 211 | python 08_skeleton_3D.py --mode hand 212 | python 08_skeleton_3D.py --mode body 213 | python 08_skeleton_3D.py --mode holistic 214 | ``` 215 | 216 | 217 | ### [9. 3D Object Detection](code/09_objectron.py) 218 | 219 | 4 different modes are available and a sample image is located in [data/sample/](data/sample/) folder. Currently supports 4 classes: Shoe / Chair / Cup / Camera. 220 | 221 | ``` 222 | python 09_objectron.py --mode shoe 223 | python 09_objectron.py --mode chair 224 | python 09_objectron.py --mode cup 225 | python 09_objectron.py --mode camera 226 | ``` 227 | 228 | ## Limitations: 229 | Estimating 3D pose from a single 2D image is an ill-posed problem and extremely challenging, thus the measurement of ROM may not be accurate! 230 | Please refer to the respective model cards for more details on other types of limitations such as lighting, motion blur, occlusions, image resolution, etc. 231 | -------------------------------------------------------------------------------- /ampere_law.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | 5 | THRESHOLD = 0.2 # 20%, 값이 클수록 손이 카메라와 가까워야 인식함 6 | 7 | gesture = { 8 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 9 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 10 | } 11 | 12 | # MediaPipe hands model 13 | mp_hands = mp.solutions.hands 14 | mp_drawing = mp.solutions.drawing_utils 15 | hands = mp_hands.Hands( 16 | max_num_hands=1, 17 | min_detection_confidence=0.5, 18 | min_tracking_confidence=0.5) 19 | 20 | # Gesture recognition model 21 | file = np.genfromtxt('data/gesture_train.csv', delimiter=',') 22 | angle = file[:,:-1].astype(np.float32) 23 | label = file[:, -1].astype(np.float32) 24 | knn = cv2.ml.KNearest_create() 25 | knn.train(angle, cv2.ml.ROW_SAMPLE, label) 26 | 27 | cap = cv2.VideoCapture(1) 28 | 29 | while cap.isOpened(): 30 | ret, img = cap.read() 31 | if not ret: 32 | continue 33 | 34 | img = cv2.flip(img, 1) # 이미지 좌우 반전 35 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 36 | 37 | result = hands.process(img) 38 | 39 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 40 | 41 | if result.multi_hand_landmarks is not None: 42 | for res in result.multi_hand_landmarks: 43 | joint = np.zeros((21, 3)) 44 | for j, lm in enumerate(res.landmark): 45 | joint[j] = [lm.x, lm.y, lm.z] 46 | 47 | # Compute angles between joints 48 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 49 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 50 | v = v2 - v1 # [20,3] 51 | # Normalize v 52 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 53 | 54 | # Get angle using arcos of dot product 55 | angle = np.arccos(np.einsum('nt,nt->n', 56 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 57 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 58 | 59 | angle = np.degrees(angle) # Convert radian to degree 60 | 61 | # Inference gesture 62 | data = np.array([angle], dtype=np.float32) 63 | ret, results, neighbours, dist = knn.findNearest(data, 3) 64 | idx = int(results[0][0]) 65 | 66 | if idx == 0 or idx == 6: # fist or six 67 | thumb_end = res.landmark[4] 68 | fist_end = res.landmark[17] 69 | 70 | text = None 71 | 72 | if thumb_end.x - fist_end.x > THRESHOLD: 73 | text = 'RIGHT' 74 | elif fist_end.x - thumb_end.x > THRESHOLD: 75 | text = 'LEFT' 76 | elif thumb_end.y - fist_end.y > THRESHOLD: 77 | text = 'DOWN' 78 | elif fist_end.y - thumb_end.y > THRESHOLD: 79 | text = 'UP' 80 | 81 | if text is not None: 82 | cv2.putText(img, text=text, org=(int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0] + 20)), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 83 | elif idx in [1, 2, 3, 4, 5, 9]: # 숫자 1,2,3,4,5 인식 84 | if idx == 9: 85 | idx = 2 86 | 87 | cv2.putText(img, text=str(idx), org=(int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0] + 20)), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 88 | 89 | mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 90 | 91 | cv2.imshow('Ampere\'s Law', img) 92 | if cv2.waitKey(1) == ord('q'): 93 | break 94 | -------------------------------------------------------------------------------- /code/00_image.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo with a single color image 3 | ### Input : Color image of face / hand / body 4 | ### Output: 2D/2.5D/3D display of face, hand, body keypoint/joint 5 | ### Usage : python 00_image.py -m face 6 | ### python 00_image.py -m hand 7 | ### python 00_image.py -m body 8 | ### python 00_image.py -m holistic 9 | ############################################################################### 10 | 11 | import cv2 12 | import sys 13 | import argparse 14 | 15 | from utils_display import DisplayFace, DisplayHand, DisplayBody, DisplayHolistic 16 | from utils_mediapipe import MediaPipeFace, MediaPipeHand, MediaPipeBody, MediaPipeHolistic 17 | 18 | 19 | # User select mode 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument('-m', '--mode', default='hand', 22 | help='Select mode: face / hand / body / holistic') 23 | args = parser.parse_args() 24 | mode = args.mode 25 | 26 | # Load mediapipe and display class 27 | if mode=='face': 28 | pipe = MediaPipeFace(static_image_mode=True, max_num_faces=1) 29 | disp = DisplayFace(draw3d=True) 30 | file = '../data/sample/mona.png' 31 | elif mode=='hand': 32 | pipe = MediaPipeHand(static_image_mode=True, max_num_hands=1) 33 | disp = DisplayHand(draw3d=True, max_num_hands=1) 34 | file = '../data/sample/hand.png' 35 | elif mode=='body': 36 | pipe = MediaPipeBody(static_image_mode=True, model_complexity=1) 37 | disp = DisplayBody(draw3d=True) 38 | file = '../data/sample/upper_limb4.png' 39 | elif mode=='holistic': 40 | pipe = MediaPipeHolistic(static_image_mode=True, model_complexity=1) 41 | disp = DisplayHolistic(draw3d=True) 42 | file = '../data/sample/lower_limb4.png' 43 | else: 44 | print('Undefined mode only the following modes are available: \nface / hand / body / holistic') 45 | sys.exit() 46 | 47 | # Read in image (Note: You can change the file path to your own test image) 48 | img = cv2.imread(file) 49 | 50 | # # Preprocess image if necessary 51 | # img = cv2.resize(img, None, fx=0.5, fy=0.5) 52 | img = cv2.flip(img, 1) 53 | # # Select ROI 54 | # r = cv2.selectROI(img) 55 | # # Crop image 56 | # img = img[r[1]:r[1]+r[3], r[0]:r[0]+r[2]] 57 | 58 | # Feedforward to extract pose param 59 | param = pipe.forward(img) 60 | 61 | # Display 2D keypoint 62 | cv2.imshow('img 2D', disp.draw2d(img.copy(), param)) 63 | # Display 2.5D keypoint 64 | cv2.imshow('img 2.5D', disp.draw2d_(img.copy(), param)) 65 | cv2.waitKey(0) # Press escape to dispay 3D view 66 | 67 | # Display 3D joint 68 | disp.draw3d(param) 69 | disp.vis.update_geometry(None) 70 | disp.vis.poll_events() 71 | disp.vis.update_renderer() 72 | disp.vis.run() 73 | 74 | pipe.pipe.close() 75 | -------------------------------------------------------------------------------- /code/01_video.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo with video input 3 | ### Input : Live video of face / hand / body 4 | ### Output: 2D/2.5D/3D display of face, hand, body keypoint/joint 5 | ### Usage : python 01_video.py -m face 6 | ### python 01_video.py -m hand 7 | ### python 01_video.py -m body 8 | ### python 01_video.py -m holistic 9 | ############################################################################### 10 | 11 | import cv2 12 | import sys 13 | import time 14 | import argparse 15 | 16 | from utils_display import DisplayFace, DisplayHand, DisplayBody, DisplayHolistic 17 | from utils_mediapipe import MediaPipeFace, MediaPipeHand, MediaPipeBody, MediaPipeHolistic 18 | 19 | 20 | # User select mode 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument('-m', '--mode', default='hand', 23 | help='Select mode: face / hand / body / holistic') 24 | args = parser.parse_args() 25 | mode = args.mode 26 | 27 | # Load mediapipe and display class 28 | if mode=='face': 29 | pipe = MediaPipeFace(static_image_mode=False, max_num_faces=1) 30 | disp = DisplayFace(draw3d=True) 31 | elif mode=='hand': 32 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=2) 33 | disp = DisplayHand(draw3d=True, max_num_hands=2) 34 | elif mode=='body': 35 | pipe = MediaPipeBody(static_image_mode=False, model_complexity=1) 36 | disp = DisplayBody(draw3d=True) 37 | elif mode=='holistic': 38 | pipe = MediaPipeHolistic(static_image_mode=False, model_complexity=1) 39 | disp = DisplayHolistic(draw3d=True) 40 | else: 41 | print('Undefined mode only the following modes are available: \nface / hand / body / holistic') 42 | sys.exit() 43 | 44 | # Start video capture 45 | cap = cv2.VideoCapture(0) # By default webcam is index 0 46 | # cap = cv2.VideoCapture('../data/video.mp4') # Read from .mp4 file 47 | # cap.set(cv2.CAP_PROP_POS_FRAMES, 1) # Set starting position of frame 48 | 49 | # # Log video 50 | # fps = 30 51 | # ret, img = cap.read() 52 | # width, height = int(cap.get(3)), int(cap.get(4)) 53 | # fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Be sure to use lower case 54 | # video = cv2.VideoWriter('../data/video_.mp4', fourcc, fps, (width, height)) 55 | 56 | prev_time = time.time() 57 | while cap.isOpened(): 58 | ret, img = cap.read() 59 | if not ret: 60 | break 61 | 62 | # Preprocess image if necessary 63 | img = cv2.flip(img, 1) # Flip image for 3rd person view 64 | # img = cv2.resize(img, None, fx=0.5, fy=0.5) 65 | 66 | # To improve performance, optionally mark image as not writeable to pass by reference 67 | img.flags.writeable = False 68 | 69 | # Feedforward to extract keypoint 70 | param = pipe.forward(img) 71 | 72 | # Compute FPS 73 | curr_time = time.time() 74 | fps = 1/(curr_time-prev_time) 75 | if mode=='body': 76 | param['fps'] = fps 77 | elif mode=='face' or mode=='hand': 78 | param[0]['fps'] = fps 79 | elif mode=='holistic': 80 | for p in param: 81 | p['fps'] = fps 82 | prev_time = curr_time 83 | 84 | img.flags.writeable = True 85 | 86 | # Display 2D keypoint 87 | cv2.imshow('img 2D', disp.draw2d(img.copy(), param)) 88 | # Display 2.5D keypoint 89 | cv2.imshow('img 2.5D', disp.draw2d_(img.copy(), param)) 90 | # Display 3D 91 | disp.draw3d(param) 92 | disp.vis.update_geometry(None) 93 | disp.vis.poll_events() 94 | disp.vis.update_renderer() 95 | 96 | # # Write to video 97 | # img = disp.draw2d(img.copy(), param) 98 | # cv2.imshow('img 2D', img) 99 | # video.write(img) 100 | 101 | key = cv2.waitKey(1) 102 | if key==27: 103 | break 104 | 105 | pipe.pipe.close() 106 | # video.release() 107 | cap.release() 108 | -------------------------------------------------------------------------------- /code/02_gesture.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo on gesture recognition 3 | ### Input : Live video of hand 4 | ### Output: 2D display of hand keypoint 5 | ### with gesture classification 6 | ### Usage : python 02_gesture.py -m train (to log data) 7 | ### : python 02_gesture.py -m eval (to perform gesture recognition) 8 | ############################################################################### 9 | 10 | import cv2 11 | import argparse 12 | 13 | from utils_display import DisplayHand 14 | from utils_mediapipe import MediaPipeHand 15 | from utils_joint_angle import GestureRecognition 16 | 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('-m', '--mode', default='eval', help='train / eval') 20 | args = parser.parse_args() 21 | mode = args.mode 22 | 23 | # Load mediapipe hand class 24 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=1) 25 | 26 | # Load display class 27 | disp = DisplayHand(max_num_hands=1) 28 | 29 | # Start video capture 30 | cap = cv2.VideoCapture(0) # By default webcam is index 0 31 | 32 | # Load gesture recognition class 33 | gest = GestureRecognition(mode) 34 | 35 | counter = 0 36 | while cap.isOpened(): 37 | ret, img = cap.read() 38 | if not ret: 39 | break 40 | 41 | # Flip image for 3rd person view 42 | img = cv2.flip(img, 1) 43 | 44 | # To improve performance, optionally mark image as not writeable to pass by reference 45 | img.flags.writeable = False 46 | 47 | # Feedforward to extract keypoint 48 | param = pipe.forward(img) 49 | if (param[0]['class'] is not None) and (mode=='eval'): 50 | param[0]['gesture'] = gest.eval(param[0]['angle']) 51 | 52 | img.flags.writeable = True 53 | 54 | # Display keypoint 55 | cv2.imshow('img 2D', disp.draw2d(img.copy(), param)) 56 | 57 | key = cv2.waitKey(1) 58 | if key==27: 59 | break 60 | if key==32 and (param[0]['class'] is not None) and (mode=='train'): 61 | # Press spacebar to log training data 62 | # Note: Need to manually change class label 63 | # 'fist','one','two','three','four','five','six', 64 | # 'rock','spiderman', 65 | # 'yeah','ok', 66 | gest.train(param[0]['angle'], gest.gesture['fist']) 67 | print('Saved', counter) # Log around 10 for each class 68 | counter += 1 69 | if key==32 and (param[0]['class'] is not None) and (mode=='eval'): 70 | cv2.waitKey(0) # Pause display until user press any key 71 | 72 | pipe.pipe.close() 73 | cap.release() 74 | -------------------------------------------------------------------------------- /code/03_game_rps.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo on playing rock paper scissor 3 | ### Input : Live video of 2 hands playing rock paper scissor 4 | ### Output: 2D display of hand keypoint 5 | ### with gesture classification (rock=fist, paper=five, scissor=three/yeah) 6 | ############################################################################### 7 | 8 | import cv2 9 | 10 | from utils_display import DisplayHand 11 | from utils_mediapipe import MediaPipeHand 12 | from utils_joint_angle import GestureRecognition 13 | 14 | 15 | # Load mediapipe hand class 16 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=2) 17 | 18 | # Load display class 19 | disp = DisplayHand(max_num_hands=2) 20 | 21 | # Start video capture 22 | cap = cv2.VideoCapture(0) # By default webcam is index 0 23 | 24 | # Load gesture recognition class 25 | gest = GestureRecognition(mode='eval') 26 | 27 | counter = 0 28 | while cap.isOpened(): 29 | ret, img = cap.read() 30 | if not ret: 31 | break 32 | 33 | # Flip image for 3rd person view 34 | img = cv2.flip(img, 1) 35 | 36 | # To improve performance, optionally mark image as not writeable to pass by reference 37 | img.flags.writeable = False 38 | 39 | # Feedforward to extract keypoint 40 | param = pipe.forward(img) 41 | # Evaluate gesture for all hands 42 | for p in param: 43 | if p['class'] is not None: 44 | p['gesture'] = gest.eval(p['angle']) 45 | 46 | img.flags.writeable = True 47 | 48 | # Display keypoint and result of rock paper scissor game 49 | cv2.imshow('Game: Rock Paper Scissor', disp.draw_game_rps(img.copy(), param)) 50 | 51 | key = cv2.waitKey(1) 52 | if key==27: 53 | break 54 | 55 | pipe.pipe.close() 56 | cap.release() 57 | -------------------------------------------------------------------------------- /code/04_hand_rom.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo on measuring hand rom 3 | ### Input : Live video of hand 4 | ### Output: 2D display of hand keypoint 5 | ### with hand rom classification and corresponding joint angle 6 | ### Usage : python 04_hand_rom.py -m train (to log data) 7 | ### : python 04_hand_rom.py -m eval (to perform hand rom recognition) 8 | ############################################################################### 9 | 10 | import cv2 11 | import argparse 12 | 13 | from utils_display import DisplayHand 14 | from utils_mediapipe import MediaPipeHand 15 | from utils_joint_angle import HandRomRecognition 16 | 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('-m', '--mode', default='eval', help='train / eval') 20 | args = parser.parse_args() 21 | mode = args.mode 22 | 23 | # Load mediapipe hand class 24 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=1) 25 | 26 | # Load display class 27 | disp = DisplayHand(max_num_hands=1) 28 | 29 | # Start video capture 30 | cap = cv2.VideoCapture(0) # By default webcam is index 0 31 | 32 | # Load hand rom recognition class 33 | gest = HandRomRecognition(mode) 34 | 35 | counter = 0 36 | while cap.isOpened(): 37 | ret, img = cap.read() 38 | if not ret: 39 | break 40 | 41 | # Flip image for 3rd person view 42 | img = cv2.flip(img, 1) 43 | 44 | # To improve performance, optionally mark image as not writeable to pass by reference 45 | img.flags.writeable = False 46 | 47 | # Feedforward to extract keypoint 48 | param = pipe.forward(img) 49 | if (param[0]['class'] is not None) and (mode=='eval'): 50 | param[0]['gesture'] = gest.eval(param[0]['angle']) 51 | 52 | img.flags.writeable = True 53 | 54 | # Display keypoint 55 | cv2.imshow('img 2D', disp.draw2d(img.copy(), param)) 56 | 57 | key = cv2.waitKey(1) 58 | if key==27: 59 | break 60 | if key==32 and (param[0]['class'] is not None) and (mode=='train'): 61 | # Press spacebar to log training data 62 | # Note: Need to manually change class label 63 | # 'Finger MCP Flexion' :0, 64 | # 'Finger PIP DIP Flexion':1, 65 | # 'Thumb MCP Flexion' :2, 66 | # 'Thumb IP Flexion' :3, 67 | # 'Thumb Radial Abduction':4, 68 | # 'Thumb Palmar Abduction':5, # From this class onwards hard to differentiate 69 | # 'Thumb Opposition' :6, 70 | # 'Forearm Pronation' :7, # Not done yet 71 | # 'Forearm Supination' :8, # Not done yet 72 | # 'Wrist Flexion' :9, # Not done yet 73 | # 'Wrist Extension' :10,# Not done yet 74 | # 'Wrist Radial Deviation':11,# Not done yet 75 | # 'Wrist Ulnar Deviation' :12,# Not done yet 76 | gest.train(param[0]['angle'], gest.gesture['Finger MCP Flexion']) 77 | print('Saved', counter) # Log around 10 for each class 78 | counter += 1 79 | if key==32 and (param[0]['class'] is not None) and (mode=='eval'): 80 | cv2.waitKey(0) # Pause display until user press any key 81 | 82 | pipe.pipe.close() 83 | cap.release() 84 | -------------------------------------------------------------------------------- /code/05_wrist_rom.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo on measuring wrist and forearm rom 3 | ### Input : Live video of hand and upper body 4 | ### Output: 2D display of hand and upper body keypoint 5 | ### with corresponding joint angle 6 | ### Usage : python 05_wrist_rom.py -m 0 -s right 7 | ### python 05_wrist_rom.py -m 1 -s right 8 | ### python 05_wrist_rom.py -m 2 -s right 9 | ### python 05_wrist_rom.py -m 0 -s left 10 | ### python 05_wrist_rom.py -m 1 -s left 11 | ### python 05_wrist_rom.py -m 2 -s left 12 | ############################################################################### 13 | 14 | import cv2 15 | import sys 16 | import argparse 17 | 18 | from utils_display import DisplayHolistic, DisplayHand 19 | from utils_mediapipe import MediaPipeHolistic, MediaPipeHand 20 | from utils_joint_angle import WristArmRom 21 | 22 | 23 | # User select mode of measurement 24 | # 0: Wrist flexion/extension 25 | # 1: Wrist radial/ulnar deviation 26 | # 2: Forearm pronation/supination 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument('-s', '--side', default='right') 29 | parser.add_argument('-m', '--mode', default=2, 30 | help='Select mode: \ 31 | 0:Wrist flex/ext, \ 32 | 1:Wrist radial/ulnar dev, \ 33 | 2:Forearm pronation/supination') 34 | args = parser.parse_args() 35 | mode = int(args.mode) 36 | side = args.side 37 | 38 | # Load mediapipe and display class 39 | if mode==0 or mode==1: # Note; To determine wrist rom need upper body elbow joint 40 | pipe = MediaPipeHolistic(static_image_mode=False, model_complexity=1) 41 | disp = DisplayHolistic(draw3d=True) 42 | elif mode==2: 43 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=1) 44 | disp = DisplayHand(draw3d=True, max_num_hands=1) 45 | else: 46 | print('Undefined mode only 3 modes are available: \n \ 47 | 0:Wrist flex/ext, \n \ 48 | 1:Wrist radial/ulnar dev, \n \ 49 | 2:Forearm pronation/supination') 50 | sys.exit() 51 | 52 | # Start video capture 53 | cap = cv2.VideoCapture(0) # By default webcam is index 0 54 | 55 | # Load wrist and forearm rom class 56 | rom = WristArmRom(mode, side) 57 | 58 | while cap.isOpened(): 59 | ret, img = cap.read() 60 | # img = cv2.imread('../data/sample/wrist_extension.png') # python 05_wrist_rom.py -m 0 -s right 61 | # img = cv2.imread('../data/sample/wrist_flexion.png') # python 05_wrist_rom.py -m 0 -s right 62 | # img = cv2.imread('../data/sample/wrist_radial.png') # python 05_wrist_rom.py -m 1 -s right 63 | # img = cv2.imread('../data/sample/wrist_ulnar.png') # python 05_wrist_rom.py -m 1 -s right 64 | if not ret: 65 | break 66 | 67 | # Flip image for 3rd person view 68 | img = cv2.flip(img, 1) 69 | 70 | # To improve performance, optionally mark image as not writeable to pass by reference 71 | img.flags.writeable = False 72 | 73 | # Feedforward to extract keypoint 74 | param = pipe.forward(img) 75 | 76 | img.flags.writeable = True 77 | 78 | # Compute wrist/forearm range of motion 79 | param = rom.eval(param) 80 | 81 | # Display keypoint 82 | cv2.imshow('img 2D', disp.draw2d(img.copy(), param)) 83 | # Display 3D 84 | disp.draw3d(param) 85 | disp.vis.update_geometry(None) 86 | disp.vis.poll_events() 87 | disp.vis.update_renderer() 88 | 89 | key = cv2.waitKey(1) 90 | if key==27: 91 | break 92 | if key==32: 93 | cv2.imwrite('img.png', disp.draw2d(img.copy(), param)) 94 | cv2.waitKey(0) # Pause display until user press any key 95 | 96 | pipe.pipe.close() 97 | cap.release() 98 | -------------------------------------------------------------------------------- /code/06_face_mask.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo with video input of face 3 | ### Input : Live video of face 4 | ### Output: Overlay 3D face mesh on image 5 | ############################################################################### 6 | 7 | import cv2 8 | import time 9 | 10 | from utils_display import DisplayFaceMask 11 | from utils_mediapipe import MediaPipeFace 12 | 13 | 14 | # Load mediapipe class 15 | pipe = MediaPipeFace(static_image_mode=False, max_num_faces=1) 16 | 17 | # Start video capture 18 | cap = cv2.VideoCapture(0) # By default webcam is index 0 19 | 20 | # Load display class 21 | ret, img = cap.read(0) # Read in a sample image 22 | disp = DisplayFaceMask(img=img, draw3d=True, max_num_faces=1) 23 | 24 | prev_time = time.time() 25 | while cap.isOpened(): 26 | ret, img = cap.read() 27 | if not ret: 28 | break 29 | 30 | # Preprocess image if necessary 31 | # img = cv2.flip(img, 1) # Flip image for 3rd person view 32 | # img = cv2.resize(img, None, fx=0.5, fy=0.5) 33 | 34 | # To improve performance, optionally mark image as not writeable to pass by reference 35 | img.flags.writeable = False 36 | 37 | # Feedforward to extract keypoint 38 | param = pipe.forward(img) 39 | 40 | # Compute FPS 41 | curr_time = time.time() 42 | fps = 1/(curr_time-prev_time) 43 | param[0]['fps'] = fps 44 | prev_time = curr_time 45 | 46 | img.flags.writeable = True 47 | cv2.imshow('img', img) 48 | 49 | # Display 3D 50 | img = disp.draw3d(param, img) 51 | 52 | key = cv2.waitKey(1) 53 | if key==27: 54 | break 55 | 56 | pipe.pipe.close() 57 | cap.release() 58 | -------------------------------------------------------------------------------- /code/07_triangulate.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo with at least 2 cameras for triangulation 3 | ### Input : Live videos of face / hand / body 4 | ### : Calibrated camera intrinsics and extrinsics 5 | ### Output: 2D/3D (triangulated) display of hand, body keypoint/joint 6 | ### Usage : python 07_triangulate.py -m body --use_panoptic_dataset 7 | ############################################################################### 8 | 9 | import cv2 10 | import sys 11 | import time 12 | import argparse 13 | import numpy as np 14 | import open3d as o3d 15 | 16 | from utils_display import DisplayHand, DisplayBody, DisplayHolistic 17 | from utils_mediapipe import MediaPipeHand, MediaPipeBody, MediaPipeHolistic 18 | from utils_3d_reconstruct import Triangulation 19 | 20 | 21 | # User select mode 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--use_panoptic_dataset', action='store_true') 24 | parser.add_argument('-m', '--mode', default='body', 25 | help='Select mode: hand / body / holistic') 26 | args = parser.parse_args() 27 | mode = args.mode 28 | 29 | # Define list of camera index 30 | # cam_idx = [4,10] # Note: Hardcoded for my setup 31 | # Read from .mp4 file 32 | if args.use_panoptic_dataset: 33 | # Test with 2 views 34 | cam_idx = ['../data/171204_pose1_sample/hdVideos/hd_00_00.mp4', 35 | '../data/171204_pose1_sample/hdVideos/hd_00_11.mp4'] 36 | 37 | # # Test with n views 38 | # num_views = 5 # Note: Maximum 31 hd cameras but processing time will be extremely slow 39 | # cam_idx = [] 40 | # for i in range(num_views): 41 | # cam_idx.append( 42 | # '../data/171204_pose1_sample/hdVideos/hd_00_'+str(i).zfill(2)+'.mp4') 43 | 44 | # Start video capture 45 | cap = [cv2.VideoCapture(cam_idx[i]) for i in range(len(cam_idx))] 46 | 47 | # Define list of other variable 48 | img = [None for i in range(len(cam_idx))] # Store image 49 | pipe = [None for i in range(len(cam_idx))] # MediaPipe class 50 | disp = [None for i in range(len(cam_idx))] # Display class 51 | param = [None for i in range(len(cam_idx))] # Store pose parameter 52 | prev_time = [time.time() for i in range(len(cam_idx))] 53 | 54 | # Open3D visualization 55 | vis = o3d.visualization.Visualizer() 56 | vis.create_window(width=640, height=480) 57 | vis.get_render_option().point_size = 5.0 58 | 59 | # Load triangulation class 60 | tri = Triangulation(cam_idx=cam_idx, vis=vis, 61 | use_panoptic_dataset=args.use_panoptic_dataset) 62 | 63 | # Load mediapipe and display class 64 | if mode=='hand': 65 | for i in range(len(cam_idx)): 66 | pipe[i] = MediaPipeHand(static_image_mode=False, max_num_hands=1) 67 | disp[i] = DisplayHand(draw3d=True, max_num_hands=1, vis=vis) 68 | elif mode=='body': 69 | for i in range(len(cam_idx)): 70 | pipe[i] = MediaPipeBody(static_image_mode=False, model_complexity=1) 71 | disp[i] = DisplayBody(draw3d=True, vis=vis) 72 | elif mode=='holistic': 73 | for i in range(len(cam_idx)): 74 | pipe[i] = MediaPipeHolistic(static_image_mode=False, model_complexity=1) 75 | disp[i] = DisplayHolistic(draw3d=True, vis=vis) 76 | else: 77 | print('Undefined mode only the following modes are available: \n hand / body / holistic') 78 | sys.exit() 79 | 80 | while True: 81 | # Loop through video capture 82 | for i, c in enumerate(cap): 83 | if not c.isOpened(): 84 | break 85 | ret, img[i] = c.read() 86 | if not ret: 87 | break 88 | 89 | # Preprocess image if necessary 90 | # img[i] = cv2.flip(img[i], 1) # Flip image for 3rd person view 91 | 92 | # To improve performance, optionally mark image as not writeable to pass by reference 93 | img[i].flags.writeable = False 94 | 95 | # Feedforward to extract keypoint 96 | param[i] = pipe[i].forward(img[i]) 97 | 98 | img[i].flags.writeable = True 99 | 100 | # Compute FPS 101 | curr_time = time.time() 102 | fps = 1/(curr_time-prev_time[i]) 103 | if mode=='body': 104 | param[i]['fps'] = fps 105 | elif mode=='hand': 106 | param[i][0]['fps'] = fps 107 | elif mode=='holistic': 108 | for p in param[i]: 109 | p['fps'] = fps 110 | prev_time[i] = curr_time 111 | 112 | # Perform triangulation 113 | if args.use_panoptic_dataset: 114 | if len(cam_idx)==2: 115 | param = tri.triangulate_2views(param, mode) 116 | else: 117 | param = tri.triangulate_nviews(param, mode) 118 | 119 | for i in range(len(cam_idx)): 120 | # Display 2D keypoint 121 | img[i] = disp[i].draw2d(img[i].copy(), param[i]) 122 | img[i] = cv2.resize(img[i], None, fx=0.5, fy=0.5) 123 | cv2.imshow('img'+str(i), img[i]) 124 | # Display 3D 125 | disp[i].draw3d(param[i]) 126 | 127 | vis.update_geometry(None) 128 | vis.poll_events() 129 | vis.update_renderer() 130 | 131 | key = cv2.waitKey(1) 132 | if key==27: 133 | break 134 | 135 | # vis.run() # Keep 3D display for visualization 136 | 137 | for p, c in zip(pipe, cap): 138 | p.pipe.close() 139 | c.release() 140 | -------------------------------------------------------------------------------- /code/08_skeleton_3D.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo on displaying 3D hand/body skeleton 3 | ### Input : Live video of hand/body 4 | ### Output: 3D display of hand/body skeleton 5 | ### Usage : python 08_skeleton_3D.py -m hand 6 | ### : python 08_skeleton_3D.py -m body 7 | ### : python 08_skeleton_3D.py -m holistic 8 | ############################################################################### 9 | 10 | import cv2 11 | import time 12 | import argparse 13 | 14 | from utils_display import DisplayHand, DisplayBody, DisplayHolistic 15 | from utils_mediapipe import MediaPipeHand, MediaPipeBody, MediaPipeHolistic 16 | 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('-m', '--mode', default='hand', help=' Select mode: hand / body / holistic') 20 | args = parser.parse_args() 21 | mode = args.mode 22 | 23 | # Start video capture 24 | cap = cv2.VideoCapture(0) # By default webcam is index 0 25 | # cap = cv2.VideoCapture('../data/video.mp4') # Read from .mp4 file 26 | 27 | # Read in sample image to estimate camera intrinsic 28 | ret, img = cap.read(0) 29 | # img = cv2.resize(img, None, fx=0.5, fy=0.5) 30 | img_width = img.shape[1] 31 | img_height = img.shape[0] 32 | intrin = { 33 | 'fx': img_width*0.9, # Approx 0.7w < f < w https://www.learnopencv.com/approximate-focal-length-for-webcams-and-cell-phone-cameras/ 34 | 'fy': img_width*0.9, 35 | 'cx': img_width*0.5, # Approx center of image 36 | 'cy': img_height*0.5, 37 | 'width': img_width, 38 | 'height': img_height, 39 | } 40 | 41 | # Load mediapipe and display class 42 | if mode=='hand': 43 | pipe = MediaPipeHand(static_image_mode=False, max_num_hands=2, intrin=intrin) 44 | disp = DisplayHand(draw3d=True, draw_camera=True, max_num_hands=2, intrin=intrin) 45 | elif mode=='body': 46 | # Note: As of version 0.8.3 3D joint estimation is only available in full body mode 47 | pipe = MediaPipeBody(static_image_mode=False, model_complexity=1, intrin=intrin) 48 | disp = DisplayBody(draw3d=True, draw_camera=True, intrin=intrin) 49 | elif mode=='holistic': 50 | # Note: As of version 0.8.3 3D joint estimation is only available in full body mode 51 | pipe = MediaPipeHolistic(static_image_mode=False, model_complexity=1, intrin=intrin) 52 | disp = DisplayHolistic(draw3d=True, draw_camera=True, intrin=intrin) 53 | 54 | prev_time = time.time() 55 | while cap.isOpened(): 56 | ret, img = cap.read() 57 | if not ret: 58 | cap.set(cv2.CAP_PROP_POS_FRAMES, 0) # Loop back 59 | ret, img = cap.read() 60 | # break 61 | 62 | # Flip image for 3rd person view 63 | img = cv2.flip(img, 1) 64 | # img = cv2.resize(img, None, fx=0.5, fy=0.5) 65 | 66 | # To improve performance, optionally mark image as not writeable to pass by reference 67 | img.flags.writeable = False 68 | 69 | # Feedforward to extract keypoint 70 | param = pipe.forward(img) 71 | 72 | # Compute FPS 73 | curr_time = time.time() 74 | fps = 1/(curr_time-prev_time) 75 | if mode=='body': 76 | param['fps'] = fps 77 | elif mode=='face' or mode=='hand': 78 | param[0]['fps'] = fps 79 | elif mode=='holistic': 80 | for p in param: 81 | p['fps'] = fps 82 | prev_time = curr_time 83 | 84 | img.flags.writeable = True 85 | 86 | # Display keypoint 87 | cv2.imshow('img 2D', disp.draw2d(img, param)) 88 | # Display 3D 89 | disp.draw3d(param,) 90 | disp.draw3d_(param, img) 91 | disp.vis.update_geometry(None) 92 | disp.vis.poll_events() 93 | disp.vis.update_renderer() 94 | 95 | key = cv2.waitKey(1) 96 | if key==27: 97 | break 98 | if key==ord('r'): # Press 'r' to reset camera view 99 | disp.camera.reset_view() 100 | 101 | pipe.pipe.close() 102 | cap.release() 103 | -------------------------------------------------------------------------------- /code/09_objectron.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Simple demo with a single color image 3 | ### Input : Color image of object 4 | ### Output: 2D/3D display of object's bounding box 5 | ### Usage : python 09_objectron.py -m shoe 6 | ### python 09_objectron.py -m chair 7 | ### python 09_objectron.py -m cup 8 | ### python 09_objectron.py -m camera 9 | ############################################################################### 10 | 11 | import cv2 12 | import argparse 13 | 14 | from utils_display import DisplayObjectron 15 | from utils_mediapipe import MediaPipeObjectron 16 | 17 | 18 | # User select mode 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('-m', '--mode', default='shoe', 21 | help='Select mode: shoe / chair / cup / camera') 22 | args = parser.parse_args() 23 | 24 | # Read in image (Note: You can change the file path to your own test image) 25 | file = '../data/sample/object.png' 26 | img = cv2.imread(file) 27 | 28 | # Estimate camera intrinsic 29 | img_width = img.shape[1] 30 | img_height = img.shape[0] 31 | intrin = { 32 | 'fx': img_width*0.9, # Approx 0.7w < f < w https://www.learnopencv.com/approximate-focal-length-for-webcams-and-cell-phone-cameras/ 33 | 'fy': img_width*0.9, 34 | 'cx': img_width*0.5, # Approx center of image 35 | 'cy': img_height*0.5, 36 | 'width': img_width, 37 | 'height': img_height, 38 | } 39 | 40 | # Load mediapipe 41 | pipe = MediaPipeObjectron(static_image_mode=True, max_num_objects=5, 42 | model_name=args.mode, intrin=intrin) 43 | 44 | # Load display class 45 | disp = DisplayObjectron(draw3d=True, draw_camera=True, intrin=intrin, max_num_objects=5) 46 | 47 | # Feedforward to extract pose param 48 | param = pipe.forward(img) 49 | 50 | # # Display 2D keypoint 51 | cv2.imshow('img', disp.draw2d(img, param)) 52 | cv2.waitKey(0) # Press escape to dispay 3D view 53 | 54 | # Display 3D joint 55 | disp.draw3d(param, img) 56 | disp.vis.update_geometry(None) 57 | disp.vis.poll_events() 58 | disp.vis.update_renderer() 59 | disp.vis.run() 60 | 61 | pipe.pipe.close() -------------------------------------------------------------------------------- /code/utils_3d_reconstruct.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Useful function for 3D reconstruction from multiple images 3 | ### 4 | ### First requirement is to obtain camera intrinsics and extrinsics parameters 5 | ### Refer to Calibration class 6 | ### 7 | ### Next use triangulation/direct linear transformation (DLT) 8 | ### To reconstruct 3D points from 2D image points 9 | ############################################################################### 10 | 11 | import cv2 12 | import glob 13 | import json 14 | import yaml 15 | import numpy as np 16 | import open3d as o3d 17 | 18 | 19 | class Calibration: 20 | def __init__(self, chessboard_size=(6,5), chessboard_sq_size=0.015): 21 | super(Calibration, self).__init__() 22 | 23 | self.chessboard_size = chessboard_size 24 | self.chessboard_sq_size = chessboard_sq_size 25 | 26 | # Prepare 3D object points in real world space 27 | self.obj_pts = np.zeros((chessboard_size[0]*chessboard_size[1],3), np.float32) 28 | # [[0,0,0], [1,0,0], [2,0,0] ....,[9,6,0]] 29 | self.obj_pts[:,:2] = np.mgrid[0:chessboard_size[0],0:chessboard_size[1]].T.reshape(-1,2) 30 | self.obj_pts *= chessboard_sq_size # Convert length of each black square to units in meter 31 | 32 | # Termination criteria for cornerSubPix 33 | self.criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 1e-3) 34 | 35 | # Flag for findChessboardCorners 36 | self.flags_findChessboard = cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK 37 | 38 | # Termination criteria for stereoCalibrate 39 | self.criteria_stereoCalibrate = (cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 100, 1e-5) 40 | 41 | # Flag for stereoCalibrate 42 | self.flags_stereoCalibrate = cv2.CALIB_FIX_INTRINSIC + cv2.CALIB_RATIONAL_MODEL + cv2.CALIB_CB_FAST_CHECK 43 | 44 | 45 | def get_intrin(self, folder): 46 | # Use chessboard pattern to calib intrinsic of color camera 47 | 48 | # Read in filename of .png from folder 49 | file = glob.glob(folder+'*.png') 50 | file.sort() 51 | 52 | # List to store object point and image point from all images 53 | objpt = [] # 3d point in real world space 54 | imgpt = [] # 2d point in image plane 55 | file_ = [] # Filename of images with success findChessboardCorners 56 | 57 | for f in file: 58 | # Read in image 59 | img = cv2.imread(f) 60 | 61 | # Convert to grayscale 62 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 63 | 64 | # Find chessboard corners 65 | ret, corners = cv2.findChessboardCorners(gray, self.chessboard_size, self.flags_findChessboard) 66 | 67 | # If found, add object points, image points (after refining them) 68 | if ret==True: 69 | corners2 = cv2.cornerSubPix(gray, corners, (5,5), (-1,-1), self.criteria) # (chessboard_size[0]*chessboard_size[1], 1, 2} 70 | imgpt.append(corners2) 71 | objpt.append(self.obj_pts) 72 | file_.append(f) 73 | print('Found ChessboardCorners', f) 74 | else: 75 | print('Cannot find ChessboardCorners', f) 76 | 77 | # Calibration 78 | if len(objpt)>0 and len(imgpt)>0: 79 | print('Calibrating ...') 80 | ret, mat, dist, rvec, tvec = cv2.calibrateCamera(objpt, imgpt, gray.shape[::-1], None, None) 81 | print('Calibrating done') 82 | 83 | # Draw projected xyz axis on the image 84 | for i, f in enumerate(file_): 85 | # Read in image 86 | img = cv2.imread(f) 87 | 88 | # Draw corners 89 | img = cv2.drawChessboardCorners(img, self.chessboard_size, imgpt[i], True) 90 | 91 | # solvePnp will return the transformation matrix to transform 3D model coordinate to 2D camera coordinate 92 | ret, rvec, tvec = cv2.solvePnP(objpt[i], imgpt[i], mat, dist) 93 | self.project_3Daxis_to_2Dimage(img, mat, dist, rvec, tvec) 94 | 95 | # Save image with new name and extension 96 | f_ = f[:-4] + '_.jpg' 97 | cv2.imwrite(f_, img) 98 | 99 | # Get reprojection error 100 | error = self.get_reprojection_error( 101 | np.asarray(objpt[i]).reshape(-1, 3), # To convert from m list of (n, 3) to (m*n, 3) 102 | np.asarray(imgpt[i]).reshape(-1, 2), # To convert from m list of (n, 1, 2) to (m*n, 2) 103 | mat, dist, rvec, tvec) 104 | print('Img', i, f, 'reprojection error', error) 105 | 106 | # Save camera intrinsic 107 | img = cv2.imread(file[0]) 108 | data = dict(intrin_mat=mat.tolist(), 109 | dist_coeff=dist.tolist(), 110 | img_height=img.shape[0], 111 | img_width=img.shape[1]) 112 | filepath = folder + 'intrin.yaml' 113 | with open(filepath, 'w') as f: 114 | yaml.dump(data, f, default_flow_style=False) 115 | print('Saved camera intrinsic to', filepath) 116 | 117 | 118 | def get_extrin(self, folder): 119 | # Use chessboard pattern to get extrinsic of color cameras 120 | # Note: Simplified calibration with only one image per camera view 121 | 122 | # Read in filename of .png from folder 123 | file = glob.glob(folder+'*.png') 124 | file.sort() 125 | 126 | for i, f in enumerate(file): 127 | # Extract camera index from last few characters of filename 128 | cam_idx = f.split('/')[-1] 129 | cam_idx = cam_idx.split('.')[0] # Note: f = '../data/calib_extrin/cam_X.png', where X is camera index 130 | 131 | # Read in camera intrinsic 132 | filepath = '../data/calib_intrin/'+cam_idx+'/intrin.yaml' 133 | param = yaml.load(open(filepath), Loader=yaml.FullLoader) 134 | mat = np.asarray(param['intrin_mat']) 135 | dist = np.asarray(param['dist_coeff']) 136 | 137 | # Read in image 138 | img = cv2.imread(f) 139 | 140 | # Convert to grayscale first 141 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 142 | 143 | # Find chessboard corners 144 | ret, corners = cv2.findChessboardCorners(gray, self.chessboard_size, self.flags_findChessboard) 145 | 146 | # If found, add object points, image points (after refining them) 147 | if ret == True: 148 | corners2 = cv2.cornerSubPix(gray, corners, (5,5), (-1,-1), self.criteria) # (chessboard_size[0]*chessboard_size[1], 1, 2} 149 | 150 | # Draw corners 151 | img = cv2.drawChessboardCorners(img, self.chessboard_size, corners2, ret) 152 | 153 | # solvePnp will return the transformation matrix to transform 3D model coordinate to 2D camera coordinate 154 | ret, rvec, tvec = cv2.solvePnP(self.obj_pts, corners2, mat, dist) 155 | self.project_3Daxis_to_2Dimage(img, mat, dist, rvec, tvec) 156 | 157 | # Save image with new name and extension 158 | f_ = f[:-4] + '_.jpg' 159 | cv2.imwrite(f_, img) 160 | # Display image 161 | cv2.imshow('img'+str(i), img) 162 | cv2.waitKey(1) 163 | 164 | # Get reprojection error 165 | error = self.get_reprojection_error( 166 | np.asarray(self.obj_pts).reshape(-1, 3), # To convert from m list of (n, 3) to (m*n, 3) 167 | np.asarray(corners2).reshape(-1, 2), # To convert from m list of (n, 1, 2) to (m*n, 2) 168 | mat, dist, rvec, tvec) 169 | print('Img', f, 'reprojection error', error) 170 | 171 | # Create 4 by 4 homo matrix [R|T] to transform 3D model coordinate to 3D camera coordinate 172 | homo_matrix = np.hstack((cv2.Rodrigues(rvec)[0], tvec)) # 3 by 4 matrix 173 | homo_matrix = np.vstack((homo_matrix, np.array([0,0,0,1]))) # 4 by 4 matrix 174 | 175 | # Save camera extrinsic 176 | data = dict(extrin_mat=homo_matrix.tolist()) 177 | filepath = f[:-4] + '_extrin.yaml' 178 | with open(filepath, 'w') as f: 179 | yaml.dump(data, f, default_flow_style=False) 180 | print('Saved camera extrinsic to', filepath) 181 | 182 | 183 | def get_extrin_mirror(self, folder, idx=0): 184 | # Use chessboard to calib extrin of color cameras 185 | # Note: Simplified calibration with only one image per camera view 186 | # Note: Assume setup contains one camera with two plane mirrors -> total of 3 camera views 187 | 188 | print('Assume setup contains one camera with two plane mirrors') 189 | print('Select region of interest (ROI) containing chessboard pattern in the following order:') 190 | print(' 1) Select ROI of actual camera view') 191 | print(' 2) Select ROI of 1st virtual camera view') 192 | print(' 3) Select ROI of 2nd virtual camera view') 193 | 194 | # Read in camera intrinsic 195 | cam_idx = 'cam_'+str(idx).zfill(2) 196 | filepath = '../data/calib_intrin/'+cam_idx+'/intrin.yaml' 197 | param = yaml.load(open(filepath), Loader=yaml.FullLoader) 198 | mat = np.asarray(param['intrin_mat']) 199 | dist = np.asarray(param['dist_coeff']) 200 | 201 | # Read in image 202 | img = cv2.imread(folder+'image.png') 203 | # Keep original image for drawing 204 | ori = cv2.imread(folder+'image.png') 205 | 206 | for i in range(3): # Note: Hardcode 3 views 207 | # Manually select ROI 208 | print('') 209 | roi = cv2.selectROI(img) # Return top left x, y, width, height 210 | # Mask out non ROI 211 | tmp = self.mask_non_roi(img, roi) 212 | 213 | # Convert to grayscale 214 | gray = cv2.cvtColor(tmp, cv2.COLOR_BGR2GRAY) 215 | 216 | # Find chessboard corners 217 | ret, corners = cv2.findChessboardCorners(gray, self.chessboard_size, self.flags_findChessboard) 218 | 219 | if ret==False: 220 | print('Cannot find ChessboardCorners') 221 | else: 222 | # If found, add object points, image points (after refining them) 223 | corners2 = cv2.cornerSubPix(gray, corners, (5,5), (-1,-1), self.criteria) # (chessboard_size[0]*chessboard_size[1], 1, 2} 224 | 225 | # For next two views need to flip corners as they are from virtual camera 226 | if i>0: 227 | corners2 = self.flip_corners(corners2) 228 | # Need to mirror reflect about the x axis also 229 | corners2[:,:,0] = img.shape[1] - corners2[:,:,0] 230 | img = cv2.flip(img, flipCode=1) 231 | ori = cv2.flip(ori, flipCode=1) 232 | 233 | # Draw corners 234 | ori = cv2.drawChessboardCorners(ori, self.chessboard_size, corners2, ret) 235 | 236 | # solvePnp will return the transformation matrix to transform 3D model coordinate to 2D camera coordinate 237 | ret, rvec, tvec = cv2.solvePnP(self.obj_pts, corners2, mat, dist) 238 | self.project_3Daxis_to_2Dimage(ori, mat, dist, rvec, tvec) 239 | 240 | # Mask out chessboard 241 | img = self.mask_chessboard(img, corners2) 242 | 243 | # Flip back image 244 | if i>0: 245 | img = cv2.flip(img, flipCode=1) 246 | ori = cv2.flip(ori, flipCode=1) 247 | 248 | # Display image 249 | cv2.imshow('ori', ori) 250 | cv2.waitKey(1) 251 | 252 | # Get reprojection error 253 | error = self.get_reprojection_error( 254 | np.asarray(self.obj_pts).reshape(-1, 3), # To convert from m list of (n, 3) to (m*n, 3) 255 | np.asarray(corners2).reshape(-1, 2), # To convert from m list of (n, 1, 2) to (m*n, 2) 256 | mat, dist, rvec, tvec) 257 | print('Img', i, 'reprojection error', error) 258 | 259 | # Create 4 by 4 homo matrix [R|T] to transform 3D model coordinate to 3D camera coordinate 260 | homo_matrix = np.hstack((cv2.Rodrigues(rvec)[0], tvec)) # 3 by 4 matrix 261 | homo_matrix = np.vstack((homo_matrix, np.array([0,0,0,1]))) # 4 by 4 matrix 262 | # Note: Inverse matrix to get transformation from image to camera frame 263 | homo_matrix = np.linalg.inv(homo_matrix) 264 | 265 | # Save camera extrinsic 266 | data = dict(extrin_mat=homo_matrix.tolist()) 267 | filepath = folder + 'cam_' + str(i).zfill(2) + '_extrin.yaml' 268 | with open(filepath, 'w') as f: 269 | yaml.dump(data, f, default_flow_style=False) 270 | print('Saved camera extrinsic to', filepath) 271 | 272 | # Save image with new name and extension 273 | cv2.imwrite(folder+'image_.jpg', ori) 274 | 275 | 276 | def project_3Daxis_to_2Dimage(self, img, mat, dist, rvec, tvec): 277 | axis_3D = np.float32([[0,0,0],[1,0,0],[0,1,0],[0,0,1]]) * 2 * self.chessboard_sq_size # Create a 3D axis that is twice the length of chessboard_sq_size 278 | axis_2D = cv2.projectPoints(axis_3D, rvec, tvec, mat, dist)[0].reshape(-1, 2) 279 | if axis_2D.shape[0]==4: 280 | colours = [(0,0,255),(0,255,0),(255,0,0)] # BGR 281 | for i in range(1,4): 282 | (x0, y0), (x1, y1) = axis_2D[0], axis_2D[i] 283 | cv2.line(img, (int(x0), int(y0)), (int(x1), int(y1)), colours[i-1], 3) 284 | 285 | 286 | def get_reprojection_error(self, p3D, p2D, mat, dist, rvec, tvec): 287 | p2D_reproj = cv2.projectPoints(p3D, rvec, tvec, mat, dist)[0].reshape(-1, 2) 288 | error = cv2.norm(p2D, p2D_reproj, cv2.NORM_L2) / len(p2D) 289 | 290 | return error # https://docs.opencv.org/3.4.3/dc/dbb/tutorial_py_calibration.html 291 | 292 | 293 | def mask_non_roi(self, img, roi): 294 | x, y, width, height = roi 295 | 296 | # Create binary mask 297 | msk = np.zeros((img.shape[0],img.shape[1]), dtype=np.uint8) 298 | msk[y:y+height,x:x+width] = 255 299 | 300 | # Mask non ROI 301 | tmp = cv2.bitwise_and(img, img, mask=msk) 302 | 303 | return tmp 304 | 305 | 306 | def mask_chessboard(self, img, corners): 307 | # Get four corners 308 | c0 = corners[0,:,:].astype(np.int32) 309 | c1 = corners[self.chessboard_size[0]-1,:,:].astype(np.int32) 310 | c2 = corners[-self.chessboard_size[0],:,:].astype(np.int32) 311 | c3 = corners[-1,:,:].astype(np.int32) 312 | c_ = np.array([c0,c1,c3,c2]) 313 | 314 | tmp = img.copy() 315 | cv2.fillPoly(tmp, [c_], color=(255,255,255)) # Mask out the corners with white polygon 316 | 317 | return tmp 318 | 319 | 320 | def flip_corners(self, corners): 321 | col, row = self.chessboard_size[0], self.chessboard_size[1] 322 | temp = corners.copy() 323 | 324 | # Note: For (col=odd,row=even) checkerboard 325 | if row%2==0: 326 | for r in range(row): 327 | temp[r*col:r*col+col, :, :] = corners[(row-r-1)*col:(row-r-1)*col+col:, :, :] 328 | 329 | # Note: For (col=even,row=odd) checkerboard 330 | elif col%2==0: 331 | for r in range(row): 332 | for c in range(col): 333 | temp[r*col+c, :, :] = corners[r*col+(col-c-1), :, :] 334 | 335 | return temp 336 | 337 | 338 | def visualize_cam_pose(self, folder): 339 | file = glob.glob(folder+'*.yaml') 340 | file.sort() 341 | color = [[1,0,0],[0,1,0],[0,0,1]] # Note: maximum 3 cameras 342 | cam_frame = [] 343 | for i, f in enumerate(file): 344 | # Read in camera extrin 345 | param = yaml.load(open(f), Loader=yaml.FullLoader) 346 | extrin = np.asarray(param['extrin_mat']) 347 | # Create camera frame 348 | frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.1) 349 | frame.transform(np.linalg.inv(extrin)) 350 | frame.paint_uniform_color(color[i]) 351 | cam_frame.append(frame) 352 | 353 | # Create world reference frame 354 | ref_frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=self.chessboard_sq_size*4) 355 | 356 | # Create chessboard pattern 357 | mesh = self.create_chessboard_pattern_open3d() 358 | 359 | o3d.visualization.draw_geometries([ref_frame, mesh] + cam_frame) 360 | 361 | 362 | def create_chessboard_pattern_open3d(self): 363 | # Create chessboard pattern for Open3D visualization 364 | 365 | # Origin at top left of the checkerboard 366 | # Axis is +ve X points to right, +ve Y points downwards 367 | # o-->X 368 | # | 369 | # Y 370 | # Vertex order is anti clockwise 371 | # 0 ---- 3 372 | # | | 373 | # 1 ---- 2 374 | vertices = [] 375 | triangles = [] # Must be anti clockwise order when view from outside of the mesh 376 | black = True # Use to alternate between black and white square when loop across row and col 377 | index = 0 # To keep track of the number of vertices 378 | size = self.chessboard_sq_size 379 | for i in range(self.chessboard_size[1]+1): # In +ve Y axis direction 380 | for j in range(self.chessboard_size[0]+1): # In +ve X axis direction 381 | if black: 382 | x0, y0 = j*size, i*size # In anti clockwise order from top left 383 | x1, y1 = j*size, i*size+size # bottom left 384 | x2, y2 = j*size+size, i*size+size # bottom right 385 | x3, y3 = j*size+size, i*size # top right 386 | vertices.append([x0, y0, 0]) 387 | vertices.append([x1, y1, 0]) 388 | vertices.append([x2, y2, 0]) 389 | vertices.append([x3, y3, 0]) 390 | triangles.append([index, index+1, index+2]) 391 | triangles.append([index, index+2, index+3]) 392 | index += 4 393 | 394 | black = not black # Toggle the flag for next square 395 | 396 | if (self.chessboard_size[0]+1)%2 == 0: # Important: Need to check if col is even else will get parallel black strips as for even col the sq in the next row follw the same color 397 | black = not black 398 | 399 | # To shift the origin to the bottom right of first top left black square 400 | vertices = np.asarray(vertices) - np.array([size, size, 0]) 401 | 402 | mesh = o3d.geometry.TriangleMesh() 403 | mesh.vertices = o3d.utility.Vector3dVector(vertices) 404 | mesh.triangles = o3d.utility.Vector3iVector(triangles) 405 | mesh.paint_uniform_color([0,0,0]) # Black color 406 | 407 | return mesh 408 | 409 | 410 | class Triangulation: 411 | def __init__(self, cam_idx, vis=None, use_panoptic_dataset=False): 412 | super(Triangulation, self).__init__() 413 | 414 | ############################# 415 | ### Load camera parameter ### 416 | ############################# 417 | if use_panoptic_dataset: 418 | data_path = '../data/' 419 | seq_name = '171204_pose1_sample' 420 | # Load camera calibration param 421 | with open(data_path+seq_name+'/calibration_{0}.json'.format(seq_name)) as f: 422 | calib = json.load(f) 423 | 424 | # Cameras are identified by a tuple of (panel#,node#) 425 | # Note: 31 HD cameras (0,0) - (0,30), where the zero in the first index means HD camera 426 | cameras = {(cam['panel'],cam['node']):cam for cam in calib['cameras']} 427 | 428 | # Convert data into numpy arrays for convenience 429 | for k, cam in cameras.items(): 430 | cam['K'] = np.matrix(cam['K']) 431 | cam['distCoef'] = np.array(cam['distCoef']) 432 | cam['R'] = np.matrix(cam['R']) 433 | cam['t'] = np.array(cam['t'])*0.01 # Convert cm to m 434 | 435 | # Extract camera index integer from video file name 436 | cam_idx_ = [] 437 | for c in cam_idx: 438 | # Example of file name 439 | # ../data/171204_pose1_sample/hdVideos/hd_00_00.mp4 440 | value = c.split('_')[-1] # Select the last split (XX.mp4) 441 | value = value.split('.')[0] # Select the first split (XX) 442 | cam_idx_.append(int(value)) 443 | 444 | # Compute projection matrix 445 | self.pmat = [] 446 | for i in range(len(cam_idx_)): 447 | cam = cameras[(0,cam_idx_[i])] 448 | extrin_mat = np.zeros((3,4)) 449 | extrin_mat[:3,:3] = cam['R'] 450 | extrin_mat[:3,3:] = cam['t'] 451 | self.pmat.append(cam['K'] @ extrin_mat) 452 | 453 | ############################# 454 | ### Visualize camera pose ### 455 | ############################# 456 | # Draw world frame 457 | frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5) 458 | vis.add_geometry(frame) 459 | 460 | # Draw camera axis 461 | hd_cam_idx = zip([0] * 30,range(0,30)) # Choose only HD cameras 462 | hd_cameras = [cameras[cam].copy() for cam in hd_cam_idx] 463 | for i, cam in enumerate(hd_cameras): 464 | if i in cam_idx_: # Show only those selected camera 465 | extrin_mat = np.eye(4) 466 | extrin_mat[:3,:3] = cam['R'] 467 | extrin_mat[:3,3:] = cam['t'] 468 | axis = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.2) 469 | axis.transform(np.linalg.inv(extrin_mat)) 470 | vis.add_geometry(axis) 471 | 472 | 473 | def triangulate_2views(self, param, mode): 474 | 475 | if mode=='body': 476 | p0 = param[0]['keypt'] # [nPt,2] 477 | p1 = param[1]['keypt'] # [nPt,2] 478 | 479 | elif mode=='holistic': 480 | _, param_lh, param_rh, param_bd = param[0] 481 | p0 = np.vstack((param_lh['keypt'], 482 | param_rh['keypt'], 483 | param_bd['keypt'])) # [21+21+33/25,2] 484 | 485 | _, param_lh, param_rh, param_bd = param[1] 486 | p1 = np.vstack((param_lh['keypt'], 487 | param_rh['keypt'], 488 | param_bd['keypt'])) # [21+21+33/25,2] 489 | 490 | 491 | # Note: OpenCV can only triangulate from 2 views 492 | p3d = cv2.triangulatePoints( 493 | self.pmat[0], self.pmat[1], 494 | p0.T, p1.T) # Note: triangulatePoints requires 2xN arrays, so transpose 495 | 496 | # However, homgeneous point is returned so need to divide by last term 497 | p3d /= p3d[3] # [4,nPt] 498 | p3d = p3d[:3,:].T # [nPt,3] 499 | 500 | # Update param 3D joint 501 | if mode=='body': 502 | param[0]['joint'] = p3d # [nPt,3] 503 | param[1]['joint'] = p3d # [nPt,3] 504 | 505 | elif mode=='holistic': 506 | _, param_lh, param_rh, param_bd = param[0] 507 | param_lh['joint'] = p3d[ :21] 508 | param_rh['joint'] = p3d[21:42] 509 | param_bd['joint'] = p3d[42:] 510 | 511 | _, param_lh, param_rh, param_bd = param[1] 512 | param_lh['joint'] = p3d[ :21] 513 | param_rh['joint'] = p3d[21:42] 514 | param_bd['joint'] = p3d[42:] 515 | 516 | return param 517 | 518 | 519 | def triangulate_nviews(self, param, mode): 520 | 521 | p2d = [] # List of len nCam to store [nPt,2] for each view 522 | if mode=='body': 523 | for p in param: 524 | p2d.append(p['keypt']) # [nPt,2] 525 | 526 | elif mode=='holistic': 527 | for p in param: 528 | _, param_lh, param_rh, param_bd = p 529 | p2d.append(np.vstack(( 530 | param_lh['keypt'], 531 | param_rh['keypt'], 532 | param_bd['keypt'])) # [21+21+33/25,2] 533 | ) 534 | 535 | # Convert list into a single array 536 | p2d = np.concatenate(p2d, axis=1) # [nPt,2*nCam] 537 | nPt = p2d.shape[0] 538 | 539 | p3d = np.zeros((nPt,3)) 540 | for i in range(nPt): 541 | p3d[i,:] = self.triangulate_point(p2d[i,:].reshape(-1,2)) 542 | 543 | # Update param 3D joint 544 | if mode=='body': 545 | for p in param: 546 | p['joint'] = p3d # [nPt,3] 547 | 548 | elif mode=='holistic': 549 | for p in param: 550 | _, param_lh, param_rh, param_bd = p 551 | param_lh['joint'] = p3d[ :21] 552 | param_rh['joint'] = p3d[21:42] 553 | param_bd['joint'] = p3d[42:] 554 | 555 | return param 556 | 557 | 558 | def triangulate_point(self, point): 559 | # Modified from 560 | # https://gist.github.com/davegreenwood/e1d2227d08e24cc4e353d95d0c18c914 561 | 562 | # Other possible python implementation 563 | # https://www.mail-archive.com/floatcanvas@mithis.com/msg00513.html 564 | 565 | # Also its worthwhile to read through the below link 566 | # http://kwon3d.com/theory/dlt/dlt.html 567 | # For indepth explanation on DLT and many other useful theories 568 | # required by multicam mocap by Prof Young-Hoo Kwon 569 | 570 | # Use DLT to triangulate a 3D point from N image points in N camera views 571 | N = len(self.pmat) # Number of camera views 572 | M = np.zeros((3*N, 4+N)) 573 | for i in range(N): 574 | M[3*i:3*i+3, :4] = self.pmat[i] # [3,4] 575 | M[3*i:3*i+2,4+i] = -point[i] # [2,1] 576 | M[3*i+2 ,4+i] = -1 # Homogeneous coordinate 577 | V = np.linalg.svd(M)[-1] # [4+N,4+N] 578 | X = V[-1,:4] # [4] 579 | X = X / X[3] # [4] 580 | 581 | return X[:3] # [3] 582 | 583 | 584 | class PanopticDataset: 585 | def __init__(self, data_path='../data/', seq_name='171204_pose1_sample'): 586 | super(PanopticDataset, self).__init__() 587 | 588 | # Load camera calibration parameters 589 | with open(data_path+seq_name+'/calibration_{0}.json'.format(seq_name)) as f: 590 | calib = json.load(f) 591 | 592 | # Cameras are identified by a tuple of (panel#,node#) 593 | cameras = {(cam['panel'],cam['node']):cam for cam in calib['cameras']} 594 | 595 | # Convert data into numpy arrays for convenience 596 | for k, cam in cameras.items(): 597 | cam['K'] = np.matrix(cam['K']) 598 | cam['distCoef'] = np.array(cam['distCoef']) 599 | cam['R'] = np.matrix(cam['R']) 600 | cam['t'] = np.array(cam['t']).reshape((3,1)) * 0.01 # Convert cm to m 601 | 602 | # Choose only HD cameras for visualization 603 | hd_cam_idx = zip([0] * 30,range(0,30)) 604 | hd_cameras = [cameras[cam].copy() for cam in hd_cam_idx] 605 | 606 | # Select an HD camera (0,0) - (0,30), where the zero in the first index means HD camera 607 | # cam = cameras[(0,5)] 608 | 609 | # Visualize 3D camera pose 610 | self.vis = o3d.visualization.Visualizer() 611 | self.vis.create_window(width=640, height=480) 612 | 613 | # Draw world frame 614 | frame = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.5) 615 | 616 | # Draw camera axis 617 | axes = [] 618 | for cam in hd_cameras: 619 | axis = o3d.geometry.TriangleMesh.create_coordinate_frame(size=0.1) 620 | 621 | tmat = np.eye(4) 622 | tmat[:3,:3] = cam['R'] 623 | tmat[:3,3:] = cam['t'] 624 | axis.transform(np.linalg.inv(tmat)) 625 | axes.append(axis) 626 | 627 | # Draw body pose 628 | hd_idx = 0 629 | hd_skel_json_path = data_path+seq_name+'/hdPose3d_stage1_coco19/' 630 | # Edges between joints in the body skeleton 631 | body_edges = np.array([[1,2],[1,4],[4,5],[5,6],[1,3],[3,7],[7,8],[8,9],[3,13],[13,14],[14,15],[1,10],[10,11],[11,12]])-1 632 | 633 | # Load the json file with this frame's skeletons 634 | skel_json_fname = hd_skel_json_path+'body3DScene_{0:08d}.json'.format(hd_idx) 635 | with open(skel_json_fname) as f: 636 | bframe = json.load(f) 637 | 638 | # Bodies 639 | for ids in range(len(bframe['bodies'])): 640 | body = bframe['bodies'][ids] 641 | skel = np.array(body['joints19']).reshape((-1,4))[:,:3] * 0.01 642 | pcd = o3d.geometry.PointCloud() 643 | pcd.points = o3d.utility.Vector3dVector(skel) 644 | 645 | bone = o3d.geometry.LineSet() 646 | bone.points = o3d.utility.Vector3dVector(skel) 647 | bone.lines = o3d.utility.Vector2iVector(body_edges) 648 | 649 | self.vis.add_geometry(pcd) 650 | self.vis.add_geometry(bone) 651 | 652 | self.vis.add_geometry(frame) 653 | for a in axes: 654 | self.vis.add_geometry(a) 655 | 656 | self.vis.run() 657 | 658 | 659 | def projectPoints(X, K, R, t, Kd): 660 | """ Projects points X (3xN) using camera intrinsics K (3x3), 661 | extrinsics (R,t) and distortion parameters Kd=[k1,k2,p1,p2,k3]. 662 | 663 | Roughly, x = K*(R*X + t) + distortion 664 | 665 | See http://docs.opencv.org/2.4/doc/tutorials/calib3d/camera_calibration/camera_calibration.html 666 | or cv2.projectPoints 667 | """ 668 | 669 | x = np.asarray(R*X + t) 670 | 671 | x[0:2,:] = x[0:2,:]/x[2,:] 672 | 673 | r = x[0,:]*x[0,:] + x[1,:]*x[1,:] 674 | 675 | x[0,:] = x[0,:]*(1 + Kd[0]*r + Kd[1]*r*r + Kd[4]*r*r*r) + 2*Kd[2]*x[0,:]*x[1,:] + Kd[3]*(r + 2*x[0,:]*x[0,:]) 676 | x[1,:] = x[1,:]*(1 + Kd[0]*r + Kd[1]*r*r + Kd[4]*r*r*r) + 2*Kd[3]*x[0,:]*x[1,:] + Kd[2]*(r + 2*x[1,:]*x[1,:]) 677 | 678 | x[0,:] = K[0,0]*x[0,:] + K[0,1]*x[1,:] + K[0,2] 679 | x[1,:] = K[1,0]*x[0,:] + K[1,1]*x[1,:] + K[1,2] 680 | 681 | return x 682 | 683 | 684 | ############################################################################### 685 | ### Simple example to test program ### 686 | ############################################################################### 687 | if __name__ == '__main__': 688 | 689 | pano = PanopticDataset() 690 | -------------------------------------------------------------------------------- /code/utils_joint_angle.py: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | ### Useful function for converting 3D joint to joint angles 3 | ### Joint angle is then used for: 4 | ### 1) Gesture recognition 5 | ### 2) Hand ROM recognition 6 | ############################################################################### 7 | 8 | import cv2 9 | import numpy as np 10 | 11 | 12 | def convert_relative_to_actual_3d_joint_(param, intrin): 13 | # Adapted from Iqbal et al. 14 | # (ECCV 2018) Hand Pose Estimation via Latent 2.5D Heatmap Regression 15 | # But there are some errors in equation: 16 | # (4) missing camera intrinsic 17 | # (6) missing a constant multiplication of 2 18 | 19 | # Select wrist joint (n) and index finger MCP (m) 20 | xn, yn = param['keypt'][0] # Wrist 21 | xm, ym = param['keypt'][9] # Index finger MCP 22 | xn = (xn-intrin['cx'])/intrin['fx'] 23 | xm = (xm-intrin['cx'])/intrin['fx'] 24 | yn = (yn-intrin['cy'])/intrin['fy'] 25 | ym = (ym-intrin['cy'])/intrin['fy'] 26 | 27 | Zn = param['joint'][0,2] # Relative Z coor of wrist 28 | Zm = param['joint'][9,2] # Relative Z coor of index finger MCP 29 | 30 | # Precalculate value for computing Zroot 31 | xx = xn-xm 32 | yy = yn-ym 33 | xZ = xn*Zn - xm*Zm 34 | yZ = yn*Zn - ym*Zm 35 | ZZ = Zn-Zm 36 | 37 | # Compute Zroot relative 38 | C = 1 39 | a = xx*xx + yy*yy 40 | b = 2*(xx*xZ + yy*yZ) 41 | c = xZ*xZ + yZ*yZ + ZZ*ZZ - C*C 42 | Zroot = (-b + np.sqrt(b*b - 4*a*c))/(2*a) 43 | 44 | # Convert to actual scale 45 | s = 0.08 # Note: Hardcode distance from wrist to index finger MCP as 8cm 46 | Zroot *= s / C 47 | 48 | # Compute actual depth 49 | param['joint_3d'][:,2] = param['joint'][:,2] + Zroot 50 | 51 | # Compute X and Y 52 | param['joint_3d'][:,0] = (param['keypt'][:,0]-intrin['cx'])/intrin['fx'] 53 | param['joint_3d'][:,1] = (param['keypt'][:,1]-intrin['cy'])/intrin['fy'] 54 | param['joint_3d'][:,:2] *= param['joint_3d'][:,2:3] 55 | 56 | return param['joint_3d'] 57 | 58 | 59 | ############################################################# 60 | ### Simple gesture recognition from joint angle using KNN ### 61 | ############################################################# 62 | class GestureRecognition: 63 | def __init__(self, mode='train'): 64 | super(GestureRecognition, self).__init__() 65 | 66 | # 11 types of gesture 'name':class label 67 | self.gesture = { 68 | 'fist':0,'one':1,'two':2,'three':3,'four':4,'five':5,'six':6, 69 | 'rock':7,'spiderman':8,'yeah':9,'ok':10, 70 | } 71 | 72 | if mode=='train': 73 | # Create .csv file to log training data 74 | self.file = open('../data/gesture_train.csv', 'a+') 75 | elif mode=='eval': 76 | # Load training data 77 | file = np.genfromtxt('../data/gesture_train.csv', delimiter=',') 78 | # Extract input joint angles 79 | angle = file[:,:-1].astype(np.float32) 80 | # Extract output class label 81 | label = file[:, -1].astype(np.float32) 82 | # Use OpenCV KNN 83 | self.knn = cv2.ml.KNearest_create() 84 | self.knn.train(angle, cv2.ml.ROW_SAMPLE, label) 85 | 86 | 87 | def train(self, angle, label): 88 | # Log training data 89 | data = np.append(angle, label) # Combine into one array 90 | np.savetxt(self.file, [data], delimiter=',', fmt='%f') 91 | 92 | 93 | def eval(self, angle): 94 | # Use KNN for gesture recognition 95 | data = np.asarray([angle], dtype=np.float32) 96 | ret, results, neighbours ,dist = self.knn.findNearest(data, 3) 97 | idx = int(results[0][0]) # Index of class label 98 | 99 | return list(self.gesture)[idx] # Return name of class label 100 | 101 | 102 | ############################################################## 103 | ### Simple hand ROM recognition from joint angle using KNN ### 104 | ############################################################## 105 | class HandRomRecognition: 106 | def __init__(self, mode='train'): 107 | super(HandRomRecognition, self).__init__() 108 | 109 | # 13 types of hand ROM 'name':class label 110 | self.gesture = { 111 | 'Finger MCP Flexion' :0, 112 | 'Finger PIP DIP Flexion':1, 113 | 'Thumb MCP Flexion' :2, 114 | 'Thumb IP Flexion' :3, 115 | 'Thumb Radial Abduction':4, 116 | 'Thumb Palmar Abduction':5, # From this class onwards hard to differentiate 117 | 'Thumb Opposition' :6, 118 | 'Wrist Flexion' :7, # Refer to WristArmRom 119 | 'Wrist Extension' :8, # Refer to WristArmRom 120 | 'Wrist Radial Deviation':9, # Refer to WristArmRom 121 | 'Wrist Ulnar Deviation' :10,# Refer to WristArmRom 122 | 'Forearm Neutral' :11,# Refer to WristArmRom 123 | 'Forearm Pronation' :12,# Refer to WristArmRom 124 | 'Forearm Supination' :13,# Refer to WristArmRom 125 | } 126 | 127 | if mode=='train': 128 | # Create .csv file to log training data 129 | self.file = open('../data/handrom_train.csv', 'a+') 130 | elif mode=='eval': 131 | # Load training data 132 | file = np.genfromtxt('../data/handrom_train.csv', delimiter=',') 133 | # Extract input joint angles 134 | angle = file[:,:-1].astype(np.float32) 135 | # Extract output class label 136 | label = file[:, -1].astype(np.float32) 137 | # Use OpenCV KNN 138 | self.knn = cv2.ml.KNearest_create() 139 | self.knn.train(angle, cv2.ml.ROW_SAMPLE, label) 140 | 141 | 142 | def train(self, angle, label): 143 | # Log training data 144 | data = np.append(angle, label) # Combine into one array 145 | np.savetxt(self.file, [data], delimiter=',', fmt='%f') 146 | 147 | 148 | def eval(self, angle): 149 | # Use KNN for gesture recognition 150 | data = np.asarray([angle], dtype=np.float32) 151 | ret, results, neighbours ,dist = self.knn.findNearest(data, 3) 152 | idx = int(results[0][0]) # Index of class label 153 | 154 | return list(self.gesture)[idx] # Return name of class label 155 | 156 | 157 | ########################################################### 158 | ### Simple measurement of wrist and forearm joint angle ### 159 | ########################################################### 160 | class WristArmRom: 161 | def __init__(self, mode=2, side='right'): 162 | super(WristArmRom, self).__init__() 163 | 164 | # 3 modes of measurement 165 | # 0: Wrist flexion/extension 166 | # 1: Wrist radial/ulnar deviation 167 | # 2: Forearm pronation/supination 168 | self.mode = mode 169 | self.side = side # left / right 170 | 171 | # 13 types of hand ROM 'name':class label 172 | self.gesture = { 173 | 'Finger MCP Flexion' :0, # Refer to HandRomRecognition 174 | 'Finger PIP DIP Flexion':1, # Refer to HandRomRecognition 175 | 'Thumb MCP Flexion' :2, # Refer to HandRomRecognition 176 | 'Thumb IP Flexion' :3, # Refer to HandRomRecognition 177 | 'Thumb Radial Abduction':4, # Refer to HandRomRecognition 178 | 'Thumb Palmar Abduction':5, # Refer to HandRomRecognition 179 | 'Thumb Opposition' :6, # Refer to HandRomRecognition 180 | 'Wrist Flex/Extension' :7, # Use mode 0 # Note: hard to differentiate 181 | # 'Wrist Extension' :8, # Note: hard to differentiate 182 | 'Wrist Radial/Ulnar Dev':9, # Use mode 1 # Note: hard to differentiate 183 | # 'Wrist Ulnar Deviation' :10, # Note: hard to differentiate 184 | 'Forearm Neutral' :11,# Use mode 2 185 | 'Forearm Pronation' :12,# Use mode 2 186 | 'Forearm Supination' :13,# Use mode 2 187 | } 188 | 189 | 190 | def eval(self, param): 191 | # Wrist flexion/extension, wrist radial/ulnar deviation 192 | if self.mode==0 or self.mode==1: 193 | # Assume camera is placed such that upper body of the subject 194 | # is visible especially the elbow joints 195 | 196 | # Wrist joint angle can be simply calculated from angle between 197 | # Vector 1: joining body elbow joint[13/14] to wrist[15/16] 198 | # Vector 2: joining hand wrist[0] to middle finger MCP joint[9] 199 | _, param_lh, param_rh, param_bd = param 200 | 201 | if self.side=='left': 202 | v1 = param_bd['joint'][15]- param_bd['joint'][13] 203 | v2 = param_lh['joint'][9] - param_lh['joint'][0] 204 | elif self.side=='right': 205 | v1 = param_bd['joint'][16]- param_bd['joint'][14] 206 | v2 = param_rh['joint'][9] - param_rh['joint'][0] 207 | 208 | # Normalize vector 209 | v1 = v1/(np.linalg.norm(v1)+1e-6) 210 | v2 = v2/(np.linalg.norm(v2)+1e-6) 211 | # Get angle using arcos of dot product between the two vectors 212 | angle = np.arccos(np.dot(v1, v2)) 213 | angle = np.degrees(angle) # Convert radian to degree 214 | 215 | if self.side=='left': 216 | param_lh['angle'][0] = angle # Note: Temporary make use hand joint angles 217 | if self.mode==0 and angle>10: 218 | param_lh['gesture'] = 'Wrist Flex/Extension' 219 | elif self.mode==1 and angle>10: 220 | param_lh['gesture'] = 'Wrist Radial/Ulnar Dev' 221 | elif self.side=='right': 222 | param_rh['angle'][0] = angle # Note: Temporary make use hand joint angles 223 | if self.mode==0 and angle>10: 224 | param_rh['gesture'] = 'Wrist Flex/Extension' 225 | elif self.mode==1 and angle>10: 226 | param_rh['gesture'] = 'Wrist Radial/Ulnar Dev' 227 | 228 | 229 | # Forearm pronation/supination 230 | elif self.mode==2: 231 | # Assume camera is placed at the same level as the hand 232 | # such that palmar side of the hand is directly facing camera 233 | # Forearm pronation/supination can be simply calculated from palm direction normal 234 | 235 | # Get normal of plane joining wrist[0], index finger MCP[5] and little finger MCP[17] 236 | # Note: Palm normal points from palmar side to dorsal side 237 | v1 = param[0]['joint'][5] - param[0]['joint'][0] # Vector pointing from wrist to index finger MCP 238 | v2 = param[0]['joint'][17]- param[0]['joint'][0] # Vector pointing from wrist to little finger MCP 239 | n = np.cross(v1,v2) # Palm normal vector 240 | # Normalize normal vector 241 | n = n/(np.linalg.norm(n)+1e-6) 242 | 243 | # When forearm is in neutral position, palm is directly facing camera 244 | # Palm normal vector shld be parallel to camera z-axis 245 | # Get angle using arcos of dot product between palm normal and z-axis 246 | angle = np.arccos(np.dot(n,np.array([0,0,1]))) 247 | angle = np.degrees(angle) # Convert radian to degree 248 | # Get direction by checking on y component of normal vector 249 | direc = n[1] 250 | 251 | if self.side=='left': 252 | # Reverse angle and direction for left hand 253 | angle = 180-angle 254 | direc = -direc 255 | 256 | if angle<30: 257 | param[0]['gesture'] = 'Forearm Neutral' 258 | elif angle>30 and direc<0: 259 | param[0]['gesture'] = 'Forearm Pronation' 260 | elif angle>30 and direc>0: 261 | param[0]['gesture'] = 'Forearm Supination' 262 | 263 | param[0]['angle'][0] = angle # Note: Temporary make use hand joint angles 264 | 265 | return param 266 | -------------------------------------------------------------------------------- /data/gesture_train.csv: -------------------------------------------------------------------------------- 1 | 32.647995,27.334458,18.777239,32.558145,149.876268,29.578624,37.800344,133.458050,45.085468,32.342115,142.727088,37.366495,26.056262,139.415512,40.519754,0.000000 2 | 45.166793,26.117019,18.645196,31.567098,138.965945,20.022370,33.176819,132.806630,21.381263,28.201580,138.854737,14.808465,26.329570,128.394781,23.400235,0.000000 3 | 20.826588,35.926397,39.943409,20.257126,88.651968,29.781060,17.812546,102.683664,16.751548,11.247996,103.264677,18.000063,15.584646,82.515342,23.045704,0.000000 4 | 31.327256,24.482902,27.073693,22.894414,64.537885,47.775269,16.256466,89.803293,40.146627,9.443338,91.327471,35.448822,14.446064,71.702363,35.047523,0.000000 5 | 28.164625,31.028619,23.568194,21.468457,141.750809,21.386416,29.323112,129.772949,38.852031,24.264641,138.309162,33.321451,23.231644,129.452243,30.819853,0.000000 6 | 39.493933,30.825017,37.798715,48.789407,101.569152,29.543310,62.408973,102.175635,65.566150,59.301602,112.161872,55.427656,60.565796,102.860590,49.489659,0.000000 7 | 46.636410,27.871526,25.808261,22.547515,69.819060,42.556670,16.971363,85.544017,36.496952,11.288500,75.542530,39.251253,16.539423,48.637377,29.019384,0.000000 8 | 45.042568,19.961871,11.893033,13.603903,32.278803,27.388893,7.747610,17.797417,5.794515,2.940872,16.492116,1.481793,9.127502,8.934992,6.483260,0.000000 9 | 47.299112,32.659637,25.455925,33.674011,52.766652,43.792789,29.175726,97.063906,24.384931,27.473441,104.583461,24.572468,32.064351,94.460182,22.230902,0.000000 10 | 46.483112,30.871050,36.277547,40.493482,96.787611,39.591184,38.159227,110.260392,30.181509,30.411843,119.519070,30.006762,26.418003,122.606342,33.310417,0.000000 11 | 27.792747,27.346865,18.358880,13.459867,12.675573,14.827358,21.851454,94.371765,35.972556,19.283318,104.185629,31.780360,26.073177,81.116557,33.292727,1.000000 12 | 35.320273,26.381613,20.191984,14.668020,7.185043,4.248832,25.126893,121.084548,26.620646,24.078772,117.775156,30.698631,30.541078,89.432634,37.903796,1.000000 13 | 26.048195,19.402587,26.436642,6.365824,9.542588,3.241575,28.086456,113.535227,26.468682,20.288474,116.216192,24.670197,26.922405,95.790580,20.527646,1.000000 14 | 27.221395,15.414500,25.939750,6.623381,7.608257,2.696879,28.052599,108.763997,27.351137,21.551970,108.780979,26.114410,25.929242,91.324413,21.103923,1.000000 15 | 32.014172,10.991831,28.415461,5.728636,6.481493,0.889479,23.010316,90.874638,34.321755,12.829574,92.857042,35.219209,11.395936,68.408045,41.629204,1.000000 16 | 35.391177,18.592058,33.007410,4.140826,6.493375,1.912261,25.824411,100.550619,29.616766,19.956852,99.451776,31.702370,24.273458,82.648994,25.282382,1.000000 17 | 33.667543,32.918064,19.468711,7.488233,2.724066,7.948471,28.506697,49.497494,13.477383,55.206680,70.580958,23.938099,54.585635,72.790658,27.310205,1.000000 18 | 34.297057,31.368970,12.617624,12.278204,1.644287,8.980053,30.469056,26.478147,7.106024,52.598374,70.892878,25.786772,42.428934,65.808129,34.571704,1.000000 19 | 37.283942,26.503899,38.901662,12.471588,1.771188,4.433059,38.654754,109.571409,23.595078,35.738030,105.880785,31.167126,25.614191,107.024708,29.663542,1.000000 20 | 37.309748,20.163841,29.692652,7.735964,3.484735,2.936218,33.402019,102.752624,25.379006,27.139984,100.369156,26.013738,25.387794,95.077939,19.624609,1.000000 21 | 27.064267,4.116995,11.001622,16.486113,10.552898,11.181537,14.660537,7.230117,6.805212,5.992467,1.279764,12.949280,2.583446,5.751485,12.190489,2.000000 22 | 31.191134,4.290191,11.017077,14.137684,3.406932,3.979965,14.172432,62.540096,5.982379,4.602033,33.828657,3.623022,7.488203,11.515055,14.876723,2.000000 23 | 32.675400,6.051246,9.601219,16.191259,6.049887,3.021566,23.574714,34.476614,52.095947,20.944558,28.680245,10.530155,21.732707,9.850545,3.993437,2.000000 24 | 36.983640,10.596499,14.782542,8.568285,5.152676,2.903961,20.725178,85.560392,20.407994,15.760408,73.182706,27.857098,16.365531,47.081374,18.958339,2.000000 25 | 28.553979,4.386473,14.044008,14.800291,8.822664,4.434532,12.653980,85.107713,5.685620,4.419841,57.803916,8.104906,9.925122,21.634206,28.194048,2.000000 26 | 43.905587,4.436173,6.406616,7.984581,3.269949,4.945221,21.120639,23.389945,11.310605,16.343349,24.295424,15.957736,9.628379,15.356038,23.239797,2.000000 27 | 47.227182,2.093504,13.185004,10.198872,9.016928,4.762649,30.488841,87.460182,19.246491,31.417656,83.437891,35.377278,26.301992,71.631192,41.428162,2.000000 28 | 38.963707,3.312926,14.764106,13.346643,2.701027,3.122325,15.783439,70.890089,24.271086,16.162595,55.105191,27.125569,14.065205,31.765137,30.163877,2.000000 29 | 38.235214,9.765067,12.204059,13.054985,3.186906,6.008764,30.496132,105.006685,19.614097,37.621662,101.843967,33.236817,36.027421,96.934316,33.416261,2.000000 30 | 44.894136,5.703140,18.672275,16.862815,7.250208,1.171276,27.349864,91.238617,15.085455,31.288754,83.884860,30.505573,32.303599,72.093530,33.306199,2.000000 31 | 33.233165,5.251370,12.361472,7.406677,2.403716,1.809087,14.060716,1.544563,4.514234,31.659243,100.935772,25.510835,35.251030,89.933031,19.148129,3.000000 32 | 52.796238,11.266403,14.081873,4.255632,1.900488,2.120873,8.555830,4.423145,2.657591,32.922917,102.226827,22.889829,35.501908,85.247861,24.733276,3.000000 33 | 45.530282,6.599183,2.705272,11.851240,11.457306,8.534295,10.935138,3.143677,7.689091,40.015122,108.650040,20.131360,44.474448,95.466861,23.438341,3.000000 34 | 40.178526,5.883763,11.293230,9.412355,5.010292,3.067079,17.325150,5.933416,5.999398,39.241775,112.753544,23.859172,37.437907,104.919452,24.762976,3.000000 35 | 41.220657,6.328683,11.057584,8.445178,5.794255,1.084988,11.758416,8.098485,5.679773,36.361657,116.513918,16.134616,37.406455,101.434675,18.718483,3.000000 36 | 58.029470,7.936083,18.771053,7.850386,3.486047,9.073247,6.786294,4.949442,3.565229,34.004830,84.326025,32.755850,41.777266,74.807435,32.737475,3.000000 37 | 47.655365,6.263117,10.347858,4.807681,4.465829,8.093180,5.107651,1.721795,4.607089,33.309515,102.655187,23.337544,38.665966,107.032195,18.660136,3.000000 38 | 37.110564,9.127239,10.668107,9.388768,7.970887,9.143346,12.781739,13.336964,6.699117,2.049043,44.085116,10.394698,5.635468,36.843559,23.718730,3.000000 39 | 38.684006,11.977840,10.109062,4.396552,1.626858,0.703276,12.755652,4.171569,4.843316,33.498154,83.193397,31.964912,32.269514,74.880485,33.148861,3.000000 40 | 37.850077,11.219031,11.298609,4.051946,5.171304,1.238764,9.662268,7.351379,3.868443,29.601931,85.170091,34.670269,24.091245,67.618521,43.369632,3.000000 41 | 33.375924,45.897643,44.959032,4.206236,2.510320,3.222605,6.785700,5.948434,5.084050,4.488179,10.262063,2.284405,10.728425,9.843519,2.926731,4.000000 42 | 42.745151,37.643024,50.160862,5.435130,4.830251,6.826986,5.556171,3.002356,0.205582,4.943730,4.228434,1.315191,14.777243,1.532173,2.396397,4.000000 43 | 36.649501,39.702291,44.822909,3.642930,8.181224,8.751886,3.456824,9.219551,4.474447,5.561067,16.236727,1.138055,18.158290,4.719707,7.470625,4.000000 44 | 33.732343,45.422586,55.822462,7.638146,2.152629,4.033472,1.464054,4.268573,1.663308,2.984584,7.626433,3.389159,6.941995,4.292772,5.763352,4.000000 45 | 38.116837,42.441980,54.622501,5.664158,2.892497,5.785950,4.736157,4.173156,2.245456,4.465982,7.503835,2.509713,12.375703,4.960036,2.874021,4.000000 46 | 24.033897,40.088233,38.787525,9.386012,9.464247,10.413310,5.538884,8.901695,6.428157,5.607933,8.132257,8.182374,12.767122,4.234681,9.934021,4.000000 47 | 22.047348,48.635157,48.085973,4.204146,8.782940,8.546689,8.453367,8.064216,4.190607,8.495972,9.646204,6.860789,15.861648,9.336959,7.072326,4.000000 48 | 33.365355,38.958717,46.813890,9.899286,7.299461,2.715544,9.758196,11.715413,5.594981,17.783596,9.339260,5.354658,18.192992,8.105073,12.362090,4.000000 49 | 38.594702,38.014552,39.186073,10.419738,4.358816,1.884457,3.774121,6.272337,6.311814,4.514871,3.737852,3.404841,16.175068,7.367417,6.986737,4.000000 50 | 34.595418,45.275342,38.231041,16.250354,8.341614,3.207451,15.948974,17.222175,0.766148,21.815026,15.449771,4.494622,21.761233,9.026763,6.234337,4.000000 51 | 35.255824,12.681438,12.162714,1.949913,4.178936,2.694380,5.003759,5.714465,4.993314,2.866081,7.437629,5.021466,6.839853,3.950859,7.966758,5.000000 52 | 35.381951,6.525500,19.199896,3.555485,5.347654,2.759508,1.414179,4.648061,6.142657,1.858605,7.096097,7.531731,6.483093,6.014813,12.057738,5.000000 53 | 41.645858,15.220567,8.432321,3.891638,2.995404,3.117573,2.547617,4.607339,4.529059,4.610331,8.582590,3.758644,12.137491,3.805865,7.507453,5.000000 54 | 31.975783,6.425621,27.088570,1.623824,0.324600,3.288059,4.708754,6.496409,2.555214,3.167269,10.568400,3.977490,5.989966,12.268311,8.626645,5.000000 55 | 52.354737,15.120519,8.159867,12.150277,3.072984,2.300643,6.897060,7.296752,5.010795,13.108232,14.797280,6.259755,33.110258,40.388509,18.435610,5.000000 56 | 42.999924,9.240689,12.167061,6.017784,5.659045,3.776685,8.656432,8.790815,2.774698,9.226938,7.683244,2.427294,5.585492,5.957392,7.547471,5.000000 57 | 43.477665,14.105631,13.982119,10.789480,3.810610,5.423428,6.094256,6.136796,5.340599,8.386305,7.252753,1.856217,5.491802,10.913479,5.328268,5.000000 58 | 20.702271,10.081805,4.982003,23.877205,7.948928,1.551362,19.273723,11.337202,3.647189,20.742387,9.034537,1.285506,19.839281,4.202442,2.836296,5.000000 59 | 17.150237,8.940540,9.556369,14.012305,6.129931,4.739453,8.193098,13.911509,5.344650,6.554458,10.177747,5.336161,3.375271,4.698845,3.420790,5.000000 60 | 39.389713,1.117582,18.518180,2.084555,9.091669,5.572484,4.861925,9.557979,1.797316,8.672537,6.123965,4.182991,5.379867,3.624339,9.622453,5.000000 61 | 32.768108,9.503798,21.593121,70.120588,79.257446,19.285731,62.786166,89.545887,25.282514,35.479257,113.421803,22.441275,2.555735,2.533934,11.763745,6.000000 62 | 46.482827,7.663155,23.567131,63.902830,93.346809,22.482054,56.086617,100.846042,23.564547,33.851183,117.178513,25.286268,6.500882,2.959386,19.501039,6.000000 63 | 38.634087,11.621667,12.321092,65.498904,85.213781,26.907686,64.852347,83.909753,32.034636,62.020283,83.323703,34.953183,29.379008,6.594299,10.074840,6.000000 64 | 36.741209,5.558563,16.938405,62.598507,83.631724,28.465062,62.411796,78.189144,32.007691,54.511854,86.981567,39.002972,30.600811,3.231690,22.656990,6.000000 65 | 32.129682,4.522638,20.253860,56.993430,92.831512,27.698204,64.360650,95.358514,25.051703,46.830764,114.904887,27.940107,11.182135,6.640856,21.113219,6.000000 66 | 44.091949,7.621931,10.947011,55.501159,89.272723,24.395429,55.948755,82.842522,30.297209,38.470464,104.984128,27.955863,14.350666,8.625918,9.942533,6.000000 67 | 27.479936,4.097992,12.034439,54.818352,83.537471,23.730609,48.670413,87.563799,21.889500,49.545667,79.728856,24.308692,24.089367,8.214144,6.436579,6.000000 68 | 36.959546,8.284973,14.947872,68.824398,94.120399,21.986314,66.216127,86.226797,19.891681,68.870759,84.502687,33.513477,39.479300,6.941457,13.541512,6.000000 69 | 31.751109,4.022216,18.241775,50.880285,84.077726,26.394814,47.129386,86.697861,22.057838,45.542544,80.530217,21.078212,23.212825,9.845619,9.567523,6.000000 70 | 32.046934,2.906928,16.507149,65.617373,85.127539,28.785882,59.764329,87.419937,28.859106,55.605317,84.626147,30.414226,24.107689,8.410958,11.024586,6.000000 71 | 44.811824,2.247344,16.590034,3.907380,1.037199,5.722256,24.259901,79.244223,50.637637,27.387484,89.464871,38.621035,18.931667,12.572601,2.455508,7.000000 72 | 37.624495,9.248881,15.595617,13.943688,6.967265,3.055396,23.551784,87.673090,25.656546,26.030782,73.236715,36.542141,17.261648,14.773418,0.548913,7.000000 73 | 46.128234,4.915813,20.525325,18.394542,5.316863,7.119637,36.039645,100.389825,28.537262,32.345505,93.461557,34.594623,16.737188,8.445271,13.111714,7.000000 74 | 44.808399,4.501318,15.921841,5.676711,2.912810,3.308981,47.062820,107.593445,23.310176,44.997130,102.337257,28.679087,22.346223,1.190159,15.552526,7.000000 75 | 26.530469,6.008313,17.595035,18.261555,3.655714,3.716061,38.781065,99.707067,33.522438,39.233322,94.661400,30.749988,17.065474,7.716530,4.521263,7.000000 76 | 25.322479,18.037882,12.067315,32.241247,4.182720,10.241490,64.298837,85.322581,35.394987,66.325426,85.916786,50.186207,33.529981,4.814649,21.073403,7.000000 77 | 26.159149,11.137073,9.392574,32.497866,5.530923,21.209995,58.690552,88.162466,44.455593,66.220628,82.889458,53.075216,26.491866,8.719384,20.402972,7.000000 78 | 27.655743,8.826697,10.426987,24.228312,4.138157,1.587366,43.504149,89.435706,34.546310,49.337679,85.551153,39.430488,23.411189,3.340633,15.438018,7.000000 79 | 36.260453,2.525784,12.413987,6.889512,1.646308,7.154511,41.815285,94.048841,25.811301,32.851651,98.766322,29.309239,13.612372,3.147535,24.566801,7.000000 80 | 24.798043,8.936647,9.570077,23.075096,20.969004,7.107377,43.418623,85.495565,23.589483,45.492214,78.922022,23.059844,20.685546,13.108988,7.381683,7.000000 81 | 24.596646,29.736908,25.081947,8.884024,6.559019,0.456588,29.827859,102.927281,18.240074,12.725527,97.401429,17.777858,15.098036,16.796076,6.547921,8.000000 82 | 50.252621,28.077263,30.040298,22.374885,2.026989,4.306113,48.841783,103.821814,30.767418,54.614023,96.278374,33.056198,26.596671,9.641538,4.516223,8.000000 83 | 46.720347,25.202566,24.210076,17.260718,9.712490,3.675929,28.947328,75.844410,28.755709,30.145959,65.809682,40.230338,16.848285,4.840667,8.473947,8.000000 84 | 28.024332,40.119232,25.496783,13.016671,7.596354,14.071081,32.813221,113.327516,27.055776,24.094514,106.255872,24.239304,2.695157,12.629440,4.089145,8.000000 85 | 34.887251,22.980568,30.863599,3.881194,2.064988,2.772694,38.137402,103.151437,25.618216,29.328686,97.400949,24.351962,5.864652,4.536465,12.272375,8.000000 86 | 44.262576,27.125702,15.256883,20.713364,3.406373,9.884314,48.267375,88.476030,29.769608,53.448143,86.022664,36.539874,34.193978,5.617906,18.224099,8.000000 87 | 34.424282,20.819976,23.608225,11.864899,2.669081,5.032476,42.855215,93.195115,35.297608,43.077306,90.292816,43.513382,13.542847,2.543531,21.420034,8.000000 88 | 23.015253,23.751870,6.535196,3.763148,11.911140,3.379227,11.032025,87.060834,28.273261,6.145853,86.565476,29.938395,12.812652,48.514888,26.460021,8.000000 89 | 25.593816,27.246954,20.144459,20.815079,5.138932,12.616756,49.827635,88.011111,33.921899,49.312024,90.373766,37.904222,21.777395,2.446046,13.132388,8.000000 90 | 25.844453,24.825664,26.760306,17.165493,1.734105,12.963703,47.661407,79.048283,33.757782,44.984442,81.283531,35.549849,6.113230,3.008991,13.152020,8.000000 91 | 37.490134,32.495884,23.745591,10.263570,5.490865,1.261483,15.089296,8.650429,9.230090,48.912789,79.448029,45.186757,67.013845,71.875879,37.987664,9.000000 92 | 29.998130,43.524030,25.121150,15.417454,5.852091,7.589172,16.529002,0.725466,7.439875,24.380792,120.390560,19.199350,15.767035,114.411769,15.472736,9.000000 93 | 34.622599,37.017378,27.669637,2.741039,8.577513,2.243323,14.292178,6.378143,11.999477,45.919511,94.120950,44.025584,55.995302,88.266768,37.774801,9.000000 94 | 46.051695,54.653762,13.225411,14.781907,10.640506,4.251865,26.399216,6.573900,13.331524,71.594673,72.859317,49.275027,84.113429,61.311140,49.315297,9.000000 95 | 42.190255,30.782101,19.135231,8.149782,0.703450,2.855704,12.872799,7.649640,10.776583,41.484759,94.828726,26.366485,53.216605,91.162535,30.145703,9.000000 96 | 38.171631,28.749776,23.954708,7.154842,3.578104,5.526701,9.363086,5.571996,12.836742,43.175990,94.345481,30.334696,56.697473,91.082536,34.231611,9.000000 97 | 33.777163,44.571014,9.712207,2.335842,12.885402,6.266991,6.820075,3.736563,1.888396,65.052365,84.113728,15.254376,46.649516,92.143424,12.844981,9.000000 98 | 33.079898,42.409666,15.798544,5.416477,5.498882,2.514304,11.149445,1.194875,2.339769,48.673194,98.541595,16.054597,34.371564,104.661793,14.501611,9.000000 99 | 40.680805,40.801298,8.909207,5.691228,5.981328,2.100842,4.616145,2.721628,1.298055,40.710221,108.934659,12.880654,35.708418,112.317369,14.130233,9.000000 100 | 36.514748,45.446515,14.760226,4.549033,8.010085,5.285616,6.942849,0.038435,0.997313,46.468994,97.612162,17.154017,36.863595,107.383317,16.551778,9.000000 101 | 35.166499,10.133535,24.003586,20.414161,65.397661,53.280209,1.070313,10.611034,5.501693,5.086899,6.810580,2.684862,12.257347,2.932739,6.857893,10.000000 102 | 27.177540,7.165712,7.382681,27.979051,59.310011,43.088365,5.833023,13.201885,5.084093,5.462635,7.353229,7.037985,20.449650,11.142431,6.311775,10.000000 103 | 34.202671,12.923296,4.436527,7.954484,69.748850,41.098671,3.179318,9.590110,1.405584,8.344494,5.966596,0.871905,10.470855,2.615677,4.257028,10.000000 104 | 25.280928,15.253511,41.613189,19.461021,79.660231,45.791824,1.437713,7.270879,2.006402,2.456731,6.494313,1.439689,7.423358,3.559971,7.069693,10.000000 105 | 20.440146,14.117206,39.528597,16.725306,73.608294,36.204705,12.769179,54.409591,18.337170,9.325091,8.840259,3.291002,10.046129,0.961772,5.568736,10.000000 106 | 42.600813,1.628398,5.641295,29.320372,59.391940,37.741437,2.664271,1.934314,5.444876,8.683744,1.608252,4.105315,14.743984,4.496544,5.945350,10.000000 107 | 29.871450,9.303312,17.007823,31.072375,81.973636,33.115093,3.079640,6.669931,2.969039,11.754539,10.396972,5.003912,17.389056,5.201686,6.154532,10.000000 108 | 38.840654,10.783952,14.997242,40.915437,51.729814,31.135298,3.212290,3.550812,4.857939,13.059346,1.997148,4.008071,22.157437,2.847960,3.501141,10.000000 109 | 36.957795,16.899360,21.456777,49.829542,76.993500,35.017241,8.948312,8.467755,1.604304,6.864062,5.030107,3.714164,20.978079,4.631371,3.630589,10.000000 110 | 36.359711,12.904628,12.775588,55.680258,67.934220,28.683606,6.375559,0.552467,4.157252,14.441847,1.584075,2.392591,33.491625,6.145185,1.497830,10.000000 111 | -------------------------------------------------------------------------------- /data/handrom_train.csv: -------------------------------------------------------------------------------- 1 | 24.980672,7.836687,12.198086,54.799112,149.431577,42.285964,60.854295,120.394795,85.025315,51.545303,130.710423,74.330390,39.330515,100.898659,79.656672,0.000000 2 | 27.778368,20.923018,15.878485,56.965238,152.964000,41.730110,60.632759,128.091563,86.006800,50.670370,135.271401,78.064358,35.431228,113.289719,72.738860,0.000000 3 | 28.253230,15.691632,22.501012,40.331309,167.285507,37.158387,47.521096,134.794214,74.192608,39.976452,148.073053,57.332697,27.969948,142.009022,48.671489,0.000000 4 | 28.675995,15.062306,20.129900,50.119760,163.817291,29.327212,56.444076,138.334334,63.220250,45.865355,146.061906,51.596180,32.044012,122.905108,28.062548,0.000000 5 | 23.972362,11.765355,18.980875,47.134513,165.459569,36.317885,51.935593,138.614626,78.865742,39.864323,149.500953,60.747533,19.683549,135.718627,33.740904,0.000000 6 | 27.783580,12.287740,17.511700,42.800175,164.590807,41.175678,55.003108,131.291313,78.773776,49.814852,138.453872,67.858153,42.445997,111.326860,83.269350,0.000000 7 | 27.582997,10.612048,16.048186,50.377670,157.103003,36.724371,54.894586,133.302963,71.439204,44.961021,138.011720,59.743336,23.581839,121.843166,20.394588,0.000000 8 | 29.247137,11.857716,16.057565,41.996109,159.189465,34.551147,52.283832,125.505391,74.721660,45.024971,128.715937,60.650899,28.101788,113.983640,15.198230,0.000000 9 | 31.642552,27.283449,22.736593,45.013501,162.378557,28.980256,51.939889,131.959796,62.784720,39.849946,139.720919,47.052839,26.589354,121.929776,16.913772,0.000000 10 | 31.956472,40.550577,59.410367,34.638097,163.310178,33.625908,45.540286,132.444856,65.782252,42.419350,140.348636,49.648872,30.346833,132.799026,32.247155,0.000000 11 | 21.477633,7.113319,5.165919,40.999855,94.061988,34.287945,29.487936,116.494854,26.713977,18.134278,126.572083,32.232763,19.350768,100.322862,41.714466,1.000000 12 | 31.554162,4.472374,8.395717,44.339242,58.615681,37.961702,33.672262,76.098864,42.454786,27.039329,89.155907,47.072395,34.521344,66.247448,63.819846,1.000000 13 | 37.063814,19.403312,7.814462,39.681587,72.236413,44.754032,32.880977,88.963974,41.390164,26.677168,95.667087,40.281745,37.337466,66.785574,54.908393,1.000000 14 | 29.528545,16.023841,0.995855,40.222523,60.191399,35.383227,33.561779,80.162904,33.384381,23.305174,90.314569,41.510099,29.058720,70.232726,53.075547,1.000000 15 | 32.660108,26.309939,16.660749,39.273520,69.309475,38.725787,34.089429,93.917276,35.435773,25.223282,97.841032,41.113381,32.729285,67.994795,53.291261,1.000000 16 | 37.763216,30.288631,24.040320,37.284819,65.682980,43.385032,33.206334,79.437373,44.157359,27.997968,83.415959,43.585994,37.392654,58.047341,50.581197,1.000000 17 | 33.084958,32.079755,29.040894,32.980005,60.613441,37.870700,30.946266,69.310785,39.405582,25.810325,78.428306,35.994690,25.375326,65.185660,38.139771,1.000000 18 | 34.405388,26.981957,13.330024,44.915726,69.493511,40.295784,38.321214,96.100643,37.940608,29.624373,107.189254,38.758777,38.693976,83.036053,46.146845,1.000000 19 | 39.828902,30.199452,18.734031,38.082253,66.499474,43.133046,35.324314,76.628554,43.577052,29.059295,83.098060,39.563511,39.051934,60.390956,40.331934,1.000000 20 | 38.662251,26.773778,9.975648,46.991586,56.555213,40.384134,40.332510,75.878567,39.579019,30.659696,89.343048,42.106874,41.475154,67.038009,54.776279,1.000000 21 | 35.832764,79.534354,22.190897,28.324440,9.907121,6.437652,9.447296,6.193965,9.334546,5.740372,12.243239,10.900193,22.920817,2.348960,12.542933,2.000000 22 | 38.510422,87.482982,17.397301,29.995844,7.935827,7.942670,10.752697,6.745110,9.255018,5.385962,12.874881,12.211182,24.112409,4.490294,17.004607,2.000000 23 | 34.375996,73.974527,24.754854,31.120657,8.546488,7.282532,11.306074,4.444392,8.370283,3.900641,8.846753,6.964540,23.286091,1.864999,8.402216,2.000000 24 | 32.503833,76.240740,26.433529,31.235630,7.507752,6.658946,10.825302,10.530689,6.249863,6.071969,12.658310,8.682494,25.756767,1.421538,10.108059,2.000000 25 | 23.744018,73.480195,23.798883,30.729704,14.363662,8.949000,12.757638,10.016690,4.634576,8.555262,2.689541,3.616634,21.963359,5.639529,9.525494,2.000000 26 | 33.486107,67.324179,26.276275,28.414760,9.230945,7.135567,9.991510,5.028491,7.717026,6.832246,7.932713,5.233004,25.583683,2.080481,7.227445,2.000000 27 | 30.168309,70.969542,32.205064,27.526369,9.944240,8.075054,9.389364,7.984605,9.331336,6.970410,8.225229,7.859249,29.443447,2.937119,9.249800,2.000000 28 | 31.194238,57.566591,39.703982,28.054722,8.811356,5.090669,9.681432,8.450576,8.181026,7.748618,7.735988,3.237065,25.603529,2.326355,4.642785,2.000000 29 | 35.587754,80.020470,20.744294,30.995421,9.068732,7.811603,13.010017,7.375602,7.932416,7.540829,3.582032,8.393649,33.172826,4.158346,14.899550,2.000000 30 | 32.470085,76.429840,22.620470,31.476825,7.977530,8.590058,13.292598,6.334827,8.417723,4.855955,9.802514,10.716954,24.611552,0.940413,13.988034,2.000000 31 | 22.479300,40.233208,67.036035,15.974295,9.905494,4.552424,1.924334,16.692569,6.737317,10.543362,6.333858,1.048378,21.835228,6.948224,7.330450,3.000000 32 | 23.290824,36.919276,59.243925,19.299222,7.550911,2.177158,2.060850,16.636044,5.726581,8.917429,10.669330,5.174115,21.070265,8.027950,9.957394,3.000000 33 | 21.024151,40.163456,59.066326,21.083884,11.146967,3.652880,4.211324,15.074664,5.587046,10.332758,8.650549,5.869982,23.683990,9.880418,11.977771,3.000000 34 | 22.235946,39.738490,60.400840,20.834795,9.333227,1.935512,3.502491,14.338161,6.693658,11.046677,6.713871,3.963731,23.501564,7.278813,7.790933,3.000000 35 | 26.304876,45.566057,64.590795,12.927208,5.189612,11.482090,6.059140,16.706563,9.143405,3.201219,11.589890,2.906998,7.145993,6.988630,7.120544,3.000000 36 | 25.312071,41.987075,71.382830,13.092798,5.035776,8.259406,6.639226,17.571916,7.818047,5.780957,11.624989,5.869027,11.049822,8.839376,10.136944,3.000000 37 | 24.971463,43.983381,62.047571,15.079330,9.676598,6.583385,5.263508,18.179005,8.468395,7.408785,12.885185,1.371180,16.253385,10.286129,8.179925,3.000000 38 | 23.019779,41.016332,58.484368,17.372242,8.946337,3.320532,5.714648,16.730802,9.692179,7.156845,8.262615,2.266728,17.625941,6.552479,7.268329,3.000000 39 | 26.189264,44.191889,62.408085,14.997343,5.004746,8.487306,4.738317,15.467748,8.345673,7.461954,8.332968,3.563215,17.037250,7.342545,7.555611,3.000000 40 | 25.666389,40.612491,71.231588,20.961764,10.391506,6.821114,8.772635,18.101853,8.962714,4.927817,10.321938,4.671901,16.085222,3.900565,1.891323,3.000000 41 | 21.202102,18.523284,9.219103,26.491315,11.670058,7.836982,11.743995,6.895257,10.854348,14.306590,5.679977,6.095020,25.925134,8.514456,9.912693,5.000000 42 | 18.214808,18.439986,6.005788,27.812686,12.163658,6.502393,15.212880,9.091534,10.530123,9.889143,8.441060,5.577102,21.553281,8.562910,11.469539,5.000000 43 | 14.757515,12.995421,7.156902,27.358220,11.535121,7.551199,13.150245,7.965087,8.490933,11.868143,6.384315,5.399020,27.711058,3.908572,10.224543,5.000000 44 | 13.634542,12.725070,10.301115,28.268059,11.028661,8.252470,14.682126,8.200386,10.500822,13.690616,7.096391,7.066143,22.785399,4.222525,10.535989,5.000000 45 | 18.450614,14.773956,10.409221,25.206606,12.311439,9.879131,11.004110,7.455125,11.947796,16.256508,4.608633,9.252899,26.217065,8.501416,12.234386,5.000000 46 | 18.297211,13.101727,10.571010,32.225272,14.600588,7.362581,17.299273,9.708418,4.335881,15.885483,10.265136,1.603028,30.632075,8.853812,4.971644,5.000000 47 | 25.331805,12.556043,15.517905,37.205811,14.114626,6.201578,21.112416,6.889460,6.273254,17.286860,2.230909,4.543966,29.082611,1.540538,0.849100,5.000000 48 | 29.076582,10.853922,10.231537,39.625616,10.033734,5.200170,23.683916,10.031595,4.855586,16.453337,7.973402,6.468697,21.057331,4.607568,7.536145,5.000000 49 | 31.605327,9.889499,14.981022,43.239472,11.730687,4.233398,25.597082,6.905698,7.466872,15.845175,4.905297,8.585843,29.361060,0.651012,5.815040,5.000000 50 | 17.378155,13.522293,10.688753,31.245072,12.905821,9.104013,16.534070,7.934471,5.576988,15.307555,5.137994,1.495535,28.773005,5.531837,5.357963,5.000000 51 | 22.540392,7.975358,14.583452,27.260001,5.960455,2.698397,7.462900,13.875745,3.934212,10.392291,8.554501,3.418969,26.460748,5.386650,11.504141,4.000000 52 | 21.252498,9.572638,13.047589,29.531380,8.157350,4.459736,9.510163,14.133387,5.132636,7.696712,7.673124,4.081244,22.410690,5.151429,9.993897,4.000000 53 | 21.615291,8.817530,14.012632,27.122992,5.883064,3.018872,7.225376,14.412378,3.772858,10.254862,8.104627,5.454421,24.762186,8.459349,13.759472,4.000000 54 | 19.919600,7.029719,12.619832,30.395712,2.861364,1.999099,10.278523,11.439746,4.179174,13.117478,3.002854,5.860521,27.097180,4.479328,16.042752,4.000000 55 | 20.592580,8.337752,13.976443,28.499198,4.194280,1.849648,10.499883,14.194021,4.233971,10.522489,9.514510,5.388564,23.240210,7.205099,14.672210,4.000000 56 | 20.847649,8.795391,13.664240,27.772561,5.613650,3.186849,9.178398,14.919599,4.319260,8.091681,8.479897,6.012374,21.909198,7.984232,13.595576,4.000000 57 | 19.792088,8.266345,13.355282,28.236172,4.849156,2.887348,8.813806,11.838671,4.304404,11.109292,3.344327,4.085917,26.464145,6.359762,13.385742,4.000000 58 | 20.485052,8.309982,14.368009,28.524444,4.820090,1.493171,9.705231,14.380765,3.099102,9.393969,7.142402,4.988604,23.038336,5.169868,14.085092,4.000000 59 | 21.046851,6.921125,15.484157,28.733976,3.023861,2.161805,9.222091,10.822185,3.907296,12.783563,5.168014,6.202989,26.686237,6.429382,16.008585,4.000000 60 | 18.594501,7.436277,12.662291,28.107907,4.367439,1.558868,8.557479,13.043491,4.057493,11.410553,4.465351,4.340432,24.763384,2.046462,13.534055,4.000000 61 | 37.149256,38.590075,13.708292,4.128614,6.374718,7.263470,18.485715,18.974774,13.231541,7.569011,26.188266,8.170061,26.491809,37.370945,7.028859,6.000000 62 | 41.320672,50.762978,14.067883,3.234356,6.283897,6.350027,19.149663,17.147302,11.743074,3.449296,36.226201,16.266229,29.561021,27.669231,17.855836,6.000000 63 | 39.752560,60.657628,12.488850,4.189097,3.015528,1.975925,12.325735,7.826493,5.383029,13.406100,20.186253,13.919919,37.262153,52.091827,39.365681,6.000000 64 | 38.819805,57.024414,9.823122,6.203312,4.396216,7.868157,8.298203,7.706825,4.986412,14.611068,25.235321,10.806337,33.182677,48.883124,35.114938,6.000000 65 | 42.655558,57.375734,10.034795,5.207012,7.849752,1.780630,9.162498,5.009651,11.324765,6.821544,40.979628,29.952271,29.868513,42.680140,33.906748,6.000000 66 | 41.139131,63.109532,12.160901,3.405289,2.356179,2.481547,17.405802,11.246595,4.185575,10.965256,27.065727,6.737836,36.633181,33.465734,20.964098,6.000000 67 | 37.433725,46.351477,13.715816,7.453569,6.443460,5.532805,21.713660,16.773146,8.149731,10.832615,30.341696,7.571042,28.177957,39.168665,16.891807,6.000000 68 | 38.512712,53.147500,13.518451,3.094810,1.955753,4.383393,17.625983,15.458993,6.095049,6.493764,31.560551,5.802108,32.641259,29.635781,16.828846,6.000000 69 | 37.671748,52.413328,16.137471,4.008517,5.835990,4.788203,17.176833,19.624005,6.472517,7.741927,36.450148,6.159748,31.891899,38.157599,21.962429,6.000000 70 | 37.064584,50.160315,18.791848,3.173202,0.566580,2.538843,16.905359,15.340525,4.229249,6.654419,30.008033,5.482276,32.792877,30.761226,16.285375,6.000000 71 | -------------------------------------------------------------------------------- /data/sample/hand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/hand.png -------------------------------------------------------------------------------- /data/sample/lower_limb1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb1.png -------------------------------------------------------------------------------- /data/sample/lower_limb2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb2.png -------------------------------------------------------------------------------- /data/sample/lower_limb3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb3.png -------------------------------------------------------------------------------- /data/sample/lower_limb4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb4.png -------------------------------------------------------------------------------- /data/sample/lower_limb5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb5.png -------------------------------------------------------------------------------- /data/sample/lower_limb6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/lower_limb6.png -------------------------------------------------------------------------------- /data/sample/mona.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/mona.png -------------------------------------------------------------------------------- /data/sample/object.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/object.png -------------------------------------------------------------------------------- /data/sample/upper_limb1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb1.png -------------------------------------------------------------------------------- /data/sample/upper_limb2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb2.png -------------------------------------------------------------------------------- /data/sample/upper_limb3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb3.png -------------------------------------------------------------------------------- /data/sample/upper_limb4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb4.png -------------------------------------------------------------------------------- /data/sample/upper_limb5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb5.png -------------------------------------------------------------------------------- /data/sample/upper_limb6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/upper_limb6.png -------------------------------------------------------------------------------- /data/sample/wrist_extension.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/wrist_extension.png -------------------------------------------------------------------------------- /data/sample/wrist_flexion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/wrist_flexion.png -------------------------------------------------------------------------------- /data/sample/wrist_radial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/wrist_radial.png -------------------------------------------------------------------------------- /data/sample/wrist_ulnar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/data/sample/wrist_ulnar.png -------------------------------------------------------------------------------- /doc/00_image.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/00_image.gif -------------------------------------------------------------------------------- /doc/02_gesture.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/02_gesture.gif -------------------------------------------------------------------------------- /doc/03_game_rps.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/03_game_rps.gif -------------------------------------------------------------------------------- /doc/04_hand_rom.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/04_hand_rom.gif -------------------------------------------------------------------------------- /doc/05_wrist_rom.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/05_wrist_rom.gif -------------------------------------------------------------------------------- /doc/06_face_mask.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/06_face_mask.gif -------------------------------------------------------------------------------- /doc/07_triangulate.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/07_triangulate.gif -------------------------------------------------------------------------------- /doc/08_skeleton_3D.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/08_skeleton_3D.gif -------------------------------------------------------------------------------- /doc/09_objectron.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/09_objectron.gif -------------------------------------------------------------------------------- /doc/rris_database.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/doc/rris_database.gif -------------------------------------------------------------------------------- /dual.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | 5 | max_num_hands = 2 6 | gesture = { 7 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 8 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 9 | } 10 | rps_gesture = {0:'rock', 5:'paper', 9:'scissors'} 11 | 12 | # MediaPipe hands model 13 | mp_hands = mp.solutions.hands 14 | mp_drawing = mp.solutions.drawing_utils 15 | hands = mp_hands.Hands( 16 | max_num_hands=max_num_hands, 17 | min_detection_confidence=0.5, 18 | min_tracking_confidence=0.5) 19 | 20 | # Gesture recognition model 21 | file = np.genfromtxt('data/gesture_train.csv', delimiter=',') 22 | angle = file[:,:-1].astype(np.float32) 23 | label = file[:, -1].astype(np.float32) 24 | knn = cv2.ml.KNearest_create() 25 | knn.train(angle, cv2.ml.ROW_SAMPLE, label) 26 | 27 | cap = cv2.VideoCapture(0) 28 | 29 | while cap.isOpened(): 30 | ret, img = cap.read() 31 | if not ret: 32 | continue 33 | 34 | img = cv2.flip(img, 1) 35 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 36 | 37 | result = hands.process(img) 38 | 39 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 40 | 41 | if result.multi_hand_landmarks is not None: 42 | rps_result = [] 43 | 44 | for res in result.multi_hand_landmarks: 45 | joint = np.zeros((21, 3)) 46 | for j, lm in enumerate(res.landmark): 47 | joint[j] = [lm.x, lm.y, lm.z] 48 | 49 | # Compute angles between joints 50 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 51 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 52 | v = v2 - v1 # [20,3] 53 | # Normalize v 54 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 55 | 56 | # Get angle using arcos of dot product 57 | angle = np.arccos(np.einsum('nt,nt->n', 58 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 59 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 60 | 61 | angle = np.degrees(angle) # Convert radian to degree 62 | 63 | # Inference gesture 64 | data = np.array([angle], dtype=np.float32) 65 | ret, results, neighbours, dist = knn.findNearest(data, 3) 66 | idx = int(results[0][0]) 67 | 68 | # Draw gesture result 69 | if idx in rps_gesture.keys(): 70 | org = (int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0])) 71 | cv2.putText(img, text=rps_gesture[idx].upper(), org=(org[0], org[1] + 20), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 72 | 73 | rps_result.append({ 74 | 'rps': rps_gesture[idx], 75 | 'org': org 76 | }) 77 | 78 | mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 79 | 80 | # Who wins? 81 | if len(rps_result) >= 2: 82 | winner = None 83 | text = '' 84 | 85 | if rps_result[0]['rps']=='rock': 86 | if rps_result[1]['rps']=='rock' : text = 'Tie' 87 | elif rps_result[1]['rps']=='paper' : text = 'Paper wins' ; winner = 1 88 | elif rps_result[1]['rps']=='scissors': text = 'Rock wins' ; winner = 0 89 | elif rps_result[0]['rps']=='paper': 90 | if rps_result[1]['rps']=='rock' : text = 'Paper wins' ; winner = 0 91 | elif rps_result[1]['rps']=='paper' : text = 'Tie' 92 | elif rps_result[1]['rps']=='scissors': text = 'Scissors wins'; winner = 1 93 | elif rps_result[0]['rps']=='scissors': 94 | if rps_result[1]['rps']=='rock' : text = 'Rock wins' ; winner = 1 95 | elif rps_result[1]['rps']=='paper' : text = 'Scissors wins'; winner = 0 96 | elif rps_result[1]['rps']=='scissors': text = 'Tie' 97 | 98 | if winner is not None: 99 | cv2.putText(img, text='Winner', org=(rps_result[winner]['org'][0], rps_result[winner]['org'][1] + 70), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=2, color=(0, 255, 0), thickness=3) 100 | cv2.putText(img, text=text, org=(int(img.shape[1] / 2), 100), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=2, color=(0, 0, 255), thickness=3) 101 | 102 | cv2.imshow('Game', img) 103 | if cv2.waitKey(1) == ord('q'): 104 | break 105 | -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- 1 | name: mp 2 | channels: 3 | - open3d-admin 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - pip=20.2.2 8 | - python=3.7.7 9 | - numpy=1.19.1 10 | - pandas=1.1.4 11 | - plyfile=0.7.2 12 | - pyyaml=5.3.1 13 | - tqdm=4.54.1 14 | - matplotlib=3.3.3 15 | - open3d=0.11.2.0 16 | - pip: 17 | - opencv-python==4.2.0.34 18 | - mediapipe==0.8.4.2 19 | - addict==2.4.0 20 | - sklearn==0.0 -------------------------------------------------------------------------------- /fan.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | from dynamikontrol import Module 5 | 6 | module = Module() 7 | 8 | max_num_hands = 1 9 | gesture = { 10 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 11 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 12 | } 13 | rps_gesture = {0:'OFF', 1:'LOW', 2:'MID', 3: 'HIGH'} 14 | 15 | # MediaPipe hands model 16 | mp_hands = mp.solutions.hands 17 | mp_drawing = mp.solutions.drawing_utils 18 | hands = mp_hands.Hands( 19 | max_num_hands=max_num_hands, 20 | min_detection_confidence=0.5, 21 | min_tracking_confidence=0.5) 22 | 23 | # Gesture recognition model 24 | file = np.genfromtxt('data/gesture_train.csv', delimiter=',') 25 | angle = file[:,:-1].astype(np.float32) 26 | label = file[:, -1].astype(np.float32) 27 | knn = cv2.ml.KNearest_create() 28 | knn.train(angle, cv2.ml.ROW_SAMPLE, label) 29 | 30 | cap = cv2.VideoCapture(0) 31 | 32 | while cap.isOpened(): 33 | ret, img = cap.read() 34 | if not ret: 35 | break 36 | 37 | img = cv2.flip(img, 1) 38 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 39 | 40 | result = hands.process(img) 41 | 42 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 43 | 44 | if result.multi_hand_landmarks is not None: 45 | for res in result.multi_hand_landmarks: 46 | joint = np.zeros((21, 3)) 47 | for j, lm in enumerate(res.landmark): 48 | joint[j] = [lm.x, lm.y, lm.z] 49 | 50 | # Compute angles between joints 51 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 52 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 53 | v = v2 - v1 # [20,3] 54 | # Normalize v 55 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 56 | 57 | # Get angle using arcos of dot product 58 | angle = np.arccos(np.einsum('nt,nt->n', 59 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 60 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 61 | 62 | angle = np.degrees(angle) # Convert radian to degree 63 | 64 | # Inference gesture 65 | data = np.array([angle], dtype=np.float32) 66 | ret, results, neighbours, dist = knn.findNearest(data, 5) 67 | idx = int(results[0][0]) 68 | 69 | # Draw gesture result 70 | if idx in rps_gesture.keys(): 71 | cv2.putText(img, text=rps_gesture[idx].upper(), org=(int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0] + 20)), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 72 | 73 | if idx > 0: 74 | module.motor.speed(idx * 1000) 75 | else: 76 | module.motor.stop() 77 | 78 | mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 79 | 80 | cv2.imshow('AI Fan', img) 81 | if cv2.waitKey(1) == ord('q'): 82 | break 83 | 84 | module.motor.stop() 85 | module.disconnect() 86 | -------------------------------------------------------------------------------- /fy_filter.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | 5 | max_num_hands = 1 6 | gesture = { 7 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 8 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 11:'fy' 9 | } 10 | 11 | # MediaPipe hands model 12 | mp_hands = mp.solutions.hands 13 | mp_drawing = mp.solutions.drawing_utils 14 | hands = mp_hands.Hands( 15 | max_num_hands=max_num_hands, 16 | min_detection_confidence=0.5, 17 | min_tracking_confidence=0.5) 18 | 19 | # Gesture recognition model 20 | file = np.genfromtxt('data/gesture_train_fy.csv', delimiter=',') 21 | angle = file[:,:-1].astype(np.float32) 22 | label = file[:, -1].astype(np.float32) 23 | knn = cv2.ml.KNearest_create() 24 | knn.train(angle, cv2.ml.ROW_SAMPLE, label) 25 | 26 | cap = cv2.VideoCapture(0) 27 | 28 | while cap.isOpened(): 29 | ret, img = cap.read() 30 | if not ret: 31 | continue 32 | 33 | img = cv2.flip(img, 1) 34 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 35 | 36 | result = hands.process(img) 37 | 38 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 39 | 40 | if result.multi_hand_landmarks is not None: 41 | for res in result.multi_hand_landmarks: 42 | joint = np.zeros((21, 3)) 43 | for j, lm in enumerate(res.landmark): 44 | joint[j] = [lm.x, lm.y, lm.z] 45 | 46 | # Compute angles between joints 47 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 48 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 49 | v = v2 - v1 # [20,3] 50 | # Normalize v 51 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 52 | 53 | # Get angle using arcos of dot product 54 | angle = np.arccos(np.einsum('nt,nt->n', 55 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 56 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 57 | 58 | angle = np.degrees(angle) # Convert radian to degree 59 | 60 | # Inference gesture 61 | data = np.array([angle], dtype=np.float32) 62 | ret, results, neighbours, dist = knn.findNearest(data, 3) 63 | idx = int(results[0][0]) 64 | 65 | if idx == 11: 66 | x1, y1 = tuple((joint.min(axis=0)[:2] * [img.shape[1], img.shape[0]] * 0.95).astype(int)) 67 | x2, y2 = tuple((joint.max(axis=0)[:2] * [img.shape[1], img.shape[0]] * 1.05).astype(int)) 68 | 69 | fy_img = img[y1:y2, x1:x2].copy() 70 | fy_img = cv2.resize(fy_img, dsize=None, fx=0.05, fy=0.05, interpolation=cv2.INTER_NEAREST) 71 | fy_img = cv2.resize(fy_img, dsize=(x2 - x1, y2 - y1), interpolation=cv2.INTER_NEAREST) 72 | 73 | img[y1:y2, x1:x2] = fy_img 74 | 75 | # mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 76 | 77 | cv2.imshow('Filter', img) 78 | if cv2.waitKey(1) == ord('q'): 79 | break 80 | -------------------------------------------------------------------------------- /gather_dataset.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | 5 | max_num_hands = 1 6 | gesture = { 7 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 8 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 11:'fy' 9 | } 10 | 11 | # MediaPipe hands model 12 | mp_hands = mp.solutions.hands 13 | mp_drawing = mp.solutions.drawing_utils 14 | hands = mp_hands.Hands( 15 | max_num_hands=max_num_hands, 16 | min_detection_confidence=0.5, 17 | min_tracking_confidence=0.5) 18 | 19 | # Gesture recognition data 20 | file = np.genfromtxt('data/gesture_train.csv', delimiter=',') 21 | print(file.shape) 22 | 23 | cap = cv2.VideoCapture(0) 24 | 25 | def click(event, x, y, flags, param): 26 | global data, file 27 | if event == cv2.EVENT_LBUTTONDOWN: 28 | file = np.vstack((file, data)) 29 | print(file.shape) 30 | 31 | cv2.namedWindow('Dataset') 32 | cv2.setMouseCallback('Dataset', click) 33 | 34 | while cap.isOpened(): 35 | ret, img = cap.read() 36 | if not ret: 37 | continue 38 | 39 | img = cv2.flip(img, 1) 40 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 41 | 42 | result = hands.process(img) 43 | 44 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 45 | 46 | if result.multi_hand_landmarks is not None: 47 | for res in result.multi_hand_landmarks: 48 | joint = np.zeros((21, 3)) 49 | for j, lm in enumerate(res.landmark): 50 | joint[j] = [lm.x, lm.y, lm.z] 51 | 52 | # Compute angles between joints 53 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 54 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 55 | v = v2 - v1 # [20,3] 56 | # Normalize v 57 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 58 | 59 | # Get angle using arcos of dot product 60 | angle = np.arccos(np.einsum('nt,nt->n', 61 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 62 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 63 | 64 | angle = np.degrees(angle) # Convert radian to degree 65 | 66 | data = np.array([angle], dtype=np.float32) 67 | data = np.append(data, 11) 68 | 69 | mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 70 | 71 | cv2.imshow('Dataset', img) 72 | if cv2.waitKey(1) == ord('q'): 73 | break 74 | 75 | np.savetxt('data/gesture_train_fy.csv', file, delimiter=',') 76 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | ## Sample application of MediaPipe using JavaScript 2 | 3 | Click on the links below to try out realtime, online, markerless pose estimation from color camera. Note: You will need to allow the website to access the camera connected to your computer, and the recommended camera resolution is HD 720p (1280 x 720). 4 | 5 | * [Extract left and right elbow flexion angles](https://codepen.io/rris/full/YzpLrVg) 6 | * [Extract hand flexion angles](https://codepen.io/rris/full/vYyVyoa) 7 | * [Fruit ninja game with wrist joint](https://codepen.io/rris/full/ExNOLNp) 8 | * [Fruit ninja game with index fingertip](https://codepen.io/rris/full/jOVQjbj) 9 | 10 | Refer to [MediaPipe project on CodePen](https://codepen.io/mediapipe) for more examples. 11 | -------------------------------------------------------------------------------- /java/elbow/codepen.css: -------------------------------------------------------------------------------- 1 | @keyframes spin { 2 | 0% { transform: rotate(0deg); } 3 | 100% { transform: rotate(360deg); } 4 | } 5 | 6 | .abs { 7 | position: absolute; 8 | } 9 | 10 | a { 11 | color: white; 12 | text-decoration: none; 13 | &:hover { 14 | color: lightblue; 15 | } 16 | } 17 | 18 | body { 19 | bottom: 0; 20 | font-family: 'Titillium Web', sans-serif; 21 | color: white; 22 | left: 0; 23 | margin: 0; 24 | position: absolute; 25 | right: 0; 26 | top: 0; 27 | transform-origin: 0px 0px; 28 | overflow: hidden; 29 | } 30 | 31 | .container { 32 | position: absolute; 33 | background-color: #596e73; 34 | height: 720px; 35 | width: 1280px; 36 | } 37 | 38 | .input_video { 39 | position:relative; 40 | top: 0; 41 | left: 0; 42 | right: 0; 43 | bottom: 0; 44 | &.selfie { 45 | transform: scale(-1, 1); 46 | } 47 | } 48 | 49 | .output_canvas { 50 | position:absolute; 51 | height: 720px; 52 | width: 1280px; 53 | left: 0; 54 | top: 0; 55 | } 56 | 57 | .logo { 58 | bottom: 10px; 59 | right: 20px; 60 | 61 | .title { 62 | color: white; 63 | font-size: 28px; 64 | } 65 | 66 | .subtitle { 67 | position: relative; 68 | color: white; 69 | font-size: 10px; 70 | left: -30px; 71 | top: 20px; 72 | } 73 | } 74 | 75 | .control-panel { 76 | position: absolute; 77 | left: 10px; 78 | top: 10px; 79 | } 80 | 81 | .loading { 82 | display: flex; 83 | position: absolute; 84 | top: 0; 85 | right: 0; 86 | bottom: 0; 87 | left: 0; 88 | align-items: center; 89 | backface-visibility: hidden; 90 | justify-content: center; 91 | opacity: 1; 92 | transition: opacity 1s; 93 | 94 | .message { 95 | font-size: x-large; 96 | } 97 | 98 | .spinner { 99 | position: absolute; 100 | width: 120px; 101 | height: 120px; 102 | animation: spin 1s linear infinite; 103 | border: 32px solid #bebebe; 104 | border-top: 32px solid #3498db; 105 | border-radius: 50%; 106 | } 107 | } 108 | 109 | .loaded .loading { 110 | opacity: 0; 111 | } 112 | 113 | .shoutout { 114 | left: 0; 115 | right: 0; 116 | bottom: 40px; 117 | text-align: center; 118 | font-size: 24px; 119 | position: absolute; 120 | background-color: black; 121 | color: rgb(255,255,255); 122 | } 123 | -------------------------------------------------------------------------------- /java/elbow/codepen.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 |
7 | Loading 8 |
9 |
10 | 17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 | -------------------------------------------------------------------------------- /java/elbow/codepen.js: -------------------------------------------------------------------------------- 1 | // Our input frames will come from here. 2 | const videoElement = document.getElementsByClassName("input_video")[0]; 3 | const canvasElement = document.getElementsByClassName("output_canvas")[0]; 4 | const controlsElement = document.getElementsByClassName("control-panel")[0]; 5 | const canvasCtx = canvasElement.getContext("2d"); 6 | 7 | // We'll add this to our control panel later, but we'll save it here so we can 8 | // call tick() each time the graph runs. 9 | const fpsControl = new FPS(); 10 | 11 | // Optimization: Turn off animated spinner after its hiding animation is done. 12 | const spinner = document.querySelector(".loading"); 13 | spinner.ontransitionend = () => { 14 | spinner.style.display = "none"; 15 | }; 16 | 17 | function zColor(data) { 18 | const z = clamp(data.from.z + 0.5, 0, 1); 19 | return `rgba(0, ${255 * z}, ${255 * (1 - z)}, 1)`; 20 | } 21 | 22 | function onResults(results) { 23 | // Hide the spinner. 24 | document.body.classList.add("loaded"); 25 | 26 | // Update the frame rate. 27 | fpsControl.tick(); 28 | 29 | // Draw the overlays. 30 | canvasCtx.save(); 31 | canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height); 32 | canvasCtx.drawImage( 33 | results.image, 34 | 0, 35 | 0, 36 | canvasElement.width, 37 | canvasElement.height 38 | ); 39 | drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS, { 40 | visibilityMin: 0.65, 41 | color: (data) => { 42 | const x0 = canvasElement.width * data.from.x; 43 | const y0 = canvasElement.height * data.from.y; 44 | const x1 = canvasElement.width * data.to.x; 45 | const y1 = canvasElement.height * data.to.y; 46 | 47 | const z0 = clamp(data.from.z + 0.5, 0, 1); 48 | const z1 = clamp(data.to.z + 0.5, 0, 1); 49 | 50 | const gradient = canvasCtx.createLinearGradient(x0, y0, x1, y1); 51 | gradient.addColorStop(0, `rgba(0, ${255 * z0}, ${255 * (1 - z0)}, 1)`); 52 | gradient.addColorStop(1.0, `rgba(0, ${255 * z1}, ${255 * (1 - z1)}, 1)`); 53 | return gradient; 54 | } 55 | }); 56 | drawLandmarks( 57 | canvasCtx, 58 | Object.values(POSE_LANDMARKS_LEFT).map( 59 | (index) => results.poseLandmarks[index] 60 | ), 61 | { visibilityMin: 0.65, color: zColor, fillColor: "#FF0000" } 62 | ); 63 | drawLandmarks( 64 | canvasCtx, 65 | Object.values(POSE_LANDMARKS_RIGHT).map( 66 | (index) => results.poseLandmarks[index] 67 | ), 68 | { visibilityMin: 0.65, color: zColor, fillColor: "#00FF00" } 69 | ); 70 | drawLandmarks( 71 | canvasCtx, 72 | Object.values(POSE_LANDMARKS_NEUTRAL).map( 73 | (index) => results.poseLandmarks[index] 74 | ), 75 | { visibilityMin: 0.65, color: zColor, fillColor: "#AAAAAA" } 76 | ); 77 | canvasCtx.restore(); 78 | 79 | // Compute elbow joint angle 80 | leftElbowAngle = getFlexionAng(results, 11,13,15); 81 | rightElbowAngle = getFlexionAng(results, 12,14,16); 82 | 83 | // Display elbow joint angle 84 | document.getElementById('left-elbow-angle').innerHTML = 'Left elbow angle: ' + leftElbowAngle + ' deg'; 85 | document.getElementById('right-elbow-angle').innerHTML = 'Right elbow angle: ' + rightElbowAngle + ' deg'; 86 | } 87 | 88 | const pose = new Pose({ 89 | locateFile: (file) => { 90 | return `https://cdn.jsdelivr.net/npm/@mediapipe/pose@0.2/${file}`; 91 | } 92 | }); 93 | pose.onResults(onResults); 94 | 95 | /** 96 | * Instantiate a camera. We'll feed each frame we receive into the solution. 97 | */ 98 | const camera = new Camera(videoElement, { 99 | onFrame: async () => { 100 | await pose.send({ image: videoElement }); 101 | }, 102 | width: 1280, 103 | height: 720 104 | }); 105 | camera.start(); 106 | 107 | // Present a control panel through which the user can manipulate the solution 108 | // options. 109 | new ControlPanel(controlsElement, { 110 | selfieMode: true, 111 | upperBodyOnly: true, 112 | smoothLandmarks: true, 113 | minDetectionConfidence: 0.5, 114 | minTrackingConfidence: 0.5, 115 | pauseCamera: false, 116 | }) 117 | .add([ 118 | new StaticText({ title: "Elbow joint angle" }), 119 | fpsControl, 120 | new Toggle({ title: "Selfie Mode", field: "selfieMode" }), 121 | new Toggle({ title: "Upper-body Only", field: "upperBodyOnly" }), 122 | new Toggle({ title: "Smooth Landmarks", field: "smoothLandmarks" }), 123 | new Toggle({ title: "Pause Camera", field: "pauseCamera" }), 124 | // new Slider({ 125 | // title: "Min Detection Confidence", 126 | // field: "minDetectionConfidence", 127 | // range: [0, 1], 128 | // step: 0.01 129 | // }), 130 | // new Slider({ 131 | // title: "Min Tracking Confidence", 132 | // field: "minTrackingConfidence", 133 | // range: [0, 1], 134 | // step: 0.01 135 | // }) 136 | ]) 137 | .on((options) => { 138 | videoElement.classList.toggle("selfie", options.selfieMode); 139 | pose.setOptions(options); 140 | // Add option to pause camera 141 | options.pauseCamera ? camera.video.pause() : camera.start(); 142 | }); 143 | 144 | // Added by GM 145 | function getFlexionAng(results, j0, j1, j2) { 146 | // j0, j1, j2 are joint indexes of poseLandmarks 147 | // 1st: rescale landmarks from relative coor to absolute 3D coor 148 | var A = rescale(results.poseLandmarks[j0]); 149 | var B = rescale(results.poseLandmarks[j1]); 150 | var C = rescale(results.poseLandmarks[j2]); 151 | // 2nd: Find the acute angle at joint j1 152 | return getAngBtw3Pts(A, B, C).toFixed(0); 153 | } 154 | 155 | function rescale(lm) { 156 | // Convert landmarks from relative coor to absolute 3D coor 157 | // Note: Assume image width:1280, height:720 158 | var A = {x:0, y:0, z:0}; 159 | A.x = lm.x * 1280 - 640; 160 | A.y = lm.y * 720 - 360; 161 | A.z = lm.z * 1280 * 0.25; // Note: Seems like need to further scale down z by 0.25 else will get elongated forearm and feet 162 | return A; 163 | } 164 | 165 | function getAngBtw3Pts(A, B, C) { 166 | // Note: Points A, B and C are 3D points with x,y,z 167 | // 1st: Find vector AB (a,b,c) and BC (d,e,f) 168 | // 2nd: Find acute angle between vector AB and BC using cosine rule 169 | 170 | // Find vector AB = OB - OA 171 | var a = B.x - A.x; 172 | var b = B.y - A.y; 173 | var c = B.z - A.z; 174 | // Find vector BC = OC - OB 175 | var d = C.x - B.x; 176 | var e = C.y - B.y; 177 | var f = C.z - B.z; 178 | 179 | // BA.BC -> dot product 180 | var num = a*d + b*e + c*f; 181 | // |BA|*|BC| -> multiply norm of BA and BC 182 | var den = Math.sqrt(a*a + b*b + c*c) * Math.sqrt(d*d + e*e + f*f); 183 | // ang = acos(BA.BC / |BA|*|BC|) 184 | var ang = Math.acos(num/den); 185 | 186 | // Convert radian to degree 187 | return ang * 180 / Math.PI; 188 | } -------------------------------------------------------------------------------- /java/elbow/head.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /java/hand-fruit-ninja/codepen.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | padding: 0; 3 | margin: 0; 4 | } 5 | 6 | .control-panel { 7 | position: absolute; 8 | left: 10px; 9 | top: 40px; 10 | } 11 | 12 | .input_video { 13 | position:absolute; 14 | top: 0px; 15 | left: 0px; 16 | opacity: 0.5; 17 | width:1280px; 18 | height:720px; 19 | &.selfie { 20 | transform: scale(-1, 1); 21 | } 22 | } 23 | 24 | div#game { 25 | width:100%; 26 | height:100%; 27 | } 28 | 29 | #mouse-circle { 30 | position: absolute; 31 | width: 30px; 32 | height: 30px; 33 | border-radius: 50%; 34 | background-color: yellow; 35 | pointer-events: none !important; 36 | } -------------------------------------------------------------------------------- /java/hand-fruit-ninja/codepen.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
-------------------------------------------------------------------------------- /java/hand-fruit-ninja/codepen.js: -------------------------------------------------------------------------------- 1 | // Note: Need to include the link below 2 | // https://cdnjs.cloudflare.com/ajax/libs/phaser/2.0.6/phaser.min.js 3 | // In Pen Settings -> JS -> Add External Scripts/Pens 4 | 5 | const videoElement = document.getElementsByClassName("input_video")[0]; 6 | const controlsElement = document.getElementsByClassName("control-panel")[0]; 7 | const mouseCircle = document.getElementById("mouse-circle"); 8 | 9 | var global_x = 0; 10 | var global_y = 0; 11 | 12 | function onResults(results) { 13 | if (results.multiHandLandmarks && results.multiHandedness) { 14 | global_x = results.multiHandLandmarks[0][8].x * 1280; // Index 8 is index fingertip 15 | global_y = results.multiHandLandmarks[0][8].y * 720; // Index 8 is index fingertip 16 | mouseCircle.style.left = (global_x-15) + "px"; 17 | mouseCircle.style.top = (global_y-15) + "px"; 18 | } 19 | } 20 | 21 | const hands = new Hands({locateFile: (file) => { 22 | return `https://cdn.jsdelivr.net/npm/@mediapipe/hands@0.1/${file}`; 23 | }}); 24 | hands.onResults(onResults); 25 | 26 | const camera = new Camera(videoElement, { 27 | onFrame: async () => { 28 | await hands.send({ image: videoElement }); 29 | }, 30 | width: 1280, 31 | height: 720 32 | }); 33 | camera.start(); 34 | 35 | // Present a control panel through which the user can manipulate the solution 36 | // options. 37 | new ControlPanel(controlsElement, { 38 | selfieMode: true, 39 | maxNumHands: 1, 40 | minDetectionConfidence: 0.5, 41 | minTrackingConfidence: 0.5, 42 | pauseCamera: false, 43 | }) 44 | .add([ 45 | new StaticText({ title: "Use your index fingertip to cut the green circles" }), 46 | // new Toggle({ title: "Selfie Mode", field: "selfieMode" }), 47 | new Toggle({ title: "Pause Camera", field: "pauseCamera" }), 48 | // new Slider( 49 | // {title: 'Max Number of Hands', field: 'maxNumHands', range: [1, 4], step: 1}), 50 | // new Slider({ 51 | // title: "Min Detection Confidence", 52 | // field: "minDetectionConfidence", 53 | // range: [0, 1], 54 | // step: 0.01 55 | // }), 56 | // new Slider({ 57 | // title: "Min Tracking Confidence", 58 | // field: "minTrackingConfidence", 59 | // range: [0, 1], 60 | // step: 0.01 61 | // }) 62 | ]) 63 | .on((options) => { 64 | videoElement.classList.toggle("selfie", options.selfieMode); 65 | hands.setOptions(options); 66 | // Add option to pause camera 67 | options.pauseCamera ? camera.video.pause() : camera.start(); 68 | }); 69 | 70 | 71 | var w = window.innerWidth; 72 | var h = window.innerHeight; 73 | 74 | var game = new Phaser.Game(w, h, Phaser.AUTO, 'game', 75 | { preload: preload, create: create, update: update, render: render }); 76 | 77 | function preload() { 78 | // this.load.image('toast', 'http://www.pngmart.com/files/5/Toast-PNG-Free-Download.png'); 79 | // this.load.image('burnt', 'http://pluspng.com/img-png/burnt-food-png-the-first-incident-involved-toast-or-to-be-more-precise-burnt-toast-246.png'); 80 | // this.load.image('toaster', 'https://purepng.com/public/uploads/large/purepng.com-toastertoastertoast-makerelectric-smalltoast-sliced-breadheat-17015284328352zoyd.png'); 81 | 82 | var bmd = game.add.bitmapData(100,100); 83 | bmd.ctx.fillStyle = '#00ff00'; 84 | bmd.ctx.arc(50,50,50, 0, Math.PI * 2); 85 | bmd.ctx.fill(); 86 | game.cache.addBitmapData('good', bmd); 87 | 88 | var bmd = game.add.bitmapData(64,64); 89 | bmd.ctx.fillStyle = '#ff0000'; 90 | bmd.ctx.arc(32,32,32, 0, Math.PI * 2); 91 | bmd.ctx.fill(); 92 | game.cache.addBitmapData('bad', bmd); 93 | } 94 | 95 | var good_objects, 96 | bad_objects, 97 | slashes, 98 | line, 99 | scoreLabel, 100 | score = 0, 101 | points = []; 102 | 103 | var fireRate = 1000; 104 | var nextFire = 0; 105 | 106 | 107 | function create() { 108 | 109 | //this.add.image(400, 300, 'toast'); 110 | 111 | game.physics.startSystem(Phaser.Physics.ARCADE); 112 | game.physics.arcade.gravity.y = 300; 113 | 114 | good_objects = createGroup(4, game.cache.getBitmapData('good')); 115 | bad_objects = createGroup(4, game.cache.getBitmapData('bad')); 116 | 117 | slashes = game.add.graphics(0, 0); 118 | 119 | scoreLabel = game.add.text(10,10,'Tip: get the green ones!'); 120 | scoreLabel.fill = 'white'; 121 | 122 | emitter = game.add.emitter(0, 0, 300); 123 | emitter.makeParticles('parts'); 124 | emitter.gravity = 300; 125 | emitter.setYSpeed(-400,400); 126 | 127 | throwObject(); 128 | } 129 | 130 | function createGroup (numItems, sprite) { 131 | var group = game.add.group(); 132 | group.enableBody = true; 133 | group.physicsBodyType = Phaser.Physics.ARCADE; 134 | group.createMultiple(numItems, sprite); 135 | group.setAll('checkWorldBounds', true); 136 | group.setAll('outOfBoundsKill', true); 137 | return group; 138 | } 139 | 140 | function throwObject() { 141 | if (game.time.now > nextFire && good_objects.countDead()>0 && bad_objects.countDead()>0) { 142 | nextFire = game.time.now + fireRate; 143 | throwGoodObject(); 144 | if (Math.random()>.5) { 145 | throwBadObject(); 146 | } 147 | } 148 | } 149 | 150 | function throwGoodObject() { 151 | var obj = good_objects.getFirstDead(); 152 | obj.reset(game.world.centerX + Math.random()*100-Math.random()*100, 600); 153 | obj.anchor.setTo(0.5, 0.5); 154 | //obj.body.angularAcceleration = 100; 155 | game.physics.arcade.moveToXY(obj, game.world.centerX, game.world.centerY, 530); 156 | } 157 | 158 | function throwBadObject() { 159 | var obj = bad_objects.getFirstDead(); 160 | obj.reset(game.world.centerX + Math.random()*100-Math.random()*100, 600); 161 | obj.anchor.setTo(0.5, 0.5); 162 | //obj.body.angularAcceleration = 100; 163 | game.physics.arcade.moveToXY(obj, game.world.centerX, game.world.centerY, 530); 164 | } 165 | 166 | function update() { 167 | throwObject(); 168 | 169 | points.push({ 170 | // x: game.input.x, 171 | // y: game.input.y 172 | x: global_x, 173 | y: global_y 174 | }); 175 | points = points.splice(points.length-10, points.length); 176 | //game.add.sprite(game.input.x, game.input.y, 'hit'); 177 | 178 | if (points.length<1 || points[0].x==0) { 179 | return; 180 | } 181 | 182 | slashes.clear(); 183 | slashes.beginFill(0xFFFFFF); 184 | slashes.alpha = .5; 185 | slashes.moveTo(points[0].x, points[0].y); 186 | for (var i=1; i 110) { 216 | return; 217 | } 218 | 219 | if (fruit.parent == good_objects) { 220 | killFruit(fruit); 221 | } else { 222 | resetScore(); 223 | } 224 | } 225 | 226 | } 227 | 228 | function resetScore() { 229 | var highscore = Math.max(score, localStorage.getItem("highscore")); 230 | localStorage.setItem("highscore", highscore); 231 | 232 | good_objects.forEachExists(killFruit); 233 | bad_objects.forEachExists(killFruit); 234 | 235 | score = 0; 236 | scoreLabel.text = 'Game Over!\nHigh Score: '+highscore; 237 | // Retrieve 238 | } 239 | 240 | function render() { 241 | } 242 | 243 | function killFruit(fruit) { 244 | 245 | emitter.x = fruit.x; 246 | emitter.y = fruit.y; 247 | emitter.start(true, 2000, null, 4); 248 | fruit.kill(); 249 | points = []; 250 | score++; 251 | scoreLabel.text = 'Score: ' + score; 252 | } 253 | -------------------------------------------------------------------------------- /java/hand/codepen.css: -------------------------------------------------------------------------------- 1 | @keyframes spin { 2 | 0% { transform: rotate(0deg); } 3 | 100% { transform: rotate(360deg); } 4 | } 5 | 6 | .abs { 7 | position: absolute; 8 | } 9 | 10 | a { 11 | color: white; 12 | text-decoration: none; 13 | &:hover { 14 | color: lightblue; 15 | } 16 | } 17 | 18 | body { 19 | bottom: 0; 20 | font-family: 'Titillium Web', sans-serif; 21 | color: white; 22 | left: 0; 23 | margin: 0; 24 | position: absolute; 25 | right: 0; 26 | top: 0; 27 | transform-origin: 0px 0px; 28 | overflow: hidden; 29 | } 30 | 31 | .container { 32 | position: absolute; 33 | background-color: #596e73; 34 | height: 720px; 35 | width: 1280px; 36 | } 37 | 38 | .input_video { 39 | position:relative; 40 | top: 0; 41 | left: 0; 42 | right: 0; 43 | bottom: 0; 44 | &.selfie { 45 | transform: scale(-1, 1); 46 | } 47 | } 48 | 49 | .output_canvas { 50 | position:absolute; 51 | height: 720px; 52 | width: 1280px; 53 | left: 0; 54 | top: 0; 55 | } 56 | 57 | .logo { 58 | bottom: 10px; 59 | right: 20px; 60 | 61 | .title { 62 | color: white; 63 | font-size: 28px; 64 | } 65 | 66 | .subtitle { 67 | position: relative; 68 | color: white; 69 | font-size: 10px; 70 | left: -30px; 71 | top: 20px; 72 | } 73 | } 74 | 75 | .control-panel { 76 | position: absolute; 77 | left: 10px; 78 | top: 10px; 79 | } 80 | 81 | .loading { 82 | display: flex; 83 | position: absolute; 84 | top: 0; 85 | right: 0; 86 | bottom: 0; 87 | left: 0; 88 | align-items: center; 89 | backface-visibility: hidden; 90 | justify-content: center; 91 | opacity: 1; 92 | transition: opacity 1s; 93 | 94 | .message { 95 | font-size: x-large; 96 | } 97 | 98 | .spinner { 99 | position: absolute; 100 | width: 120px; 101 | height: 120px; 102 | animation: spin 1s linear infinite; 103 | border: 32px solid #bebebe; 104 | border-top: 32px solid #3498db; 105 | border-radius: 50%; 106 | } 107 | } 108 | 109 | .loaded .loading { 110 | opacity: 0; 111 | } 112 | 113 | .shoutout { 114 | left: 0px; 115 | bottom: 160px; 116 | position: absolute; 117 | width: 50%; 118 | text-align: left; 119 | font-size: 24px; 120 | font-family: "Lucida Console", monospace; 121 | color: white; 122 | background-color: black; 123 | } 124 | 125 | .progressBar { 126 | left: 0px; 127 | bottom: 40px; 128 | position: absolute; 129 | width: 50%; 130 | text-align: right; 131 | font-size: 24px; 132 | line-height: 24px; 133 | font-family: "Lucida Console", monospace; 134 | background: linear-gradient(90deg, #D71440, #FFDD00, #07B152); 135 | direction: rtl; 136 | } 137 | 138 | #pThumb { 139 | color: white; 140 | background-color: black; 141 | } 142 | #pIndex { 143 | color: white; 144 | background-color: black; 145 | } 146 | #pMiddle { 147 | color: white; 148 | background-color: black; 149 | } 150 | #pRing { 151 | color: white; 152 | background-color: black; 153 | } 154 | #pLittle { 155 | color: white; 156 | background-color: black; 157 | } -------------------------------------------------------------------------------- /java/hand/codepen.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | 6 |
7 | Loading 8 |
9 |
10 | 17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
100%
29 |
100%
30 |
100%
31 |
100%
32 |
100%
33 |
-------------------------------------------------------------------------------- /java/hand/codepen.js: -------------------------------------------------------------------------------- 1 | // Our input frames will come from here. 2 | const videoElement = document.getElementsByClassName('input_video')[0]; 3 | const canvasElement = document.getElementsByClassName('output_canvas')[0]; 4 | const controlsElement = document.getElementsByClassName('control-panel')[0]; 5 | const canvasCtx = canvasElement.getContext('2d'); 6 | 7 | // We'll add this to our control panel later, but we'll save it here so we can 8 | // call tick() each time the graph runs. 9 | const fpsControl = new FPS(); 10 | 11 | // Optimization: Turn off animated spinner after its hiding animation is done. 12 | const spinner = document.querySelector('.loading'); 13 | spinner.ontransitionend = () => { 14 | spinner.style.display = 'none'; 15 | }; 16 | 17 | // Exponential filter to filter out high frequency noise 18 | var alpha = 0.5; 19 | var thumb1 = 0; 20 | var thumb2 = 0; 21 | var thumb3 = 0; 22 | var index1 = 0; 23 | var index2 = 0; 24 | var index3 = 0; 25 | var middle1 = 0; 26 | var middle2 = 0; 27 | var middle3 = 0; 28 | var ring1 = 0; 29 | var ring2 = 0; 30 | var ring3 = 0; 31 | var little1 = 0; 32 | var little2 = 0; 33 | var little3 = 0; 34 | 35 | // For progress display 36 | var progress1 = 0; 37 | var progress2 = 0; 38 | var progress3 = 0; 39 | var progress4 = 0; 40 | var progress5 = 0; 41 | 42 | // For playing sound 43 | var enableSound = false; 44 | var soundThres = 20; 45 | var sound1 = new Audio('https://assets.codepen.io/5628206/beat_1.mp3'); 46 | var sound2 = new Audio('https://assets.codepen.io/5628206/beat_2.mp3'); 47 | var sound3 = new Audio('https://assets.codepen.io/5628206/beat_3.mp3'); 48 | var sound4 = new Audio('https://assets.codepen.io/5628206/beat_4.mp3'); 49 | var sound5 = new Audio('https://assets.codepen.io/5628206/beat_5.mp3'); 50 | var play1 = false; 51 | var play2 = false; 52 | var play3 = false; 53 | var play4 = false; 54 | var play5 = false; 55 | 56 | function onResults(results) { 57 | // Hide the spinner. 58 | document.body.classList.add('loaded'); 59 | 60 | // Update the frame rate. 61 | fpsControl.tick(); 62 | 63 | // Draw the overlays. 64 | canvasCtx.save(); 65 | canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height); 66 | canvasCtx.drawImage( 67 | results.image, 0, 0, canvasElement.width, canvasElement.height); 68 | if (results.multiHandLandmarks && results.multiHandedness) { 69 | for (let index = 0; index < results.multiHandLandmarks.length; index++) { 70 | const classification = results.multiHandedness[index]; 71 | const isRightHand = classification.label === 'Right'; 72 | const landmarks = results.multiHandLandmarks[index]; 73 | drawConnectors( 74 | canvasCtx, landmarks, HAND_CONNECTIONS, 75 | {color: isRightHand ? '#00FF00' : '#FF0000'}), 76 | drawLandmarks(canvasCtx, landmarks, { 77 | color: isRightHand ? '#00FF00' : '#FF0000', 78 | fillColor: isRightHand ? '#FF0000' : '#00FF00', 79 | radius: (x) => { 80 | return lerp(x.from.z, -0.15, .1, 10, 1); 81 | } 82 | }); 83 | } 84 | 85 | // Compute hand joint flexion angles 86 | // thumb1 = alpha*getFlexionAng(results, 0,1,2) + (1-alpha)*thumb1; // CMC hide this value as it is not accurate 87 | thumb2 = alpha*getFlexionAng(results, 1,2,3) + (1-alpha)*thumb2; // MCP 88 | thumb3 = alpha*getFlexionAng(results, 2,3,4) + (1-alpha)*thumb3; // IP 89 | 90 | index1 = alpha*getFlexionAng(results, 0,5,6) + (1-alpha)*index1; // MCP 91 | index2 = alpha*getFlexionAng(results, 5,6,7) + (1-alpha)*index2; // PIP 92 | index3 = alpha*getFlexionAng(results, 6,7,8) + (1-alpha)*index3; // DIP 93 | 94 | middle1 = alpha*getFlexionAng(results, 0, 9,10) + (1-alpha)*middle1; // MCP 95 | middle2 = alpha*getFlexionAng(results, 9,10,11) + (1-alpha)*middle2; // PIP 96 | middle3 = alpha*getFlexionAng(results, 10,11,12) + (1-alpha)*middle3; // DIP 97 | 98 | ring1 = alpha*getFlexionAng(results, 0,13,14) + (1-alpha)*ring1; // MCP 99 | ring2 = alpha*getFlexionAng(results, 13,14,15) + (1-alpha)*ring2; // PIP 100 | ring3 = alpha*getFlexionAng(results, 14,15,16) + (1-alpha)*ring3; // DIP 101 | 102 | little1 = alpha*getFlexionAng(results, 0,17,18) + (1-alpha)*little1; // MCP 103 | little2 = alpha*getFlexionAng(results, 17,18,19) + (1-alpha)*little2; // PIP 104 | little3 = alpha*getFlexionAng(results, 18,19,20) + (1-alpha)*little3; // DIP 105 | 106 | // Display hand joint angles 107 | document.getElementById('Thumb' ).innerHTML = 108 | 'T. MCP: ' + 109 | padLeadingZeros(thumb2.toFixed(0), 3) + 110 | ' IP: ' + 111 | padLeadingZeros(thumb3.toFixed(0), 3) + ' deg'; 112 | document.getElementById('Index' ).innerHTML = 113 | 'I. MCP: ' + 114 | padLeadingZeros(index1.toFixed(0), 3) + 115 | ' PIP: ' + 116 | padLeadingZeros(index2.toFixed(0), 3) + 117 | ' DIP: ' + 118 | padLeadingZeros(index3.toFixed(0), 3) + ' deg'; 119 | document.getElementById('Middle').innerHTML = 120 | 'M. MCP: ' + 121 | padLeadingZeros(middle1.toFixed(0), 3) + 122 | ' PIP: ' + 123 | padLeadingZeros(middle2.toFixed(0), 3) + 124 | ' DIP: ' + 125 | padLeadingZeros(middle3.toFixed(0), 3) + ' deg'; 126 | document.getElementById('Ring' ).innerHTML = 127 | 'R. MCP: ' + 128 | padLeadingZeros(ring1.toFixed(0), 3) + 129 | ' PIP: ' + 130 | padLeadingZeros(ring2.toFixed(0), 3) + 131 | ' DIP: ' + 132 | padLeadingZeros(ring3.toFixed(0), 3) + ' deg'; 133 | document.getElementById('Little').innerHTML = 134 | 'L. MCP: ' + 135 | padLeadingZeros(little1.toFixed(0), 3) + 136 | ' PIP: ' + 137 | padLeadingZeros(little2.toFixed(0), 3) + 138 | ' DIP: ' + 139 | padLeadingZeros(little3.toFixed(0), 3) + ' deg'; 140 | 141 | // Display progress bar 142 | progress1 = ((thumb2 + thumb3)/180*100).toFixed(0); 143 | document.getElementById("pThumb").style.width = progress1 + "%"; 144 | document.getElementById("pThumb").innerHTML = progress1 + "%"; 145 | progress2 = ((index1 + index2 + index3)/270*100).toFixed(0); 146 | document.getElementById("pIndex").style.width = progress2 + "%"; 147 | document.getElementById("pIndex").innerHTML = progress2 + "%"; 148 | progress3 = ((middle1 + middle2 + middle3)/270*100).toFixed(0); 149 | document.getElementById("pMiddle").style.width = progress3 + "%"; 150 | document.getElementById("pMiddle").innerHTML = progress3 + "%"; 151 | progress4 = ((ring1 + ring2 + ring3)/270*100).toFixed(0); 152 | document.getElementById("pRing").style.width = progress4 + "%"; 153 | document.getElementById("pRing").innerHTML = progress4 + "%"; 154 | progress5 = ((little1 + little2 + little3)/270*100).toFixed(0); 155 | document.getElementById("pLittle").style.width = progress5 + "%"; 156 | document.getElementById("pLittle").innerHTML = progress5 + "%"; 157 | 158 | // Play sound 159 | if(enableSound) { 160 | if((thumb3 > soundThres) && (!play1)) { 161 | sound1.currentTime = 0; 162 | sound1.play(); 163 | play1 = true; 164 | } 165 | if(thumb3 <= soundThres) { 166 | play1 = false; 167 | } 168 | if((index1 > soundThres) && (!play2)) { 169 | sound2.currentTime = 0; 170 | sound2.play(); 171 | play2 = true; 172 | } 173 | if(index1 <= soundThres) { 174 | play2 = false; 175 | } 176 | if((middle1 > soundThres) && (!play3)) { 177 | sound3.currentTime = 0; 178 | sound3.play(); 179 | play3 = true; 180 | } 181 | if(middle1 <= soundThres) { 182 | play3 = false; 183 | } 184 | if((ring1 > soundThres) && (!play4)) { 185 | sound4.currentTime = 0; 186 | sound4.play(); 187 | play4 = true; 188 | } 189 | if(ring1 <= soundThres) { 190 | play4 = false; 191 | } 192 | if((little1 > soundThres) && (!play5)) { 193 | sound5.currentTime = 0; 194 | sound5.play(); 195 | play5 = true; 196 | } 197 | if(little1 <= soundThres) { 198 | play5 = false; 199 | } 200 | } 201 | } 202 | canvasCtx.restore(); 203 | } 204 | 205 | const hands = new Hands({locateFile: (file) => { 206 | return `https://cdn.jsdelivr.net/npm/@mediapipe/hands@0.1/${file}`; 207 | }}); 208 | hands.onResults(onResults); 209 | 210 | /** 211 | * Instantiate a camera. We'll feed each frame we receive into the solution. 212 | */ 213 | const camera = new Camera(videoElement, { 214 | onFrame: async () => { 215 | await hands.send({image: videoElement}); 216 | }, 217 | width: 1280, 218 | height: 720 219 | }); 220 | camera.start(); 221 | 222 | // Present a control panel through which the user can manipulate the solution 223 | // options. 224 | new ControlPanel(controlsElement, { 225 | selfieMode: true, 226 | maxNumHands: 1, 227 | minDetectionConfidence: 0.5, 228 | minTrackingConfidence: 0.5, 229 | pauseCamera: false, 230 | playSound: false, 231 | }) 232 | .add([ 233 | new StaticText({title: 'Hand flexion angle'}), 234 | fpsControl, 235 | new Toggle({title: 'Selfie Mode', field: 'selfieMode'}), 236 | new Toggle({ title: "Pause Camera", field: "pauseCamera" }), 237 | new Toggle({ title: "Play Sound", field: "playSound" }), 238 | // new Slider( 239 | // {title: 'Max Number of Hands', field: 'maxNumHands', range: [1, 4], step: 1}), 240 | // new Slider({ 241 | // title: 'Min Detection Confidence', 242 | // field: 'minDetectionConfidence', 243 | // range: [0, 1], 244 | // step: 0.01 245 | // }), 246 | // new Slider({ 247 | // title: 'Min Tracking Confidence', 248 | // field: 'minTrackingConfidence', 249 | // range: [0, 1], 250 | // step: 0.01 251 | // }), 252 | ]) 253 | .on(options => { 254 | videoElement.classList.toggle('selfie', options.selfieMode); 255 | hands.setOptions(options); 256 | // Add option to pause camera 257 | options.pauseCamera ? camera.video.pause() : camera.start(); 258 | // Add option to play sound 259 | options.playSound ? enableSound=true : enableSound=false; 260 | }); 261 | 262 | // Added by GM 263 | function getFlexionAng(results, j0, j1, j2) { 264 | // j0, j1, j2 are joint indexes of multiHandLandmarks 265 | // 1st: rescale landmarks from relative coor to absolute 3D coor 266 | var A = rescale(results.multiHandLandmarks[0][j0]); 267 | var B = rescale(results.multiHandLandmarks[0][j1]); 268 | var C = rescale(results.multiHandLandmarks[0][j2]); 269 | // 2nd: Find the acute angle at joint j1 270 | return getAngBtw3Pts(A, B, C); 271 | } 272 | 273 | function rescale(lm) { 274 | // Convert landmarks from relative coor to absolute 3D coor 275 | // Note: Assume image width:1280, height:720 276 | var A = {x:0, y:0, z:0}; 277 | A.x = lm.x * 1280 - 640; 278 | A.y = lm.y * 720 - 360; 279 | A.z = lm.z * 1280; 280 | return A; 281 | } 282 | 283 | function getAngBtw3Pts(A, B, C) { 284 | // Note: Points A, B and C are 3D points with x,y,z 285 | // 1st: Find vector AB (a,b,c) and BC (d,e,f) 286 | // 2nd: Find acute angle between vector AB and BC using cosine rule 287 | 288 | // Find vector AB = OB - OA 289 | var a = B.x - A.x; 290 | var b = B.y - A.y; 291 | var c = B.z - A.z; 292 | // Find vector BC = OC - OB 293 | var d = C.x - B.x; 294 | var e = C.y - B.y; 295 | var f = C.z - B.z; 296 | 297 | // BA.BC -> dot product 298 | var num = a*d + b*e + c*f; 299 | // |BA|*|BC| -> multiply norm of BA and BC 300 | var den = Math.sqrt(a*a + b*b + c*c) * Math.sqrt(d*d + e*e + f*f); 301 | // ang = acos(BA.BC / |BA|*|BC|) 302 | var ang = Math.acos(num/den); 303 | 304 | // Convert radian to degree 305 | return ang * 180 / Math.PI; 306 | } 307 | 308 | function padLeadingZeros(num, size) { 309 | // Adapted from https://www.codegrepper.com/code-examples/javascript/convert+to+number+keep+leading+zeros+javascript 310 | var s = num+""; 311 | while (s.length < size) s = "0" + s; 312 | return s; 313 | } -------------------------------------------------------------------------------- /java/hand/head.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /java/pose-fruit-ninja/codepen.css: -------------------------------------------------------------------------------- 1 | html, body { 2 | padding: 0; 3 | margin: 0; 4 | } 5 | 6 | .control-panel { 7 | position: absolute; 8 | left: 10px; 9 | top: 40px; 10 | } 11 | 12 | .input_video { 13 | position:absolute; 14 | top: 0px; 15 | left: 0px; 16 | opacity: 0.5; 17 | width:1280px; 18 | height:720px; 19 | &.selfie { 20 | transform: scale(-1, 1); 21 | } 22 | } 23 | 24 | div#game { 25 | width:100%; 26 | height:100%; 27 | } 28 | 29 | #mouse-circle { 30 | position: absolute; 31 | width: 30px; 32 | height: 30px; 33 | border-radius: 50%; 34 | background-color: yellow; 35 | pointer-events: none !important; 36 | } -------------------------------------------------------------------------------- /java/pose-fruit-ninja/codepen.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
-------------------------------------------------------------------------------- /java/pose-fruit-ninja/codepen.js: -------------------------------------------------------------------------------- 1 | // Note: Need to include the link below 2 | // https://cdnjs.cloudflare.com/ajax/libs/phaser/2.0.6/phaser.min.js 3 | // In Pen Settings -> JS -> Add External Scripts/Pens 4 | 5 | const videoElement = document.getElementsByClassName("input_video")[0]; 6 | const controlsElement = document.getElementsByClassName("control-panel")[0]; 7 | const mouseCircle = document.getElementById("mouse-circle"); 8 | 9 | var global_x = 0; 10 | var global_y = 0; 11 | 12 | function onResults(results) { 13 | global_x = results.poseLandmarks[15].x * 1280; 14 | global_y = results.poseLandmarks[15].y * 720; 15 | mouseCircle.style.left = (global_x-15) + "px"; 16 | mouseCircle.style.top = (global_y-15) + "px"; 17 | } 18 | 19 | const pose = new Pose({ 20 | locateFile: (file) => { 21 | return `https://cdn.jsdelivr.net/npm/@mediapipe/pose@0.2/${file}`; 22 | } 23 | }); 24 | pose.onResults(onResults); 25 | 26 | const camera = new Camera(videoElement, { 27 | onFrame: async () => { 28 | await pose.send({ image: videoElement }); 29 | }, 30 | width: 1280, 31 | height: 720 32 | }); 33 | camera.start(); 34 | 35 | // Present a control panel through which the user can manipulate the solution 36 | // options. 37 | new ControlPanel(controlsElement, { 38 | selfieMode: true, 39 | upperBodyOnly: true, 40 | smoothLandmarks: true, 41 | minDetectionConfidence: 0.5, 42 | minTrackingConfidence: 0.5, 43 | pauseCamera: false, 44 | }) 45 | .add([ 46 | new StaticText({ title: "Use your left hand to cut the green circles" }), 47 | // new Toggle({ title: "Selfie Mode", field: "selfieMode" }), 48 | // new Toggle({ title: "Upper-body Only", field: "upperBodyOnly" }), 49 | // new Toggle({ title: "Smooth Landmarks", field: "smoothLandmarks" }), 50 | new Toggle({ title: "Pause Camera", field: "pauseCamera" }), 51 | // new Slider({ 52 | // title: "Min Detection Confidence", 53 | // field: "minDetectionConfidence", 54 | // range: [0, 1], 55 | // step: 0.01 56 | // }), 57 | // new Slider({ 58 | // title: "Min Tracking Confidence", 59 | // field: "minTrackingConfidence", 60 | // range: [0, 1], 61 | // step: 0.01 62 | // }) 63 | ]) 64 | .on((options) => { 65 | videoElement.classList.toggle("selfie", options.selfieMode); 66 | pose.setOptions(options); 67 | // Add option to pause camera 68 | options.pauseCamera ? camera.video.pause() : camera.start(); 69 | }); 70 | 71 | 72 | var w = window.innerWidth; 73 | var h = window.innerHeight; 74 | 75 | var game = new Phaser.Game(w, h, Phaser.AUTO, 'game', 76 | { preload: preload, create: create, update: update, render: render }); 77 | 78 | function preload() { 79 | // this.load.image('toast', 'http://www.pngmart.com/files/5/Toast-PNG-Free-Download.png'); 80 | // this.load.image('burnt', 'http://pluspng.com/img-png/burnt-food-png-the-first-incident-involved-toast-or-to-be-more-precise-burnt-toast-246.png'); 81 | // this.load.image('toaster', 'https://purepng.com/public/uploads/large/purepng.com-toastertoastertoast-makerelectric-smalltoast-sliced-breadheat-17015284328352zoyd.png'); 82 | 83 | var bmd = game.add.bitmapData(100,100); 84 | bmd.ctx.fillStyle = '#00ff00'; 85 | bmd.ctx.arc(50,50,50, 0, Math.PI * 2); 86 | bmd.ctx.fill(); 87 | game.cache.addBitmapData('good', bmd); 88 | 89 | var bmd = game.add.bitmapData(64,64); 90 | bmd.ctx.fillStyle = '#ff0000'; 91 | bmd.ctx.arc(32,32,32, 0, Math.PI * 2); 92 | bmd.ctx.fill(); 93 | game.cache.addBitmapData('bad', bmd); 94 | } 95 | 96 | var good_objects, 97 | bad_objects, 98 | slashes, 99 | line, 100 | scoreLabel, 101 | score = 0, 102 | points = []; 103 | 104 | var fireRate = 1000; 105 | var nextFire = 0; 106 | 107 | 108 | function create() { 109 | 110 | //this.add.image(400, 300, 'toast'); 111 | 112 | game.physics.startSystem(Phaser.Physics.ARCADE); 113 | game.physics.arcade.gravity.y = 300; 114 | 115 | good_objects = createGroup(4, game.cache.getBitmapData('good')); 116 | bad_objects = createGroup(4, game.cache.getBitmapData('bad')); 117 | 118 | slashes = game.add.graphics(0, 0); 119 | 120 | scoreLabel = game.add.text(10,10,'Tip: get the green ones!'); 121 | scoreLabel.fill = 'white'; 122 | 123 | emitter = game.add.emitter(0, 0, 300); 124 | emitter.makeParticles('parts'); 125 | emitter.gravity = 300; 126 | emitter.setYSpeed(-400,400); 127 | 128 | throwObject(); 129 | } 130 | 131 | function createGroup (numItems, sprite) { 132 | var group = game.add.group(); 133 | group.enableBody = true; 134 | group.physicsBodyType = Phaser.Physics.ARCADE; 135 | group.createMultiple(numItems, sprite); 136 | group.setAll('checkWorldBounds', true); 137 | group.setAll('outOfBoundsKill', true); 138 | return group; 139 | } 140 | 141 | function throwObject() { 142 | if (game.time.now > nextFire && good_objects.countDead()>0 && bad_objects.countDead()>0) { 143 | nextFire = game.time.now + fireRate; 144 | throwGoodObject(); 145 | if (Math.random()>.5) { 146 | throwBadObject(); 147 | } 148 | } 149 | } 150 | 151 | function throwGoodObject() { 152 | var obj = good_objects.getFirstDead(); 153 | obj.reset(game.world.centerX + Math.random()*100-Math.random()*100, 600); 154 | obj.anchor.setTo(0.5, 0.5); 155 | //obj.body.angularAcceleration = 100; 156 | game.physics.arcade.moveToXY(obj, game.world.centerX, game.world.centerY, 530); 157 | } 158 | 159 | function throwBadObject() { 160 | var obj = bad_objects.getFirstDead(); 161 | obj.reset(game.world.centerX + Math.random()*100-Math.random()*100, 600); 162 | obj.anchor.setTo(0.5, 0.5); 163 | //obj.body.angularAcceleration = 100; 164 | game.physics.arcade.moveToXY(obj, game.world.centerX, game.world.centerY, 530); 165 | } 166 | 167 | function update() { 168 | throwObject(); 169 | 170 | points.push({ 171 | // x: game.input.x, 172 | // y: game.input.y 173 | x: global_x, 174 | y: global_y 175 | }); 176 | points = points.splice(points.length-10, points.length); 177 | //game.add.sprite(game.input.x, game.input.y, 'hit'); 178 | 179 | if (points.length<1 || points[0].x==0) { 180 | return; 181 | } 182 | 183 | slashes.clear(); 184 | slashes.beginFill(0xFFFFFF); 185 | slashes.alpha = .5; 186 | slashes.moveTo(points[0].x, points[0].y); 187 | for (var i=1; i 110) { 217 | return; 218 | } 219 | 220 | if (fruit.parent == good_objects) { 221 | killFruit(fruit); 222 | } else { 223 | resetScore(); 224 | } 225 | } 226 | 227 | } 228 | 229 | function resetScore() { 230 | var highscore = Math.max(score, localStorage.getItem("highscore")); 231 | localStorage.setItem("highscore", highscore); 232 | 233 | good_objects.forEachExists(killFruit); 234 | bad_objects.forEachExists(killFruit); 235 | 236 | score = 0; 237 | scoreLabel.text = 'Game Over!\nHigh Score: '+highscore; 238 | // Retrieve 239 | } 240 | 241 | function render() { 242 | } 243 | 244 | function killFruit(fruit) { 245 | 246 | emitter.x = fruit.x; 247 | emitter.y = fruit.y; 248 | emitter.start(true, 2000, null, 4); 249 | fruit.kill(); 250 | points = []; 251 | score++; 252 | scoreLabel.text = 'Score: ' + score; 253 | } 254 | -------------------------------------------------------------------------------- /result.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/result.gif -------------------------------------------------------------------------------- /result_fan.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/result_fan.gif -------------------------------------------------------------------------------- /result_fy_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kairess/Rock-Paper-Scissors-Machine/fd5f4f2f6541206675d0833e096b66db90a9a61c/result_fy_filter.png -------------------------------------------------------------------------------- /single.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import mediapipe as mp 3 | import numpy as np 4 | 5 | max_num_hands = 1 6 | gesture = { 7 | 0:'fist', 1:'one', 2:'two', 3:'three', 4:'four', 5:'five', 8 | 6:'six', 7:'rock', 8:'spiderman', 9:'yeah', 10:'ok', 9 | } 10 | rps_gesture = {0:'rock', 5:'paper', 9:'scissors'} 11 | 12 | # MediaPipe hands model 13 | mp_hands = mp.solutions.hands 14 | mp_drawing = mp.solutions.drawing_utils 15 | hands = mp_hands.Hands( 16 | max_num_hands=max_num_hands, 17 | min_detection_confidence=0.5, 18 | min_tracking_confidence=0.5) 19 | 20 | # Gesture recognition model 21 | file = np.genfromtxt('data/gesture_train.csv', delimiter=',') 22 | angle = file[:,:-1].astype(np.float32) 23 | label = file[:, -1].astype(np.float32) 24 | knn = cv2.ml.KNearest_create() 25 | knn.train(angle, cv2.ml.ROW_SAMPLE, label) 26 | 27 | cap = cv2.VideoCapture(0) 28 | 29 | while cap.isOpened(): 30 | ret, img = cap.read() 31 | if not ret: 32 | continue 33 | 34 | img = cv2.flip(img, 1) 35 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 36 | 37 | result = hands.process(img) 38 | 39 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 40 | 41 | if result.multi_hand_landmarks is not None: 42 | for res in result.multi_hand_landmarks: 43 | joint = np.zeros((21, 3)) 44 | for j, lm in enumerate(res.landmark): 45 | joint[j] = [lm.x, lm.y, lm.z] 46 | 47 | # Compute angles between joints 48 | v1 = joint[[0,1,2,3,0,5,6,7,0,9,10,11,0,13,14,15,0,17,18,19],:] # Parent joint 49 | v2 = joint[[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],:] # Child joint 50 | v = v2 - v1 # [20,3] 51 | # Normalize v 52 | v = v / np.linalg.norm(v, axis=1)[:, np.newaxis] 53 | 54 | # Get angle using arcos of dot product 55 | angle = np.arccos(np.einsum('nt,nt->n', 56 | v[[0,1,2,4,5,6,8,9,10,12,13,14,16,17,18],:], 57 | v[[1,2,3,5,6,7,9,10,11,13,14,15,17,18,19],:])) # [15,] 58 | 59 | angle = np.degrees(angle) # Convert radian to degree 60 | 61 | # Inference gesture 62 | data = np.array([angle], dtype=np.float32) 63 | ret, results, neighbours, dist = knn.findNearest(data, 3) 64 | idx = int(results[0][0]) 65 | 66 | # Draw gesture result 67 | if idx in rps_gesture.keys(): 68 | cv2.putText(img, text=rps_gesture[idx].upper(), org=(int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0] + 20)), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 69 | 70 | # Other gestures 71 | # cv2.putText(img, text=gesture[idx].upper(), org=(int(res.landmark[0].x * img.shape[1]), int(res.landmark[0].y * img.shape[0] + 20)), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255, 255, 255), thickness=2) 72 | 73 | mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS) 74 | 75 | cv2.imshow('Game', img) 76 | if cv2.waitKey(1) == ord('q'): 77 | break 78 | --------------------------------------------------------------------------------