├── MoveNet Refined.ipynb └── MoveNet Tutorial.ipynb /MoveNet Refined.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 0. Install and Import Dependencies" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "!pip install tensorflow==2.4.1 tensorflow-gpu==2.4.1 opencv-python matplotlib" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import tensorflow as tf\n", 26 | "import numpy as np\n", 27 | "from matplotlib import pyplot as plt\n", 28 | "import cv2" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "# 1. Load Model" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "interpreter = tf.lite.Interpreter(model_path='lite-model_movenet_singlepose_lightning_3.tflite')\n", 45 | "interpreter.allocate_tensors()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "# 2. Make Detections" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "cap = cv2.VideoCapture(0)\n", 62 | "while cap.isOpened():\n", 63 | " ret, frame = cap.read()\n", 64 | " \n", 65 | " # Reshape image\n", 66 | " img = frame.copy()\n", 67 | " img = tf.image.resize_with_pad(np.expand_dims(img, axis=0), 192,192)\n", 68 | " input_image = tf.cast(img, dtype=tf.float32)\n", 69 | " \n", 70 | " # Setup input and output \n", 71 | " input_details = interpreter.get_input_details()\n", 72 | " output_details = interpreter.get_output_details()\n", 73 | " \n", 74 | " # Make predictions \n", 75 | " interpreter.set_tensor(input_details[0]['index'], np.array(input_image))\n", 76 | " interpreter.invoke()\n", 77 | " keypoints_with_scores = interpreter.get_tensor(output_details[0]['index'])\n", 78 | " \n", 79 | " # Rendering \n", 80 | " draw_connections(frame, keypoints_with_scores, EDGES, 0.4)\n", 81 | " draw_keypoints(frame, keypoints_with_scores, 0.4)\n", 82 | " \n", 83 | " cv2.imshow('MoveNet Lightning', frame)\n", 84 | " \n", 85 | " if cv2.waitKey(10) & 0xFF==ord('q'):\n", 86 | " break\n", 87 | " \n", 88 | "cap.release()\n", 89 | "cv2.destroyAllWindows()" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "# 3. Draw Keypoints" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "def draw_keypoints(frame, keypoints, confidence_threshold):\n", 106 | " y, x, c = frame.shape\n", 107 | " shaped = np.squeeze(np.multiply(keypoints, [y,x,1]))\n", 108 | " \n", 109 | " for kp in shaped:\n", 110 | " ky, kx, kp_conf = kp\n", 111 | " if kp_conf > confidence_threshold:\n", 112 | " cv2.circle(frame, (int(kx), int(ky)), 4, (0,255,0), -1) " 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "# 4. Draw Edges" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "EDGES = {\n", 129 | " (0, 1): 'm',\n", 130 | " (0, 2): 'c',\n", 131 | " (1, 3): 'm',\n", 132 | " (2, 4): 'c',\n", 133 | " (0, 5): 'm',\n", 134 | " (0, 6): 'c',\n", 135 | " (5, 7): 'm',\n", 136 | " (7, 9): 'm',\n", 137 | " (6, 8): 'c',\n", 138 | " (8, 10): 'c',\n", 139 | " (5, 6): 'y',\n", 140 | " (5, 11): 'm',\n", 141 | " (6, 12): 'c',\n", 142 | " (11, 12): 'y',\n", 143 | " (11, 13): 'm',\n", 144 | " (13, 15): 'm',\n", 145 | " (12, 14): 'c',\n", 146 | " (14, 16): 'c'\n", 147 | "}" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "def draw_connections(frame, keypoints, edges, confidence_threshold):\n", 157 | " y, x, c = frame.shape\n", 158 | " shaped = np.squeeze(np.multiply(keypoints, [y,x,1]))\n", 159 | " \n", 160 | " for edge, color in edges.items():\n", 161 | " p1, p2 = edge\n", 162 | " y1, x1, c1 = shaped[p1]\n", 163 | " y2, x2, c2 = shaped[p2]\n", 164 | " \n", 165 | " if (c1 > confidence_threshold) & (c2 > confidence_threshold): \n", 166 | " cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), (0,0,255), 2)" 167 | ] 168 | } 169 | ], 170 | "metadata": { 171 | "kernelspec": { 172 | "display_name": "movenet", 173 | "language": "python", 174 | "name": "movenet" 175 | }, 176 | "language_info": { 177 | "codemirror_mode": { 178 | "name": "ipython", 179 | "version": 3 180 | }, 181 | "file_extension": ".py", 182 | "mimetype": "text/x-python", 183 | "name": "python", 184 | "nbconvert_exporter": "python", 185 | "pygments_lexer": "ipython3", 186 | "version": "3.7.3" 187 | } 188 | }, 189 | "nbformat": 4, 190 | "nbformat_minor": 2 191 | } 192 | -------------------------------------------------------------------------------- /MoveNet Tutorial.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# 0. Install and Import Dependencies" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "!pip install tensorflow==2.4.1 tensorflow-gpu==2.4.1 opencv-python matplotlib" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import tensorflow as tf\n", 26 | "import numpy as np\n", 27 | "from matplotlib import pyplot as plt\n", 28 | "import cv2" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "# 1. Load Model" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "interpreter = tf.lite.Interpreter(model_path='lite-model_movenet_singlepose_lightning_3.tflite')\n", 45 | "interpreter.allocate_tensors()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "# 2. Make Detections" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": null, 58 | "metadata": { 59 | "scrolled": true 60 | }, 61 | "outputs": [], 62 | "source": [ 63 | "plt.imshow(tf.cast(np.squeeze(img), dtype=tf.int32))" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": null, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "img = frame.copy()" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "img.shape" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "interpreter.get_output_details()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "cap = cv2.VideoCapture(0)\n", 100 | "while cap.isOpened():\n", 101 | " ret, frame = cap.read()\n", 102 | " \n", 103 | " # Reshape image\n", 104 | " img = frame.copy()\n", 105 | " img = tf.image.resize_with_pad(np.expand_dims(img, axis=0), 192,192)\n", 106 | " input_image = tf.cast(img, dtype=tf.float32)\n", 107 | " \n", 108 | " # Setup input and output \n", 109 | " input_details = interpreter.get_input_details()\n", 110 | " output_details = interpreter.get_output_details()\n", 111 | " \n", 112 | " # Make predictions \n", 113 | " interpreter.set_tensor(input_details[0]['index'], np.array(input_image))\n", 114 | " interpreter.invoke()\n", 115 | " keypoints_with_scores = interpreter.get_tensor(output_details[0]['index'])\n", 116 | " print(keypoints_with_scores)\n", 117 | " \n", 118 | " # Rendering \n", 119 | " draw_connections(frame, keypoints_with_scores, EDGES, 0.4)\n", 120 | " draw_keypoints(frame, keypoints_with_scores, 0.4)\n", 121 | " \n", 122 | " cv2.imshow('MoveNet Lightning', frame)\n", 123 | " \n", 124 | " if cv2.waitKey(10) & 0xFF==ord('q'):\n", 125 | " break\n", 126 | " \n", 127 | "cap.release()\n", 128 | "cv2.destroyAllWindows()" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "right_eye = keypoints_with_scores[0][0][2]\n", 138 | "left_elbow = keypoints_with_scores[0][0][7]" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": null, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "shaped = np.squeeze(np.multiply(interpreter.get_tensor(interpreter.get_output_details()[0]['index']), [480,640,1]))" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "for kp in shaped:\n", 157 | " ky, kx, kp_conf = kp\n", 158 | " print(int(ky), int(kx), kp_conf)" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "# 3. Draw Keypoints" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [ 174 | "def draw_keypoints(frame, keypoints, confidence_threshold):\n", 175 | " y, x, c = frame.shape\n", 176 | " shaped = np.squeeze(np.multiply(keypoints, [y,x,1]))\n", 177 | " \n", 178 | " for kp in shaped:\n", 179 | " ky, kx, kp_conf = kp\n", 180 | " if kp_conf > confidence_threshold:\n", 181 | " cv2.circle(frame, (int(kx), int(ky)), 4, (0,255,0), -1) " 182 | ] 183 | }, 184 | { 185 | "cell_type": "markdown", 186 | "metadata": {}, 187 | "source": [ 188 | "# 4. Draw Edges" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": null, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "EDGES = {\n", 198 | " (0, 1): 'm',\n", 199 | " (0, 2): 'c',\n", 200 | " (1, 3): 'm',\n", 201 | " (2, 4): 'c',\n", 202 | " (0, 5): 'm',\n", 203 | " (0, 6): 'c',\n", 204 | " (5, 7): 'm',\n", 205 | " (7, 9): 'm',\n", 206 | " (6, 8): 'c',\n", 207 | " (8, 10): 'c',\n", 208 | " (5, 6): 'y',\n", 209 | " (5, 11): 'm',\n", 210 | " (6, 12): 'c',\n", 211 | " (11, 12): 'y',\n", 212 | " (11, 13): 'm',\n", 213 | " (13, 15): 'm',\n", 214 | " (12, 14): 'c',\n", 215 | " (14, 16): 'c'\n", 216 | "}" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "shaped[0], shaped[1]\n" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "for edge, color in EDGES.items():\n", 235 | " p1, p2 = edge\n", 236 | " y1, x1, c1 = shaped[p1]\n", 237 | " y2, x2, c2 = shaped[p2]\n", 238 | " print((int(x2), int(y2)))" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": {}, 245 | "outputs": [], 246 | "source": [ 247 | "def draw_connections(frame, keypoints, edges, confidence_threshold):\n", 248 | " y, x, c = frame.shape\n", 249 | " shaped = np.squeeze(np.multiply(keypoints, [y,x,1]))\n", 250 | " \n", 251 | " for edge, color in edges.items():\n", 252 | " p1, p2 = edge\n", 253 | " y1, x1, c1 = shaped[p1]\n", 254 | " y2, x2, c2 = shaped[p2]\n", 255 | " \n", 256 | " if (c1 > confidence_threshold) & (c2 > confidence_threshold): \n", 257 | " cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), (0,0,255), 2)" 258 | ] 259 | } 260 | ], 261 | "metadata": { 262 | "kernelspec": { 263 | "display_name": "movenet", 264 | "language": "python", 265 | "name": "movenet" 266 | }, 267 | "language_info": { 268 | "codemirror_mode": { 269 | "name": "ipython", 270 | "version": 3 271 | }, 272 | "file_extension": ".py", 273 | "mimetype": "text/x-python", 274 | "name": "python", 275 | "nbconvert_exporter": "python", 276 | "pygments_lexer": "ipython3", 277 | "version": "3.7.3" 278 | } 279 | }, 280 | "nbformat": 4, 281 | "nbformat_minor": 2 282 | } 283 | --------------------------------------------------------------------------------