├── _config.yml ├── .gitignore ├── favicon.ico ├── License.txt ├── hand-dataset.js ├── README.md ├── index.html ├── webcam.js ├── model.js ├── style.css └── LICENSE /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-hacker -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac OS files 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oscar-defelice/handgesture.github.io/main/favicon.ico -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the \"License\"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | https://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an \"AS IS\" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /hand-dataset.js: -------------------------------------------------------------------------------- 1 | class HandDataset { 2 | constructor() { 3 | this.labels = [] 4 | } 5 | 6 | addExample(example, label) { 7 | if (this.xs == null) { 8 | this.xs = tf.keep(example); 9 | this.labels.push(label); 10 | } else { 11 | const oldX = this.xs; 12 | this.xs = tf.keep(oldX.concat(example, 0)); 13 | this.labels.push(label); 14 | oldX.dispose(); 15 | } 16 | } 17 | 18 | encodeLabels(numClasses) { 19 | for (var i = 0; i < this.labels.length; i++) { 20 | if (this.ys == null) { 21 | this.ys = tf.keep(tf.tidy( 22 | () => { 23 | return tf.oneHot( 24 | tf.tensor1d([this.labels[i]]).toInt(), numClasses) 25 | })); 26 | } else { 27 | const y = tf.tidy( 28 | () => { 29 | return tf.oneHot( 30 | tf.tensor1d([this.labels[i]]).toInt(), numClasses) 31 | }); 32 | const oldY = this.ys; 33 | this.ys = tf.keep(oldY.concat(y, 0)); 34 | oldY.dispose(); 35 | y.dispose(); 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # handgesture.github.io 2 | Repository for a browser based tensorflow model to classify hand gestures. 3 | 4 | ## Introduction 5 | 6 | We make use of trasfer learning and of [MobileNet](https://arxiv.org/abs/1704.04861), a set of light and efficient CNNs designed specifically for mobile and embedded computer vision applications. MobileNets are based on a streamlined architecture that uses depth-wise separable convolutions to build light weight deep neural networks. 7 | 8 | ## How to use 9 | 10 | Simply, go to the [webpage](https://oscar-defelice.github.io/handgesture.github.io/). 11 | It might be required to allow the browser to access your webcam. 12 | Hence, you build a little dataset (50/60 images per class should be enough). 13 | 14 | ### Example of dataset construction 15 | 16 | You put your hand in front of the camera, indicating for instance 2. 17 | Click on the `two` button would add the image and the corresponding label to the dataset. 18 | 19 | ### Training 20 | Once you are happy with the examples you have added, train your network. 21 | A browser alert would notify when training is done, allowing you starting the predictions. 22 | 23 | ### Predictions 24 | Again, you put your hand in front of the camera, miming the gesture of three, and you should visualise the message _I see three_. 25 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Train and Predict Hand gestures 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |

Train and Predict Hand Gestures

22 |
23 |
24 | 25 |
26 |
27 | 28 | 29 | 30 | 31 | 32 | 33 |
Zero Samples:
34 |
One Samples:
35 |
Two Samples:
36 |
Three Samples:
37 |
Four Samples:
38 |
Five Samples:
39 | 40 |
Once training is complete, click 'Start Predicting' to see predictions, and 'Stop Predicting' to end. Once you are happy with your model, click 'Download Model' to save the model (and its weights) to your local disk.
41 | 42 | 43 | 44 |
45 |


46 |


47 |
48 | 49 |
50 |
51 | Powered by 52 | Oscar de Felice 53 |
54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /webcam.js: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // ============================================================================= 15 | 16 | /** 17 | * A class that wraps webcam video elements to capture Tensor4Ds. 18 | */ 19 | 20 | 21 | class Webcam { 22 | /** 23 | * @param {HTMLVideoElement} webcamElement A HTMLVideoElement representing the 24 | * webcam feed. 25 | */ 26 | constructor(webcamElement) { 27 | this.webcamElement = webcamElement; 28 | } 29 | 30 | /** 31 | * Captures a frame from the webcam and normalizes it between -1 and 1. 32 | * Returns a batched image (1-element batch) of shape [1, w, h, c]. 33 | */ 34 | 35 | 36 | capture() { 37 | return tf.tidy(() => { 38 | // Reads the image as a Tensor from the webcam