14 |
15 | ## 💪 Try it
16 | If you are not convinced yet just try it out here: https://woody.pizza/tensorflow/hand-detection/
17 |
18 | There is also an example where you can control the mouse by using your hand: https://woody.pizza/tensorflow/hand-detection/mouse
19 |
20 | ## 🌐 Multiple browser support
21 | Probably this will work with the most browsers, but here is a list which browsers I have tested:
22 |
23 | | Browser | supported |
24 | |:-----------------:|:---------:|
25 | | Firefox | ✅ |
26 | | Chrome | ✅ |
27 | | Edge | ✅ |
28 | | Internet Explorer | ❌ |
29 |
30 | | Mobile Browser | supported |
31 | |:--------------:|:---------:|
32 | | Firefox | ✅ |
33 | | Chrome | ✅ |
34 |
35 | ## ✌️ Credits
36 | - [Materialize](https://materializecss.com/)
37 | - [ml5js](https://ml5js.org/)
38 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | | AI: | 23 |
24 |
25 |
31 |
32 | |
33 |
| FPS: | 36 |
37 | 38 | 39 | 40 | |
41 |
| AI: | 24 |
25 |
26 |
32 |
33 | |
34 |
| Border: | 37 |
38 | 39 | 40 | 41 | |
42 |
| FPS: | 45 |
46 | 47 | 48 | 49 | |
50 |
46 |
47 |
65 |
66 |
67 |
68 |
69 |
--------------------------------------------------------------------------------
/mouse/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 
55 |
56 |
74 |
75 |
76 |
77 |
78 |
--------------------------------------------------------------------------------
/video.js:
--------------------------------------------------------------------------------
1 | document.getElementById("ai").addEventListener("change", toggleAi)
2 | document.getElementById("fps").addEventListener("input", changeFps)
3 |
4 | const video = document.getElementById("video");
5 | const c1 = document.getElementById('c1');
6 | const ctx1 = c1.getContext('2d');
7 | var cameraAvailable = false;
8 | var aiEnabled = false;
9 | var fps = 16;
10 |
11 | /* Setting up the constraint */
12 | var facingMode = "user"; // Can be 'user' or 'environment' to access back or front camera (NEAT!)
13 | var constraints = {
14 | audio: false,
15 | video: {
16 | facingMode: facingMode
17 | }
18 | };
19 |
20 | /* Stream it to video element */
21 | camera();
22 | function camera() {
23 | if (!cameraAvailable) {
24 | console.log("camera")
25 | navigator.mediaDevices.getUserMedia(constraints).then(function (stream) {
26 | cameraAvailable = true;
27 | video.srcObject = stream;
28 | }).catch(function (err) {
29 | cameraAvailable = false;
30 | if (modelIsLoaded) {
31 | if (err.name === "NotAllowedError") {
32 | document.getElementById("loadingText").innerText = "Waiting for camera permission";
33 | }
34 | }
35 | setTimeout(camera, 1000);
36 | });
37 | }
38 | }
39 |
40 | window.onload = function () {
41 | timerCallback();
42 | }
43 |
44 | function timerCallback() {
45 | if (isReady()) {
46 | setResolution();
47 | ctx1.drawImage(video, 0, 0, c1.width, c1.height);
48 | if (aiEnabled) {
49 | ai();
50 | }
51 | }
52 | setTimeout(timerCallback, fps);
53 | }
54 |
55 | function isReady() {
56 | if (modelIsLoaded && cameraAvailable) {
57 | document.getElementById("loadingText").style.display = "none";
58 | document.getElementById("ai").disabled = false;
59 | return true;
60 | } else {
61 | return false;
62 | }
63 | }
64 |
65 | function setResolution() {
66 | if (window.screen.width < video.videoWidth) {
67 | c1.width = window.screen.width * 0.9;
68 | let factor = c1.width / video.videoWidth;
69 | c1.height = video.videoHeight * factor;
70 | } else if (window.screen.height < video.videoHeight) {
71 | c1.height = window.screen.height * 0.50;
72 | let factor = c1.height / video.videoHeight;
73 | c1.width = video.videoWidth * factor;
74 | }
75 | else {
76 | c1.width = video.videoWidth;
77 | c1.height = video.videoHeight;
78 | }
79 | };
80 |
81 | function toggleAi() {
82 | aiEnabled = document.getElementById("ai").checked;
83 | }
84 |
85 | function changeFps() {
86 | fps = 1000 / document.getElementById("fps").value;
87 | }
88 |
89 | function ai() {
90 | // Detect hand in the canvas element
91 | handpose.predict(c1, results => {
92 | // do something with the results
93 | console.log(results);
94 | for (let index = 0; index < results.length; index++) {
95 | const element = results[index];
96 | // box
97 | ctx1.beginPath();
98 | ctx1.strokeStyle = "green";
99 | ctx1.rect(element.boundingBox.topLeft[0], element.boundingBox.topLeft[1], element.boundingBox.bottomRight[0] - element.boundingBox.topLeft[0], element.boundingBox.bottomRight[1] - element.boundingBox.topLeft[1]);
100 | ctx1.stroke();
101 | ctx1.beginPath();
102 | ctx1.strokeStyle = "red";
103 | // thumb
104 | const thumb = element.annotations.thumb;
105 | ctx1.moveTo(thumb[0][0], thumb[0][1]);
106 | for (let i = 1; i < thumb.length; i++) {
107 | const ele = thumb[i];
108 | ctx1.lineTo(ele[0], ele[1]);
109 | ctx1.moveTo(ele[0], ele[1]);
110 | }
111 | // indexFinger
112 | const indexFinger = element.annotations.indexFinger;
113 | ctx1.moveTo(indexFinger[0][0], indexFinger[0][1]);
114 | for (let i = 1; i < indexFinger.length; i++) {
115 | const ele = indexFinger[i];
116 | ctx1.lineTo(ele[0], ele[1]);
117 | ctx1.moveTo(ele[0], ele[1]);
118 | }
119 | // middleFinger
120 | const middleFinger = element.annotations.middleFinger;
121 | ctx1.moveTo(middleFinger[0][0], middleFinger[0][1]);
122 | for (let i = 1; i < middleFinger.length; i++) {
123 | const ele = middleFinger[i];
124 | ctx1.lineTo(ele[0], ele[1]);
125 | ctx1.moveTo(ele[0], ele[1]);
126 | }
127 | // ringFinger
128 | const ringFinger = element.annotations.ringFinger;
129 | ctx1.moveTo(ringFinger[0][0], ringFinger[0][1]);
130 | for (let i = 1; i < ringFinger.length; i++) {
131 | const ele = ringFinger[i];
132 | ctx1.lineTo(ele[0], ele[1]);
133 | ctx1.moveTo(ele[0], ele[1]);
134 | }
135 | // pinky
136 | const pinky = element.annotations.pinky;
137 | ctx1.moveTo(pinky[0][0], pinky[0][1]);
138 | for (let i = 1; i < pinky.length; i++) {
139 | const ele = pinky[i];
140 | ctx1.lineTo(ele[0], ele[1]);
141 | ctx1.moveTo(ele[0], ele[1]);
142 | }
143 | ctx1.stroke();
144 | // palmBase
145 | const palmBase = element.annotations.palmBase;
146 | for (let i = 0; i < palmBase.length; i++) {
147 | ctx1.beginPath();
148 | ctx1.strokeStyle = "blue";
149 | const ele = palmBase[i];
150 | ctx1.arc(ele[0], ele[1], 10, 0, 2 * Math.PI);
151 | ctx1.stroke();
152 | }
153 |
154 | // landmarks
155 | const landmarks = element.landmarks;
156 | for (let i = 0; i < landmarks.length; i++) {
157 | ctx1.beginPath();
158 | ctx1.strokeStyle = "blue";
159 | const ele = landmarks[i];
160 | ctx1.arc(ele[0], ele[1], 2, 0, 2 * Math.PI);
161 | ctx1.fillStyle = "blue";
162 | ctx1.fill();
163 | ctx1.stroke();
164 | }
165 | }
166 | });
167 | }
--------------------------------------------------------------------------------
/mouse/video.js:
--------------------------------------------------------------------------------
1 | document.getElementById("ai").addEventListener("change", toggleAi)
2 | document.getElementById("fps").addEventListener("input", changeFps)
3 | document.getElementById("border").addEventListener("input", changeBorder)
4 |
5 | const video = document.getElementById("video");
6 | const c1 = document.getElementById('c1');
7 | const ctx1 = c1.getContext('2d');
8 | var cameraAvailable = false;
9 | var aiEnabled = false;
10 | var fps = 16;
11 | var border = 200; // mouse border
12 |
13 | /* Setting up the constraint */
14 | var facingMode = "user"; // Can be 'user' or 'environment' to access back or front camera (NEAT!)
15 | var constraints = {
16 | audio: false,
17 | video: {
18 | facingMode: facingMode
19 | }
20 | };
21 |
22 | /* Stream it to video element */
23 | camera();
24 | function camera() {
25 | if (!cameraAvailable) {
26 | console.log("camera")
27 | navigator.mediaDevices.getUserMedia(constraints).then(function (stream) {
28 | cameraAvailable = true;
29 | video.srcObject = stream;
30 | }).catch(function (err) {
31 | cameraAvailable = false;
32 | if (modelIsLoaded) {
33 | if (err.name === "NotAllowedError") {
34 | document.getElementById("loadingText").innerText = "Waiting for camera permission";
35 | }
36 | }
37 | setTimeout(camera, 1000);
38 | });
39 | }
40 | }
41 |
42 | window.onload = function () {
43 | timerCallback();
44 | }
45 |
46 | function timerCallback() {
47 | if (isReady()) {
48 | setResolution();
49 | ctx1.drawImage(video, 0, 0, c1.width, c1.height);
50 | setBorder();
51 | if (aiEnabled) {
52 | lockCheck();
53 | ai();
54 | }
55 | }
56 | setTimeout(timerCallback, fps);
57 | }
58 |
59 | function isReady() {
60 | if (modelIsLoaded && cameraAvailable) {
61 | document.getElementById("loadingText").style.display = "none";
62 | document.getElementById("ai").disabled = false;
63 | return true;
64 | } else {
65 | return false;
66 | }
67 | }
68 |
69 | function setResolution() {
70 | if (window.screen.width < video.videoWidth) {
71 | c1.width = window.screen.width * 0.9;
72 | let factor = c1.width / video.videoWidth;
73 | c1.height = video.videoHeight * factor;
74 | } else if (window.screen.height < video.videoHeight) {
75 | c1.height = window.screen.height * 0.50;
76 | let factor = c1.height / video.videoHeight;
77 | c1.width = video.videoWidth * factor;
78 | }
79 | else {
80 | c1.width = video.videoWidth;
81 | c1.height = video.videoHeight;
82 | }
83 | };
84 |
85 | function toggleAi() {
86 | let enabled = document.getElementById("ai").checked;
87 | aiEnabled = enabled;
88 | if (enabled) {
89 | c1.requestPointerLock();
90 | } else {
91 | setFakeMouse(0, 0); // hide mouse
92 | }
93 | }
94 |
95 | function lockCheck() {
96 | if (!/Mobi|Android/i.test(navigator.userAgent)) {
97 | if (document.pointerLockElement !== c1) {
98 | document.getElementById("ai").checked = false;
99 | toggleAi();
100 | }
101 | }
102 | }
103 |
104 | function changeFps() {
105 | fps = 1000 / document.getElementById("fps").value;
106 | }
107 |
108 | function changeBorder() {
109 | border = document.getElementById("border").value;
110 | }
111 |
112 | function setFakeMouse(x, y) {
113 | let cMouse = document.getElementById("mouse");
114 | let ctxMouse = cMouse.getContext("2d");
115 | cMouse.width = window.innerWidth;
116 | cMouse.height = window.innerHeight;
117 | x = cMouse.width * x;
118 | x = cMouse.width - x; // mirror
119 | y = cMouse.height * y;
120 | // console.log("x: " + x + " - y: " + y);
121 | let img = new Image;
122 | img.src = "pointer.png";
123 | ctxMouse.drawImage(img, x, y, img.width * 0.5, img.height * 0.5);
124 | }
125 |
126 | // draw mouse border
127 | function setBorder() {
128 | ctx1.beginPath();
129 | ctx1.strokeStyle = "yellow";
130 | ctx1.rect(border / 2, border / 2, c1.width - border, c1.height - border);
131 | ctx1.stroke();
132 | ctx1.beginPath();
133 | }
134 |
135 | function ai() {
136 | // Detect hand in the canvas element
137 | handpose.predict(c1, results => {
138 | // do something with the results
139 | console.log(results);
140 | for (let index = 0; index < results.length; index++) {
141 | const element = results[index];
142 | // box
143 | ctx1.beginPath();
144 | ctx1.strokeStyle = "green";
145 | ctx1.rect(element.boundingBox.topLeft[0], element.boundingBox.topLeft[1], element.boundingBox.bottomRight[0] - element.boundingBox.topLeft[0], element.boundingBox.bottomRight[1] - element.boundingBox.topLeft[1]);
146 | ctx1.stroke();
147 | ctx1.beginPath();
148 | ctx1.strokeStyle = "red";
149 | // thumb
150 | const thumb = element.annotations.thumb;
151 | ctx1.moveTo(thumb[0][0], thumb[0][1]);
152 | for (let i = 1; i < thumb.length; i++) {
153 | const ele = thumb[i];
154 | ctx1.lineTo(ele[0], ele[1]);
155 | ctx1.moveTo(ele[0], ele[1]);
156 | }
157 | // indexFinger
158 | const indexFinger = element.annotations.indexFinger;
159 | ctx1.moveTo(indexFinger[0][0], indexFinger[0][1]);
160 | for (let i = 1; i < indexFinger.length; i++) {
161 | const ele = indexFinger[i];
162 | ctx1.lineTo(ele[0], ele[1]);
163 | ctx1.moveTo(ele[0], ele[1]);
164 | }
165 | // middleFinger
166 | const middleFinger = element.annotations.middleFinger;
167 | ctx1.moveTo(middleFinger[0][0], middleFinger[0][1]);
168 | for (let i = 1; i < middleFinger.length; i++) {
169 | const ele = middleFinger[i];
170 | ctx1.lineTo(ele[0], ele[1]);
171 | ctx1.moveTo(ele[0], ele[1]);
172 | }
173 | // ringFinger
174 | const ringFinger = element.annotations.ringFinger;
175 | ctx1.moveTo(ringFinger[0][0], ringFinger[0][1]);
176 | for (let i = 1; i < ringFinger.length; i++) {
177 | const ele = ringFinger[i];
178 | ctx1.lineTo(ele[0], ele[1]);
179 | ctx1.moveTo(ele[0], ele[1]);
180 | }
181 | // pinky
182 | const pinky = element.annotations.pinky;
183 | ctx1.moveTo(pinky[0][0], pinky[0][1]);
184 | for (let i = 1; i < pinky.length; i++) {
185 | const ele = pinky[i];
186 | ctx1.lineTo(ele[0], ele[1]);
187 | ctx1.moveTo(ele[0], ele[1]);
188 | }
189 | ctx1.stroke();
190 | // palmBase
191 | const palmBase = element.annotations.palmBase;
192 | // mouse
193 | let x = (middleFinger[0][0] - border / 2) / (c1.width - border);
194 | let y = (middleFinger[0][1] - border / 2) / (c1.height - border);
195 | // create pointer
196 | if (aiEnabled) { // check again if enabled to prevent mouse override
197 | setFakeMouse(x, y);
198 | }
199 | for (let i = 0; i < palmBase.length; i++) {
200 | ctx1.beginPath();
201 | ctx1.strokeStyle = "blue";
202 | const ele = palmBase[i];
203 | ctx1.arc(ele[0], ele[1], 10, 0, 2 * Math.PI);
204 | ctx1.stroke();
205 | }
206 |
207 | // landmarks
208 | const landmarks = element.landmarks;
209 | for (let i = 0; i < landmarks.length; i++) {
210 | ctx1.beginPath();
211 | ctx1.strokeStyle = "blue";
212 | const ele = landmarks[i];
213 | ctx1.arc(ele[0], ele[1], 2, 0, 2 * Math.PI);
214 | ctx1.fillStyle = "blue";
215 | ctx1.fill();
216 | ctx1.stroke();
217 | }
218 | }
219 | });
220 | }
--------------------------------------------------------------------------------