├── .gitignore
├── .prettierignore
├── .prettierrc
├── .vscode
└── settings.json
├── LICENSE
├── README.md
├── components
├── about.js
├── handimage
│ ├── A.svg
│ ├── Ahand.svg
│ ├── B.svg
│ ├── Bhand.svg
│ ├── C.svg
│ ├── Chand.svg
│ ├── D.svg
│ ├── Dhand.svg
│ ├── E.svg
│ ├── Ehand.svg
│ ├── F.svg
│ ├── Fhand.svg
│ ├── G.svg
│ ├── Ghand.svg
│ ├── H.svg
│ ├── Hhand.svg
│ ├── I.svg
│ ├── Ihand.svg
│ ├── J.svg
│ ├── Jhand.svg
│ ├── K.svg
│ ├── Khand.svg
│ ├── L.svg
│ ├── Lhand.svg
│ ├── M.svg
│ ├── Mhand.svg
│ ├── N.svg
│ ├── Nhand.svg
│ ├── O.svg
│ ├── Ohand.svg
│ ├── P.svg
│ ├── Phand.svg
│ ├── Q.svg
│ ├── Qhand.svg
│ ├── R.svg
│ ├── Rhand.svg
│ ├── S.svg
│ ├── Shand.svg
│ ├── T.svg
│ ├── Thand.svg
│ ├── U.svg
│ ├── Uhand.svg
│ ├── V.svg
│ ├── Vhand.svg
│ ├── W.svg
│ ├── Whand.svg
│ ├── X.svg
│ ├── Xhand.svg
│ ├── Y.svg
│ ├── Yhand.svg
│ ├── Z.svg
│ ├── Zhand.svg
│ └── index.js
├── handposeutil.js
├── handsigns
│ ├── Asign.js
│ ├── Bsign.js
│ ├── Csign.js
│ ├── Dsign.js
│ ├── Esign.js
│ ├── Fsign.js
│ ├── Gsign.js
│ ├── Hsign.js
│ ├── Isign.js
│ ├── Jsign.js
│ ├── Ksign.js
│ ├── Lsign.js
│ ├── Msign.js
│ ├── Nsign.js
│ ├── Osign.js
│ ├── Psign.js
│ ├── Qsign.js
│ ├── Rsign.js
│ ├── Ssign.js
│ ├── Tsign.js
│ ├── Usign.js
│ ├── Vsign.js
│ ├── Wsign.js
│ ├── Xsign.js
│ ├── Ysign.js
│ ├── Zsign.js
│ └── index.js
├── image.js
└── metatags.js
├── package.json
├── pages
├── 404.js
├── _app.js
└── index.js
├── public
├── 2764.svg
├── Handsigns.svg
├── favicon.ico
├── handImages.svg
└── loveyou_emoji.svg
├── styles
├── Home.module.css
└── globals.css
└── yarn.lock
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 |
8 | # Runtime data
9 | pids
10 | *.pid
11 | *.seed
12 | *.pid.lock
13 |
14 | # Directory for instrumented libs generated by jscoverage/JSCover
15 | lib-cov
16 |
17 | # Coverage directory used by tools like istanbul
18 | coverage
19 |
20 | # nyc test coverage
21 | .nyc_output
22 |
23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
24 | .grunt
25 |
26 | # Bower dependency directory (https://bower.io/)
27 | bower_components
28 |
29 | # node-waf configuration
30 | .lock-wscript
31 |
32 | # Compiled binary addons (http://nodejs.org/api/addons.html)
33 | build/Release
34 |
35 | # Dependency directories
36 | node_modules/
37 | jspm_packages/
38 |
39 | # Typescript v1 declaration files
40 | typings/
41 |
42 | # Optional npm cache directory
43 | .npm
44 |
45 | # Optional eslint cache
46 | .eslintcache
47 |
48 | # Optional REPL history
49 | .node_repl_history
50 |
51 | # Output of 'npm pack'
52 | *.tgz
53 |
54 | # dotenv environment variable files
55 | .env*
56 |
57 | #nextJS
58 | .next
59 |
60 | # Mac files
61 | .DS_Store
62 |
63 | # Yarn
64 | yarn-error.log
65 | .pnp/
66 | .pnp.js
67 | # Yarn Integrity file
68 | .yarn-integrity
69 |
--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | .cache
2 | package.json
3 | package-lock.json
4 | public
5 |
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "arrowParens": "avoid",
3 | "semi": false
4 | }
5 |
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "workbench.colorCustomizations": {
3 | "activityBar.activeBackground": "#93e6fc",
4 | "activityBar.activeBorder": "#fa45d4",
5 | "activityBar.background": "#93e6fc",
6 | "activityBar.foreground": "#15202b",
7 | "activityBar.inactiveForeground": "#15202b99",
8 | "activityBarBadge.background": "#fa45d4",
9 | "activityBarBadge.foreground": "#15202b",
10 | "sash.hoverBorder": "#93e6fc",
11 | "statusBar.background": "#61dafb",
12 | "statusBar.foreground": "#15202b",
13 | "statusBarItem.hoverBackground": "#2fcefa",
14 | "statusBarItem.remoteBackground": "#61dafb",
15 | "statusBarItem.remoteForeground": "#15202b",
16 | "titleBar.activeBackground": "#61dafb",
17 | "titleBar.activeForeground": "#15202b",
18 | "titleBar.inactiveBackground": "#61dafb99",
19 | "titleBar.inactiveForeground": "#15202b99"
20 | },
21 | "peacock.color": "#61dafb"
22 | }
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | BSD-2-Clause (BSD-2)
2 |
3 | Copyright (c) 2021 Syauqy Nurul Aziz
4 |
5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6 |
7 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8 |
9 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
10 |
11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
12 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
6 |
7 |
8 | Handsign - ASL Hand Gesture Detection Using TensorFlow
9 |
10 |
11 | Handsign is a simple AI-based hand gesture recognition that translates a hand pose into the American Sign Language (ASL) alphabet. Using Tensorflow JS and its Handpose preloaded model to detect the hand object and its parts. Handsign also uses an additional library called Fingerpose to classify certain of custom hand gestures based on the finger position.
12 |
13 | `#dohackathon` `#madewithTFJS`
14 |
15 | 
16 |
17 | ## [See Demo](https://handsign-tensorflow.vercel.app)
18 |
19 | # Installation
20 |
21 | 1. **Clone the repository**
22 |
23 | ```shell
24 | # copy the repo to your machine
25 |
26 | git clone https://github.com/syauqy/handsign-tensorflow.git
27 | ```
28 |
29 | 2. **Start the project**
30 |
31 | ```shell
32 | # move to the project folder and install all dependencies
33 |
34 | cd handsign-tensorflow
35 | yarn install
36 | ```
37 |
38 | 3. **Run the project on your local machine**
39 |
40 | ```shell
41 | # run Next
42 |
43 | yarn dev
44 | ```
45 |
46 | 4. **The project is live 🚀**
47 |
48 | Your project is live and running at `http://localhost:3000`
49 |
50 | You can edit the core program at `/pages/index.js`
51 |
52 | # What's inside the project
53 |
54 | ## Extract the fingerpose data
55 |
56 | uncomment the `` component
57 |
58 | ```js
59 | Pose data */}
63 |
64 |
65 | ```
66 |
67 | uncomment the `estimatedGestures` data to change `'.pose-data'` innerHTML
68 |
69 | ```js
70 | // document.querySelector('.pose-data').innerHTML =JSON.stringify(estimatedGestures.poseData, null, 2);
71 | ```
72 |
73 | the `estimatedGestures` data will render on your screen.
74 |
75 | # References & Libraries
76 |
77 | - [Tensorflow JS](https://www.tensorflow.org/js) - A Library for ML in JS.
78 |
79 | - [Handpose](https://github.com/tensorflow/tfjs-models/tree/master/handpose) - A lightweight ML pipeline consisting of two models: A palm detector and a hand-skeleton finger tracking model.
80 |
81 | - [Fingerpose](https://github.com/andypotato/fingerpose) - A pose classifier for hand landmarks detected by TensorFlow.js Handpose's model.
82 |
83 | - Sign language illustration is created by [Pelin Kahraman](https://thenounproject.com/pelodrome/)
84 |
85 | If you want to learn more about Tensorflow JS and custom gesture handpose, please kindly check these amazing videos
86 |
87 | - [Machine Learning with TensorFlow in JavaScript](https://www.youtube.com/watch?v=WIHZ7kjJ35o) - by [Jason Lengstorf](https://github.com/jlengstorf) and [Jason Mayes](https://github.com/jasonmayes)
88 |
89 | - [Building a Real Time Sign Language Detection App with React.JS and Tensorflow.JS](https://www.youtube.com/watch?v=ZTSRZt04JkY) - by [Nicholas Renotte](https://github.com/nicknochnack)
90 |
91 |
92 |
--------------------------------------------------------------------------------
/components/about.js:
--------------------------------------------------------------------------------
1 | import React from "react"
2 | import handImages from "../public/handImages.svg"
3 | import {
4 | Text,
5 | Button,
6 | Image,
7 | Modal,
8 | ModalOverlay,
9 | ModalContent,
10 | ModalHeader,
11 | ModalFooter,
12 | ModalBody,
13 | ModalCloseButton,
14 | useDisclosure,
15 | Link,
16 | } from "@chakra-ui/react"
17 |
18 | export default function About() {
19 | const { isOpen, onOpen, onClose } = useDisclosure()
20 |
21 | return (
22 |
23 |
24 | Learn More
25 |
26 |
27 |
28 |
30 | American Sign Language (ASL)
31 |
33 |
34 | American Sign Language (ASL) is a visual language that serves as
35 | the predominant sign language of Deaf communities in the United
36 | States and most of Canada.
39 |
41 | This sign language illustration is created by{" "}
42 |
50 |
51 |
52 | Close
53 |
54 |
55 |
56 |
57 | )
58 | }
59 |
--------------------------------------------------------------------------------
/components/handimage/A.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Ahand.svg:
--------------------------------------------------------------------------------
1 |
2 |
11 |
12 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/components/handimage/B.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Bhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
10 |
11 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/components/handimage/C.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Chand.svg:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/components/handimage/D.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Dhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/E.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Ehand.svg:
--------------------------------------------------------------------------------
1 |
2 |
14 |
15 |
18 |
19 |
20 |
--------------------------------------------------------------------------------
/components/handimage/F.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Fhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
9 |
10 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/components/handimage/G.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Ghand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/H.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Hhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
11 |
12 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/components/handimage/I.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Ihand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/J.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Jhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/K.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Khand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/L.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Lhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
11 |
12 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/components/handimage/M.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Mhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
14 |
15 |
18 |
19 |
20 |
--------------------------------------------------------------------------------
/components/handimage/N.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Nhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
11 |
12 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/components/handimage/O.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Ohand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/P.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Phand.svg:
--------------------------------------------------------------------------------
1 |
2 |
14 |
15 |
18 |
19 |
20 |
--------------------------------------------------------------------------------
/components/handimage/Q.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Qhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/R.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Rhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/S.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Shand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/T.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Thand.svg:
--------------------------------------------------------------------------------
1 |
2 |
11 |
12 |
15 |
16 |
17 |
--------------------------------------------------------------------------------
/components/handimage/U.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Uhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/V.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Vhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
9 |
10 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/components/handimage/W.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Whand.svg:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/components/handimage/X.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Xhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
16 |
17 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/components/handimage/Y.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Yhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
10 |
11 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/components/handimage/Z.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/components/handimage/Zhand.svg:
--------------------------------------------------------------------------------
1 |
2 |
13 |
14 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/components/handimage/index.js:
--------------------------------------------------------------------------------
1 | import A_sign from "./A.svg"
2 | import B_sign from "./B.svg"
3 | import C_sign from "./C.svg"
4 | import D_sign from "./D.svg"
5 | import E_sign from "./E.svg"
6 | import F_sign from "./F.svg"
7 | import G_sign from "./G.svg"
8 | import H_sign from "./H.svg"
9 | import I_sign from "./I.svg"
10 | import J_sign from "./J.svg"
11 | import K_sign from "./K.svg"
12 | import L_sign from "./L.svg"
13 | import M_sign from "./M.svg"
14 | import N_sign from "./N.svg"
15 | import O_sign from "./O.svg"
16 | import P_sign from "./P.svg"
17 | import Q_sign from "./Q.svg"
18 | import R_sign from "./R.svg"
19 | import S_sign from "./S.svg"
20 | import T_sign from "./T.svg"
21 | import U_sign from "./U.svg"
22 | import V_sign from "./V.svg"
23 | import W_sign from "./W.svg"
24 | import X_sign from "./X.svg"
25 | import Y_sign from "./Y.svg"
26 | import Z_sign from "./Z.svg"
27 |
28 | import A_hand from "./Ahand.svg"
29 | import B_hand from "./Bhand.svg"
30 | import C_hand from "./Chand.svg"
31 | import D_hand from "./Dhand.svg"
32 | import E_hand from "./Ehand.svg"
33 | import F_hand from "./Fhand.svg"
34 | import G_hand from "./Ghand.svg"
35 | import H_hand from "./Hhand.svg"
36 | import I_hand from "./Ihand.svg"
37 | import J_hand from "./Jhand.svg"
38 | import K_hand from "./Khand.svg"
39 | import L_hand from "./Lhand.svg"
40 | import M_hand from "./Mhand.svg"
41 | import N_hand from "./Nhand.svg"
42 | import O_hand from "./Ohand.svg"
43 | import P_hand from "./Phand.svg"
44 | import Q_hand from "./Qhand.svg"
45 | import R_hand from "./Rhand.svg"
46 | import S_hand from "./Shand.svg"
47 | import T_hand from "./Thand.svg"
48 | import U_hand from "./Uhand.svg"
49 | import V_hand from "./Vhand.svg"
50 | import W_hand from "./Whand.svg"
51 | import X_hand from "./Xhand.svg"
52 | import Y_hand from "./Yhand.svg"
53 | import Z_hand from "./Zhand.svg"
54 |
55 | export const Signimage = {
56 | A: A_sign,
57 | B: B_sign,
58 | C: C_sign,
59 | D: D_sign,
60 | E: E_sign,
61 | F: F_sign,
62 | G: G_sign,
63 | H: H_sign,
64 | I: I_sign,
65 | J: J_sign,
66 | K: K_sign,
67 | L: L_sign,
68 | M: M_sign,
69 | N: N_sign,
70 | O: O_sign,
71 | P: P_sign,
72 | Q: Q_sign,
73 | R: R_sign,
74 | S: S_sign,
75 | T: T_sign,
76 | U: U_sign,
77 | V: V_sign,
78 | W: W_sign,
79 | X: X_sign,
80 | Y: Y_sign,
81 | Z: Z_sign,
82 | }
83 |
84 | export const Signpass = [
85 | {
86 | src: A_hand,
87 | alt: "A",
88 | },
89 | {
90 | src: B_hand,
91 | alt: "B",
92 | },
93 | {
94 | src: C_hand,
95 | alt: "C",
96 | },
97 | {
98 | src: D_hand,
99 | alt: "D",
100 | },
101 | {
102 | src: E_hand,
103 | alt: "E",
104 | },
105 | {
106 | src: F_hand,
107 | alt: "F",
108 | },
109 | {
110 | src: G_hand,
111 | alt: "G",
112 | },
113 | {
114 | src: H_hand,
115 | alt: "H",
116 | },
117 | {
118 | src: I_hand,
119 | alt: "I",
120 | },
121 | {
122 | src: J_hand,
123 | alt: "J",
124 | },
125 | {
126 | src: K_hand,
127 | alt: "K",
128 | },
129 | {
130 | src: L_hand,
131 | alt: "L",
132 | },
133 | {
134 | src: M_hand,
135 | alt: "M",
136 | },
137 | {
138 | src: N_hand,
139 | alt: "N",
140 | },
141 | {
142 | src: O_hand,
143 | alt: "O",
144 | },
145 | {
146 | src: P_hand,
147 | alt: "P",
148 | },
149 | {
150 | src: Q_hand,
151 | alt: "Q",
152 | },
153 | {
154 | src: R_hand,
155 | alt: "R",
156 | },
157 | {
158 | src: S_hand,
159 | alt: "S",
160 | },
161 | {
162 | src: T_hand,
163 | alt: "T",
164 | },
165 | {
166 | src: U_hand,
167 | alt: "U",
168 | },
169 | {
170 | src: V_hand,
171 | alt: "V",
172 | },
173 | {
174 | src: W_hand,
175 | alt: "W",
176 | },
177 | {
178 | src: X_hand,
179 | alt: "X",
180 | },
181 | {
182 | src: Y_hand,
183 | alt: "Y",
184 | },
185 | {
186 | src: Z_hand,
187 | alt: "Z",
188 | },
189 | ]
190 |
--------------------------------------------------------------------------------
/components/handposeutil.js:
--------------------------------------------------------------------------------
1 | //finger points
2 | const fingerJoints = {
3 | thumb:[0,1,2,3,4],
4 | index:[0,5,6,7,8],
5 | mid:[0,9,10,11,12],
6 | ring:[0,13,14,15,16],
7 | pinky:[0,17,18,19,20]
8 | };
9 |
10 | //drawing function
11 | export const drawHand = (prediction, ctx) => {
12 | //check the prediction
13 | if(prediction.length > 0){
14 | //loop to the preditions
15 | prediction.forEach((prediction) =>{
16 | //grab landmarks
17 | const landmarks = prediction.landmarks;
18 |
19 | //loop the finger joints
20 | for(let j = 0; j {
17 | const data = useStaticQuery(graphql`
18 | query {
19 | placeholderImage: file(relativePath: { eq: "gatsby-astronaut.png" }) {
20 | childImageSharp {
21 | fluid(maxWidth: 300) {
22 | ...GatsbyImageSharpFluid
23 | }
24 | }
25 | }
26 | }
27 | `)
28 |
29 | if (!data?.placeholderImage?.childImageSharp?.fluid) {
30 | return Picture not found
31 | }
32 |
33 | return
7 |
10 | Handsign | Learn ASL using AI camera
12 |
31 |
32 | )
33 | }
34 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "handsign-alphabet",
3 | "private": true,
4 | "description": "A simple ASL (American Sign Language) alphabet detection using TensorFlow and Handpose model",
5 | "version": "0.1.0",
6 | "author": "Syauqy ",
7 | "dependencies": {
8 | "@chakra-ui/gatsby-plugin": "^1.0.0",
9 | "@chakra-ui/react": "^1.0.4",
10 | "@emotion/react": "^11.1.4",
11 | "@emotion/styled": "^11.0.0",
12 | "@tensorflow-models/handpose": "^0.0.6",
13 | "@tensorflow/tfjs": "^2.4.0",
14 | "@tensorflow/tfjs-backend-webgl": "^2.4.0",
15 | "fingerpose": "^0.0.2",
16 | "framer-motion": "^3.1.1",
17 | "prop-types": "^15.7.2",
18 | "next": "latest",
19 | "react": "latest",
20 | "react-dom": "latest",
21 | "react-helmet": "^6.1.0",
22 | "react-icons": "^4.1.0",
23 | "react-webcam": "^5.2.2"
24 | },
25 | "devDependencies": {
26 | "prettier": "2.1.2"
27 | },
28 | "keywords": [
29 | "gatsby"
30 | ],
31 | "license": "0BSD",
32 | "scripts": {
33 | "dev": "next dev",
34 | "build": "next build",
35 | "start": "next start"
36 | },
37 | "repository": {
38 | "type": "git",
39 | "url": "https://github.com/syauqy/handsign-tensorflow-gatsby"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/pages/404.js:
--------------------------------------------------------------------------------
1 | import React from "react"
2 |
3 | // import SEO from "../components/seo"
4 |
5 | const NotFoundPage = () => (
6 |
7 | {/*
*/}
8 |
404: Not Found
9 |
You just hit a route that doesn't exist... the sadness.
10 |
8 |
10 | )
11 | }
12 |
13 | export default MyApp
14 |
--------------------------------------------------------------------------------
/pages/index.js:
--------------------------------------------------------------------------------
1 | import React, { useRef, useState, useEffect } from "react"
2 | import * as tf from "@tensorflow/tfjs"
3 | import * as handpose from "@tensorflow-models/handpose"
4 | import Webcam from "react-webcam"
5 | import { drawHand } from "../components/handposeutil"
6 | import * as fp from "fingerpose"
7 | import Handsigns from "../components/handsigns"
8 |
9 | import {
10 | Text,
11 | Heading,
12 | Button,
13 | Image,
14 | Stack,
15 | Container,
16 | Box,
17 | VStack,
18 | ChakraProvider,
19 | } from "@chakra-ui/react"
20 |
21 | import { Signimage, Signpass } from "../components/handimage"
22 |
23 | import About from "../components/about"
24 | import Metatags from "../components/metatags"
25 |
26 | // import "../styles/App.css"
27 |
28 | // import "@tensorflow/tfjs-backend-webgl"
29 |
30 | import { RiCameraFill, RiCameraOffFill } from "react-icons/ri"
31 |
32 | export default function Home() {
33 | const webcamRef = useRef(null)
34 | const canvasRef = useRef(null)
35 |
36 | const [camState, setCamState] = useState("on")
37 |
38 | const [sign, setSign] = useState(null)
39 |
40 | let signList = []
41 | let currentSign = 0
42 |
43 | let gamestate = "started"
44 |
45 | // let net;
46 |
47 | async function runHandpose() {
48 | const net = await handpose.load()
49 | _signList()
50 |
51 | // window.requestAnimationFrame(loop);
52 |
53 | setInterval(() => {
54 | detect(net)
55 | }, 150)
56 | }
57 |
58 | function _signList() {
59 | signList = generateSigns()
60 | }
61 |
62 | function shuffle(a) {
63 | for (let i = a.length - 1; i > 0; i--) {
64 | const j = Math.floor(Math.random() * (i + 1))
65 | ;[a[i], a[j]] = [a[j], a[i]]
66 | }
67 | return a
68 | }
69 |
70 | function generateSigns() {
71 | const password = shuffle(Signpass)
72 | return password
73 | }
74 |
75 | async function detect(net) {
76 | // Check data is available
77 | if (
78 | typeof webcamRef.current !== "undefined" &&
79 | webcamRef.current !== null &&
80 | webcamRef.current.video.readyState === 4
81 | ) {
82 | // Get Video Properties
83 | const video = webcamRef.current.video
84 | const videoWidth = webcamRef.current.video.videoWidth
85 | const videoHeight = webcamRef.current.video.videoHeight
86 |
87 | // Set video width
88 | webcamRef.current.video.width = videoWidth
89 | webcamRef.current.video.height = videoHeight
90 |
91 | // Set canvas height and width
92 | canvasRef.current.width = videoWidth
93 | canvasRef.current.height = videoHeight
94 |
95 | // Make Detections
96 | const hand = await net.estimateHands(video)
97 |
98 | if (hand.length > 0) {
99 | //loading the fingerpose model
100 | const GE = new fp.GestureEstimator([
101 | fp.Gestures.ThumbsUpGesture,
102 | Handsigns.aSign,
103 | Handsigns.bSign,
104 | Handsigns.cSign,
105 | Handsigns.dSign,
106 | Handsigns.eSign,
107 | Handsigns.fSign,
108 | Handsigns.gSign,
109 | Handsigns.hSign,
110 | Handsigns.iSign,
111 | Handsigns.jSign,
112 | Handsigns.kSign,
113 | Handsigns.lSign,
114 | Handsigns.mSign,
115 | Handsigns.nSign,
116 | Handsigns.oSign,
117 | Handsigns.pSign,
118 | Handsigns.qSign,
119 | Handsigns.rSign,
120 | Handsigns.sSign,
121 | Handsigns.tSign,
122 | Handsigns.uSign,
123 | Handsigns.vSign,
124 | Handsigns.wSign,
125 | Handsigns.xSign,
126 | Handsigns.ySign,
127 | Handsigns.zSign,
128 | ])
129 |
130 | const estimatedGestures = await GE.estimate(hand[0].landmarks, 6.5)
131 | // document.querySelector('.pose-data').innerHTML =JSON.stringify(estimatedGestures.poseData, null, 2);
132 |
133 | if (gamestate === "started") {
134 | document.querySelector("#app-title").innerText =
135 | "Make a 👍 gesture with your hand to start"
136 | }
137 |
138 | if (
139 | estimatedGestures.gestures !== undefined &&
140 | estimatedGestures.gestures.length > 0
141 | ) {
142 | const confidence = estimatedGestures.gestures.map(p => p.confidence)
143 | const maxConfidence = confidence.indexOf(
144 | Math.max.apply(undefined, confidence)
145 | )
146 |
147 | //setting up game state, looking for thumb emoji
148 | if (
149 | estimatedGestures.gestures[maxConfidence].name === "thumbs_up" &&
150 | gamestate !== "played"
151 | ) {
152 | _signList()
153 | gamestate = "played"
154 | document.getElementById("emojimage").classList.add("play")
155 | document.querySelector(".tutor-text").innerText =
156 | "make a hand gesture based on letter shown below"
157 | } else if (gamestate === "played") {
158 | document.querySelector("#app-title").innerText = ""
159 |
160 | //looping the sign list
161 | if (currentSign === signList.length) {
162 | _signList()
163 | currentSign = 0
164 | return
165 | }
166 |
167 | // console.log(signList[currentSign].src.src)
168 |
169 | //game play state
170 |
171 | if (
172 | typeof signList[currentSign].src.src === "string" ||
173 | signList[currentSign].src.src instanceof String
174 | ) {
175 | document
176 | .getElementById("emojimage")
177 | .setAttribute("src", signList[currentSign].src.src)
178 | if (
179 | signList[currentSign].alt ===
180 | estimatedGestures.gestures[maxConfidence].name
181 | ) {
182 | currentSign++
183 | }
184 | setSign(estimatedGestures.gestures[maxConfidence].name)
185 | }
186 | } else if (gamestate === "finished") {
187 | return
188 | }
189 | }
190 | }
191 | // Draw hand lines
192 | const ctx = canvasRef.current.getContext("2d")
193 | drawHand(hand, ctx)
194 | }
195 | }
196 |
197 | // if (sign) {
198 | // console.log(sign, Signimage[sign])
199 | // }
200 |
201 | useEffect(() => {
202 | runHandpose()
203 | }, [])
204 |
205 | function turnOffCamera() {
206 | if (camState === "on") {
207 | setCamState("off")
208 | } else {
209 | setCamState("on")
210 | }
211 | }
212 |
213 | return (
214 |
215 |
217 |
218 |
219 |
229 |
230 |
237 | 🧙♀️ Loading the Magic 🧙♂️
238 |
239 |
240 |
241 | {camState === "on" ? (
242 |
245 | )}
246 |
247 | {sign ? (
248 |
258 |
259 | detected gestures
260 |
261 |
272 |
277 |
278 | Pose data */}
292 |
293 |
294 |
295 |
310 |
311 |
312 | )
313 | }
314 |
--------------------------------------------------------------------------------
/public/2764.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 |
9 |
11 |
12 |
--------------------------------------------------------------------------------
/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/syauqy/handsign-tensorflow/8d31c89037fd20f79741050f81f341d0c8d3702c/public/favicon.ico
--------------------------------------------------------------------------------
/public/loveyou_emoji.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
7 |
9 |
10 |
--------------------------------------------------------------------------------
/styles/Home.module.css:
--------------------------------------------------------------------------------
1 | .container {
2 | min-height: 100vh;
3 | padding: 0 0.5rem;
4 | display: flex;
5 | flex-direction: column;
6 | justify-content: center;
7 | align-items: center;
8 | height: 100vh;
9 | }
10 |
11 | .main {
12 | padding: 5rem 0;
13 | flex: 1;
14 | display: flex;
15 | flex-direction: column;
16 | justify-content: center;
17 | align-items: center;
18 | }
19 |
20 | .footer {
21 | width: 100%;
22 | height: 100px;
23 | border-top: 1px solid #eaeaea;
24 | display: flex;
25 | justify-content: center;
26 | align-items: center;
27 | }
28 |
29 | .footer a {
30 | display: flex;
31 | justify-content: center;
32 | align-items: center;
33 | flex-grow: 1;
34 | }
35 |
36 | .title a {
37 | color: #0070f3;
38 | text-decoration: none;
39 | }
40 |
41 | .title a:hover,
42 | .title a:focus,
43 | .title a:active {
44 | text-decoration: underline;
45 | }
46 |
47 | .title {
48 | margin: 0;
49 | line-height: 1.15;
50 | font-size: 4rem;
51 | }
52 |
53 | .title,
54 | .description {
55 | text-align: center;
56 | }
57 |
58 | .description {
59 | line-height: 1.5;
60 | font-size: 1.5rem;
61 | }
62 |
63 | .code {
64 | background: #fafafa;
65 | border-radius: 5px;
66 | padding: 0.75rem;
67 | font-size: 1.1rem;
68 | font-family: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono,
69 | Bitstream Vera Sans Mono, Courier New, monospace;
70 | }
71 |
72 | .grid {
73 | display: flex;
74 | align-items: center;
75 | justify-content: center;
76 | flex-wrap: wrap;
77 | max-width: 800px;
78 | margin-top: 3rem;
79 | }
80 |
81 | .card {
82 | margin: 1rem;
83 | padding: 1.5rem;
84 | text-align: left;
85 | color: inherit;
86 | text-decoration: none;
87 | border: 1px solid #eaeaea;
88 | border-radius: 10px;
89 | transition: color 0.15s ease, border-color 0.15s ease;
90 | width: 45%;
91 | }
92 |
93 | .card:hover,
94 | .card:focus,
95 | .card:active {
96 | color: #0070f3;
97 | border-color: #0070f3;
98 | }
99 |
100 | .card h2 {
101 | margin: 0 0 1rem 0;
102 | font-size: 1.5rem;
103 | }
104 |
105 | .card p {
106 | margin: 0;
107 | font-size: 1.25rem;
108 | line-height: 1.5;
109 | }
110 |
111 | .logo {
112 | height: 1em;
113 | margin-left: 0.5rem;
114 | }
115 |
116 | @media (max-width: 600px) {
117 | .grid {
118 | width: 100%;
119 | flex-direction: column;
120 | }
121 | }
122 |
--------------------------------------------------------------------------------
/styles/globals.css:
--------------------------------------------------------------------------------
1 | html,
2 | body {
3 | padding: 0;
4 | margin: 0;
5 | font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen,
6 | Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
7 | }
8 |
9 | a {
10 | color: inherit;
11 | text-decoration: none;
12 | }
13 |
14 | * {
15 | box-sizing: border-box;
16 | }
17 |
18 | .pabrik-tablist {
19 | overflow: scroll;
20 | }
21 |
22 | .pabrik-tablist::-webkit-scrollbar {
23 | display: none;
24 | }
25 |
26 | img {
27 | max-width: 100%;
28 | }
29 |
30 | .boops {
31 | display: block;
32 | overflow: hidden;
33 | position: fixed;
34 | top: 0;
35 | z-index: 9;
36 | height: 100%;
37 | width: 100%;
38 | object-fit: cover;
39 | }
40 |
41 | #webcam {
42 | position: fixed;
43 | height: 100%;
44 | width: 100%;
45 | object-fit: cover;
46 | transform: scaleX(-1);
47 | filter: FlipH;
48 | }
49 |
50 | #gesture-canvas {
51 | position: fixed;
52 | /* top:0;
53 | left:0;
54 | right: 0;
55 | bottom: 0; */
56 | height: 100%;
57 | width: 100%;
58 | object-fit: cover;
59 | transform: scaleX(-1);
60 | filter: FlipH;
61 | z-index: 9;
62 | }
63 |
64 | .pose-data {
65 | font-size: 10px;
66 | }
67 |
68 | #webcam-container {
69 | position: fixed;
70 | top: 0;
71 | left: 0;
72 | right: 0;
73 | bottom: 0;
74 | width: 100%;
75 | height: 100%;
76 | /* background: royalblue; */
77 | }
78 |
79 | #app-title {
80 | /* position: absolute; */
81 | /* top:0;
82 | left:0;
83 | right: 0;
84 | bottom: 0; */
85 | z-index: 5;
86 | }
87 |
88 | .tutor-text {
89 | z-index: 5;
90 | }
91 |
92 | #start-button {
93 | width: 400px;
94 | /* background-color: black;*/
95 | /* color: white; */
96 | /* font-size: 16px; */
97 | border-radius: 30px;
98 | border: none;
99 | padding: 15px 20px;
100 | text-align: center;
101 | /* box-shadow: 0 5px 10px 0 rgba(0,0,0,0.2); */
102 | position: fixed;
103 | bottom: 30px;
104 | left: calc(50% - 150px);
105 | z-index: 10;
106 | }
107 |
108 | #emojimage {
109 | /* position: fixed; */
110 | top: 60px;
111 | left: calc(50% - 50px);
112 | border-style: none;
113 | }
114 |
115 | #emojimage.play {
116 | /* position: fixed; */
117 | z-index: 10;
118 | }
119 |
120 | #emojis {
121 | /* display: grid;
122 | grid-template-columns: repeat(9, 1fr);
123 | height: 50px; */
124 | position: absolute;
125 | top: 0;
126 | left: 0;
127 | right: 0;
128 | bottom: 0;
129 | text-align: -webkit-center;
130 | }
131 |
132 | #emojis img {
133 | opacity: 0.5;
134 | transition: opacity 200ms linear;
135 | }
136 |
137 | #emojis img.found {
138 | opacity: 1;
139 | }
140 |
--------------------------------------------------------------------------------
5 |
6 |