├── gitignore.txt
├── package.json
├── README.md
├── index.html
└── main.js
/gitignore.txt:
--------------------------------------------------------------------------------
1 | /node_modules
2 | /.DS_Store
3 | package-lock.json
4 | yarn.lock
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sign-language-translator",
3 | "version": "0.0.1",
4 | "license": "do what you want but try to give back",
5 | "dependencies": {
6 | "deeplearn": "~0.5.0",
7 | "deeplearn-knn-image-classifier": "^0.3.0",
8 | "pre-commit": "^1.2.2"
9 | },
10 | "scripts": {
11 | "start": "budo main.js:dist/build.js --live --host localhost",
12 | "build": "browserify main.js -o dist/build.js"
13 | },
14 | "pre-commit": [
15 | "build"
16 | ],
17 | "browserify": {
18 | "transform": [
19 | [
20 | "babelify",
21 | {
22 | "presets": [
23 | "es2015"
24 | ],
25 | "plugins": [
26 | "syntax-async-functions",
27 | "transform-regenerator"
28 | ]
29 | }
30 | ]
31 | ]
32 | },
33 | "devDependencies": {
34 | "babel-core": "^6.26.0",
35 | "babel-loader": "^7.1.2",
36 | "babel-plugin-syntax-async-functions": "^6.13.0",
37 | "babel-plugin-transform-regenerator": "^6.26.0",
38 | "babel-preset-es2015": "^6.24.1",
39 | "babelify": "^8.0.0",
40 | "budo": "^10.0.4"
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Sign-Language-Translator
2 | Using TensorFlow to translate Sign Language in real-time. **_Final Release_**
3 |
4 | The Sign Language Translator is my final project for Grade 12 Computer Science and the ⚡#PoweredByTF 2.0 Challenge!. Its purpose is to allow users to communicate more effectively with their computers and other people. To be specific, using this program, you can sign multiple words with one gesture and copy the translated text with the click of a button. Additionally, users can video call each other and talk using gestures that get converted into Computer Speech.
5 |
6 | ## Features
7 | - [x] Hand Gesture Training and Classification
8 | - [x] Prediction works in varying Lighting Conditions
9 | - [x] Retrainable Image Classes
10 | - [x] Translated Text can be Copied to Clipboard
11 | - [x] Cards that display Information about each Gesture
12 | - [x] Video Call functionality
13 | - [x] Text to Speech of translated text
14 | - [x] Minimal stress on memory
15 | - [x] Cohesive Text Styling
16 | - [x] Simple User Interface
17 | - [x] Comprehensive Commenting
18 |
19 | ## Live Sign Translator on https://bit.ly/2CwPvgP
20 |
21 | ## To Run locally
22 | Open `index.html` in Chrome.
23 | *NOTE:* This will disable video call functionality.
24 |
25 | ## To Run on http://localhost:9966/
26 | ```
27 | npm install
28 | npm start
29 | ```
30 |
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | Sign Translator
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
Welcome to Sign Language Translator
29 |
30 |
31 |
36 |
Train Gestures
37 | Train about 30 samples of your Start Gesture and 30 for your idle, Stop Gesture.
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
Status: Not Ready
47 |
52 |
53 |
54 |
55 |
![checkmark]()
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
![checkmark]()
64 |
65 |
66 |
68 |
69 |
70 |
72 |
73 |
74 |
75 |
76 |
Add Gesture
77 |
78 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
Start Signing!
92 |
93 |
98 |
99 |
100 |
101 |
--------------------------------------------------------------------------------
/main.js:
--------------------------------------------------------------------------------
1 |
2 | // Importing the k-Nearest Neighbors Algorithm
3 | import {
4 | KNNImageClassifier
5 | } from 'deeplearn-knn-image-classifier';
6 | import * as dl from 'deeplearn';
7 |
8 | // Webcam Image size. Must be 227.
9 | const IMAGE_SIZE = 227;
10 | // K value for KNN. 10 means that we will take votes from 10 data points to classify each tensor.
11 | const TOPK = 10;
12 | // Percent confidence above which prediction needs to be to return a prediction.
13 | const confidenceThreshold = 0.98
14 |
15 | // Initial Gestures that need to be trained.
16 | // The start gesture is for signalling when to start prediction
17 | // The stop gesture is for signalling when to stop prediction
18 | var words = ["start", "stop"];
19 |
20 | /*
21 | The Main class is responsible for the training and prediction of words.
22 | It controls the webcam, user interface, as well as initiates the output of predicted words.
23 | */
24 | class Main {
25 | constructor() {
26 | // Initialize variables for display as well as prediction purposes
27 | this.exampleCountDisplay = [];
28 | this.checkMarks = [];
29 | this.gestureCards = [];
30 | this.training = -1; // -1 when no class is being trained
31 | this.videoPlaying = false;
32 | this.previousPrediction = -1;
33 | this.currentPredictedWords = [];
34 |
35 | // Variables to restrict prediction rate
36 | this.now;
37 | this.then = Date.now();
38 | this.startTime = this.then;
39 | this.fps = 5; //framerate - number of prediction per second
40 | this.fpsInterval = 1000 / this.fps;
41 | this.elapsed = 0;
42 |
43 | // Initalizing kNN model to none.
44 | this.knn = null;
45 | /* Initalizing previous kNN model that we trained when training of the current model
46 | is stopped or prediction has begun. */
47 | this.previousKnn = this.knn;
48 |
49 | // Storing all elements that from the User Interface that need to be altered into variables.
50 | this.welcomeContainer = document.getElementById("welcomeContainer");
51 | this.proceedBtn = document.getElementById("proceedButton");
52 | this.proceedBtn.style.display = "block";
53 | this.proceedBtn.classList.add("animated");
54 | this.proceedBtn.classList.add("flash");
55 | this.proceedBtn.addEventListener('click', () => {
56 | this.welcomeContainer.classList.add("slideOutUp");
57 | })
58 |
59 | this.stageTitle = document.getElementById("stage");
60 | this.stageInstruction = document.getElementById("steps");
61 | this.predButton = document.getElementById("predictButton");
62 | this.backToTrainButton = document.getElementById("backButton");
63 | this.nextButton = document.getElementById('nextButton');
64 |
65 | this.statusContainer = document.getElementById("status");
66 | this.statusText = document.getElementById("status-text");
67 |
68 | this.translationHolder = document.getElementById("translationHolder");
69 | this.translationText = document.getElementById("translationText");
70 | this.translatedCard = document.getElementById("translatedCard");
71 |
72 | this.initialTrainingHolder = document.getElementById('initialTrainingHolder');
73 |
74 | this.videoContainer = document.getElementById("videoHolder");
75 | this.video = document.getElementById("video");
76 |
77 | this.trainingContainer = document.getElementById("trainingHolder");
78 | this.addGestureTitle = document.getElementById("add-gesture");
79 | this.plusImage = document.getElementById("plus_sign");
80 | this.addWordForm = document.getElementById("add-word");
81 | this.newWordInput = document.getElementById("new-word");
82 | this.doneRetrain = document.getElementById("doneRetrain");
83 | this.trainingCommands = document.getElementById("trainingCommands");
84 |
85 | this.videoCallBtn = document.getElementById("videoCallBtn");
86 | this.videoCall = document.getElementById("videoCall");
87 |
88 | this.trainedCardsHolder = document.getElementById("trainedCardsHolder");
89 |
90 | // Start Translator function is called
91 | this.initializeTranslator();
92 |
93 | // Instantiate Prediction Output
94 | this.predictionOutput = new PredictionOutput();
95 | }
96 |
97 | /*This function starts the webcam and initial training process. It also loads the kNN
98 | classifier*/
99 | initializeTranslator() {
100 | this.startWebcam();
101 | this.initialTraining();
102 | this.loadKNN();
103 | }
104 |
105 | //This function sets up the webcam
106 | startWebcam() {
107 | navigator.mediaDevices.getUserMedia({
108 | video: {
109 | facingMode: 'user'
110 | },
111 | audio: false
112 | })
113 | .then((stream) => {
114 | this.video.srcObject = stream;
115 | this.video.width = IMAGE_SIZE;
116 | this.video.height = IMAGE_SIZE;
117 | this.video.addEventListener('playing', () => this.videoPlaying = true);
118 | this.video.addEventListener('paused', () => this.videoPlaying = false);
119 | })
120 | }
121 |
122 | /*This function initializes the training for Start and Stop Gestures. It also
123 | sets a click listener for the next button.*/
124 | initialTraining() {
125 | // if next button on initial training page is pressed, setup the custom gesture training UI.
126 | this.nextButton.addEventListener('click', () => {
127 | const exampleCount = this.knn.getClassExampleCount();
128 | if (Math.max(...exampleCount) > 0) {
129 | // if start gesture has not been trained
130 | if (exampleCount[0] == 0) {
131 | alert('You haven\'t added examples for the Start Gesture');
132 | return;
133 | }
134 |
135 | // if stop gesture has not been trained
136 | if (exampleCount[1] == 0) {
137 | alert('You haven\'t added examples for the Stop Gesture.\n\nCapture yourself in idle states e.g hands by your side, empty background etc.');
138 | return;
139 | }
140 |
141 | this.nextButton.style.display = "none";
142 | this.stageTitle.innerText = "Continue Training";
143 | this.stageInstruction.innerText = "Add Gesture Name and Train.";
144 |
145 | //Start custom gesture training process
146 | this.setupTrainingUI();
147 | }
148 | });
149 |
150 | //Create initial training buttons
151 | this.initialGestures(0, "startButton");
152 | this.initialGestures(1, "stopButton");
153 | }
154 |
155 | //This function loads the kNN classifier
156 | loadKNN() {
157 | this.knn = new KNNImageClassifier(words.length, TOPK);
158 |
159 | // Load knn model
160 | this.knn.load().then(() => this.initializeTraining());
161 | }
162 |
163 | /*This creates the training and clear buttons for the initial Start and Stop gesture.
164 | It also creates the Gesture Card.*/
165 | initialGestures(i, btnType) {
166 | // Get specified training button
167 | var trainBtn = document.getElementById(btnType);
168 |
169 | // Call training function for this gesture on click
170 | trainBtn.addEventListener('click', () => {
171 | this.train(i);
172 | });
173 |
174 | // Clear button to remove training examples on click
175 | var clearBtn = document.getElementById('clear_' + btnType);
176 | clearBtn.addEventListener('click', () => {
177 | this.knn.clearClass(i);
178 | this.exampleCountDisplay[i].innerText = " 0 examples";
179 | this.gestureCards[i].removeChild(this.gestureCards[i].childNodes[1]);
180 | this.checkMarks[i].src = "Images\\loader.gif";
181 | });
182 |
183 | // Variables for training information for the user
184 | var exampleCountDisplay = document.getElementById('counter_' + btnType);
185 | var checkMark = document.getElementById('checkmark_' + btnType);
186 |
187 | // Create Gesture Card
188 | var gestureCard = document.createElement("div");
189 | gestureCard.className = "trained-gestures";
190 |
191 | var gestName = "";
192 | if (i == 0) {
193 | gestName = "Start";
194 | } else {
195 | gestName = "Stop";
196 | }
197 | var gestureName = document.createElement("h5");
198 | gestureName.innerText = gestName;
199 | gestureCard.appendChild(gestureName);
200 | this.trainedCardsHolder.appendChild(gestureCard);
201 |
202 | exampleCountDisplay.innerText = " 0 examples";
203 | checkMark.src = 'Images\\loader.gif';
204 | this.exampleCountDisplay.push(exampleCountDisplay);
205 | this.checkMarks.push(checkMark);
206 | this.gestureCards.push(gestureCard);
207 | }
208 |
209 | /*This function sets up the custom gesture training UI.*/
210 | setupTrainingUI() {
211 | const exampleCount = this.knn.getClassExampleCount();
212 | // check if training is complete
213 | if (Math.max(...exampleCount) > 0) {
214 | // if start gesture has not been trained
215 | if (exampleCount[0] == 0) {
216 | alert('You haven\'t added examples for the wake word');
217 | return;
218 | }
219 |
220 | // if stop gesture has not been trained
221 | if (exampleCount[1] == 0) {
222 | alert('You haven\'t added examples for the Stop Gesture.\n\nCapture yourself in idle states e.g hands by your side, empty background etc.');
223 | return;
224 | }
225 |
226 | // Remove Initial Training Screen
227 | this.initialTrainingHolder.style.display = "none";
228 |
229 | // Add the Custom Gesture Training UI
230 | this.trainingContainer.style.display = "block";
231 | this.trainedCardsHolder.style.display = "block";
232 |
233 | // Add Gesture on Submission of new gesture form
234 | this.addWordForm.addEventListener('submit', (e) => {
235 | this.trainingCommands.innerHTML = "";
236 |
237 | e.preventDefault(); // preventing default submission action
238 | var word = this.newWordInput.value.trim(); // returns new word without whitespace
239 |
240 | // if a new word is entered, add it to the gesture classes and start training
241 | if (word && !words.includes(word)) {
242 | //Add word to words array
243 | words.push(word);
244 |
245 | // Create train and clear buttons for new gesture and set reset form
246 | this.createTrainingBtns(words.indexOf(word));
247 | this.newWordInput.value = '';
248 |
249 | // Increase the amount of classes and array length in the kNN model
250 | this.knn.numClasses += 1;
251 | this.knn.classLogitsMatrices.push(null);
252 | this.knn.classExampleCount.push(0);
253 |
254 | // Start training the word and create the translate button
255 | this.initializeTraining();
256 | this.createTranslateBtn();
257 | } else {
258 | alert("Duplicate word or no word entered");
259 | }
260 | return;
261 | });
262 | } else {
263 | alert('You haven\'t added any examples yet.\n\nAdd a Gesture, then perform the sign in front of the webcam.');
264 | }
265 | }
266 |
267 | /*This creates the training and clear buttons for the new gesture. It also creates the
268 | Gesture Card.*/
269 | createTrainingBtns(i) { //i is the index of the new word
270 | // Create Train and Clear Buttons
271 | var trainBtn = document.createElement('button');
272 | trainBtn.className = "trainBtn";
273 | trainBtn.innerText = "Train";
274 | this.trainingCommands.appendChild(trainBtn);
275 |
276 | var clearBtn = document.createElement('button');
277 | clearBtn.className = "clearButton";
278 | clearBtn.innerText = "Clear";
279 | this.trainingCommands.appendChild(clearBtn);
280 |
281 | // Change training class from none to specified class if training button is pressed
282 | trainBtn.addEventListener('mousedown', () => {
283 | this.train(i);
284 | });
285 |
286 | // Create clear button to remove training examples on click
287 | clearBtn.addEventListener('click', () => {
288 | this.knn.clearClass(i);
289 | this.exampleCountDisplay[i].innerText = " 0 examples";
290 | this.gestureCards[i].removeChild(this.gestureCards[i].childNodes[1]);
291 | this.checkMarks[i].src = 'Images\\loader.gif';
292 | });
293 |
294 | // Create elements to display training information for the user
295 | var exampleCountDisplay = document.createElement('h3');
296 | exampleCountDisplay.style.color = "black";
297 | this.trainingCommands.appendChild(exampleCountDisplay);
298 |
299 | var checkMark = document.createElement('img');
300 | checkMark.className = "checkMark";
301 | this.trainingCommands.appendChild(checkMark);
302 |
303 | //Create Gesture Card
304 | var gestureCard = document.createElement("div");
305 | gestureCard.className = "trained-gestures";
306 |
307 | var gestName = words[i];
308 | var gestureName = document.createElement("h5");
309 | gestureName.innerText = gestName;
310 | gestureCard.appendChild(gestureName);
311 | this.trainedCardsHolder.appendChild(gestureCard);
312 |
313 | exampleCountDisplay.innerText = " 0 examples";
314 | checkMark.src = 'Images\\loader.gif';
315 | this.exampleCountDisplay.push(exampleCountDisplay);
316 | this.checkMarks.push(checkMark);
317 | this.gestureCards.push(gestureCard);
318 |
319 | // Retrain/Continue Training gesture on click of the gesture card
320 | gestureCard.addEventListener('click', () => { //create btn
321 | /* If gesture card was not already pressed display the specific gesture card's
322 | training buttons to train it*/
323 | if (gestureCard.style.marginTop == "17px" || gestureCard.style.marginTop == "") {
324 | this.addWordForm.style.display = "none";
325 | this.addGestureTitle.innerText = gestName;
326 | this.plusImage.src = "Images/retrain.svg";
327 | this.plusImage.classList.add("rotateIn");
328 |
329 | // Display done retraining button and the training buttons for the specific gesture
330 | this.doneRetrain.style.display = "block";
331 | this.trainingCommands.innerHTML = "";
332 | this.trainingCommands.appendChild(trainBtn);
333 | this.trainingCommands.appendChild(clearBtn);
334 | this.trainingCommands.appendChild(exampleCountDisplay);
335 | this.trainingCommands.appendChild(checkMark);
336 | gestureCard.style.marginTop = "-10px";
337 | }
338 | // if gesture card is pressed again, change the add gesture card back to add gesture mode instead of retrain mode
339 | else {
340 | this.addGestureTitle.innerText = "Add Gesture";
341 | this.addWordForm.style.display = "block";
342 | gestureCard.style.marginTop = "17px";
343 |
344 | this.trainingCommands.innerHTML = "";
345 | this.addWordForm.style.display = "block";
346 | this.doneRetrain.style.display = "none";
347 | this.plusImage.src = "Images/plus_sign.svg";
348 | this.plusImage.classList.add("rotateInLeft");
349 | }
350 | });
351 |
352 | // if done retrain button is pressed again, change the add gesture card back to add gesture mode instead of retrain mode
353 | this.doneRetrain.addEventListener('click', () => {
354 | this.addGestureTitle.innerText = "Add Gesture";
355 | this.addWordForm.style.display = "block";
356 | gestureCard.style.marginTop = "17px";
357 |
358 | this.trainingCommands.innerHTML = "";
359 | this.addWordForm.style.display = "block";
360 | this.plusImage.src = "Images/plus_sign.svg";
361 | this.plusImage.classList.add("rotateInLeft");
362 | this.doneRetrain.style.display = "none";
363 | });
364 | }
365 |
366 | // This function starts the training process.
367 | initializeTraining() {
368 | if (this.timer) {
369 | this.stopTraining();
370 | }
371 | var promise = this.video.play();
372 |
373 | if (promise !== undefined) {
374 | promise.then(_ => {
375 | console.log("Autoplay started")
376 | }).catch(error => {
377 | console.log("Autoplay prevented")
378 | })
379 | }
380 | }
381 |
382 | // This function adds examples for the gesture to the kNN model
383 | train(gestureIndex) {
384 | console.log(this.videoPlaying);
385 | if (this.videoPlaying) {
386 | console.log("entered training");
387 | // Get image data from video element
388 | const image = dl.fromPixels(this.video);
389 |
390 | // Add current image to classifier
391 | this.knn.addImage(image, gestureIndex);
392 |
393 | // Get example count
394 | const exampleCount = this.knn.getClassExampleCount()[gestureIndex];
395 |
396 | if (exampleCount > 0) {
397 | //if example count for this particular gesture is more than 0, update it
398 | this.exampleCountDisplay[gestureIndex].innerText = ' ' + exampleCount + ' examples';
399 |
400 | //if example count for this particular gesture is 1, add a capture of the gesture to gesture cards
401 | if (exampleCount == 1 && this.gestureCards[gestureIndex].childNodes[1] == null) {
402 | var gestureImg = document.createElement("canvas");
403 | gestureImg.className = "trained_image";
404 | gestureImg.getContext('2d').drawImage(video, 0, 0, 400, 180);
405 | this.gestureCards[gestureIndex].appendChild(gestureImg);
406 | }
407 |
408 | // if 30 examples are trained, show check mark to the user
409 | if (exampleCount == 30) {
410 | this.checkMarks[gestureIndex].src = "Images//checkmark.svg";
411 | this.checkMarks[gestureIndex].classList.add("animated");
412 | this.checkMarks[gestureIndex].classList.add("rotateIn");
413 | }
414 | }
415 | }
416 | }
417 |
418 | /*This function creates the button that goes to the Translate Page. It also initializes the UI
419 | of the translate page and starts or stops prediction on click.*/
420 | createTranslateBtn() {
421 | this.predButton.style.display = "block";
422 | this.createVideoCallBtn(); // create video call button that displays on translate page
423 | this.createBackToTrainBtn(); // create back to train button that will go back to training page
424 |
425 | this.predButton.addEventListener('click', () => {
426 | // Change the styling of video display and start prediction
427 | console.log("go to translate");
428 | const exampleCount = this.knn.getClassExampleCount();
429 | // check if training is complete
430 | if (Math.max(...exampleCount) > 0) {
431 | this.video.style.display = "inline-block"; // turn on video from webscam in case it's off
432 |
433 | this.videoCall.style.display = "none"; // turn off video call in case it's on
434 | this.videoCallBtn.style.display = "block";
435 |
436 | this.backToTrainButton.style.display = "block";
437 |
438 | // Change style of video display
439 | this.video.className = "videoPredict";
440 | this.videoContainer.style.display = "inline-block";
441 | this.videoContainer.style.width = "";
442 | this.videoContainer.style.height = "";
443 | this.videoContainer.className = "videoContainerPredict";
444 | this.videoContainer.style.border = "8px solid black";
445 |
446 |
447 | // Update stage and instruction info
448 | this.stageTitle.innerText = "Translate";
449 | this.stageInstruction.innerText = "Start Translating with your Start Gesture.";
450 |
451 | // Remove training UI
452 | this.trainingContainer.style.display = "none";
453 | this.trainedCardsHolder.style.marginTop = "130px";
454 |
455 | // Display translation holder that contains translated text
456 | this.translationHolder.style.display = "block";
457 |
458 | this.predButton.style.display = "none";
459 | // Start Translation
460 | this.setUpTranslation();
461 | } else {
462 | alert('You haven\'t added any examples yet.\n\nPress and hold on the "Add Example" button next to each word while performing the sign in front of the webcam.');
463 | }
464 | })
465 | }
466 |
467 | /*This function stops the training process and allows user's to copy text on the click of
468 | the translation text.*/
469 | setUpTranslation() {
470 | // stop training
471 | if (this.timer) {
472 | this.stopTraining();
473 | }
474 |
475 | // Set status to predict, call copy translated text listener and start prediction
476 | this.setStatusText("Status: Ready to Predict!", "predict");
477 | this.video.play();
478 | this.pred = requestAnimationFrame(this.predict.bind(this));
479 | }
480 |
481 | /*This function predicts the class of the gesture and returns the predicted text if its above a set threshold.*/
482 | predict() {
483 | this.now = Date.now();
484 | this.elapsed = this.now - this.then;
485 |
486 | if (this.elapsed > this.fpsInterval) {
487 | this.then = this.now - this.elapsed % this.fpsInterval;
488 | if (this.videoPlaying) {
489 | const exampleCount = this.knn.getClassExampleCount();
490 | const image = dl.fromPixels(this.video);
491 |
492 | if (Math.max(...exampleCount) > 0) {
493 | this.knn.predictClass(image)
494 | .then((res) => {
495 | for (let i = 0; i < words.length; i++) {
496 | /*if gesture matches this word & is above threshold & isn't same as prev prediction
497 | and is not stop gesture, return that word to the user*/
498 | if (res.classIndex == i && res.confidences[i] > confidenceThreshold && res.classIndex != this.previousPrediction) { // && res.classIndex != 1) {
499 | this.setStatusText("Status: Predicting!", "predict");
500 |
501 | // Send word to Prediction Output so it will display or speak out the word.
502 | this.predictionOutput.textOutput(words[i], this.gestureCards[i], res.confidences[i] * 100);
503 |
504 | // set previous prediction so it doesnt get called again
505 | this.previousPrediction = res.classIndex;
506 | }
507 | }
508 | }).then(() => image.dispose())
509 | } else {
510 | image.dispose();
511 | }
512 | }
513 | }
514 |
515 | // Recursion on predict method
516 | this.pred = requestAnimationFrame(this.predict.bind(this));
517 | }
518 |
519 | /*This function pauses the predict method*/
520 | pausePredicting() {
521 | console.log("pause predicting");
522 | this.setStatusText("Status: Paused Predicting", "predict");
523 | cancelAnimationFrame(this.pred);
524 | this.previousKnn = this.knn;
525 | }
526 |
527 | // if predict button is actually a back to training button, stop translation and recreate training UI
528 | createBackToTrainBtn() {
529 | this.backToTrainButton.addEventListener('click', () => {
530 | main.pausePredicting();
531 |
532 | this.stageTitle.innerText = "Continue Training";
533 | this.stageInstruction.innerText = "Add Gesture Name and Train.";
534 |
535 | this.predButton.innerText = "Translate";
536 | this.predButton.style.display = "block";
537 | this.backToTrainButton.style.display = "none";
538 | this.statusContainer.style.display = "none";
539 |
540 | // Remove all elements from translation mode
541 | this.video.className = "videoTrain";
542 | this.videoContainer.className = "videoContainerTrain";
543 | this.videoCallBtn.style.display = "none";
544 |
545 | this.translationHolder.style.display = "none";
546 | this.statusContainer.style.display = "none";
547 |
548 | // Show elements from training mode
549 | this.trainingContainer.style.display = "block";
550 | this.trainedCardsHolder.style.marginTop = "0px";
551 | this.trainedCardsHolder.style.display = "block";
552 | });
553 | }
554 |
555 | /*This function stops the training process*/
556 | stopTraining() {
557 | this.video.pause();
558 | cancelAnimationFrame(this.timer);
559 | console.log("Knn for start: " + this.knn.getClassExampleCount()[0]);
560 | this.previousKnn = this.knn; // saves current knn model so it can be used later
561 | }
562 |
563 | /*This function displays the button that start video call.*/
564 | createVideoCallBtn() {
565 | // Display video call feed instead of normal webcam feed when video call btn is clicked
566 | videoCallBtn.addEventListener('click', () => {
567 | this.stageTitle.innerText = "Video Call";
568 | this.stageInstruction.innerText = "Translate Gestures to talk to people on Video Call";
569 |
570 | this.video.style.display = "none";
571 | this.videoContainer.style.borderStyle = "none";
572 | this.videoContainer.style.overflow = "hidden";
573 | this.videoContainer.style.width = "630px";
574 | this.videoContainer.style.height = "355px";
575 |
576 | this.videoCall.style.display = "block";
577 | this.videoCallBtn.style.display = "none";
578 | this.backToTrainButton.style.display = "none";
579 | this.predButton.innerText = "Local Translation";
580 | this.predButton.style.display = "block";
581 |
582 | this.setStatusText("Status: Video Call Activated");
583 | })
584 | }
585 | /*This function sets the status text*/
586 | setStatusText(status, type) { //make default type thing
587 | this.statusContainer.style.display = "block";
588 | this.statusText.innerText = status;
589 | if (type == "copy") {
590 | console.log("copy");
591 | this.statusContainer.style.backgroundColor = "blue";
592 | } else {
593 | this.statusContainer.style.backgroundColor = "black";
594 | }
595 | }
596 | }
597 |
598 | /*
599 | The PredictionOutput class is responsible for turning the translated gesture into text, gesture card, and speech output.
600 | */
601 | class PredictionOutput {
602 | constructor() {
603 | //Initializing variables for speech synthesis and output
604 | this.synth = window.speechSynthesis;
605 | this.voices = [];
606 | this.pitch = 1.0;
607 | this.rate = 0.9;
608 |
609 | this.statusContainer = document.getElementById("status");
610 | this.statusText = document.getElementById("status-text");
611 |
612 | this.translationHolder = document.getElementById("translationHolder");
613 | this.translationText = document.getElementById("translationText");
614 | this.translatedCard = document.getElementById("translatedCard");
615 | this.trainedCardsHolder = document.getElementById("trainedCardsHolder");
616 |
617 | this.selectedVoice = 48; // this is Google-US en. Can set voice and language of choice
618 |
619 | this.currentPredictedWords = [];
620 | this.waitTimeForQuery = 10000;
621 |
622 | this.synth.onvoiceschanged = () => {
623 | this.populateVoiceList()
624 | };
625 |
626 | //Set up copy translation event listener
627 | this.copyTranslation();
628 | }
629 |
630 | // Checks if speech synthesis is possible and if selected voice is available
631 | populateVoiceList() {
632 | if (typeof speechSynthesis === 'undefined') {
633 | console.log("no synth");
634 | return;
635 | }
636 | this.voices = this.synth.getVoices();
637 |
638 | if (this.voices.indexOf(this.selectedVoice) > 0) {
639 | console.log(this.voices[this.selectedVoice].name + ':' + this.voices[this.selectedVoice].lang);
640 | }
641 | }
642 |
643 | /*This function outputs the word using text and gesture cards*/
644 | textOutput(word, gestureCard, gestureAccuracy) {
645 | // If the word is start, clear translated text content
646 | if (word == 'start') {
647 | this.clearPara();
648 |
649 | setTimeout(() => {
650 | // if no query detected after start is signed, clear para
651 | if (this.currentPredictedWords.length == 1) {
652 | this.clearPara();
653 | }
654 | }, this.waitTimeForQuery);
655 | }
656 |
657 | // If first word is not start, return
658 | if (word != 'start' && this.currentPredictedWords.length == 0) {
659 | return;
660 | }
661 |
662 | // If word was already said in this query, return
663 | if (this.currentPredictedWords.includes(word)) {
664 | return;
665 | }
666 |
667 | // Add word to predicted words in this query
668 | this.currentPredictedWords.push(word);
669 |
670 | // Depending on the word, display the text output
671 | if (word == "start") {
672 | this.translationText.innerText += ' ';
673 | } else if (word == "stop") {
674 | this.translationText.innerText += '.';
675 | } else {
676 | this.translationText.innerText += ' ' + word;
677 | }
678 |
679 | //Clone Gesture Card
680 | this.translatedCard.innerHTML = " ";
681 | var clonedCard = document.createElement("div");
682 | clonedCard.className = "trained-gestures";
683 |
684 | var gestName = gestureCard.childNodes[0].innerText;
685 | var gestureName = document.createElement("h5");
686 | gestureName.innerText = gestName;
687 | clonedCard.appendChild(gestureName);
688 |
689 | var gestureImg = document.createElement("canvas");
690 | gestureImg.className = "trained_image";
691 | gestureImg.getContext('2d').drawImage(gestureCard.childNodes[1], 0, 0, 400, 180);
692 | clonedCard.appendChild(gestureImg);
693 |
694 | var gestAccuracy = document.createElement("h7");
695 | gestAccuracy.innerText = "Confidence: " + gestureAccuracy + "%";
696 | clonedCard.appendChild(gestAccuracy);
697 |
698 | this.translatedCard.appendChild(clonedCard);
699 |
700 | // If its not video call mode, speak out the user's word
701 | if (word != "start" && word != "stop") {
702 | this.speak(word);
703 | }
704 | }
705 |
706 | /*This functions clears translation text and cards. Sets the previous predicted words to null*/
707 | clearPara() {
708 | this.translationText.innerText = '';
709 | main.previousPrediction = -1;
710 | this.currentPredictedWords = []; // empty words in this query
711 | this.translatedCard.innerHTML = " ";
712 | }
713 |
714 | /*The function below is adapted from https://stackoverflow.com/questions/45071353/javascript-copy-text-string-on-click/53977796#53977796
715 | It copies the translated text to the user's clipboard*/
716 | copyTranslation() {
717 | this.translationHolder.addEventListener('mousedown', () => {
718 | main.setStatusText("Text Copied!", "copy");
719 | const el = document.createElement('textarea'); // Create a