├── src ├── assets │ ├── .gitkeep │ ├── i18n │ │ ├── es.json │ │ └── en.json │ ├── images │ │ ├── google │ │ │ ├── sign-in.png │ │ │ └── sign-in.webp │ │ └── conference │ │ │ └── hangoutsMeet.png │ └── models │ │ └── sign-detector │ │ ├── group1-shard1of1.bin │ │ └── model.json ├── app │ ├── components │ │ ├── audio │ │ │ ├── audio.component.html │ │ │ ├── broadcast-test │ │ │ │ ├── broadcast-test.component.css │ │ │ │ ├── broadcast-test.component.ts │ │ │ │ ├── broadcast-test.component.spec.ts │ │ │ │ └── broadcast-test.component.html │ │ │ ├── audio.component.css │ │ │ ├── audio-instructions │ │ │ │ ├── audio-instructions.component.css │ │ │ │ ├── audio-instructions.component.ts │ │ │ │ ├── audio-instructions.component.spec.ts │ │ │ │ └── audio-instructions.component.html │ │ │ ├── audio.component.spec.ts │ │ │ └── audio.component.ts │ │ ├── settings │ │ │ ├── settings.component.scss │ │ │ ├── settings.component.html │ │ │ ├── settings.component.spec.ts │ │ │ └── settings.component.ts │ │ ├── video │ │ │ ├── video-help │ │ │ │ ├── video-help.component.scss │ │ │ │ ├── video-help.component.ts │ │ │ │ ├── video-help.component.html │ │ │ │ └── video-help.component.spec.ts │ │ │ ├── video-pose │ │ │ │ ├── video-pose.component.css │ │ │ │ ├── video-pose.component.html │ │ │ │ ├── video-pose.component.spec.ts │ │ │ │ └── video-pose.component.ts │ │ │ ├── video-controls │ │ │ │ ├── video-controls.component.scss │ │ │ │ ├── video-controls.component.spec.ts │ │ │ │ ├── video-controls.component.ts │ │ │ │ └── video-controls.component.html │ │ │ ├── video.component.html │ │ │ ├── video.component.scss │ │ │ ├── video.component.spec.ts │ │ │ └── video.component.ts │ │ ├── header │ │ │ ├── header.component.html │ │ │ ├── header.component.ts │ │ │ ├── header.component.scss │ │ │ └── header.component.spec.ts │ │ └── base │ │ │ └── base.component.ts │ ├── app.component.scss │ ├── pages │ │ ├── help │ │ │ ├── help.component.css │ │ │ ├── help.component.html │ │ │ ├── help.component.ts │ │ │ └── help.component.spec.ts │ │ └── main │ │ │ ├── main.component.html │ │ │ ├── main.component.ts │ │ │ ├── main.component.scss │ │ │ └── main.component.spec.ts │ ├── core │ │ ├── helpers │ │ │ └── wait │ │ │ │ └── wait.ts │ │ ├── modules │ │ │ ├── ngxs │ │ │ │ ├── store │ │ │ │ │ ├── video │ │ │ │ │ │ ├── video.actions.ts │ │ │ │ │ │ ├── video.state.ts │ │ │ │ │ │ └── video.spec.ts │ │ │ │ │ ├── audio │ │ │ │ │ │ ├── audio.actions.ts │ │ │ │ │ │ └── audio.state.ts │ │ │ │ │ ├── settings │ │ │ │ │ │ ├── settings.actions.ts │ │ │ │ │ │ └── settings.state.ts │ │ │ │ │ ├── models │ │ │ │ │ │ ├── models.actions.ts │ │ │ │ │ │ └── models.state.ts │ │ │ │ │ └── app │ │ │ │ │ │ ├── app.actions.ts │ │ │ │ │ │ └── app.state.ts │ │ │ │ ├── ngxs.module.ts │ │ │ │ └── ngxs-interceptor │ │ │ │ │ └── ngxs-interceptor.plugin.ts │ │ │ ├── transloco │ │ │ │ ├── transloco.loader.ts │ │ │ │ └── transloco.module.ts │ │ │ ├── angular-fire │ │ │ │ └── angular-fire.module.ts │ │ │ ├── shared.module.ts │ │ │ └── angular-material │ │ │ │ └── angular-material.module.ts │ │ └── services │ │ │ ├── pose │ │ │ ├── pose.service.spec.ts │ │ │ ├── models │ │ │ │ ├── base.pose-model.ts │ │ │ │ └── posenet.pose-model.ts │ │ │ └── pose.service.ts │ │ │ ├── detector │ │ │ ├── detector.service.spec.ts │ │ │ └── detector.service.ts │ │ │ └── navigator │ │ │ ├── navigator.service.spec.ts │ │ │ └── navigator.service.ts │ ├── app.component.html │ ├── app-routing.module.ts │ ├── app.component.spec.ts │ ├── app.module.ts │ └── app.component.ts ├── favicon.ico ├── main.ts ├── environments │ ├── environment.prod.ts │ └── environment.ts ├── index.html ├── test.ts ├── theme │ ├── styles.scss │ └── variables.scss └── polyfills.ts ├── .firebaserc ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ └── build_client.yml ├── firebase.json ├── tsconfig.app.json ├── .editorconfig ├── e2e ├── src │ ├── app.po.ts │ └── app.e2e-spec.ts ├── tsconfig.json └── protractor.conf.js ├── tsconfig.spec.json ├── tsconfig.base.json ├── tsconfig.json ├── .gitignore ├── .browserslistrc ├── LICENSE ├── karma.conf.js ├── README.md ├── package.json ├── tslint.json └── angular.json /src/assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/components/audio/audio.component.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/components/settings/settings.component.scss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/components/video/video-help/video-help.component.scss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/components/video/video-pose/video-pose.component.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/components/audio/broadcast-test/broadcast-test.component.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/app.component.scss: -------------------------------------------------------------------------------- 1 | mat-sidenav-container { 2 | height: 100%; 3 | } 4 | -------------------------------------------------------------------------------- /src/app/components/audio/audio.component.css: -------------------------------------------------------------------------------- 1 | :host { 2 | display: none; 3 | } 4 | -------------------------------------------------------------------------------- /src/app/components/video/video-pose/video-pose.component.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "default": "sign-language-detector" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/assets/i18n/es.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "transloco es", 3 | "dynamic": "transloco {{value}}" 4 | } 5 | -------------------------------------------------------------------------------- /src/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/detection-app/HEAD/src/favicon.ico -------------------------------------------------------------------------------- /src/app/components/audio/audio-instructions/audio-instructions.component.css: -------------------------------------------------------------------------------- 1 | mat-dialog-actions { 2 | justify-content: flex-end; 3 | } 4 | -------------------------------------------------------------------------------- /src/app/pages/help/help.component.css: -------------------------------------------------------------------------------- 1 | :host { 2 | display: block; 3 | max-width: 1280px; 4 | margin: auto; 5 | padding: 16px; 6 | } 7 | -------------------------------------------------------------------------------- /src/assets/images/google/sign-in.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/detection-app/HEAD/src/assets/images/google/sign-in.png -------------------------------------------------------------------------------- /src/assets/images/google/sign-in.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/detection-app/HEAD/src/assets/images/google/sign-in.webp -------------------------------------------------------------------------------- /src/app/core/helpers/wait/wait.ts: -------------------------------------------------------------------------------- 1 | export function wait(ms: number): Promise { 2 | return new Promise(resolve => setTimeout(resolve, ms)); 3 | } 4 | -------------------------------------------------------------------------------- /src/app/pages/main/main.component.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
5 |
6 | -------------------------------------------------------------------------------- /src/assets/images/conference/hangoutsMeet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/detection-app/HEAD/src/assets/images/conference/hangoutsMeet.png -------------------------------------------------------------------------------- /src/assets/models/sign-detector/group1-shard1of1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sign-language-processing/detection-app/HEAD/src/assets/models/sign-detector/group1-shard1of1.bin -------------------------------------------------------------------------------- /src/app/components/header/header.component.html: -------------------------------------------------------------------------------- 1 | 2 | hearing 3 | 4 | {{'header.title' | transloco}} 5 | 6 | -------------------------------------------------------------------------------- /src/app/pages/help/help.component.html: -------------------------------------------------------------------------------- 1 |

Help

2 |

This is the help page where structure has not yet been decided.

3 |

In my opinion, it needs a written + video explanation of this app.

4 | -------------------------------------------------------------------------------- /src/app/app.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Expected Behavior 2 | 3 | 4 | ## Actual Behavior 5 | 6 | 7 | ## Steps to Reproduce the Problem 8 | 9 | 1. 10 | 1. 11 | 1. 12 | 13 | ## Specifications 14 | 15 | - Version: 16 | - Platform: 17 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/video/video.actions.ts: -------------------------------------------------------------------------------- 1 | export class StartCamera { 2 | static readonly type = '[Video] Start Camera'; 3 | } 4 | 5 | export class StopCamera { 6 | static readonly type = '[Video] Stop Camera'; 7 | } 8 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/audio/audio.actions.ts: -------------------------------------------------------------------------------- 1 | export class EnableTransmission { 2 | static readonly type = '[Audio] Enable Transmission'; 3 | } 4 | 5 | export class DisableTransmission { 6 | static readonly type = '[Audio] Disable Transmission'; 7 | } 8 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/settings/settings.actions.ts: -------------------------------------------------------------------------------- 1 | export class SetSetting { 2 | static readonly type = '[Settings] Set Setting'; 3 | static readonly eventParams = ['setting', 'value']; 4 | 5 | constructor(public setting: string, public value: any) { 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/app/components/header/header.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | 3 | 4 | @Component({ 5 | selector: 'app-header', 6 | templateUrl: './header.component.html', 7 | styleUrls: ['./header.component.scss'] 8 | }) 9 | export class HeaderComponent { 10 | } 11 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/models/models.actions.ts: -------------------------------------------------------------------------------- 1 | export class PoseVideoFrame { 2 | static readonly type = '[Models] Pose Video Frame'; 3 | 4 | constructor(public video: HTMLVideoElement) { 5 | } 6 | } 7 | 8 | export class DetectSigning { 9 | static readonly type = '[Models] Detect Signing'; 10 | } 11 | -------------------------------------------------------------------------------- /firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "hosting": { 3 | "public": "dist/sign-language-detector", 4 | "ignore": [ 5 | "firebase.json", 6 | "**/.*", 7 | "**/node_modules/**" 8 | ], 9 | "rewrites": [ 10 | { 11 | "source": "**", 12 | "destination": "/index.html" 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "outDir": "./out-tsc/app", 5 | "types": [] 6 | }, 7 | "files": [ 8 | "src/main.ts", 9 | "src/polyfills.ts" 10 | ], 11 | "include": [ 12 | "src/**/*.d.ts", 13 | "node_modules/@types/chrome/index.d.ts" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see https://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.ts] 12 | quote_type = single 13 | 14 | [*.md] 15 | max_line_length = off 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /e2e/src/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | navigateTo(): Promise { 5 | return browser.get(browser.baseUrl) as Promise; 6 | } 7 | 8 | getTitleText(): Promise { 9 | return element(by.css('app-root .content span')).getText() as Promise; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /e2e/tsconfig.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "../tsconfig.base.json", 4 | "compilerOptions": { 5 | "outDir": "../out-tsc/e2e", 6 | "module": "commonjs", 7 | "target": "es2018", 8 | "types": [ 9 | "jasmine", 10 | "jasminewd2", 11 | "node" 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/app/pages/help/help.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-help', 5 | templateUrl: './help.component.html', 6 | styleUrls: ['./help.component.css'] 7 | }) 8 | export class HelpPageComponent implements OnInit { 9 | 10 | constructor() { 11 | } 12 | 13 | ngOnInit(): void { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/app/pages/main/main.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-main', 5 | templateUrl: './main.component.html', 6 | styleUrls: ['./main.component.scss'] 7 | }) 8 | export class MainPageComponent implements OnInit { 9 | 10 | constructor() { 11 | } 12 | 13 | ngOnInit(): void { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/app/components/settings/settings.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | {{t('shareData')}} 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import {enableProdMode} from '@angular/core'; 2 | import {platformBrowserDynamic} from '@angular/platform-browser-dynamic'; 3 | 4 | import {AppModule} from './app/app.module'; 5 | import {environment} from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule) 12 | .catch(err => console.error(err)); 13 | -------------------------------------------------------------------------------- /tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | /* To learn more about this file see: https://angular.io/config/tsconfig. */ 2 | { 3 | "extends": "./tsconfig.base.json", 4 | "compilerOptions": { 5 | "outDir": "./out-tsc/spec", 6 | "types": [ 7 | "jasmine" 8 | ] 9 | }, 10 | "files": [ 11 | "src/test.ts", 12 | "src/polyfills.ts" 13 | ], 14 | "include": [ 15 | "src/**/*.spec.ts", 16 | "src/**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /src/app/core/services/pose/pose.service.spec.ts: -------------------------------------------------------------------------------- 1 | import {TestBed} from '@angular/core/testing'; 2 | 3 | import {PoseService} from './pose.service'; 4 | 5 | describe('PoseService', () => { 6 | let service: PoseService; 7 | 8 | beforeEach(() => { 9 | TestBed.configureTestingModule({}); 10 | service = TestBed.inject(PoseService); 11 | }); 12 | 13 | it('should be created', () => { 14 | expect(service).toBeTruthy(); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "downlevelIteration": true, 9 | "experimentalDecorators": true, 10 | "moduleResolution": "node", 11 | "importHelpers": true, 12 | "target": "es2015", 13 | "module": "es2020", 14 | "lib": [ 15 | "es2018", 16 | "dom" 17 | ] 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/app/core/services/detector/detector.service.spec.ts: -------------------------------------------------------------------------------- 1 | import {TestBed} from '@angular/core/testing'; 2 | 3 | import {DetectorService} from './detector.service'; 4 | 5 | describe('DetectorService', () => { 6 | let service: DetectorService; 7 | 8 | beforeEach(() => { 9 | TestBed.configureTestingModule({}); 10 | service = TestBed.inject(DetectorService); 11 | }); 12 | 13 | it('should be created', () => { 14 | expect(service).toBeTruthy(); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/app/core/services/navigator/navigator.service.spec.ts: -------------------------------------------------------------------------------- 1 | import {TestBed} from '@angular/core/testing'; 2 | 3 | import {NavigatorService} from './navigator.service'; 4 | 5 | describe('NavigatorService', () => { 6 | let service: NavigatorService; 7 | 8 | beforeEach(() => { 9 | TestBed.configureTestingModule({}); 10 | service = TestBed.inject(NavigatorService); 11 | }); 12 | 13 | it('should be created', () => { 14 | expect(service).toBeTruthy(); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/app/components/base/base.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnDestroy} from '@angular/core'; 2 | import {Subject} from 'rxjs'; 3 | 4 | @Component({ 5 | selector: 'app-base', 6 | template: ` 7 |

8 | base works! 9 |

10 | `, 11 | styles: [] 12 | }) 13 | export abstract class BaseComponent implements OnDestroy { 14 | ngUnsubscribe: Subject = new Subject(); 15 | 16 | ngOnDestroy(): void { 17 | this.ngUnsubscribe.next(); 18 | this.ngUnsubscribe.complete(); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true, 3 | firebase: { 4 | apiKey: 'AIzaSyDG-av_fhucDCL_cQv0z45q3g1u2lCJ2QA', 5 | authDomain: 'sign-language-detector.firebaseapp.com', 6 | databaseURL: 'https://sign-language-detector.firebaseio.com', 7 | projectId: 'sign-language-detector', 8 | storageBucket: 'sign-language-detector.appspot.com', 9 | messagingSenderId: '1012541058868', 10 | appId: '1:1012541058868:web:e25196b4c789d52eebff61', 11 | measurementId: 'G-ES4X804B85' 12 | } 13 | }; 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | /* 2 | This is a "Solution Style" tsconfig.json file, and is used by editors and TypeScript’s language server to improve development experience. 3 | It is not intended to be used to perform a compilation. 4 | 5 | To learn more about this file see: https://angular.io/config/solution-tsconfig. 6 | */ 7 | { 8 | "files": [], 9 | "references": [ 10 | { 11 | "path": "./tsconfig.app.json" 12 | }, 13 | { 14 | "path": "./tsconfig.spec.json" 15 | }, 16 | { 17 | "path": "./e2e/tsconfig.json" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Sign Language Detector 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/app/app-routing.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {RouterModule, Routes} from '@angular/router'; 3 | import {MainPageComponent} from './pages/main/main.component'; 4 | import {HelpPageComponent} from './pages/help/help.component'; 5 | 6 | 7 | const routes: Routes = [ 8 | { 9 | path: '', 10 | component: MainPageComponent 11 | }, 12 | { 13 | path: 'help', 14 | component: HelpPageComponent 15 | } 16 | ]; 17 | 18 | @NgModule({ 19 | imports: [RouterModule.forRoot(routes)], 20 | exports: [RouterModule] 21 | }) 22 | export class AppRoutingModule { 23 | } 24 | -------------------------------------------------------------------------------- /src/app/components/video/video-help/video-help.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {MatDialog} from '@angular/material/dialog'; 3 | import {BroadcastTestComponent} from '../../audio/broadcast-test/broadcast-test.component'; 4 | 5 | @Component({ 6 | selector: 'app-video-help', 7 | templateUrl: './video-help.component.html', 8 | styleUrls: ['./video-help.component.scss'] 9 | }) 10 | export class VideoHelpComponent { 11 | 12 | constructor(private dialog: MatDialog) { 13 | } 14 | 15 | broadcastTest(): void { 16 | this.dialog.open(BroadcastTestComponent); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/app/core/services/pose/models/base.pose-model.ts: -------------------------------------------------------------------------------- 1 | import {Hand} from '../../../modules/ngxs/store/settings/settings.state'; 2 | 3 | export type Pose = { x: number, y: number }[]; 4 | export const POSE_LENGTH = 25; 5 | 6 | export abstract class BasePoseModel { 7 | width: number; 8 | height: number; 9 | 10 | async load(): Promise { 11 | /*** 12 | * Load model to memory 13 | */ 14 | } 15 | 16 | async unload(): Promise { 17 | /*** 18 | * Free GPU memory and other resources 19 | */ 20 | } 21 | 22 | abstract async predict(video: HTMLVideoElement, dominantHand: Hand): Promise; 23 | } 24 | -------------------------------------------------------------------------------- /src/app/components/audio/audio-instructions/audio-instructions.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {Store} from '@ngxs/store'; 3 | import {SetSetting} from '../../../core/modules/ngxs/store/settings/settings.actions'; 4 | 5 | @Component({ 6 | selector: 'app-audio-instructions', 7 | templateUrl: './audio-instructions.component.html', 8 | styleUrls: ['./audio-instructions.component.css'] 9 | }) 10 | export class AudioInstructionsComponent { 11 | 12 | constructor(private store: Store) { 13 | } 14 | 15 | activateMicrophone(): void { 16 | this.store.dispatch(new SetSetting('transmitAudio', true)); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/app/components/audio/broadcast-test/broadcast-test.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {BehaviorSubject, Observable} from 'rxjs'; 3 | import {Select} from '@ngxs/store'; 4 | import {AudioStateModel} from '../../../core/modules/ngxs/store/audio/audio.state'; 5 | 6 | @Component({ 7 | selector: 'app-broadcast-test', 8 | templateUrl: './broadcast-test.component.html', 9 | styleUrls: ['./broadcast-test.component.css'] 10 | }) 11 | export class BroadcastTestComponent { 12 | @Select(state => state.audio) audioState$: Observable; 13 | 14 | play$: BehaviorSubject = new BehaviorSubject(false); 15 | } 16 | -------------------------------------------------------------------------------- /src/app/core/modules/transloco/transloco.loader.ts: -------------------------------------------------------------------------------- 1 | import {HttpClient} from '@angular/common/http'; 2 | import {Translation, TRANSLOCO_LOADER, TranslocoLoader} from '@ngneat/transloco'; 3 | import {Injectable} from '@angular/core'; 4 | import {Observable} from 'rxjs'; 5 | 6 | @Injectable({providedIn: 'root'}) 7 | export class HttpLoader implements TranslocoLoader { 8 | constructor(private http: HttpClient) { 9 | } 10 | 11 | getTranslation(langPath: string): Observable { 12 | return this.http.get(`/assets/i18n/${langPath}.json`); 13 | } 14 | } 15 | 16 | export const translocoLoader = {provide: TRANSLOCO_LOADER, useClass: HttpLoader}; 17 | 18 | -------------------------------------------------------------------------------- /src/app/components/video/video-controls/video-controls.component.scss: -------------------------------------------------------------------------------- 1 | button[mat-fab] { 2 | margin: 0 6px; 3 | 4 | &.transparent { 5 | box-shadow: 0 0 0 1px inset white; 6 | background: transparent; 7 | } 8 | } 9 | 10 | #signing-indicator { 11 | position: absolute; 12 | left: 16px; 13 | bottom: 0; 14 | color: #64ffda; 15 | 16 | &.active { 17 | animation: spin 0.2s linear infinite; 18 | } 19 | } 20 | 21 | #hand-selector { 22 | position: absolute; 23 | right: 16px; 24 | 25 | &.dominant-left { 26 | transform: scaleX(-1); 27 | } 28 | } 29 | 30 | @keyframes spin { 31 | 100% { 32 | -webkit-transform: rotate(360deg); 33 | transform: rotate(360deg); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /src/app/core/modules/angular-fire/angular-fire.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {environment} from '../../../../environments/environment'; 3 | import {AngularFireAnalyticsModule, ScreenTrackingService, UserTrackingService} from '@angular/fire/analytics'; 4 | import {AngularFireModule} from '@angular/fire'; 5 | import {AngularFirestoreModule} from '@angular/fire/firestore'; 6 | 7 | 8 | @NgModule({ 9 | imports: [ 10 | AngularFireModule.initializeApp(environment.firebase), 11 | AngularFireAnalyticsModule, 12 | AngularFirestoreModule, 13 | ], 14 | providers: [ 15 | UserTrackingService, 16 | ScreenTrackingService 17 | ] 18 | }) 19 | export class AppAngularFireModule { 20 | } 21 | -------------------------------------------------------------------------------- /src/app/components/video/video.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

{{t('errors.' + video.error)}}

4 | 5 | 6 |
7 | 8 |
9 |
10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /e2e/src/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { AppPage } from './app.po'; 2 | import { browser, logging } from 'protractor'; 3 | 4 | describe('workspace-project App', () => { 5 | let page: AppPage; 6 | 7 | beforeEach(() => { 8 | page = new AppPage(); 9 | }); 10 | 11 | it('should display welcome message', () => { 12 | page.navigateTo(); 13 | expect(page.getTitleText()).toEqual('sign-language-detector app is running!'); 14 | }); 15 | 16 | afterEach(async () => { 17 | // Assert that there are no errors emitted from the browser 18 | const logs = await browser.manage().logs().get(logging.Type.BROWSER); 19 | expect(logs).not.toContain(jasmine.objectContaining({ 20 | level: logging.Level.SEVERE, 21 | } as logging.Entry)); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /src/app/components/video/video-help/video-help.component.html: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 15 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/app/components/audio/audio.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {AudioComponent} from './audio.component'; 4 | 5 | describe('AudioComponent', () => { 6 | let component: AudioComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [AudioComponent] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(AudioComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/app/pages/help/help.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {HelpPageComponent} from './help.component'; 4 | 5 | describe('HelpPageComponent', () => { 6 | let component: HelpPageComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [HelpPageComponent] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(HelpPageComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/app/core/modules/shared.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {AppAngularMaterialModule} from './angular-material/angular-material.module'; 3 | import {AppNgxsModule} from './ngxs/ngxs.module'; 4 | import {AppAngularFireModule} from './angular-fire/angular-fire.module'; 5 | import {AppTranslocoModule} from './transloco/transloco.module'; 6 | import {CommonModule} from '@angular/common'; 7 | 8 | const components = []; 9 | 10 | const modules = [ 11 | AppNgxsModule, 12 | AppAngularFireModule, 13 | AppTranslocoModule, 14 | AppAngularMaterialModule, 15 | CommonModule, 16 | ]; 17 | 18 | @NgModule({ 19 | declarations: components, 20 | imports: modules, 21 | exports: [ 22 | ...components, 23 | ...modules 24 | ] 25 | }) 26 | export class AppSharedModule { 27 | } 28 | -------------------------------------------------------------------------------- /src/app/components/video/video-pose/video-pose.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {VideoPoseComponent} from './video-pose.component'; 4 | 5 | describe('VideoPoseComponent', () => { 6 | let component: VideoPoseComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [VideoPoseComponent] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(VideoPoseComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/app/pages/main/main.component.scss: -------------------------------------------------------------------------------- 1 | @import "../../../theme/variables"; 2 | 3 | :host { 4 | display: flex; 5 | height: calc(100vh - 64px); 6 | } 7 | 8 | app-video { 9 | min-height: 216px; 10 | height: 720px; 11 | max-height: 45vw; 12 | 13 | &.aspect-4-3 { 14 | min-width: 288px; 15 | width: 960px; 16 | max-width: 60vw; 17 | } 18 | 19 | &.aspect-16-9 { 20 | min-width: 384px; 21 | width: 1280px; 22 | max-width: 80vw; 23 | } 24 | 25 | &.aspect-2-1 { 26 | min-width: 432px; 27 | width: 1440px; 28 | max-width: 90vw; 29 | } 30 | } 31 | 32 | #main-vertical { 33 | display: flex; 34 | flex-direction: column; 35 | margin: auto; 36 | 37 | @media #{$mat-lt-md} { 38 | margin: 16px auto 0; 39 | } 40 | } 41 | 42 | 43 | #main-horizontal { 44 | display: flex; 45 | } 46 | -------------------------------------------------------------------------------- /src/app/components/video/video.component.scss: -------------------------------------------------------------------------------- 1 | :host { 2 | background: #202124; 3 | display: flex; 4 | border-radius: 8px; 5 | position: relative; 6 | 7 | color: white; 8 | text-align: center; 9 | align-items: center; 10 | overflow: hidden; 11 | } 12 | 13 | p { 14 | margin: auto; 15 | font-size: 36px; 16 | } 17 | 18 | #video-container { 19 | height: 100%; 20 | 21 | video { 22 | position: absolute; 23 | } 24 | } 25 | 26 | .flip { 27 | transform: scaleX(-1); 28 | } 29 | 30 | app-video-pose { 31 | width: 100%; 32 | height: 100%; 33 | z-index: 0; 34 | position: absolute; 35 | } 36 | 37 | app-video-controls { 38 | position: absolute; 39 | bottom: 16px; 40 | left: 0; 41 | right: 0; 42 | } 43 | 44 | app-video-help { 45 | position: absolute; 46 | 47 | top: 16px; 48 | right: 16px; 49 | } 50 | -------------------------------------------------------------------------------- /src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/zone-testing'; 4 | import {getTestBed} from '@angular/core/testing'; 5 | import {BrowserDynamicTestingModule, platformBrowserDynamicTesting} from '@angular/platform-browser-dynamic/testing'; 6 | 7 | declare const require: { 8 | context(path: string, deep?: boolean, filter?: RegExp): { 9 | keys(): string[]; 10 | (id: string): T; 11 | }; 12 | }; 13 | 14 | // First, initialize the Angular testing environment. 15 | getTestBed().initTestEnvironment( 16 | BrowserDynamicTestingModule, 17 | platformBrowserDynamicTesting() 18 | ); 19 | // Then we find all the tests. 20 | const context = require.context('./', true, /\.spec\.ts$/); 21 | // And load the modules. 22 | context.keys().map(context); 23 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/app/app.actions.ts: -------------------------------------------------------------------------------- 1 | export class StartLoading { 2 | static readonly type = '[App] Start Loading'; 3 | } 4 | 5 | export class StopLoading { 6 | static readonly type = '[App] Stop Loading'; 7 | } 8 | 9 | export class ResetError { 10 | static readonly type = '[App] Reset Error'; 11 | } 12 | 13 | export class DisplayError { 14 | static readonly type = '[App] Display Error'; 15 | static readonly eventParams = ['message']; 16 | 17 | public message: string; 18 | 19 | constructor(public error: any) { 20 | console.error(error); 21 | 22 | if (error.error && error.error.message) { 23 | this.message = error.error.message; 24 | } else if (typeof error.message === 'string') { 25 | this.message = error.message; 26 | } else { 27 | this.message = error; 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/app/components/header/header.component.scss: -------------------------------------------------------------------------------- 1 | .spacer { 2 | flex: 1 1 auto; 3 | } 4 | 5 | .auth-avatar { 6 | border-radius: 100%; 7 | cursor: pointer; 8 | width: 36px; 9 | height: 36px; 10 | overflow: hidden; 11 | 12 | :focus { 13 | outline: 0; 14 | } 15 | } 16 | 17 | #sign-in { 18 | text-transform: uppercase; 19 | } 20 | 21 | #auth-details { 22 | display: flex; 23 | margin-top: 4px; // mimicking the Google Meet UI 24 | 25 | div { 26 | display: flex; 27 | flex-direction: column; 28 | text-align: right; 29 | margin-right: 12px; 30 | font-weight: 400; 31 | padding-top: 2px; 32 | 33 | span { 34 | font-size: 14px; 35 | line-height: 16px; 36 | } 37 | 38 | a { 39 | font-size: 13px; 40 | line-height: 16px; 41 | cursor: pointer; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/app/components/video/video.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {VideoComponent} from './video.component'; 4 | import {AppModule} from '../../app.module'; 5 | 6 | describe('VideoComponent', () => { 7 | let component: VideoComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [VideoComponent], 13 | imports: [AppModule] 14 | }).compileComponents(); 15 | })); 16 | 17 | beforeEach(() => { 18 | fixture = TestBed.createComponent(VideoComponent); 19 | component = fixture.componentInstance; 20 | fixture.detectChanges(); 21 | }); 22 | 23 | it('should create', () => { 24 | expect(component).toBeTruthy(); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/app/components/audio/broadcast-test/broadcast-test.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {BroadcastTestComponent} from './broadcast-test.component'; 4 | 5 | describe('BroadcastTestComponent', () => { 6 | let component: BroadcastTestComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [BroadcastTestComponent] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(BroadcastTestComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/app/components/header/header.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {HeaderComponent} from './header.component'; 4 | import {AppModule} from '../../app.module'; 5 | 6 | describe('HeaderComponent', () => { 7 | let component: HeaderComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [HeaderComponent], 13 | imports: [AppModule] 14 | }).compileComponents(); 15 | })); 16 | 17 | beforeEach(() => { 18 | fixture = TestBed.createComponent(HeaderComponent); 19 | component = fixture.componentInstance; 20 | fixture.detectChanges(); 21 | }); 22 | 23 | it('should create', () => { 24 | expect(component).toBeTruthy(); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/app/pages/main/main.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {MainPageComponent} from './main.component'; 4 | import {AppModule} from '../../app.module'; 5 | 6 | describe('MainPageComponent', () => { 7 | let component: MainPageComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [MainPageComponent], 13 | imports: [AppModule] 14 | }).compileComponents(); 15 | })); 16 | 17 | beforeEach(() => { 18 | fixture = TestBed.createComponent(MainPageComponent); 19 | component = fixture.componentInstance; 20 | fixture.detectChanges(); 21 | }); 22 | 23 | it('should create', () => { 24 | expect(component).toBeTruthy(); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | # Only exists if Bazel was run 8 | /bazel-out 9 | 10 | # dependencies 11 | /node_modules 12 | 13 | # profiling files 14 | chrome-profiler-events*.json 15 | speed-measure-plugin*.json 16 | 17 | # IDEs and editors 18 | /.idea 19 | .project 20 | .classpath 21 | .c9/ 22 | *.launch 23 | .settings/ 24 | *.sublime-workspace 25 | 26 | # IDE - VSCode 27 | .vscode/* 28 | !.vscode/settings.json 29 | !.vscode/tasks.json 30 | !.vscode/launch.json 31 | !.vscode/extensions.json 32 | .history/* 33 | 34 | # misc 35 | /.sass-cache 36 | /connect.lock 37 | /coverage 38 | /libpeerconnection.log 39 | npm-debug.log 40 | yarn-error.log 41 | testem.log 42 | /typings 43 | 44 | # System Files 45 | .DS_Store 46 | Thumbs.db 47 | 48 | .firebase 49 | -------------------------------------------------------------------------------- /src/app/components/settings/settings.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {SettingsComponent} from './settings.component'; 4 | import {AppModule} from '../../app.module'; 5 | 6 | describe('SettingsComponent', () => { 7 | let component: SettingsComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [SettingsComponent], 13 | imports: [AppModule] 14 | }).compileComponents(); 15 | })); 16 | 17 | beforeEach(() => { 18 | fixture = TestBed.createComponent(SettingsComponent); 19 | component = fixture.componentInstance; 20 | fixture.detectChanges(); 21 | }); 22 | 23 | it('should create', () => { 24 | expect(component).toBeTruthy(); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/settings/settings.state.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {Action, State, StateContext} from '@ngxs/store'; 3 | import {SetSetting} from './settings.actions'; 4 | 5 | export type Hand = 'right' | 'left'; 6 | 7 | export interface SettingsStateModel { 8 | dominantHand: Hand; 9 | transmitAudio: boolean; 10 | receiveVideo: boolean; 11 | } 12 | 13 | const initialState: SettingsStateModel = { 14 | dominantHand: 'right', 15 | transmitAudio: true, 16 | receiveVideo: true 17 | }; 18 | 19 | @Injectable() 20 | @State({ 21 | name: 'settings', 22 | defaults: initialState 23 | }) 24 | export class SettingsState { 25 | @Action(SetSetting) 26 | setSetting({patchState}: StateContext, {setting, value}: SetSetting): void { 27 | patchState({[setting]: value}); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/app/components/audio/broadcast-test/broadcast-test.component.html: -------------------------------------------------------------------------------- 1 | 2 |

{{t('title')}}

3 | 4 | 5 |

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |

{{ 'audio.errors.' + audioState.error | transloco }}

14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/app/components/audio/audio-instructions/audio-instructions.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {AudioInstructionsComponent} from './audio-instructions.component'; 4 | 5 | describe('AudioInstructionsComponent', () => { 6 | let component: AudioInstructionsComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | declarations: [AudioInstructionsComponent] 12 | }) 13 | .compileComponents(); 14 | })); 15 | 16 | beforeEach(() => { 17 | fixture = TestBed.createComponent(AudioInstructionsComponent); 18 | component = fixture.componentInstance; 19 | fixture.detectChanges(); 20 | }); 21 | 22 | it('should create', () => { 23 | expect(component).toBeTruthy(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/app/components/video/video-help/video-help.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {VideoHelpComponent} from './video-help.component'; 4 | import {AppModule} from '../../../app.module'; 5 | 6 | describe('VideoHelpComponent', () => { 7 | let component: VideoHelpComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [ 13 | VideoHelpComponent 14 | ], 15 | imports: [AppModule] 16 | }).compileComponents(); 17 | })); 18 | 19 | beforeEach(() => { 20 | fixture = TestBed.createComponent(VideoHelpComponent); 21 | component = fixture.componentInstance; 22 | fixture.detectChanges(); 23 | }); 24 | 25 | it('should create', () => { 26 | expect(component).toBeTruthy(); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /.browserslistrc: -------------------------------------------------------------------------------- 1 | # This file is used by the build system to adjust CSS and JS output to support the specified browsers below. 2 | # For additional information regarding the format and rule options, please see: 3 | # https://github.com/browserslist/browserslist#queries 4 | 5 | # For the full list of supported browsers by the Angular framework, please see: 6 | # https://angular.io/guide/browser-support 7 | 8 | # You can see what browsers were selected by your queries by running: 9 | # npx browserslist 10 | 11 | last 1 Chrome version 12 | last 1 Firefox version 13 | last 2 Edge major versions 14 | last 2 Safari major version 15 | last 2 iOS major versions 16 | Firefox ESR 17 | not IE 9-10 # Angular support for IE 9-10 has been deprecated and will be removed as of Angular v11. To opt-in, remove the 'not' prefix on this line. 18 | not IE 11 # Angular supports IE 11 only as an opt-in. To opt-in, remove the 'not' prefix on this line. 19 | -------------------------------------------------------------------------------- /src/app/components/video/video-controls/video-controls.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | 3 | import {VideoControlsComponent} from './video-controls.component'; 4 | import {AppModule} from '../../../app.module'; 5 | 6 | describe('VideoControlsComponent', () => { 7 | let component: VideoControlsComponent; 8 | let fixture: ComponentFixture; 9 | 10 | beforeEach(waitForAsync(() => { 11 | TestBed.configureTestingModule({ 12 | declarations: [VideoControlsComponent], 13 | imports: [AppModule] 14 | }).compileComponents(); 15 | })); 16 | 17 | beforeEach(() => { 18 | fixture = TestBed.createComponent(VideoControlsComponent); 19 | component = fixture.componentInstance; 20 | fixture.detectChanges(); 21 | }); 22 | 23 | it('should create', () => { 24 | expect(component).toBeTruthy(); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/app/core/modules/transloco/transloco.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {environment} from '../../../../environments/environment'; 3 | import {TRANSLOCO_CONFIG, TranslocoConfig, TranslocoModule} from '@ngneat/transloco'; 4 | import {TranslocoMessageFormatModule} from '@ngneat/transloco-messageformat'; 5 | import {translocoLoader} from './transloco.loader'; 6 | import {HttpClientModule} from '@angular/common/http'; 7 | 8 | 9 | @NgModule({ 10 | imports: [ 11 | HttpClientModule, 12 | TranslocoMessageFormatModule.init(), 13 | ], 14 | exports: [ 15 | TranslocoModule, 16 | ], 17 | providers: [ 18 | { 19 | provide: TRANSLOCO_CONFIG, 20 | useValue: { 21 | availableLangs: ['en', 'es'], 22 | defaultLang: 'en', 23 | prodMode: environment.production, 24 | } as TranslocoConfig 25 | }, 26 | translocoLoader 27 | ], 28 | }) 29 | export class AppTranslocoModule { 30 | } 31 | -------------------------------------------------------------------------------- /src/app/components/settings/settings.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {Select, Store} from '@ngxs/store'; 3 | import {Observable} from 'rxjs'; 4 | import {SettingsStateModel} from '../../core/modules/ngxs/store/settings/settings.state'; 5 | import {SetSetting} from '../../core/modules/ngxs/store/settings/settings.actions'; 6 | 7 | export class BaseSettingsComponent { 8 | @Select(state => state.settings) settingsState$: Observable; 9 | 10 | constructor(private store: Store) { 11 | } 12 | 13 | 14 | applySetting(setting: string, value: any): void { 15 | this.store.dispatch(new SetSetting(setting, value)); 16 | } 17 | } 18 | 19 | @Component({ 20 | selector: 'app-settings', 21 | templateUrl: './settings.component.html', 22 | styleUrls: ['./settings.component.scss'] 23 | }) 24 | export class SettingsComponent extends BaseSettingsComponent { 25 | constructor(store: Store) { 26 | super(store); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/app/components/video/video-controls/video-controls.component.ts: -------------------------------------------------------------------------------- 1 | import {Component} from '@angular/core'; 2 | import {Select, Store} from '@ngxs/store'; 3 | import {BaseSettingsComponent} from '../../settings/settings.component'; 4 | import {Hand} from '../../../core/modules/ngxs/store/settings/settings.state'; 5 | import {Observable} from 'rxjs'; 6 | 7 | @Component({ 8 | selector: 'app-video-controls', 9 | templateUrl: './video-controls.component.html', 10 | styleUrls: ['./video-controls.component.scss'] 11 | }) 12 | export class VideoControlsComponent extends BaseSettingsComponent { 13 | @Select(state => state.models.isSigning) isSigning$: Observable; 14 | 15 | constructor(store: Store) { 16 | super(store); 17 | } 18 | 19 | flipDominantHand(currentHand: Hand): void { 20 | if (currentHand === 'right') { 21 | this.applySetting('dominantHand', 'left'); 22 | } else { 23 | this.applySetting('dominantHand', 'right'); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /e2e/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // Protractor configuration file, see link for more information 3 | // https://github.com/angular/protractor/blob/master/lib/config.ts 4 | 5 | const { SpecReporter, StacktraceOption } = require('jasmine-spec-reporter'); 6 | 7 | /** 8 | * @type { import("protractor").Config } 9 | */ 10 | exports.config = { 11 | allScriptsTimeout: 11000, 12 | specs: [ 13 | './src/**/*.e2e-spec.ts' 14 | ], 15 | capabilities: { 16 | browserName: 'chrome' 17 | }, 18 | directConnect: true, 19 | baseUrl: 'http://localhost:4200/', 20 | framework: 'jasmine', 21 | jasmineNodeOpts: { 22 | showColors: true, 23 | defaultTimeoutInterval: 30000, 24 | print: function() {} 25 | }, 26 | onPrepare() { 27 | require('ts-node').register({ 28 | project: require('path').join(__dirname, './tsconfig.json') 29 | }); 30 | jasmine.getEnv().addReporter(new SpecReporter({ 31 | spec: { 32 | displayStacktrace: StacktraceOption.PRETTY 33 | } 34 | })); 35 | } 36 | }; -------------------------------------------------------------------------------- /src/theme/styles.scss: -------------------------------------------------------------------------------- 1 | // Custom Theming for Angular Material 2 | // For more information: https://material.angular.io/guide/theming 3 | @import '~@angular/material/theming'; 4 | // Plus imports for other components in your app. 5 | 6 | // Include the common styles for Angular Material. We include this here so that you only 7 | // have to load a single css file for Angular Material in your app. 8 | // Be sure that you only ever include this mixin once! 9 | @include mat-core(); 10 | 11 | /* You can add global styles to this file, and also import other style files */ 12 | @import "./variables"; 13 | 14 | // Include theme styles for core and each component used in your app. 15 | // Alternatively, you can import and @include the theme mixins for each component 16 | // that you are using. 17 | @include angular-material-theme($app-theme); 18 | 19 | html, body { 20 | height: 100%; 21 | } 22 | 23 | body { 24 | margin: 0; 25 | font-family: Roboto, "Helvetica Neue", sans-serif; 26 | background-color: mat-color($mat-gray, 50); 27 | } 28 | 29 | app-root { 30 | height: 100% 31 | } 32 | -------------------------------------------------------------------------------- /src/app/app.component.spec.ts: -------------------------------------------------------------------------------- 1 | import {ComponentFixture, TestBed, waitForAsync} from '@angular/core/testing'; 2 | import {AppComponent} from './app.component'; 3 | import {AppModule} from './app.module'; 4 | 5 | describe('AppComponent', () => { 6 | let component: AppComponent; 7 | let fixture: ComponentFixture; 8 | 9 | beforeEach(waitForAsync(() => { 10 | TestBed.configureTestingModule({ 11 | imports: [AppModule] 12 | }).compileComponents(); 13 | })); 14 | 15 | beforeEach(() => { 16 | fixture = TestBed.createComponent(AppComponent); 17 | component = fixture.componentInstance; 18 | fixture.detectChanges(); 19 | }); 20 | 21 | it('should create', () => { 22 | expect(component).toBeTruthy(); 23 | }); 24 | 25 | // it('should render app', () => { 26 | // const fixture = TestBed.createComponent(AppComponent); 27 | // fixture.detectChanges(); 28 | // const compiled = fixture.nativeElement; 29 | // expect(compiled.querySelector('.content span').textContent).toContain('sign-language-detector app is running!'); 30 | // }); 31 | }); 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Amit Moryossef 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular-devkit/build-angular'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage-istanbul-reporter'), 13 | require('@angular-devkit/build-angular/plugins/karma') 14 | ], 15 | client: { 16 | clearContext: false // leave Jasmine Spec Runner output visible in browser 17 | }, 18 | coverageIstanbulReporter: { 19 | dir: require('path').join(__dirname, './coverage/sign-language-detector'), 20 | reports: ['html', 'lcovonly', 'text-summary'], 21 | fixWebpackSourcePaths: true 22 | }, 23 | reporters: ['progress', 'kjhtml'], 24 | port: 9876, 25 | colors: true, 26 | logLevel: config.LOG_INFO, 27 | autoWatch: true, 28 | browsers: ['Chrome'], 29 | singleRun: false, 30 | restartOnFileChange: true 31 | }); 32 | }; 33 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/app/app.state.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {Action, State, StateContext} from '@ngxs/store'; 3 | import {DisplayError, ResetError, StartLoading, StopLoading} from './app.actions'; 4 | 5 | export interface AppStateModel { 6 | isLoading: boolean; 7 | error: string; 8 | } 9 | 10 | const initialState: AppStateModel = { 11 | isLoading: false, 12 | error: null 13 | }; 14 | 15 | @Injectable() 16 | @State({ 17 | name: 'app', 18 | defaults: initialState 19 | }) 20 | export class AppState { 21 | @Action(StartLoading) 22 | startLoading({patchState}: StateContext): void { 23 | patchState({isLoading: true}); 24 | } 25 | 26 | @Action(StopLoading) 27 | stopLoading({patchState}: StateContext): void { 28 | patchState({isLoading: false}); 29 | } 30 | 31 | @Action(DisplayError) 32 | displayError({patchState}: StateContext, {message}: DisplayError): void { 33 | patchState({error: message}); 34 | } 35 | 36 | @Action(ResetError) 37 | resetError({patchState}: StateContext): void { 38 | patchState({error: null}); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // This file can be replaced during build by using the `fileReplacements` array. 2 | // `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. 3 | // The list of file replacements can be found in `angular.json`. 4 | 5 | export const environment = { 6 | production: false, 7 | firebase: { 8 | apiKey: 'AIzaSyDG-av_fhucDCL_cQv0z45q3g1u2lCJ2QA', 9 | authDomain: 'sign-language-detector.firebaseapp.com', 10 | databaseURL: 'https://sign-language-detector.firebaseio.com', 11 | projectId: 'sign-language-detector', 12 | storageBucket: 'sign-language-detector.appspot.com', 13 | messagingSenderId: '1012541058868', 14 | appId: '1:1012541058868:web:e25196b4c789d52eebff61', 15 | measurementId: 'G-ES4X804B85' 16 | } 17 | }; 18 | 19 | /* 20 | * For easier debugging in development mode, you can import the following file 21 | * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. 22 | * 23 | * This import should be commented out in production mode because it will have a negative impact 24 | * on performance if an error is thrown. 25 | */ 26 | // import 'zone.js/dist/zone-error'; // Included with Angular CLI. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sign Language Detector for Video Conferencing 2 | 3 | This project contains the demo application formulated in [Real-Time Sign Language Detection using Human Pose Estimation](https://research.google/pubs/pub49425/) published in [SLRTP 2020](https://slrtp.com/) and presented in the [ECCV 2020](https://eccv2020.eu/) demo track. 4 | 5 | We use the [tf.js models](https://github.com/google-research/google-research/tree/master/sign_language_detection) open-sourced by Google research. 6 | 7 | This demo app is available to try at [sign-language-detector.web.app](https://sign-language-detector.web.app/) 8 | 9 | For an overview of this project, please watch our demo video: 10 | 11 | [![Explanation Video](https://img.youtube.com/vi/nozz2pvbG_Q/0.jpg)](https://www.youtube.com/watch?v=nozz2pvbG_Q) 12 | 13 | ## Citation 14 | 15 | ```bibtex 16 | @article{moryossef2020real, 17 | title = {Real-Time Sign-Language Detection using Human Pose Estimation}, 18 | author = {Moryossef, Amit and Tsochantaridis, Ioannis and Aharoni, Roee Yosef and Ebling, Sarah and Narayanan, Srini}, 19 | year = {2020}, 20 | booktitle = {SLRTP 2020: The Sign Language Recognition, Translation & Production Workshop}, 21 | } 22 | ``` 23 | -------------------------------------------------------------------------------- /.github/workflows/build_client.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Client Build Test 4 | 5 | # Controls when the action will run. Triggers the workflow on push or pull request 6 | # events but only for the master branch 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | # This workflow contains a single job called "build" 17 | build: 18 | # The type of runner that the job will run on 19 | runs-on: ubuntu-latest 20 | 21 | # Steps represent a sequence of tasks that will be executed as part of the job 22 | steps: 23 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 24 | - uses: actions/checkout@v2 25 | 26 | - name: Cache node_modules 27 | uses: actions/cache@v2 28 | with: 29 | path: | 30 | node_modules 31 | key: ${{ runner.os }}-${{ hashFiles('**/package.json') }} 32 | 33 | - name: Install Dependencies 34 | run: npm install 35 | 36 | - name: Lint code 37 | run: npm run lint 38 | 39 | - name: Build project 40 | run: npm run build 41 | -------------------------------------------------------------------------------- /src/app/core/services/navigator/navigator.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | 3 | @Injectable({ 4 | providedIn: 'root' 5 | }) 6 | export class NavigatorService { 7 | 8 | async getCamera(options: MediaTrackConstraints): Promise { 9 | try { 10 | return await navigator.mediaDevices.getUserMedia({audio: false, video: options}); 11 | } catch (e) { 12 | if (e.message.includes('Permission denied')) { 13 | throw new Error('permissionDenied'); 14 | } else { 15 | throw new Error('notConnected'); 16 | } 17 | } 18 | } 19 | 20 | async getMicrophone(): Promise { 21 | try { 22 | return await navigator.mediaDevices.getUserMedia({audio: true}); 23 | } catch (e) { 24 | if (e.message.includes('Permission denied')) { 25 | throw new Error('permissionDenied'); 26 | } else { 27 | throw new Error('notConnected'); 28 | } 29 | } 30 | } 31 | 32 | async getSpeaker(allowedLabels: Set): Promise { 33 | const devices = await navigator.mediaDevices.enumerateDevices(); 34 | 35 | const device = devices.find(d => d.kind === 'audiooutput' && allowedLabels.has(d.label)); 36 | if (!device) { 37 | throw new Error('missingSpeaker'); 38 | } 39 | return device; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/ngxs.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {environment} from '../../../../environments/environment'; 3 | import {NgxsModule, NgxsModuleOptions} from '@ngxs/store'; 4 | import {NgxsInterceptorPluginModule} from './ngxs-interceptor/ngxs-interceptor.plugin'; 5 | import {NgxsRouterPluginModule} from '@ngxs/router-plugin'; 6 | import {AppState} from './store/app/app.state'; 7 | import {SettingsState} from './store/settings/settings.state'; 8 | import {VideoState} from './store/video/video.state'; 9 | import {ModelsState} from './store/models/models.state'; 10 | import {AudioState} from './store/audio/audio.state'; 11 | 12 | 13 | export const ngxsConfig: NgxsModuleOptions = { 14 | developmentMode: !environment.production, 15 | selectorOptions: { 16 | // These Selector Settings are recommended in preparation for NGXS v4 17 | // (See above for their effects) 18 | suppressErrors: false, 19 | injectContainerState: false 20 | }, 21 | compatibility: { 22 | strictContentSecurityPolicy: true 23 | } 24 | }; 25 | 26 | @NgModule({ 27 | imports: [ 28 | NgxsModule.forRoot([AppState, SettingsState, VideoState, AudioState, ModelsState], ngxsConfig), 29 | NgxsRouterPluginModule.forRoot(), 30 | NgxsInterceptorPluginModule.forRoot(), 31 | ] 32 | }) 33 | export class AppNgxsModule { 34 | } 35 | -------------------------------------------------------------------------------- /src/assets/i18n/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "header": { 3 | "title": "Sign Language Detector" 4 | }, 5 | "video": { 6 | "errors": { 7 | "starting": "Camera starting", 8 | "turnedOff": "Camera turned off", 9 | "notConnected": "Camera not connected", 10 | "permissionDenied": "Camera permission denied" 11 | }, 12 | "controls": { 13 | "transmitAudio": { 14 | "on": "Turn off virtual microphone", 15 | "off": "Turn on virtual microphone" 16 | }, 17 | "receiveVideo": { 18 | "on": "Turn off camera", 19 | "off": "Turn on camera" 20 | }, 21 | "dominantHand": { 22 | "left": "Left handed", 23 | "right": "Right handed" 24 | } 25 | }, 26 | "help": { 27 | "menu": { 28 | "help": "Help", 29 | "feedback": "Report a problem", 30 | "micTest": "Test Microphone" 31 | } 32 | } 33 | }, 34 | "audio": { 35 | "errors": { 36 | "permissionDenied": "Microphone permission denied" 37 | }, 38 | "broadcast": { 39 | "title": "Microphone Test", 40 | "description": "Please make sure you are using the \"{{mic}}\" microphone in your videoconferencing app of choice." 41 | }, 42 | "instructions": { 43 | "title": "Audio Setup Instructions", 44 | "close": "Close", 45 | "done": "Done" 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/theme/variables.scss: -------------------------------------------------------------------------------- 1 | @import '~@angular/material/theming'; 2 | 3 | // Define the palettes for your theme using the Material Design palettes available in palette.scss 4 | // (imported above). For each palette, you can optionally specify a default, lighter, and darker 5 | // hue. Available color palettes: https://material.io/design/color/ 6 | $app-primary: mat-palette($mat-indigo); 7 | $app-accent: mat-palette($mat-pink); 8 | 9 | // The warn palette is optional (defaults to red). 10 | $app-warn: mat-palette($mat-red); 11 | 12 | // Create the theme object (a Sass map containing all of the palettes). 13 | $app-theme: mat-light-theme($app-primary, $app-accent, $app-warn); 14 | 15 | 16 | $mat-xs: "screen and (max-width: 599px)"; 17 | $mat-sm: "screen and (min-width: 600px) and (max-width: 959px)"; 18 | $mat-md: "screen and (min-width: 960px) and (max-width: 1279px)"; 19 | $mat-lg: "screen and (min-width: 1280px) and (max-width: 1919px)"; 20 | $mat-xl: "screen and (min-width: 1920px) and (max-width: 5000px)"; 21 | $mat-lt-sm: "screen and (max-width: 599px)"; 22 | $mat-lt-md: "screen and (max-width: 959px)"; 23 | $mat-lt-lg: "screen and (max-width: 1279px)"; 24 | $mat-lt-xl: "screen and (max-width: 1919px)"; 25 | $mat-gt-xs: "screen and (min-width: 600px)"; 26 | $mat-gt-sm: "screen and (min-width: 960px)"; 27 | $mat-gt-md: "screen and (min-width: 1280px)"; 28 | $mat-gt-xl: "screen and (min-width: 1920px)"; 29 | -------------------------------------------------------------------------------- /src/app/core/modules/angular-material/angular-material.module.ts: -------------------------------------------------------------------------------- 1 | import {NgModule} from '@angular/core'; 2 | import {MatToolbarModule} from '@angular/material/toolbar'; 3 | import {MatButtonModule} from '@angular/material/button'; 4 | import {MatIconModule} from '@angular/material/icon'; 5 | import {MatDialogModule} from '@angular/material/dialog'; 6 | import {MatSnackBarModule} from '@angular/material/snack-bar'; 7 | import {MatProgressSpinnerModule} from '@angular/material/progress-spinner'; 8 | import {MatMenuModule} from '@angular/material/menu'; 9 | import {BrowserAnimationsModule} from '@angular/platform-browser/animations'; 10 | import {MatSidenavModule} from '@angular/material/sidenav'; 11 | import {MatListModule} from '@angular/material/list'; 12 | import {MatFormFieldModule} from '@angular/material/form-field'; 13 | import {MatCheckboxModule} from '@angular/material/checkbox'; 14 | import {MatTooltipModule} from '@angular/material/tooltip'; 15 | 16 | const materialModules = [ 17 | MatToolbarModule, 18 | MatButtonModule, 19 | MatIconModule, 20 | MatDialogModule, 21 | MatSnackBarModule, 22 | MatProgressSpinnerModule, 23 | MatMenuModule, 24 | MatTooltipModule, 25 | MatSidenavModule, 26 | MatListModule, 27 | MatFormFieldModule, 28 | MatCheckboxModule, 29 | BrowserAnimationsModule 30 | ]; 31 | 32 | @NgModule({ 33 | imports: materialModules, 34 | exports: materialModules 35 | }) 36 | export class AppAngularMaterialModule { 37 | } 38 | -------------------------------------------------------------------------------- /src/app/core/services/pose/pose.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {BasePoseModel, Pose} from './models/base.pose-model'; 3 | import {PoseNetPoseModel} from './models/posenet.pose-model'; 4 | import {Hand} from '../../modules/ngxs/store/settings/settings.state'; 5 | 6 | @Injectable({ 7 | providedIn: 'root' 8 | }) 9 | export class PoseService { 10 | 11 | models: { [key: string]: BasePoseModel } = { 12 | fastPoseNet: new PoseNetPoseModel({ 13 | architecture: 'MobileNetV1', 14 | outputStride: 16, 15 | inputResolution: {width: 266, height: 200}, 16 | multiplier: 0.75 17 | }), 18 | slowPoseNet: new PoseNetPoseModel({ 19 | architecture: 'ResNet50', 20 | outputStride: 32, 21 | inputResolution: {width: 266, height: 200}, 22 | quantBytes: 2 23 | }), 24 | }; 25 | 26 | model: BasePoseModel; 27 | 28 | constructor() { 29 | this.setModel('fastPoseNet'); 30 | } 31 | 32 | async setModel(modelId: string): Promise { 33 | if (this.model) { 34 | await this.model.unload(); 35 | } 36 | if (!(modelId in this.models)) { 37 | throw new Error(`Specified model "${modelId}" is not configured`); 38 | } 39 | const model = this.models[modelId]; 40 | await model.load(); 41 | this.model = model; 42 | } 43 | 44 | predict(video: HTMLVideoElement, dominantHand: Hand): Promise { 45 | if (!this.model) { 46 | return Promise.resolve(null); 47 | } 48 | return this.model.predict(video, dominantHand); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/app/app.module.ts: -------------------------------------------------------------------------------- 1 | import {BrowserModule} from '@angular/platform-browser'; 2 | import {NgModule} from '@angular/core'; 3 | 4 | import {AppRoutingModule} from './app-routing.module'; 5 | import {AppComponent} from './app.component'; 6 | import {AppSharedModule} from './core/modules/shared.module'; 7 | import {HeaderComponent} from './components/header/header.component'; 8 | import {VideoComponent} from './components/video/video.component'; 9 | import {MainPageComponent} from './pages/main/main.component'; 10 | import {SettingsComponent} from './components/settings/settings.component'; 11 | import {VideoControlsComponent} from './components/video/video-controls/video-controls.component'; 12 | import {VideoHelpComponent} from './components/video/video-help/video-help.component'; 13 | import {NavigatorService} from './core/services/navigator/navigator.service'; 14 | import {HelpPageComponent} from './pages/help/help.component'; 15 | import {BroadcastTestComponent} from './components/audio/broadcast-test/broadcast-test.component'; 16 | import {VideoPoseComponent} from './components/video/video-pose/video-pose.component'; 17 | import {AudioComponent} from './components/audio/audio.component'; 18 | import {AudioInstructionsComponent} from './components/audio/audio-instructions/audio-instructions.component'; 19 | 20 | 21 | @NgModule({ 22 | declarations: [ 23 | AppComponent, 24 | HeaderComponent, 25 | VideoComponent, 26 | SettingsComponent, 27 | VideoControlsComponent, 28 | VideoHelpComponent, 29 | MainPageComponent, 30 | HelpPageComponent, 31 | BroadcastTestComponent, 32 | VideoPoseComponent, 33 | AudioComponent, 34 | AudioInstructionsComponent 35 | ], 36 | imports: [ 37 | BrowserModule, 38 | AppRoutingModule, 39 | AppSharedModule, 40 | ], 41 | providers: [ 42 | NavigatorService 43 | ], 44 | bootstrap: [AppComponent] 45 | }) 46 | export class AppModule { 47 | } 48 | -------------------------------------------------------------------------------- /src/app/components/audio/audio-instructions/audio-instructions.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

{{t('title')}}

4 | 5 |

6 | This app requires you to have a Virtual Audio 7 | Cable. 8 | Please install one depending on your operating system. 9 |

10 |

11 | This app currently only works with the following cables. 12 | If you are using something else, it won't work. 13 |

14 | 15 |

Windows

16 |

17 | Muzychenko VAC works in Windows XP, Vista, 7, 8, 8.1 and 10. 18 |

19 |

20 | Download the free version HERE. 21 |

22 | 23 |

MacOS

24 |

25 | VB-Audio works in MacOS. 26 |

27 |

28 | Download the free version HERE. 30 |

31 |

32 | After installing, please go to your sound settings -> microphone, 33 | and make sure that the audio level for the VB-Cable microphone is full volume. 34 |

35 | 36 |

Linux

37 |

Coming Soon

38 | 39 |

iOS

40 |

Not Supported

41 | 42 |

Android

43 |

Not Supported

44 | 45 | 46 | 47 | 50 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/models/models.state.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {Action, NgxsOnInit, Select, State, StateContext} from '@ngxs/store'; 3 | import {Pose} from '../../../../services/pose/models/base.pose-model'; 4 | import {DetectSigning, PoseVideoFrame} from './models.actions'; 5 | import {PoseService} from '../../../../services/pose/pose.service'; 6 | import {Observable} from 'rxjs'; 7 | import {Hand} from '../settings/settings.state'; 8 | import {tap} from 'rxjs/operators'; 9 | import {DetectorService} from '../../../../services/detector/detector.service'; 10 | 11 | export interface ModelsStateModel { 12 | pose: Pose; 13 | signingProbability: number; 14 | isSigning: boolean; 15 | } 16 | 17 | const initialState: ModelsStateModel = { 18 | pose: null, 19 | signingProbability: 0, 20 | isSigning: false 21 | }; 22 | 23 | @Injectable() 24 | @State({ 25 | name: 'models', 26 | defaults: initialState 27 | }) 28 | export class ModelsState implements NgxsOnInit { 29 | @Select(state => state.settings.dominantHand) dominantHand$: Observable; 30 | dominantHand: Hand = 'right'; 31 | 32 | constructor(private pose: PoseService, private detector: DetectorService) { 33 | 34 | } 35 | 36 | ngxsOnInit(ctx?: StateContext): void { 37 | this.dominantHand$.pipe( 38 | tap(dominantHand => this.dominantHand = dominantHand) 39 | ).subscribe(); 40 | } 41 | 42 | @Action(PoseVideoFrame) 43 | async poseFrame({patchState, dispatch}: StateContext, {video}: PoseVideoFrame): Promise { 44 | const pose = await this.pose.predict(video, this.dominantHand); 45 | 46 | patchState({pose}); 47 | if (pose) { // If person detected 48 | dispatch(DetectSigning); 49 | } else { 50 | patchState({isSigning: false}); 51 | } 52 | } 53 | 54 | @Action(DetectSigning) 55 | async detectSigning({getState, patchState}: StateContext): Promise { 56 | const {pose} = getState(); 57 | 58 | const signingProbability = await this.detector.detect(pose); 59 | patchState({ 60 | signingProbability, 61 | isSigning: signingProbability > 0.5 62 | }); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/ngxs-interceptor/ngxs-interceptor.plugin.ts: -------------------------------------------------------------------------------- 1 | import {Inject, Injectable, InjectionToken, ModuleWithProviders, NgModule} from '@angular/core'; 2 | import {AngularFireAnalytics} from '@angular/fire/analytics'; 3 | import {NGXS_PLUGINS, NgxsPlugin} from '@ngxs/store'; 4 | 5 | export const NGXS_INTERCEPTOR_PLUGIN_OPTIONS = new InjectionToken('NGXS_INTERCEPTOR_PLUGIN_OPTIONS'); 6 | 7 | @Injectable() 8 | export class NgxsInterceptorPlugin implements NgxsPlugin { 9 | private ignoredActions = [ 10 | '@@INIT', 11 | '@@UPDATE_STATE', 12 | '[Models] Pose Video Frame', 13 | '[Models] Detect Signing' 14 | ]; 15 | 16 | constructor(@Inject(NGXS_INTERCEPTOR_PLUGIN_OPTIONS) private options: any, 17 | private analytics: AngularFireAnalytics) { 18 | } 19 | 20 | handle(before, action, next): any { 21 | 22 | const type = action.type || action.constructor.type; 23 | const params = this.getEventParams(action, type); 24 | 25 | if (this.ignoredActions.indexOf(type) === -1) { 26 | this.analytics.logEvent(type, params) 27 | .then(() => console.debug('logEvent', {type, params})) 28 | .catch(err => console.error(err)); 29 | } 30 | return next(before, action); 31 | } 32 | 33 | private getEventParams(action, type: string): any { 34 | const ret: any = {}; 35 | if (type.indexOf('[Router]') > -1) { 36 | for (const k of ['id', 'url', 'urlAfterRedirects']) { 37 | ret[k] = action.event[k]; 38 | } 39 | } 40 | 41 | if (action.constructor.eventParams) { 42 | for (const k of action.constructor.eventParams) { 43 | ret[k] = action[k]; 44 | } 45 | } 46 | 47 | return ret; 48 | } 49 | } 50 | 51 | @NgModule() 52 | export class NgxsInterceptorPluginModule { 53 | static forRoot(config?: any): ModuleWithProviders { 54 | return { 55 | ngModule: NgxsInterceptorPluginModule, 56 | providers: [ 57 | { 58 | provide: NGXS_PLUGINS, 59 | useClass: NgxsInterceptorPlugin, 60 | multi: true 61 | }, 62 | { 63 | provide: NGXS_INTERCEPTOR_PLUGIN_OPTIONS, 64 | useValue: config 65 | } 66 | ] 67 | }; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sign-language-detector", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "ng": "ng", 6 | "start": "ng serve", 7 | "build": "ng build", 8 | "test": "ng test", 9 | "lint": "ng lint", 10 | "e2e": "ng e2e", 11 | "build:deploy": "ng build --configuration=production && firebase deploy" 12 | }, 13 | "private": true, 14 | "dependencies": { 15 | "@angular/animations": "~10.1.1", 16 | "@angular/cdk": "~10.1.1", 17 | "@angular/common": "~10.1.1", 18 | "@angular/compiler": "~10.1.1", 19 | "@angular/core": "~10.1.1", 20 | "@angular/fire": "^6.0.2", 21 | "@angular/forms": "~10.1.1", 22 | "@angular/material": "~10.1.1", 23 | "@angular/platform-browser": "~10.1.1", 24 | "@angular/platform-browser-dynamic": "~10.1.1", 25 | "@angular/router": "~10.1.1", 26 | "@ngneat/transloco": "^2.19.1", 27 | "@ngneat/transloco-messageformat": "^1.3.0", 28 | "@ngxs/router-plugin": "^3.7.0", 29 | "@ngxs/store": "^3.7.0", 30 | "@tensorflow-models/posenet": "^2.2.1", 31 | "@tensorflow/tfjs": "^2.3.0", 32 | "@tensorflow/tfjs-backend-webgl": "^2.3.0", 33 | "@tensorflow/tfjs-converter": "^2.3.0", 34 | "@tensorflow/tfjs-core": "^2.3.0", 35 | "@tensorflow/tfjs-layers": "^2.3.0", 36 | "firebase": "^7.20.0", 37 | "rxjs": "~6.6.3", 38 | "stats.js": "^0.17.0", 39 | "tslib": "^2.0.1", 40 | "zone.js": "~0.11.1" 41 | }, 42 | "devDependencies": { 43 | "@angular-devkit/architect": "^0.1001.1", 44 | "@angular-devkit/build-angular": "^0.1000.8", 45 | "@angular/cli": "~10.1.1", 46 | "@angular/compiler-cli": "~10.1.1", 47 | "@types/jasmine": "^3.5.14", 48 | "@types/jasminewd2": "~2.0.3", 49 | "@types/webgl2": "0.0.5", 50 | "codelyzer": "^6.0.0-next.1", 51 | "firebase-tools": "^8.10.0", 52 | "fuzzy": "^0.1.3", 53 | "inquirer": "^6.2.2", 54 | "inquirer-autocomplete-prompt": "^1.1.0", 55 | "jasmine-core": "~3.5.0", 56 | "jasmine-spec-reporter": "~5.0.0", 57 | "karma": "~5.0.0", 58 | "karma-chrome-launcher": "~3.1.0", 59 | "karma-coverage-istanbul-reporter": "~3.0.2", 60 | "karma-jasmine": "~3.3.0", 61 | "karma-jasmine-html-reporter": "^1.5.0", 62 | "open": "^7.2.1", 63 | "protractor": "~7.0.0", 64 | "ts-node": "~8.3.0", 65 | "tslint": "~6.1.3", 66 | "typescript": "~4.0.2" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/app/components/video/video-pose/video-pose.component.ts: -------------------------------------------------------------------------------- 1 | import {AfterViewInit, Component, ElementRef, ViewChild} from '@angular/core'; 2 | import {Select} from '@ngxs/store'; 3 | import {Observable} from 'rxjs'; 4 | import {Pose} from '../../../core/services/pose/models/base.pose-model'; 5 | import {BaseComponent} from '../../base/base.component'; 6 | import {filter, takeUntil, tap} from 'rxjs/operators'; 7 | import {PoseService} from '../../../core/services/pose/pose.service'; 8 | 9 | const POSE_LIMBS = [ 10 | [1, 2], [2, 3], [3, 4], [1, 5], [5, 6], [6, 7], [1, 8], [0, 16], [0, 15], [0, 18], [0, 17], [1, 0], [8, 9], 11 | [9, 10], [10, 11], [8, 12], [12, 13], [13, 14], [11, 24], [11, 22], [22, 23], [14, 21], [14, 19], [19, 20] 12 | ]; 13 | 14 | @Component({ 15 | selector: 'app-video-pose', 16 | templateUrl: './video-pose.component.html', 17 | styleUrls: ['./video-pose.component.css'] 18 | }) 19 | export class VideoPoseComponent extends BaseComponent implements AfterViewInit { 20 | @ViewChild('canvas') canvasEl: ElementRef; 21 | @Select(state => state.models.pose) pose$: Observable; 22 | 23 | ctx: CanvasRenderingContext2D; 24 | 25 | constructor(private elementRef: ElementRef, private poseService: PoseService) { 26 | super(); 27 | } 28 | 29 | ngAfterViewInit(): void { 30 | this.ctx = this.canvasEl.nativeElement.getContext('2d'); 31 | 32 | this.pose$.pipe( 33 | filter(Boolean), 34 | tap((pose: Pose) => this.draw(pose)), 35 | takeUntil(this.ngUnsubscribe) 36 | ).subscribe(); 37 | } 38 | 39 | draw(pose: Pose): void { 40 | const canvas = this.canvasEl.nativeElement; 41 | const {width, height} = this.elementRef.nativeElement.getBoundingClientRect(); 42 | // const {width, height} = this.poseService.model; 43 | if (canvas.width !== width) { 44 | canvas.width = width; 45 | } 46 | if (canvas.height !== height) { 47 | canvas.height = height; 48 | } 49 | this.ctx.clearRect(0, 0, canvas.width, canvas.height); 50 | this.ctx.fillStyle = 'red'; 51 | 52 | for (const [p1i, p2i] of POSE_LIMBS) { 53 | const p1 = pose[p1i]; 54 | const p2 = pose[p2i]; 55 | if (p1.x !== 0 && p2.x !== 0) { 56 | this.ctx.beginPath(); 57 | this.ctx.moveTo(p1.x, p1.y); 58 | this.ctx.lineTo(p2.x, p2.y); 59 | this.ctx.stroke(); 60 | } 61 | } 62 | 63 | for (const keypoint of pose) { 64 | if (keypoint.x !== 0) { 65 | this.ctx.beginPath(); 66 | this.ctx.arc(keypoint.x, keypoint.y, 5, 0, 2 * Math.PI); 67 | this.ctx.fill(); 68 | } 69 | } 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/app/components/video/video-controls/video-controls.component.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 11 | 12 | 20 | 21 | 29 | 30 | 31 | 32 | 33 | 34 | 42 | 43 | 51 | 52 | 53 | 54 | 55 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/app/app.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, OnInit} from '@angular/core'; 2 | import {Select, Store} from '@ngxs/store'; 3 | import {Observable} from 'rxjs'; 4 | import {filter, tap} from 'rxjs/operators'; 5 | import {MatProgressSpinner, MatSpinner} from '@angular/material/progress-spinner'; 6 | import {MatDialog, MatDialogRef} from '@angular/material/dialog'; 7 | import {DisplayError, ResetError} from './core/modules/ngxs/store/app/app.actions'; 8 | import {MatSnackBar} from '@angular/material/snack-bar'; 9 | import {AudioInstructionsComponent} from './components/audio/audio-instructions/audio-instructions.component'; 10 | import {TranslocoService} from '@ngneat/transloco'; 11 | 12 | @Component({ 13 | selector: 'app-root', 14 | templateUrl: './app.component.html', 15 | styleUrls: ['./app.component.scss'] 16 | }) 17 | export class AppComponent implements OnInit { 18 | @Select(state => state.app.isLoading) isLoading$: Observable; 19 | @Select(state => state.app.error) error$: Observable; 20 | @Select(state => state.audio.error) audioError$: Observable; 21 | 22 | loaderDialog: MatDialogRef; 23 | 24 | constructor(private dialog: MatDialog, 25 | private transloco: TranslocoService, 26 | private snackBar: MatSnackBar, 27 | private store: Store) { 28 | } 29 | 30 | ngOnInit(): void { 31 | this.manageLoading(); 32 | this.manageAppErrors(); 33 | this.manageAudioErrors(); 34 | } 35 | 36 | manageLoading(): void { 37 | this.isLoading$.pipe( 38 | filter(isLoading => isLoading || Boolean(this.loaderDialog)), 39 | tap(isLoading => { 40 | if (isLoading) { 41 | this.loaderDialog = this.dialog.open(MatSpinner, {panelClass: 'app-loader-dialog'}); 42 | } else { 43 | this.loaderDialog.close(); 44 | delete this.loaderDialog; 45 | } 46 | }) 47 | ).subscribe(); 48 | } 49 | 50 | manageAppErrors(): void { 51 | this.error$.pipe( 52 | filter(Boolean), 53 | tap((error: string) => { 54 | this.store.dispatch(new ResetError()); 55 | 56 | this.snackBar.open(error, null, { 57 | panelClass: 'mat-warn', 58 | duration: 10000 59 | }); 60 | }) 61 | ).subscribe(); 62 | } 63 | 64 | manageAudioErrors(): void { 65 | this.audioError$.pipe( 66 | filter(Boolean), 67 | tap(async (error: string) => { 68 | switch (error) { 69 | case 'missingSpeaker': 70 | this.dialog.open(AudioInstructionsComponent); 71 | break; 72 | default: 73 | const translation = await this.transloco.translate('audio.errors.' + error); 74 | this.store.dispatch(new DisplayError(translation)); 75 | } 76 | }) 77 | ).subscribe(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/polyfills.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file includes polyfills needed by Angular and is loaded before the app. 3 | * You can add your own extra polyfills to this file. 4 | * 5 | * This file is divided into 2 sections: 6 | * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. 7 | * 2. Application imports. Files imported after ZoneJS that should be loaded before your main 8 | * file. 9 | * 10 | * The current setup is for so-called "evergreen" browsers; the last versions of browsers that 11 | * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), 12 | * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. 13 | * 14 | * Learn more in https://angular.io/guide/browser-support 15 | */ 16 | 17 | /*************************************************************************************************** 18 | * BROWSER POLYFILLS 19 | */ 20 | 21 | /** IE10 and IE11 requires the following for NgClass support on SVG elements */ 22 | // import 'classlist.js'; // Run `npm install --save classlist.js`. 23 | 24 | /** 25 | * Web Animations `@angular/platform-browser/animations` 26 | * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. 27 | * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). 28 | */ 29 | // import 'web-animations-js'; // Run `npm install --save web-animations-js`. 30 | 31 | /** 32 | * By default, zone.js will patch all possible macroTask and DomEvents 33 | * user can disable parts of macroTask/DomEvents patch by setting following flags 34 | * because those flags need to be set before `zone.js` being loaded, and webpack 35 | * will put import in the top of bundle, so user need to create a separate file 36 | * in this directory (for example: zone-flags.ts), and put the following flags 37 | * into that file, and then add the following code before importing zone.js. 38 | * import './zone-flags'; 39 | * 40 | * The flags allowed in zone-flags.ts are listed here. 41 | * 42 | * The following flags will work for all browsers. 43 | * 44 | * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame 45 | * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick 46 | * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames 47 | * 48 | * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js 49 | * with the following flag, it will bypass `zone.js` patch for IE/Edge 50 | * 51 | * (window as any).__Zone_enable_cross_context_check = true; 52 | * 53 | */ 54 | 55 | /*************************************************************************************************** 56 | * Zone JS is required by default for Angular itself. 57 | */ 58 | import 'zone.js/dist/zone'; // Included with Angular CLI. 59 | 60 | 61 | /*************************************************************************************************** 62 | * APPLICATION IMPORTS 63 | */ 64 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/audio/audio.state.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {Action, NgxsOnInit, Select, State, StateContext} from '@ngxs/store'; 3 | import {tap} from 'rxjs/operators'; 4 | import {Observable} from 'rxjs'; 5 | import {SetSetting} from '../settings/settings.actions'; 6 | import {NavigatorService} from '../../../../services/navigator/navigator.service'; 7 | import {DisableTransmission, EnableTransmission} from './audio.actions'; 8 | 9 | const allowedVACDevices = new Set([ 10 | 'Line 1 (Virtual Audio Cable)', 11 | 'VB-Cable (Virtual)' 12 | ]); 13 | 14 | interface SpeakerSink { 15 | id: string; 16 | label: string; 17 | } 18 | 19 | export interface AudioStateModel { 20 | microphone: MediaStream; 21 | speakerSink: SpeakerSink; 22 | error: string; 23 | } 24 | 25 | const initialState: AudioStateModel = { 26 | microphone: null, 27 | speakerSink: null, 28 | error: null 29 | }; 30 | 31 | @Injectable() 32 | @State({ 33 | name: 'audio', 34 | defaults: initialState 35 | }) 36 | export class AudioState implements NgxsOnInit { 37 | 38 | @Select(state => state.settings.transmitAudio) transmitAudio$: Observable; 39 | 40 | constructor(private navigator: NavigatorService) { 41 | } 42 | 43 | ngxsOnInit({dispatch}: StateContext): void { 44 | this.transmitAudio$.pipe( 45 | tap((state) => { 46 | if (state) { 47 | dispatch(EnableTransmission); 48 | } else { 49 | dispatch(DisableTransmission); 50 | } 51 | }) 52 | ).subscribe(); 53 | } 54 | 55 | @Action(DisableTransmission) 56 | disableTransmission({patchState, getState}: StateContext): void { 57 | // Stop microphone stream if its open 58 | const {microphone} = getState(); 59 | if (microphone) { 60 | microphone.getTracks().forEach(track => track.stop()); 61 | } 62 | 63 | patchState({ 64 | ...initialState, 65 | error: null 66 | }); 67 | } 68 | 69 | @Action(EnableTransmission) 70 | async enableTransmission(context: StateContext): Promise { 71 | const {patchState, dispatch} = context; 72 | 73 | this.disableTransmission(context); 74 | 75 | const turnOffAudio = () => dispatch(new SetSetting('transmitAudio', false)); 76 | 77 | try { 78 | const microphone = await this.navigator.getMicrophone(); 79 | const audioTrack = microphone.getAudioTracks()[0]; 80 | audioTrack.addEventListener('ended', turnOffAudio); 81 | 82 | const speakerSink = await this.getSpeaker(); 83 | 84 | patchState({microphone, speakerSink, error: null}); 85 | } catch (e) { 86 | patchState({error: e.message}); 87 | turnOffAudio(); 88 | } 89 | } 90 | 91 | private async getSpeaker(mandatory: boolean = true): Promise { 92 | try { 93 | const speakerSinkDevice = await this.navigator.getSpeaker(allowedVACDevices); 94 | return { 95 | id: speakerSinkDevice.deviceId, 96 | label: speakerSinkDevice.label, 97 | }; 98 | } catch (e) { 99 | if (mandatory) { 100 | throw e; 101 | } 102 | return null; 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/app/core/services/pose/models/posenet.pose-model.ts: -------------------------------------------------------------------------------- 1 | import {BasePoseModel, Pose, POSE_LENGTH} from './base.pose-model'; 2 | import '@tensorflow/tfjs-backend-webgl'; // Adds the WebGL backend to the global backend registry. 3 | import * as posenet from '@tensorflow-models/posenet'; 4 | import {Keypoint, PoseNet} from '@tensorflow-models/posenet'; 5 | import {Hand} from '../../../modules/ngxs/store/settings/settings.state'; 6 | import {ModelConfig, SinglePersonInterfaceConfig} from '@tensorflow-models/posenet/dist/posenet_model'; 7 | 8 | const POSE_MAPPING = { 9 | nose: 0, 10 | leftEye: 16, 11 | rightEye: 15, 12 | leftEar: 18, 13 | rightEar: 17, 14 | leftShoulder: 5, 15 | rightShoulder: 2, 16 | leftElbow: 6, 17 | rightElbow: 3, 18 | leftWrist: 7, 19 | rightWrist: 4, 20 | leftHip: 12, 21 | rightHip: 9, 22 | leftKnee: 13, 23 | rightKnee: 10, 24 | leftAnkle: 14, 25 | rightAnkle: 11, 26 | }; 27 | 28 | const POSE_ADITIONAL: [number, [number, number]][] = [ 29 | [1, [2, 5]], 30 | [8, [9, 12]] 31 | ]; 32 | 33 | 34 | export class PoseNetPoseModel extends BasePoseModel { 35 | net: PoseNet; 36 | 37 | bbox: number[]; // Boundaries of pose estimation 38 | 39 | constructor(private modelConfig: ModelConfig) { 40 | super(); 41 | // TODO unclear why need to divide by 2 42 | this.width = (modelConfig.inputResolution as any).width / 2; 43 | this.height = (modelConfig.inputResolution as any).height / 2; 44 | 45 | this.bbox = [ 46 | this.width * 0.05, // Min X 47 | this.height * 0.05, // Min Y 48 | this.width * 0.95, // Max X 49 | this.height * 0.95, // Max Y 50 | ]; 51 | } 52 | 53 | async load(): Promise { 54 | if (this.net) { 55 | this.unload(); 56 | } 57 | 58 | this.net = await posenet.load(this.modelConfig); 59 | } 60 | 61 | async unload(): Promise { 62 | this.net.dispose(); 63 | delete this.net; 64 | } 65 | 66 | processKeypoints(keypoints: Keypoint[]): Pose { 67 | const emptyPoint = {x: 0, y: 0}; 68 | const pose: Pose = new Array(POSE_LENGTH).fill(emptyPoint); 69 | 70 | keypoints.forEach(({position, part}) => { 71 | const i = POSE_MAPPING[part]; 72 | pose[i] = position; 73 | }); 74 | 75 | // Additional calculated poses 76 | POSE_ADITIONAL.forEach(([i, [a, b]]) => { 77 | if (pose[a].x > 0 && pose[b].x > 0) { 78 | pose[i] = { 79 | x: (pose[a].x + pose[b].x) / 2, 80 | y: (pose[a].y + pose[b].y) / 2, 81 | }; 82 | } 83 | }); 84 | 85 | return pose; 86 | } 87 | 88 | keypointFilter({score, position}): boolean { 89 | return score > 0.8 && 90 | position.x > this.bbox[0] && position.y > this.bbox[1] && 91 | position.x < this.bbox[2] && position.y < this.bbox[3]; 92 | } 93 | 94 | 95 | async predict(video: HTMLVideoElement, dominantHand: Hand): Promise { 96 | const options: SinglePersonInterfaceConfig = {flipHorizontal: dominantHand === 'left'}; 97 | const {keypoints} = await this.net.estimateSinglePose(video, options); 98 | const filteredKeypoints = keypoints.filter(this.keypointFilter.bind(this)); 99 | return this.processKeypoints(filteredKeypoints); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/video/video.state.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {Action, NgxsOnInit, Select, State, StateContext} from '@ngxs/store'; 3 | import {tap} from 'rxjs/operators'; 4 | import {Observable} from 'rxjs'; 5 | import {StartCamera, StopCamera} from './video.actions'; 6 | import {SetSetting} from '../settings/settings.actions'; 7 | import {NavigatorService} from '../../../../services/navigator/navigator.service'; 8 | import {Pose} from '../../../../services/pose/models/base.pose-model'; 9 | 10 | 11 | export type AspectRatio = '16-9' | '4-3' | '2-1'; 12 | 13 | export interface CameraSettings { 14 | aspectRatio: AspectRatio; 15 | frameRate: number; 16 | height: number; 17 | width: number; 18 | } 19 | 20 | export interface VideoStateModel { 21 | camera: MediaStream; 22 | cameraSettings: CameraSettings; 23 | error: string; 24 | pose: Pose; 25 | } 26 | 27 | const initialState: VideoStateModel = { 28 | camera: null, 29 | cameraSettings: null, 30 | error: null, 31 | pose: undefined, 32 | }; 33 | 34 | @Injectable() 35 | @State({ 36 | name: 'video', 37 | defaults: initialState 38 | }) 39 | export class VideoState implements NgxsOnInit { 40 | 41 | @Select(state => state.settings.receiveVideo) receiveVideo$: Observable; 42 | 43 | constructor(private navigator: NavigatorService) { 44 | } 45 | 46 | ngxsOnInit({dispatch}: StateContext): void { 47 | this.receiveVideo$.pipe( 48 | tap((state) => { 49 | if (state) { 50 | dispatch(StartCamera); 51 | } else { 52 | dispatch(StopCamera); 53 | } 54 | }) 55 | ).subscribe(); 56 | } 57 | 58 | @Action(StopCamera) 59 | stopCamera({patchState, getState}: StateContext): void { 60 | // Stop camera stream if its open 61 | const {camera, error} = getState(); 62 | if (camera) { 63 | camera.getTracks().forEach(track => track.stop()); 64 | } 65 | 66 | patchState({ 67 | ...initialState, 68 | error: error || 'turnedOff' 69 | }); 70 | } 71 | 72 | @Action(StartCamera) 73 | async startCamera(context: StateContext): Promise { 74 | const {patchState, dispatch} = context; 75 | 76 | patchState({error: 'starting'}); 77 | this.stopCamera(context); 78 | 79 | const turnOffVideo = () => dispatch(new SetSetting('receiveVideo', false)); 80 | 81 | try { 82 | const camera = await this.navigator.getCamera({facingMode: 'user'}); 83 | if (!camera) { 84 | throw new Error('notConnected'); 85 | } 86 | 87 | const videoTrack = camera.getVideoTracks()[0]; 88 | const trackSettings = videoTrack.getSettings(); 89 | const aspectRatio = trackSettings.aspectRatio > 1.9 ? '2-1' : trackSettings.aspectRatio < 1.5 ? '4-3' : '16-9'; 90 | const cameraSettings: CameraSettings = { 91 | aspectRatio, 92 | frameRate: trackSettings.frameRate, 93 | width: trackSettings.width, 94 | height: trackSettings.height 95 | }; 96 | videoTrack.addEventListener('ended', turnOffVideo); 97 | 98 | patchState({camera, cameraSettings, error: null}); 99 | } catch (e) { 100 | patchState({error: e.message}); 101 | turnOffVideo(); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/app/components/audio/audio.component.ts: -------------------------------------------------------------------------------- 1 | import {Component, Input, OnDestroy, OnInit} from '@angular/core'; 2 | import {BaseComponent} from '../base/base.component'; 3 | import {Select} from '@ngxs/store'; 4 | import {Observable} from 'rxjs'; 5 | import {AudioStateModel} from '../../core/modules/ngxs/store/audio/audio.state'; 6 | import {distinctUntilChanged, filter, map, takeUntil, tap} from 'rxjs/operators'; 7 | 8 | @Component({ 9 | selector: 'app-audio', 10 | templateUrl: './audio.component.html', 11 | styleUrls: ['./audio.component.css'] 12 | }) 13 | export class AudioComponent extends BaseComponent implements OnInit, OnDestroy { 14 | @Select(state => state.audio) audioState$: Observable; 15 | 16 | @Input() playState: Observable; 17 | 18 | backgroundPlayer: HTMLAudioElement = document.createElement('audio'); 19 | foregroundPlayer: HTMLAudioElement = document.createElement('audio'); 20 | 21 | 22 | ngOnInit(): void { 23 | this.createBackgroundWave(); 24 | 25 | this.audioState$.pipe( 26 | tap(state => this.setMicrophone(state.microphone)), // Set Microphone 27 | map(state => state.speakerSink), 28 | filter(Boolean), 29 | tap((sink: any) => this.setVAC(sink.id)), 30 | takeUntil(this.ngUnsubscribe) 31 | ).subscribe(); 32 | 33 | // Play/pause 34 | this.playState.pipe( 35 | distinctUntilChanged(), 36 | tap(play => { 37 | if (play) { 38 | this.play(); 39 | } else { 40 | this.pause(); 41 | } 42 | }), 43 | takeUntil(this.ngUnsubscribe) 44 | ).subscribe(); 45 | } 46 | 47 | ngOnDestroy(): void { 48 | super.ngOnDestroy(); 49 | this.pause(); 50 | this.foregroundPlayer.pause(); 51 | } 52 | 53 | createBackgroundWave(): OscillatorNode { 54 | const audioCtx = new AudioContext(); 55 | window.addEventListener('load', () => { 56 | audioCtx.resume(); 57 | setTimeout(() => audioCtx.resume(), 5000); 58 | }); 59 | 60 | const oscillator = audioCtx.createOscillator(); 61 | oscillator.type = 'sine'; 62 | oscillator.frequency.value = 20000; 63 | 64 | const gainNode = audioCtx.createGain(); 65 | const streamNode = audioCtx.createMediaStreamDestination(); 66 | 67 | oscillator.connect(gainNode); 68 | gainNode.connect(streamNode); 69 | gainNode.gain.value = 0.6; 70 | 71 | oscillator.start(0); 72 | this.backgroundPlayer.srcObject = streamNode.stream; 73 | console.log('Background', this.backgroundPlayer); 74 | 75 | 76 | return oscillator; 77 | } 78 | 79 | async setVAC(sinkId: string): Promise { 80 | console.log('setVAC', sinkId); 81 | return Promise.all([ 82 | (this.backgroundPlayer as any).setSinkId(sinkId), 83 | // (this.foregroundPlayer as any).setSinkId(sinkId), 84 | ]); 85 | } 86 | 87 | async setMicrophone(microphone: MediaStream): Promise { 88 | // this.foregroundPlayer.srcObject = microphone; 89 | if (microphone) { 90 | return this.foregroundPlayer.play(); 91 | } else { 92 | return this.foregroundPlayer.pause(); 93 | } 94 | } 95 | 96 | play(): Promise { 97 | // if (!this.foregroundPlayer.srcObject) { 98 | // return; 99 | // } 100 | 101 | console.log('PLAY'); 102 | 103 | return this.backgroundPlayer.play(); 104 | } 105 | 106 | pause(): void { 107 | console.log('PAUSE'); 108 | 109 | return this.backgroundPlayer.pause(); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/app/core/services/detector/detector.service.ts: -------------------------------------------------------------------------------- 1 | import {Injectable} from '@angular/core'; 2 | import {LayersModel} from '@tensorflow/tfjs-layers/src/engine/training'; 3 | import * as tf from '@tensorflow/tfjs'; 4 | import {Tensor} from '@tensorflow/tfjs-core'; 5 | import {Vector2D} from '@tensorflow-models/posenet/dist/types'; 6 | import {Pose} from '../pose/models/base.pose-model'; 7 | 8 | 9 | const WINDOW_SIZE = 20; 10 | const LINEAR_WEIGHTS = [0.04518767, 0.06754455, 0.12843487, 0.057285164, 0.0738754, 0.0016703978, 0.038430076, 0.05914564, 0.006238494, 11 | -0.030242687, -0.001319781, -0.0072753574, -0.024136841, 1.38366595e-05, -0.017613031, 0.035724297, 0.03492076, 12 | 0.084389575, 0.08676544, 0.0062924633, -0.008905508, -0.011594881, -0.0061611193, -0.025734566, -0.0126820095]; 13 | 14 | @Injectable({ 15 | providedIn: 'root' 16 | }) 17 | export class DetectorService { 18 | lastPose: Pose; 19 | lastTimestamp: number; 20 | 21 | shoulderWidth: Float32Array = new Float32Array(WINDOW_SIZE).fill(0); 22 | shoulderWidthIndex = 0; 23 | 24 | sequentialModel: LayersModel; 25 | 26 | constructor() { 27 | tf.loadLayersModel('assets/models/sign-detector/model.json') 28 | .then(model => this.sequentialModel = model as unknown as LayersModel); 29 | } 30 | 31 | distance(p1: Vector2D, p2: Vector2D): number { 32 | const xs = p1.x - p2.x; 33 | const ys = p1.y - p2.y; 34 | return Math.sqrt(xs * xs + ys * ys); 35 | } 36 | 37 | normalizeTensor(pose: Pose): Pose { 38 | const p1 = pose[2]; 39 | const p2 = pose[5]; 40 | 41 | if (p1.x > 0 && p2.x > 0) { 42 | this.shoulderWidth[this.shoulderWidthIndex % WINDOW_SIZE] = this.distance(p1, p2); 43 | this.shoulderWidthIndex++; 44 | } 45 | 46 | if (this.shoulderWidthIndex < WINDOW_SIZE) { 47 | return null; 48 | } 49 | 50 | const meanShoulders = this.shoulderWidth.reduce((a, b) => a + b, 0) / WINDOW_SIZE; 51 | const newPose = new Array(pose.length); 52 | pose.forEach((v, i) => { 53 | newPose[i] = { 54 | x: v.x / meanShoulders, 55 | y: v.y / meanShoulders, 56 | }; 57 | }); 58 | 59 | return newPose; 60 | } 61 | 62 | distance2DTensors(p1: Pose, p2: Pose, multiplier = 1): Float32Array { 63 | const d = new Float32Array(p1.length).fill(0); 64 | for (let i = 0; i < d.length; i += 1) { 65 | const a = p1[i]; 66 | const b = p2[i]; 67 | 68 | if (a.x > 0 && b.x > 0) { 69 | d[i] = this.distance(a, b) * multiplier; 70 | } 71 | } 72 | return d; 73 | } 74 | 75 | getSequentialConfidence(opticalFlow: Float32Array): number { 76 | const pred: Tensor = this.sequentialModel.predict(tf.tensor(opticalFlow).reshape([1, 1, 25])) as Tensor; 77 | const probs = tf.softmax(pred).dataSync(); 78 | return probs[1]; 79 | } 80 | 81 | getLinearConfidence(opticalFlow: Float32Array): number { 82 | let sum = 0; 83 | for (let i = 0; i < LINEAR_WEIGHTS.length; i++) { 84 | sum += opticalFlow[i] * LINEAR_WEIGHTS[i]; 85 | } 86 | return sum; 87 | } 88 | 89 | getConfidence(opticalFlow: Float32Array): number { 90 | if (this.sequentialModel) { 91 | return this.getSequentialConfidence(opticalFlow); 92 | } else { 93 | return this.getLinearConfidence(opticalFlow); 94 | } 95 | } 96 | 97 | detect(pose: Pose): number { 98 | const timestamp = performance.now() / 1000; 99 | let confidence = 0; 100 | 101 | const normalized = this.normalizeTensor(pose); 102 | 103 | if (this.lastPose && normalized) { 104 | const fps = 1 / (timestamp - this.lastTimestamp); 105 | const distance = this.distance2DTensors(normalized, this.lastPose, fps); 106 | confidence = this.getConfidence(distance); 107 | } 108 | 109 | this.lastTimestamp = timestamp; 110 | this.lastPose = normalized; 111 | 112 | return confidence; 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "tslint:recommended", 3 | "rules": { 4 | "align": { 5 | "options": [ 6 | "parameters", 7 | "statements" 8 | ] 9 | }, 10 | "array-type": false, 11 | "arrow-return-shorthand": true, 12 | "curly": true, 13 | "deprecation": { 14 | "severity": "warning" 15 | }, 16 | "component-class-suffix": true, 17 | "contextual-lifecycle": true, 18 | "directive-class-suffix": true, 19 | "directive-selector": [ 20 | true, 21 | "attribute", 22 | "app", 23 | "camelCase" 24 | ], 25 | "component-selector": [ 26 | true, 27 | "element", 28 | "app", 29 | "kebab-case" 30 | ], 31 | "eofline": true, 32 | "import-blacklist": [ 33 | true, 34 | "rxjs/Rx" 35 | ], 36 | "import-spacing": true, 37 | "indent": { 38 | "options": [ 39 | "spaces" 40 | ] 41 | }, 42 | "max-classes-per-file": false, 43 | "max-line-length": [ 44 | true, 45 | 140 46 | ], 47 | "member-ordering": [ 48 | true, 49 | { 50 | "order": [ 51 | "static-field", 52 | "instance-field", 53 | "static-method", 54 | "instance-method" 55 | ] 56 | } 57 | ], 58 | "no-console": [ 59 | false, 60 | "debug", 61 | "info", 62 | "time", 63 | "timeEnd", 64 | "trace" 65 | ], 66 | "no-empty": false, 67 | "no-inferrable-types": [ 68 | true, 69 | "ignore-params" 70 | ], 71 | "no-non-null-assertion": true, 72 | "no-redundant-jsdoc": true, 73 | "no-switch-case-fall-through": true, 74 | "no-var-requires": false, 75 | "object-literal-key-quotes": [ 76 | true, 77 | "as-needed" 78 | ], 79 | "quotemark": [ 80 | true, 81 | "single" 82 | ], 83 | "semicolon": { 84 | "options": [ 85 | "always" 86 | ] 87 | }, 88 | "space-before-function-paren": { 89 | "options": { 90 | "anonymous": "never", 91 | "asyncArrow": "always", 92 | "constructor": "never", 93 | "method": "never", 94 | "named": "never" 95 | } 96 | }, 97 | "typedef": [ 98 | true, 99 | "call-signature" 100 | ], 101 | "typedef-whitespace": { 102 | "options": [ 103 | { 104 | "call-signature": "nospace", 105 | "index-signature": "nospace", 106 | "parameter": "nospace", 107 | "property-declaration": "nospace", 108 | "variable-declaration": "nospace" 109 | }, 110 | { 111 | "call-signature": "onespace", 112 | "index-signature": "onespace", 113 | "parameter": "onespace", 114 | "property-declaration": "onespace", 115 | "variable-declaration": "onespace" 116 | } 117 | ] 118 | }, 119 | "variable-name": { 120 | "options": [ 121 | "ban-keywords", 122 | "check-format", 123 | "allow-pascal-case" 124 | ] 125 | }, 126 | "whitespace": { 127 | "options": [ 128 | "check-branch", 129 | "check-decl", 130 | "check-operator", 131 | "check-separator", 132 | "check-type", 133 | "check-typecast" 134 | ] 135 | }, 136 | "no-conflicting-lifecycle": true, 137 | "no-host-metadata-property": true, 138 | "no-input-rename": true, 139 | "no-inputs-metadata-property": true, 140 | "no-output-native": true, 141 | "no-output-on-prefix": true, 142 | "no-output-rename": true, 143 | "no-outputs-metadata-property": true, 144 | "template-banana-in-box": true, 145 | "template-no-negated-async": true, 146 | "use-lifecycle-interface": true, 147 | "use-pipe-transform-interface": true 148 | }, 149 | "rulesDirectory": [ 150 | "codelyzer" 151 | ] 152 | } 153 | -------------------------------------------------------------------------------- /src/app/core/modules/ngxs/store/video/video.spec.ts: -------------------------------------------------------------------------------- 1 | import {TestBed} from '@angular/core/testing'; 2 | import {NgxsModule, Store} from '@ngxs/store'; 3 | import {VideoState, VideoStateModel} from './video.state'; 4 | import {StartCamera, StopCamera} from './video.actions'; 5 | import {NavigatorService} from '../../../../services/navigator/navigator.service'; 6 | 7 | 8 | describe('VideoState', () => { 9 | let store: Store; 10 | let navigatorService: NavigatorService; 11 | let snapshot: { video: VideoStateModel }; 12 | 13 | let mockCamera: MediaStream; 14 | let mockSettings: MediaTrackSettings; 15 | let mockTrack: MediaStreamTrack; 16 | 17 | beforeAll(() => { 18 | // Setup mock camera 19 | mockCamera = new MediaStream(); 20 | mockSettings = { 21 | aspectRatio: 2, 22 | frameRate: 30, 23 | height: 720, 24 | width: 1280 25 | }; 26 | mockTrack = { 27 | getSettings: () => mockSettings, 28 | addEventListener: (() => { 29 | }) as any 30 | } as MediaStreamTrack; 31 | }); 32 | 33 | beforeEach(() => { 34 | TestBed.configureTestingModule({ 35 | imports: [NgxsModule.forRoot([VideoState])], 36 | providers: [NavigatorService] 37 | }); 38 | 39 | store = TestBed.inject(Store); 40 | navigatorService = TestBed.inject(NavigatorService); 41 | snapshot = store.snapshot(); 42 | }); 43 | 44 | it('StopCamera should stop camera', () => { 45 | snapshot.video.camera = new MediaStream(); 46 | store.reset(snapshot); 47 | 48 | store.dispatch(new StopCamera()); 49 | 50 | const camera = store.selectSnapshot(state => state.video.camera); 51 | expect(camera).toBe(null); 52 | }); 53 | 54 | it('StopCamera should set error to turnedOff', () => { 55 | snapshot.video.error = null; 56 | store.reset(snapshot); 57 | 58 | store.dispatch(new StopCamera()); 59 | 60 | const error = store.selectSnapshot(state => state.video.error); 61 | expect(error).toBe('turnedOff'); 62 | }); 63 | 64 | it('StopCamera should keep error if exists', () => { 65 | const testError = 'testError'; 66 | snapshot.video.error = testError; 67 | store.reset(snapshot); 68 | 69 | store.dispatch(new StopCamera()); 70 | 71 | const error = store.selectSnapshot(state => state.video.error); 72 | expect(error).toBe(testError); 73 | }); 74 | 75 | it('StartCamera error should update store', () => { 76 | const testError = 'testError'; 77 | const spy = spyOn(navigatorService, 'getCamera').and.throwError(new Error(testError)); 78 | 79 | snapshot.video.error = null; 80 | store.reset(snapshot); 81 | 82 | store.dispatch(new StartCamera()); 83 | 84 | const error = store.selectSnapshot(state => state.video.error); 85 | expect(error).toBe(testError); 86 | }); 87 | 88 | it('StartCamera should get camera from navigator', async () => { 89 | // Setup mock camera 90 | const tracksSpy = spyOn(mockCamera, 'getVideoTracks').and.returnValue([mockTrack]); 91 | const cameraSpy = spyOn(navigatorService, 'getCamera').and.returnValue(Promise.resolve(mockCamera)); 92 | const listenerSpy = spyOn(mockTrack, 'addEventListener'); 93 | 94 | await store.dispatch(new StartCamera()).toPromise(); 95 | 96 | expect(tracksSpy).toHaveBeenCalled(); 97 | expect(cameraSpy).toHaveBeenCalled(); 98 | expect(listenerSpy).toHaveBeenCalled(); 99 | 100 | const {camera, error} = store.selectSnapshot(state => state.video); 101 | 102 | expect(error).toBe(null); 103 | expect(camera).toBe(mockCamera); 104 | }); 105 | 106 | it('StartCamera should set camera settings', async () => { 107 | const tracksSpy = spyOn(mockCamera, 'getVideoTracks').and.returnValue([mockTrack]); 108 | const cameraSpy = spyOn(navigatorService, 'getCamera').and.returnValue(Promise.resolve(mockCamera)); 109 | 110 | await store.dispatch(new StartCamera()).toPromise(); 111 | 112 | expect(tracksSpy).toHaveBeenCalled(); 113 | expect(cameraSpy).toHaveBeenCalled(); 114 | 115 | const {cameraSettings, error} = store.selectSnapshot(state => state.video); 116 | 117 | expect(error).toBe(null); 118 | expect(cameraSettings.aspectRatio).toBe('2-1'); 119 | expect(cameraSettings.height).toBe(mockSettings.height); 120 | expect(cameraSettings.width).toBe(mockSettings.width); 121 | expect(cameraSettings.frameRate).toBe(mockSettings.frameRate); 122 | }); 123 | }); 124 | -------------------------------------------------------------------------------- /angular.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@angular/cli/lib/config/schema.json", 3 | "version": 1, 4 | "newProjectRoot": "projects", 5 | "projects": { 6 | "sign-language-detector": { 7 | "projectType": "application", 8 | "schematics": {}, 9 | "root": "", 10 | "sourceRoot": "src", 11 | "prefix": "app", 12 | "architect": { 13 | "build": { 14 | "builder": "@angular-devkit/build-angular:browser", 15 | "options": { 16 | "outputPath": "dist/sign-language-detector", 17 | "index": "src/index.html", 18 | "main": "src/main.ts", 19 | "polyfills": "src/polyfills.ts", 20 | "tsConfig": "tsconfig.app.json", 21 | "aot": true, 22 | "assets": [ 23 | "src/favicon.ico", 24 | "src/assets" 25 | ], 26 | "styles": [ 27 | "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", 28 | "src/theme/styles.scss" 29 | ], 30 | "scripts": [] 31 | }, 32 | "configurations": { 33 | "production": { 34 | "fileReplacements": [ 35 | { 36 | "replace": "src/environments/environment.ts", 37 | "with": "src/environments/environment.prod.ts" 38 | } 39 | ], 40 | "optimization": true, 41 | "outputHashing": "all", 42 | "sourceMap": false, 43 | "extractCss": true, 44 | "namedChunks": false, 45 | "extractLicenses": true, 46 | "vendorChunk": false, 47 | "buildOptimizer": true, 48 | "budgets": [ 49 | { 50 | "type": "initial", 51 | "maximumWarning": "2mb", 52 | "maximumError": "5mb" 53 | }, 54 | { 55 | "type": "anyComponentStyle", 56 | "maximumWarning": "6kb", 57 | "maximumError": "10kb" 58 | } 59 | ] 60 | } 61 | } 62 | }, 63 | "serve": { 64 | "builder": "@angular-devkit/build-angular:dev-server", 65 | "options": { 66 | "browserTarget": "sign-language-detector:build" 67 | }, 68 | "configurations": { 69 | "production": { 70 | "browserTarget": "sign-language-detector:build:production" 71 | } 72 | } 73 | }, 74 | "extract-i18n": { 75 | "builder": "@angular-devkit/build-angular:extract-i18n", 76 | "options": { 77 | "browserTarget": "sign-language-detector:build" 78 | } 79 | }, 80 | "test": { 81 | "builder": "@angular-devkit/build-angular:karma", 82 | "options": { 83 | "main": "src/test.ts", 84 | "polyfills": "src/polyfills.ts", 85 | "tsConfig": "tsconfig.spec.json", 86 | "karmaConfig": "karma.conf.js", 87 | "assets": [ 88 | "src/favicon.ico", 89 | "src/assets" 90 | ], 91 | "styles": [ 92 | "./node_modules/@angular/material/prebuilt-themes/deeppurple-amber.css", 93 | "src/theme/styles.scss" 94 | ], 95 | "scripts": [] 96 | } 97 | }, 98 | "lint": { 99 | "builder": "@angular-devkit/build-angular:tslint", 100 | "options": { 101 | "tsConfig": [ 102 | "tsconfig.app.json", 103 | "tsconfig.spec.json", 104 | "e2e/tsconfig.json" 105 | ], 106 | "exclude": [ 107 | "**/node_modules/**" 108 | ] 109 | } 110 | }, 111 | "e2e": { 112 | "builder": "@angular-devkit/build-angular:protractor", 113 | "options": { 114 | "protractorConfig": "e2e/protractor.conf.js", 115 | "devServerTarget": "sign-language-detector:serve" 116 | }, 117 | "configurations": { 118 | "production": { 119 | "devServerTarget": "sign-language-detector:serve:production" 120 | } 121 | } 122 | } 123 | } 124 | } 125 | }, 126 | "defaultProject": "sign-language-detector", 127 | "cli": { 128 | "analytics": "da7f7e1b-ea69-47b0-a971-5d7f28b86a48" 129 | }, 130 | "schematics": { 131 | "@schematics/angular:component": { 132 | "styleext": "scss" 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/app/components/video/video.component.ts: -------------------------------------------------------------------------------- 1 | import {AfterViewInit, Component, ElementRef, HostBinding, ViewChild} from '@angular/core'; 2 | import {Select, Store} from '@ngxs/store'; 3 | import {Observable} from 'rxjs'; 4 | import {AspectRatio, VideoStateModel} from '../../core/modules/ngxs/store/video/video.state'; 5 | import Stats from 'stats.js'; 6 | import {filter, map, takeUntil, tap} from 'rxjs/operators'; 7 | import {BaseComponent} from '../base/base.component'; 8 | import {wait} from '../../core/helpers/wait/wait'; 9 | import {PoseVideoFrame} from '../../core/modules/ngxs/store/models/models.actions'; 10 | import {Hand} from '../../core/modules/ngxs/store/settings/settings.state'; 11 | 12 | @Component({ 13 | selector: 'app-video', 14 | templateUrl: './video.component.html', 15 | styleUrls: ['./video.component.scss'] 16 | }) 17 | export class VideoComponent extends BaseComponent implements AfterViewInit { 18 | @Select(state => state.video) videoState$: Observable; 19 | @Select(state => state.models.signingProbability) signingProbability$: Observable; 20 | @Select(state => state.models.isSigning) isSigning$: Observable; 21 | @Select(state => state.settings.dominantHand) dominantHand$: Observable; 22 | 23 | @ViewChild('video') videoEl: ElementRef; 24 | @ViewChild('stats') statsEl: ElementRef; 25 | 26 | @HostBinding('class') aspectRatio = 'aspect-16-9'; 27 | 28 | fpsStats = new Stats(); 29 | signingStats = new Stats(); 30 | 31 | constructor(private store: Store) { 32 | super(); 33 | } 34 | 35 | ngAfterViewInit(): void { 36 | this.setCamera(); 37 | this.setStats(); 38 | 39 | this.videoEl.nativeElement.addEventListener('loadeddata', this.appLoop.bind(this)); 40 | } 41 | 42 | async appLoop(): Promise { 43 | const fps = this.store.snapshot().video.cameraSettings.frameRate; 44 | const video = this.videoEl.nativeElement; 45 | const poseAction = new PoseVideoFrame(this.videoEl.nativeElement); 46 | 47 | const fpsWait = 1000 / fps - 1; 48 | 49 | while (true) { 50 | if (video.readyState !== 4) { 51 | break; 52 | } 53 | 54 | this.fpsStats.begin(); 55 | const startTime = performance.now(); 56 | await this.store.dispatch(poseAction).toPromise(); 57 | // 58 | // if (this.poseNet) { 59 | // const {keypoints} = await this.poseNet.estimateSinglePose(this.videoEl.nativeElement, {flipHorizontal: this.flipPose}); 60 | // this.processKeypoints(keypoints); 61 | // } else { 62 | // await wait(1000 / 60); 63 | // } 64 | const endTime = performance.now(); 65 | 66 | const timePassed = endTime - startTime; // Time passed in milliseconds 67 | if (timePassed < fpsWait) { 68 | await wait(fpsWait - timePassed); 69 | } 70 | this.fpsStats.end(); 71 | } 72 | } 73 | 74 | setCamera(): void { 75 | const video = this.videoEl.nativeElement; 76 | video.addEventListener('loadedmetadata', e => video.play()); 77 | 78 | this.videoState$.pipe( 79 | map(state => state.camera), 80 | tap(camera => video.srcObject = camera), 81 | // tap(camera => { 82 | // video.src = 'assets/videos/example_maayan.mp4'; 83 | // video.muted = true; 84 | // setTimeout(() => this.aspectRatio = 'aspect-16-9', 0); 85 | // }), 86 | takeUntil(this.ngUnsubscribe) 87 | ).subscribe(); 88 | 89 | this.videoState$.pipe( 90 | map(state => state.cameraSettings && state.cameraSettings.aspectRatio), 91 | filter(Boolean), 92 | tap((aspectRatio: AspectRatio) => this.aspectRatio = 'aspect-' + aspectRatio), 93 | takeUntil(this.ngUnsubscribe) 94 | ).subscribe(); 95 | } 96 | 97 | setStats(): void { 98 | this.fpsStats.showPanel(0); 99 | this.fpsStats.domElement.style.position = 'absolute'; 100 | this.statsEl.nativeElement.appendChild(this.fpsStats.dom); 101 | 102 | const signingPanel = new Stats.Panel('Signing', '#ff8', '#221'); 103 | this.signingStats.dom.innerHTML = ''; 104 | this.signingStats.addPanel(signingPanel); 105 | this.signingStats.showPanel(0); 106 | this.signingStats.domElement.style.position = 'absolute'; 107 | this.signingStats.domElement.style.left = '80px'; 108 | this.statsEl.nativeElement.appendChild(this.signingStats.dom); 109 | 110 | this.setDetectorListener(signingPanel); 111 | } 112 | 113 | setDetectorListener(panel: Stats.Panel): void { 114 | this.signingProbability$.pipe( 115 | tap(v => panel.update(v * 100, 100)), 116 | takeUntil(this.ngUnsubscribe) 117 | ).subscribe(); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/assets/models/sign-detector/model.json: -------------------------------------------------------------------------------- 1 | { 2 | "format": "layers-model", 3 | "generatedBy": "keras v2.3.0-tf", 4 | "convertedBy": "TensorFlow.js Converter v2.0.1.post1", 5 | "modelTopology": { 6 | "keras_version": "2.3.0-tf", 7 | "backend": "tensorflow", 8 | "model_config": { 9 | "class_name": "Sequential", 10 | "config": { 11 | "name": "sequential", 12 | "layers": [ 13 | { 14 | "class_name": "Dropout", 15 | "config": { 16 | "batch_input_shape": [ 17 | 1, 18 | 1, 19 | 25 20 | ], 21 | "name": "dropout", 22 | "trainable": true, 23 | "dtype": "float32", 24 | "rate": 0.5, 25 | "noise_shape": null, 26 | "seed": null 27 | } 28 | }, 29 | { 30 | "class_name": "LSTM", 31 | "config": { 32 | "name": "lstm", 33 | "trainable": true, 34 | "dtype": "float32", 35 | "return_sequences": true, 36 | "return_state": false, 37 | "go_backwards": false, 38 | "stateful": true, 39 | "unroll": false, 40 | "time_major": false, 41 | "units": 64, 42 | "activation": "tanh", 43 | "recurrent_activation": "sigmoid", 44 | "use_bias": true, 45 | "kernel_initializer": { 46 | "class_name": "GlorotUniform", 47 | "config": { 48 | "seed": null 49 | } 50 | }, 51 | "recurrent_initializer": { 52 | "class_name": "Orthogonal", 53 | "config": { 54 | "gain": 1.0, 55 | "seed": null 56 | } 57 | }, 58 | "bias_initializer": { 59 | "class_name": "Zeros", 60 | "config": {} 61 | }, 62 | "unit_forget_bias": true, 63 | "kernel_regularizer": null, 64 | "recurrent_regularizer": null, 65 | "bias_regularizer": null, 66 | "activity_regularizer": null, 67 | "kernel_constraint": null, 68 | "recurrent_constraint": null, 69 | "bias_constraint": null, 70 | "dropout": 0.0, 71 | "recurrent_dropout": 0.0, 72 | "implementation": 2 73 | } 74 | }, 75 | { 76 | "class_name": "Dense", 77 | "config": { 78 | "name": "dense", 79 | "trainable": true, 80 | "dtype": "float32", 81 | "units": 2, 82 | "activation": "linear", 83 | "use_bias": true, 84 | "kernel_initializer": { 85 | "class_name": "GlorotUniform", 86 | "config": { 87 | "seed": null 88 | } 89 | }, 90 | "bias_initializer": { 91 | "class_name": "Zeros", 92 | "config": {} 93 | }, 94 | "kernel_regularizer": null, 95 | "bias_regularizer": null, 96 | "activity_regularizer": null, 97 | "kernel_constraint": null, 98 | "bias_constraint": null 99 | } 100 | } 101 | ], 102 | "build_input_shape": [ 103 | 1, 104 | 1, 105 | 25 106 | ] 107 | } 108 | } 109 | }, 110 | "weightsManifest": [ 111 | { 112 | "paths": [ 113 | "group1-shard1of1.bin" 114 | ], 115 | "weights": [ 116 | { 117 | "name": "dense/kernel", 118 | "shape": [ 119 | 64, 120 | 2 121 | ], 122 | "dtype": "float32" 123 | }, 124 | { 125 | "name": "dense/bias", 126 | "shape": [ 127 | 2 128 | ], 129 | "dtype": "float32" 130 | }, 131 | { 132 | "name": "lstm/lstm_cell/kernel", 133 | "shape": [ 134 | 25, 135 | 256 136 | ], 137 | "dtype": "float32" 138 | }, 139 | { 140 | "name": "lstm/lstm_cell/recurrent_kernel", 141 | "shape": [ 142 | 64, 143 | 256 144 | ], 145 | "dtype": "float32" 146 | }, 147 | { 148 | "name": "lstm/lstm_cell/bias", 149 | "shape": [ 150 | 256 151 | ], 152 | "dtype": "float32" 153 | } 154 | ] 155 | } 156 | ] 157 | } 158 | --------------------------------------------------------------------------------