Title

├── README.md ├── src ├── libs │ ├── types │ │ └── math.ts │ ├── math │ │ ├── distance.ts │ │ └── extrapolation.ts │ ├── TypedEventEmitter │ │ └── index.ts │ ├── EstimatteHand │ │ ├── features │ │ │ ├── gestureUnderstanding.ts │ │ │ └── handCursor.ts │ │ ├── index.ts │ │ └── eventEmitter.ts │ └── EstimateFace │ │ ├── index.ts │ │ └── eventEmitter.ts ├── main.ts └── index.html ├── .gitignore ├── index.html ├── tsconfig.json ├── package.json ├── webpack.config.js └── tslint.json /README.md: -------------------------------------------------------------------------------- 1 | ### Управление интерфейсами с помощью жестов 2 | 3 | Библиотека для управления web-браузером с помощью камеры. 4 | -------------------------------------------------------------------------------- /src/libs/types/math.ts: -------------------------------------------------------------------------------- 1 | export type V3 = [number, number, number]; 2 | 3 | export interface Position2D { 4 | x: number; 5 | y: number; 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | src/**/*.js 4 | src/**/*.js.map 5 | typings/ 6 | dist/ 7 | coverage/ 8 | *.log 9 | package-lock.json 10 | *.lock -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Title 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/libs/math/distance.ts: -------------------------------------------------------------------------------- 1 | import { V3 } from "../types/math"; 2 | 3 | export const distance3D = (a: V3, b: V3) => 4 | Math.sqrt( 5 | Math.pow(b[0] - a[0], 2) + 6 | Math.pow(b[1] - a[1], 2) + 7 | Math.pow(b[2] - a[2], 2) 8 | ); 9 | -------------------------------------------------------------------------------- /src/libs/TypedEventEmitter/index.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from "rxjs"; 2 | 3 | export interface ITypedEventEmitter { 4 | emit: (type: string, payload: any) => void; 5 | removeListener: (event: any, handler: (payload: any) => void) => void; 6 | on: (event: string, handler: (payload: any) => void) => void; 7 | createObserverOn(type: any): Observable; 8 | } 9 | -------------------------------------------------------------------------------- /src/libs/math/extrapolation.ts: -------------------------------------------------------------------------------- 1 | import { Position2D } from "../types/math"; 2 | 3 | export const interpolation2dPoints = (window: number) => { 4 | let store: [number, number][] = []; 5 | return function (x: number, y: number): Position2D | null { 6 | store.push([x, y]); 7 | if (store.length < window) { 8 | return null; 9 | } 10 | const sum = store.reduce((acc, val) => [acc[0] + val[0], acc[1] + val[1]], [ 11 | 0, 12 | 0, 13 | ]); 14 | store = store.slice(-window - 1); 15 | return { 16 | x: sum[0] / window, 17 | y: sum[1] / window, 18 | }; 19 | }; 20 | }; 21 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "sourceMap": true, 5 | "declaration": false, 6 | "moduleResolution": "node", 7 | "emitDecoratorMetadata": true, 8 | "experimentalDecorators": true, 9 | "target": "es5", 10 | "module": "esnext", 11 | "typeRoots": [ 12 | "./node_modules/@types" 13 | ], 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "lib": [ 17 | "es2017", 18 | "es2015", 19 | "dom" 20 | ], 21 | "noImplicitAny": true, 22 | "noImplicitThis": true, 23 | "noUnusedParameters": true, 24 | "forceConsistentCasingInFileNames": true, 25 | "allowJs": true, 26 | "strictNullChecks": false 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/libs/EstimatteHand/features/gestureUnderstanding.ts: -------------------------------------------------------------------------------- 1 | import { V3 } from "../../types/math"; 2 | 3 | import { distance3D } from "../../math/distance"; 4 | 5 | export enum Gesture { 6 | POINT = "point", 7 | } 8 | 9 | interface HandInterface { 10 | thumb: [V3, V3, V3, V3]; 11 | indexFinger: [V3, V3, V3, V3]; 12 | middleFinger: [V3, V3, V3, V3]; 13 | ringFinger: [V3, V3, V3, V3]; 14 | pinky: [V3, V3, V3, V3]; 15 | palmBase: [V3, V3, V3, V3]; 16 | } 17 | 18 | function isPoint(hand: HandInterface): boolean { 19 | const lengthIndex = distance3D(hand.palmBase[0], hand.indexFinger[3]); 20 | const lengthMiddle = distance3D(hand.palmBase[0], hand.middleFinger[3]); 21 | return lengthIndex / lengthMiddle > 1.5; 22 | } 23 | 24 | export function understandGesture(hand: HandInterface): Gesture | null { 25 | if (isPoint(hand)) { 26 | return Gesture.POINT; 27 | } 28 | return null; 29 | } 30 | -------------------------------------------------------------------------------- /src/libs/EstimatteHand/features/handCursor.ts: -------------------------------------------------------------------------------- 1 | import { Position2D } from "../../types/math"; 2 | 3 | export class HandCursor { 4 | private readonly innerElement: HTMLSpanElement; 5 | private _position: Position2D; 6 | 7 | constructor() { 8 | this.innerElement = HandCursor.createHtmlCursor(); 9 | document.body.appendChild(this.innerElement); 10 | } 11 | 12 | public move(pos: Position2D) { 13 | this.innerElement.style.left = pos.x + "px"; 14 | this.innerElement.style.top = pos.y + "px"; 15 | this._position = pos; 16 | } 17 | 18 | private static createHtmlCursor(): HTMLSpanElement { 19 | const cursor = document.createElement("span"); 20 | cursor.style.position = "fixed"; 21 | cursor.style.left = "0"; 22 | cursor.style.top = "0"; 23 | cursor.style.backgroundColor = "red"; 24 | cursor.style.height = "25px"; 25 | cursor.style.width = "25px"; 26 | cursor.style.opacity = ".7"; 27 | cursor.style.borderRadius = "50%"; 28 | cursor.style.zIndex = "999"; 29 | return cursor; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "camera-browser-interfacee", 3 | "version": "0.0.1", 4 | "scripts": { 5 | "serve": "webpack-dev-server --progress --color", 6 | "build": "webpack -w", 7 | "build:prod": "cross-env NODE_ENV=production webpack" 8 | }, 9 | "devDependencies": { 10 | "@types/node": "9.6.5", 11 | "cross-env": "5.1.4", 12 | "html-webpack-plugin": "3.2.0", 13 | "source-map-loader": "0.2.3", 14 | "ts-loader": "4.2.0", 15 | "tslint": "5.9.1", 16 | "tslint-loader": "3.6.0", 17 | "typescript": "2.8.1", 18 | "uglifyjs-webpack-plugin": "1.2.4", 19 | "webpack": "4.5.0", 20 | "webpack-cli": "2.0.14", 21 | "webpack-dev-server": "3.1.3", 22 | "webpack-merge": "4.1.2" 23 | }, 24 | "dependencies": { 25 | "@tensorflow-models/facemesh": "0.0.1", 26 | "@tensorflow-models/handpose": "0.0.3", 27 | "@tensorflow/tfjs-node": "^1.7.3", 28 | "@tensorflow/tfjs-node-gpu": "^1.7.3", 29 | "prettier": "^2.0.5", 30 | "rxjs": "^6.5.5" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/libs/EstimateFace/index.ts: -------------------------------------------------------------------------------- 1 | import * as facemesh from "@tensorflow-models/facemesh"; 2 | import { FaceMesh } from "@tensorflow-models/facemesh"; 3 | 4 | import { FaceEstimatorEvent, TypedEventEmitter } from "./eventEmitter"; 5 | 6 | export class FaceEstimator { 7 | private readonly video: HTMLVideoElement; 8 | private model: FaceMesh; 9 | private readonly eventEmitter: TypedEventEmitter; 10 | 11 | private readonly updateTime: number = 1000; 12 | 13 | constructor(video: HTMLVideoElement, options?: { updateTime: number }) { 14 | this.video = video; 15 | if (options) { 16 | options.updateTime = this.updateTime = options.updateTime; 17 | } 18 | this.eventEmitter = new TypedEventEmitter(); 19 | } 20 | 21 | public async init() { 22 | this.model = await facemesh.load(); 23 | this.eventEmitter.emit(FaceEstimatorEvent.LOAD, null); 24 | setInterval(this.eventLoop, this.updateTime); 25 | } 26 | 27 | public getEventEmitter = () => this.eventEmitter; 28 | 29 | eventLoop = async () => { 30 | const faces = (await this.model.estimateFaces(this.video)) as any; 31 | 32 | const topMesh = faces[0].mesh[10]; 33 | const bottomMesh = faces[0].mesh[6]; 34 | const headPitch = bottomMesh[2] - topMesh[2]; 35 | 36 | this.eventEmitter.emit(FaceEstimatorEvent.UPDATE, { 37 | headPitch, 38 | }); 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /src/libs/EstimatteHand/index.ts: -------------------------------------------------------------------------------- 1 | import * as handtrack from "@tensorflow-models/handpose"; 2 | import { HandPose } from "@tensorflow-models/handpose"; 3 | 4 | import { HandEstimatorEvent, TypedEventEmitter } from "./eventEmitter"; 5 | import { understandGesture } from "./features/gestureUnderstanding"; 6 | 7 | import { V3 } from "../types/math"; 8 | 9 | export class HandEstimator { 10 | private readonly video: HTMLVideoElement; 11 | private model: HandPose; 12 | private readonly eventEmitter: TypedEventEmitter; 13 | 14 | private readonly updateTime: number = 1000; 15 | 16 | constructor(video: HTMLVideoElement, options?: { updateTime: number }) { 17 | this.video = video; 18 | if (options) { 19 | options.updateTime = this.updateTime = options.updateTime; 20 | } 21 | this.eventEmitter = new TypedEventEmitter(); 22 | } 23 | 24 | public async init() { 25 | this.model = await handtrack.load(); 26 | this.eventEmitter.emit(HandEstimatorEvent.LOAD, null); 27 | setInterval(this.eventLoop, this.updateTime); 28 | } 29 | 30 | public getEventEmitter = () => this.eventEmitter; 31 | 32 | eventLoop = async () => { 33 | const [hand] = (await this.model.estimateHands(this.video)) as any; 34 | if (!hand) return; 35 | 36 | this.eventEmitter.emit(HandEstimatorEvent.GESTURE_UPDATE, { 37 | gestureType: understandGesture(hand.annotations), 38 | }); 39 | 40 | const indexFinger = hand.annotations.indexFinger as V3[]; 41 | this.eventEmitter.emit(HandEstimatorEvent.UPDATE, { 42 | indexFingerPoint: indexFinger[3], 43 | }); 44 | }; 45 | } 46 | -------------------------------------------------------------------------------- /src/libs/EstimateFace/eventEmitter.ts: -------------------------------------------------------------------------------- 1 | import { Observable } from "rxjs"; 2 | import { EventEmitter } from "events"; 3 | 4 | import { ITypedEventEmitter } from "../TypedEventEmitter"; 5 | 6 | export enum FaceEstimatorEvent { 7 | LOAD = "load", 8 | UPDATE = "update", 9 | } 10 | 11 | export interface IUpdateInfo { 12 | headPitch: number; 13 | } 14 | 15 | export interface FaceEstimatorEventsMap { 16 | [FaceEstimatorEvent.LOAD]: any; 17 | [FaceEstimatorEvent.UPDATE]: IUpdateInfo; 18 | } 19 | 20 | export class TypedEventEmitter implements ITypedEventEmitter { 21 | private eventEmitter: EventEmitter; 22 | 23 | constructor() { 24 | this.eventEmitter = new EventEmitter(); 25 | this.eventEmitter.setMaxListeners(0); 26 | } 27 | 28 | emit = ( 29 | type: TYPE, 30 | payload: FaceEstimatorEventsMap[TYPE] 31 | ) => { 32 | this.eventEmitter.emit(type, payload); 33 | }; 34 | 35 | removeListener( 36 | event: TYPE, 37 | handler: (payload: FaceEstimatorEventsMap[TYPE]) => void 38 | ) { 39 | this.eventEmitter.removeListener(event, handler); 40 | } 41 | 42 | public on( 43 | type: TYPE, 44 | handler: (payload: FaceEstimatorEventsMap[TYPE]) => void 45 | ) { 46 | this.eventEmitter.addListener(type, handler); 47 | } 48 | 49 | public createObserverOn( 50 | type: TYPE 51 | ): Observable { 52 | return new Observable((observer) => 53 | this.on(type, (data) => observer.next(data)) 54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/libs/EstimatteHand/eventEmitter.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from "events"; 2 | import { Observable } from "rxjs"; 3 | 4 | import { ITypedEventEmitter } from "../TypedEventEmitter"; 5 | import { V3 } from "../types/math"; 6 | import { Gesture } from "./features/gestureUnderstanding"; 7 | 8 | export enum HandEstimatorEvent { 9 | LOAD = "load", 10 | UPDATE = "update", 11 | GESTURE_UPDATE = "gesture_update", 12 | } 13 | 14 | export interface HandEstimatorEventsMap { 15 | [HandEstimatorEvent.LOAD]: any; 16 | [HandEstimatorEvent.UPDATE]: { 17 | indexFingerPoint: V3; 18 | }; 19 | [HandEstimatorEvent.GESTURE_UPDATE]: { 20 | gestureType: Gesture; 21 | }; 22 | } 23 | 24 | export class TypedEventEmitter implements ITypedEventEmitter { 25 | private eventEmitter: EventEmitter; 26 | 27 | constructor() { 28 | this.eventEmitter = new EventEmitter(); 29 | this.eventEmitter.setMaxListeners(0); 30 | } 31 | 32 | emit = ( 33 | type: TYPE, 34 | payload: HandEstimatorEventsMap[TYPE] 35 | ) => { 36 | this.eventEmitter.emit(type, payload); 37 | }; 38 | 39 | removeListener( 40 | event: TYPE, 41 | handler: (payload: HandEstimatorEventsMap[TYPE]) => void 42 | ) { 43 | this.eventEmitter.removeListener(event, handler); 44 | } 45 | 46 | public on( 47 | type: TYPE, 48 | handler: (payload: HandEstimatorEventsMap[TYPE]) => void 49 | ) { 50 | this.eventEmitter.addListener(type, handler); 51 | } 52 | 53 | public createObserverOn( 54 | type: TYPE 55 | ): Observable { 56 | return new Observable((observer) => { 57 | this.on(type, (data) => observer.next(data)); 58 | }); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const webpack = require('webpack'); 3 | const HtmlWebpackPlugin = require('html-webpack-plugin'); 4 | const merge = require('webpack-merge'); 5 | const isProduction = process.env.NODE_ENV == 'production'; 6 | 7 | const modes = { 8 | [true]: 'production', 9 | [false]: 'development' 10 | }; 11 | 12 | const config = { 13 | context: __dirname, 14 | 15 | entry: './src/main.ts', 16 | 17 | output: { 18 | filename: '[name].[hash].js', 19 | chunkFilename: '[name].[chunkhash].chunk.js', 20 | pathinfo: true 21 | }, 22 | 23 | target: 'web', 24 | 25 | mode: modes[isProduction], 26 | 27 | resolve: { 28 | extensions: ['.js', '.ts'] 29 | }, 30 | 31 | module: { 32 | rules: [{ 33 | enforce: 'pre', 34 | test: /\.js$/, 35 | loader: 'source-map-loader' 36 | }, { 37 | enforce: 'pre', 38 | test: /\.ts$/, 39 | exclude: /node_modules/, 40 | loader: 'tslint-loader' 41 | }, { 42 | test: /\.ts$/, 43 | loader: 'ts-loader' 44 | }] 45 | }, 46 | 47 | plugins: [ 48 | new HtmlWebpackPlugin({ 49 | template: './src/index.html' 50 | }) 51 | ] 52 | }; 53 | 54 | if (isProduction) { 55 | const UglifyJsPlugin = require('uglifyjs-webpack-plugin'); 56 | 57 | module.exports = merge(config, { 58 | optimization: { 59 | minimize: true, 60 | 61 | minimizer: [ 62 | new UglifyJsPlugin({ 63 | parallel: require('os').cpus().length, 64 | 65 | uglifyOptions: { 66 | ie8: false, 67 | 68 | output: { 69 | ecma: 8, 70 | beautify: false, 71 | comments: false 72 | } 73 | } 74 | }) 75 | ] 76 | } 77 | }); 78 | } else { 79 | module.exports = merge(config, { 80 | devServer: { 81 | port: 4200, 82 | open: true, 83 | watchContentBase: true, 84 | historyApiFallback: true 85 | } 86 | }); 87 | } 88 | 89 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { filter, map } from "rxjs/operators"; 2 | 3 | import { FaceEstimator } from "./libs/EstimateFace"; 4 | import { FaceEstimatorEvent } from "./libs/EstimateFace/eventEmitter"; 5 | import { HandEstimator } from "./libs/EstimatteHand"; 6 | import { HandEstimatorEvent } from "./libs/EstimatteHand/eventEmitter"; 7 | import { HandCursor } from "./libs/EstimatteHand/features/handCursor"; 8 | 9 | import { interpolation2dPoints } from "./libs/math/extrapolation"; 10 | 11 | import { Position2D } from "./libs/types/math"; 12 | 13 | async function runFaceWebInterface(video: HTMLVideoElement) { 14 | const estimator = new FaceEstimator(video, { updateTime: 1000 }); 15 | await estimator.init(); 16 | 17 | estimator 18 | .getEventEmitter() 19 | .createObserverOn(FaceEstimatorEvent.UPDATE) 20 | .pipe( 21 | filter((payload) => payload.headPitch < -15 || payload.headPitch > 15) 22 | ) 23 | .subscribe((payload) => { 24 | document.body.scrollTo({ 25 | behavior: "smooth", 26 | top: document.body.scrollTop + payload.headPitch * 10, 27 | }); 28 | }); 29 | } 30 | 31 | async function runHandWebInterface(video: HTMLVideoElement) { 32 | const estimator = new HandEstimator(video, { updateTime: 100 }); 33 | await estimator.init(); 34 | 35 | const cursor = new HandCursor(); 36 | const interpolation = interpolation2dPoints(5); 37 | 38 | estimator 39 | .getEventEmitter() 40 | .createObserverOn(HandEstimatorEvent.UPDATE) 41 | .pipe( 42 | map((handEvent) => 43 | interpolation( 44 | handEvent.indexFingerPoint[0], 45 | handEvent.indexFingerPoint[1] 46 | ) 47 | ), 48 | filter(Boolean), 49 | map((point: Position2D) => ({ 50 | x: window.outerWidth - (window.outerWidth / video.videoWidth) * point.x, 51 | y: (window.outerHeight / video.videoHeight) * point.y, 52 | })) 53 | ) 54 | .subscribe((point: Position2D) => cursor.move(point)); 55 | 56 | const gestureContainer = document.querySelector("#gesture"); 57 | estimator 58 | .getEventEmitter() 59 | .createObserverOn(HandEstimatorEvent.GESTURE_UPDATE) 60 | .subscribe((gesture) => { 61 | gestureContainer.innerHTML = gesture.gestureType; 62 | }); 63 | } 64 | 65 | document.addEventListener("DOMContentLoaded", async () => { 66 | const video = document.querySelector("video"); 67 | if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { 68 | return; 69 | } 70 | video.srcObject = await navigator.mediaDevices.getUserMedia({ video: true }); 71 | await video.play(); 72 | 73 | runFaceWebInterface(video); 74 | runHandWebInterface(video); 75 | }); 76 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "jsRules": { 3 | "class-name": true, 4 | "comment-format": [ 5 | true, 6 | "check-space" 7 | ], 8 | "indent": [ 9 | true, 10 | "spaces" 11 | ], 12 | "no-duplicate-variable": true, 13 | "no-eval": true, 14 | "no-trailing-whitespace": true, 15 | "no-unsafe-finally": true, 16 | "one-line": [ 17 | true, 18 | "check-open-brace", 19 | "check-whitespace" 20 | ], 21 | "quotemark": [ 22 | true, 23 | "single" 24 | ], 25 | "semicolon": [ 26 | true, 27 | "always" 28 | ], 29 | "triple-equals": [ 30 | true, 31 | "allow-null-check" 32 | ], 33 | "variable-name": [ 34 | true, 35 | "ban-keywords" 36 | ], 37 | "whitespace": [ 38 | true, 39 | "check-branch", 40 | "check-decl", 41 | "check-operator", 42 | "check-separator", 43 | "check-type" 44 | ] 45 | }, 46 | "rules": { 47 | "class-name": true, 48 | "comment-format": [ 49 | true, 50 | "check-space" 51 | ], 52 | "indent": [ 53 | true, 54 | "spaces" 55 | ], 56 | "no-eval": true, 57 | "no-internal-module": true, 58 | "no-trailing-whitespace": true, 59 | "no-unsafe-finally": true, 60 | "no-var-keyword": false, 61 | "one-line": [ 62 | true, 63 | "check-open-brace", 64 | "check-whitespace" 65 | ], 66 | "quotemark": [ 67 | true, 68 | "single" 69 | ], 70 | "semicolon": [ 71 | true, 72 | "always" 73 | ], 74 | "triple-equals": [ 75 | true, 76 | "allow-null-check" 77 | ], 78 | "typedef-whitespace": [ 79 | true, 80 | { 81 | "call-signature": "nospace", 82 | "index-signature": "nospace", 83 | "parameter": "nospace", 84 | "property-declaration": "nospace", 85 | "variable-declaration": "nospace" 86 | } 87 | ], 88 | "variable-name": [ 89 | true, 90 | "ban-keywords" 91 | ], 92 | "whitespace": [ 93 | true, 94 | "check-branch", 95 | "check-decl", 96 | "check-operator", 97 | "check-separator", 98 | "check-type" 99 | ] 100 | } 101 | } -------------------------------------------------------------------------------- /src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 68 | 69 | 70 | 71 |

72 |

73 |

74 |

75 | 76 |

77 | 1 78 | blue 79 |

80 |

81 | 2 82 | blue 83 |

84 |

85 | 3 86 | blue 87 |

88 | 89 |

90 | 4 91 | blue 92 |

93 |