├── .gitignore ├── LICENSE ├── README.md ├── add.test.js ├── index.js ├── js ├── opencv.js ├── opencv_js.js └── opencv_js.wasm ├── lib ├── download.js ├── fr_age.js ├── fr_detect.js ├── fr_expression.js ├── fr_eye.js ├── fr_feature.js ├── fr_gender.js ├── fr_landmark.js ├── fr_liveness.js ├── fr_pose.js └── load_opencv.js ├── model ├── fr_age.onnx ├── fr_detect.onnx ├── fr_expression.onnx ├── fr_eye.onnx ├── fr_feature.onnx ├── fr_gender.onnx ├── fr_landmark.onnx ├── fr_liveness.onnx └── fr_pose.onnx ├── package-lock.json ├── package.json └── webpack.config.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | 84 | # Gatsby files 85 | .cache/ 86 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 87 | # https://nextjs.org/blog/next-9-1#public-directory-support 88 | # public 89 | 90 | # vuepress build output 91 | .vuepress/dist 92 | 93 | # Serverless directories 94 | .serverless/ 95 | 96 | # FuseBox cache 97 | .fusebox/ 98 | 99 | # DynamoDB Local files 100 | .dynamodb/ 101 | 102 | # TernJS port file 103 | .tern-port 104 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 gitguanqi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 | 5 | # Face Recognition SDK Javascript - Fully On Premise 6 | ## Overview 7 | Experience the epitome of speed and fairness with our `face recognition model` **Top-ranked on NIST FRVT**, coupled with an advanced **iBeta level 2 liveness detection** engine that effectively safeguards against **printed photos, video replay, 3D masks, and deepfake threats**, ensuring top-tier security. 8 |
This is `on-premise SDK` which means everything is processed on the browser and **NO** data leaves the device 9 |

10 | 11 | ## Installation 12 | 13 | ```bash 14 | npm install faceplugin 15 | ``` 16 | 17 | ## Table of Contents 18 | 19 | * **[Face Detection](#face-detection)** 20 | * **[Face Landmark Extraction](#face-landmark-extraction)** 21 | * **[Face Liveness Detection](#face-expression-detection)** 22 | * **[Face Expression Detection](#face-expression-detection)** 23 | * **[Face Pose Estimation](#face-pose-estimation)** 24 | * **[Eye Closeness Detection](#eye-closeness-detection)** 25 | * **[Gender Detection](#gender-detection)** 26 | * **[Age Detection](#age-detection)** 27 | * **[Face Feature Embedding](#face-recognition)** 28 | 29 | ## Examples 30 | 31 | https://github.com/kby-ai/FaceRecognition-Javascript/assets/125717930/551b6964-0fef-4483-85a7-76792c0f3b56 32 | 33 | * [Vue.js Demo](https://github.com/Faceplugin-ltd/FacePlugin-FaceRecognition-Vue) 34 | * [React.js Demo](https://github.com/Faceplugin-ltd/FacePlugin-FaceRecognition-React) 35 | 36 | 37 | Watch the video 38 | 39 | 40 | ## List of our Products 41 | 42 | * **[Face Recognition with Liveness Detection-Android (Java, Kotlin)](https://github.com/Faceplugin-ltd/FaceRecognition-Android)** 43 | * **[Face Recognition with Liveness Detection-iOS (Objective C, Swift)](https://github.com/Faceplugin-ltd/FaceRecognition-iOS)** 44 | * **[Face Recognition with Liveness Detection-React Native](https://github.com/Faceplugin-ltd/FaceRecognition-React-Native)** 45 | * **[Face Recognition with Liveness Detection-Flutter](https://github.com/Faceplugin-ltd/FaceRecognition-Flutter)** 46 | * **[Face Recognition with Liveness Detection-Ionic Cordova](https://github.com/Faceplugin-ltd/FaceRecognition-Ionic-Cordova)** 47 | * **[Face Recognition with Liveness Detection-.Net MAUI](https://github.com/Faceplugin-ltd/FaceRecognition-.Net)** 48 | * **[Face Recognition with Liveness Detection-.Net WPF](https://github.com/Faceplugin-ltd/FaceRecognition-WPF-.Net)** 49 | * **[Face Recognition with Liveness Detection-Javascript](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript)** 50 | * **[Face Recognition with LivenessDetection-React](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-React)** 51 | * **[Face Recognition with LivenessDetection-Vue](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Vue)** 52 | * **[Face Liveness Detection-Android (Java, Kotlin)](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Android)** 53 | * **[Face Liveness Detection-iOS (Objective C, Swift)](https://github.com/Faceplugin-ltd/FaceLivenessDetection-iOS)** 54 | * **[Face Liveness Detection-Linux](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Linux)** 55 | * **[Face Liveness Detection-Docker](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Docker)** 56 | * **[Open Source Face Recognition SDK](https://github.com/Faceplugin-ltd/Open-Source-Face-Recognition-SDK)** 57 | * **[Face Recognition SDK](https://github.com/Faceplugin-ltd/Face-Recognition-SDK)** 58 | * **[Liveness Detection SDK](https://github.com/Faceplugin-ltd/Face-Liveness-Detection-SDK)** 59 | * **[Palm Recognition SDK](https://github.com/Faceplugin-ltd/Palm-Recognition)** 60 | * **[ID Card Recognition](https://github.com/Faceplugin-ltd/ID-Card-Recognition)** 61 | * **[ID Document Liveness Detection](https://github.com/Faceplugin-ltd/ID-Document-Liveness-Detection)** 62 | 63 | ## Documentation 64 | 65 | Here are some useful documentation 66 | 67 | 68 | ### Face Detection 69 | Load detection model 70 | ``` 71 | loadDetectionModel() 72 | ``` 73 | Detect face in the image 74 | ``` 75 | detectFace(session, canvas_id) 76 | ``` 77 | 78 | 79 | ### Face Landmark Extraction 80 | Load landmark extraction model 81 | ``` 82 | loadLandmarkModel() 83 | ``` 84 | Extract face landmark in the image using detection result 85 | ``` 86 | predictLandmark(session, canvas_id, bbox) 87 | ``` 88 | 89 | 90 | ### Face Liveness Detection 91 | Load liveness detection model 92 | ``` 93 | loadLivenessModel() 94 | ``` 95 | Detect face liveness in the image using detection result. (Anti-spoofing) 96 | ``` 97 | predictLiveness(session, canvas_id, bbox) 98 | ``` 99 | 100 | 101 | ### Face Expression Detection 102 | Load expression detection model 103 | ``` 104 | loadExpressionModel() 105 | ``` 106 | Detect face expression 107 | ``` 108 | predictExpression(session, canvas_id, bbox) 109 | ``` 110 | 111 | 112 | ### Face Pose Estimation 113 | Load pose estimation model 114 | ``` 115 | loadPoseModel() 116 | ``` 117 | Predict facial pose 118 | ``` 119 | predictPose(session, canvas_id, bbox, question) 120 | ``` 121 | 122 | 123 | ### Eye Closeness Detection 124 | Load eye closeness model 125 | ``` 126 | loadEyeModel() 127 | ``` 128 | Predict eye closeness 129 | ``` 130 | predictEye(session, canvas_id, landmark) 131 | ``` 132 | 133 | 134 | ### Gender Detection 135 | Load gender detection model 136 | ``` 137 | loadGenderModel() 138 | ``` 139 | Predict gender using face image 140 | ``` 141 | predictGender(session, canvas_id, landmark) 142 | ``` 143 | 144 | 145 | ### Age Detection 146 | Load age detection model 147 | ``` 148 | loadAgeModel() 149 | ``` 150 | Predict age using face image 151 | ``` 152 | predictAge(session, canvas_id, landmark) 153 | ``` 154 | 155 | 156 | ### Face Recognition 157 | Load feature extraction model 158 | ``` 159 | loadFeatureModel() 160 | ``` 161 | Extract face feature vector in 512 dimension 162 | ``` 163 | extractFeature(session, canvas_id, landmarks) 164 | ``` 165 | 166 | ## Contact 167 | If you want to get better model, please contact us 168 | 169 |
170 | faceplugin.com  171 | faceplugin.com  172 | faceplugin.com 173 |
174 | 175 | 176 | -------------------------------------------------------------------------------- /add.test.js: -------------------------------------------------------------------------------- 1 | const add = require('./index.js').add; 2 | 3 | test('adds 1 + 2 to equal 3', () => { 4 | expect(add(1,2)).toBe(3); 5 | }) -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | export * from "./lib/fr_detect"; 2 | export * from "./lib/fr_expression"; 3 | export * from "./lib/fr_eye"; 4 | export * from "./lib/fr_landmark"; 5 | export * from "./lib/fr_liveness"; 6 | export * from "./lib/fr_pose"; 7 | export * from "./lib/fr_age"; 8 | export * from "./lib/fr_gender"; 9 | export * from "./lib/fr_feature"; 10 | export * from "./lib/load_opencv"; -------------------------------------------------------------------------------- /js/opencv_js.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/js/opencv_js.wasm -------------------------------------------------------------------------------- /lib/download.js: -------------------------------------------------------------------------------- 1 | export const download = (url, logger = null) => { 2 | return new Promise((resolve, reject) => { 3 | const request = new XMLHttpRequest(); 4 | request.open("GET", url, true); 5 | request.responseType = "arraybuffer"; 6 | if (logger) { 7 | const [log, setState] = logger; 8 | request.onprogress = (e) => { 9 | const progress = (e.loaded / e.total) * 100; 10 | setState({ text: log, progress: progress.toFixed(2) }); 11 | }; 12 | } 13 | request.onload = function () { 14 | if (this.status >= 200 && this.status < 300) { 15 | resolve(request.response); 16 | } else { 17 | reject({ 18 | status: this.status, 19 | statusText: request.statusText, 20 | }); 21 | } 22 | resolve(request.response); 23 | }; 24 | request.onerror = function () { 25 | reject({ 26 | status: this.status, 27 | statusText: request.statusText, 28 | }); 29 | }; 30 | request.send(); 31 | }); 32 | }; 33 | -------------------------------------------------------------------------------- /lib/fr_age.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | 5 | async function loadAgeModel() { 6 | var feature_session = null; 7 | await InferenceSession.create("../model/fr_age.onnx", {executionProviders: ['wasm']}) 8 | .then((session) => { 9 | feature_session = session 10 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]); 11 | for (let i = 0; i < 64 * 64 * 3; i++) { 12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 13 | } 14 | const feeds = {"input": input_tensor}; 15 | const output_tensor = feature_session.run(feeds) 16 | console.log("initialize the age session.") 17 | }) 18 | return feature_session; 19 | } 20 | 21 | function alignAgeImage(image, bbox, scale_value) { 22 | var src_h = image.rows, 23 | src_w = image.cols; 24 | 25 | var x = bbox[0] 26 | var y = bbox[1] 27 | var box_w = bbox[2] 28 | var box_h = bbox[3] 29 | 30 | var scale = Math.min((src_h - 1) / box_h, Math.min((src_w - 1) / box_w, scale_value)) 31 | 32 | var new_width = box_w * scale 33 | var new_height = box_h * scale 34 | var center_x = box_w / 2 + x, 35 | center_y = box_h / 2 + y 36 | 37 | var left_top_x = center_x - new_width / 2 38 | var left_top_y = center_y - new_height / 2 39 | var right_bottom_x = center_x + new_width / 2 40 | var right_bottom_y = center_y + new_height / 2 41 | 42 | if (left_top_x < 0) { 43 | right_bottom_x -= left_top_x 44 | left_top_x = 0 45 | } 46 | 47 | if (left_top_y < 0) { 48 | right_bottom_y -= left_top_y 49 | left_top_y = 0 50 | } 51 | 52 | if (right_bottom_x > src_w - 1) { 53 | left_top_x -= right_bottom_x - src_w + 1 54 | right_bottom_x = src_w - 1 55 | } 56 | 57 | if (right_bottom_y > src_h - 1) { 58 | left_top_y -= right_bottom_y - src_h + 1 59 | right_bottom_y = src_h - 1 60 | } 61 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0), 62 | Math.min(parseInt(right_bottom_x - left_top_x), src_w - 1), Math.min(parseInt(right_bottom_y - left_top_y), src_h - 1)) 63 | 64 | var face_image = new cv.Mat() 65 | face_image = image.roi(rect) 66 | 67 | var dsize = new cv.Size(64, 64); 68 | var resize_image = new cv.Mat(); 69 | cv.resize(face_image, resize_image, dsize); 70 | 71 | face_image.delete() 72 | return resize_image 73 | } 74 | 75 | function mergeAge(x, s1, s2, s3, lambda_local, lambda_d) { 76 | let a = 0; 77 | let b = 0; 78 | let c = 0; 79 | 80 | const V = 101; 81 | 82 | for (let i = 0; i < s1; i++) 83 | a = a + (i + lambda_local * x[12 + i]) * x[i]; 84 | // console.log("a = ", a) 85 | 86 | a = a / (s1 * (1 + lambda_d * x[9])); 87 | 88 | for (let i = 0; i < s2; i++) 89 | b = b + (i + lambda_local * x[15 + i]) * x[3 + i]; 90 | //console.log("b = ", b) 91 | 92 | b = b / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])); 93 | 94 | for (let i = 0; i < s3; i++) 95 | c = c + (i + lambda_local * x[18 + i]) * x[6 + i]; 96 | //console.log("c = ", c) 97 | 98 | c = c / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])) / (s3 * (1 + lambda_d * x[11])); 99 | return (a + b + c) * V; 100 | } 101 | 102 | function preprocessAge(img) { 103 | var cols = img.cols; 104 | var rows = img.rows; 105 | var channels = 3; 106 | 107 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 108 | 109 | for (var y = 0; y < rows; y++) 110 | for (var x = 0; x < cols; x++) { 111 | let pixel = img.ucharPtr(y, x); 112 | // if(x == 0 && y == 0) 113 | // console.log(pixel); 114 | for (var c = 0; c < channels; c++) { 115 | var pixel_value = 0 116 | if (c === 0) // R 117 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229 118 | if (c === 1) // G 119 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224 120 | if (c === 2) // B 121 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225 122 | 123 | img_data.set(y, x, c, pixel_value) 124 | } 125 | } 126 | 127 | var preprocesed = ndarray(new Float32Array(3 * 64 * 64), [1, 3, 64, 64]) 128 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 129 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 130 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 131 | 132 | return preprocesed 133 | } 134 | 135 | async function predictAge(session, canvas_id, bbox) { 136 | var img = cv.imread(canvas_id); 137 | 138 | var face_size = bbox.shape[0]; 139 | var bbox_size = bbox.shape[1]; 140 | 141 | const result = []; 142 | for (let i = 0; i < face_size; i++) { 143 | var x1 = parseInt(bbox.data[i * bbox_size]), 144 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 145 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 146 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 147 | width = Math.abs(x2 - x1), 148 | height = Math.abs(y2 - y1); 149 | 150 | var face_img = alignAgeImage(img, [x1, y1, width, height], 1.4); 151 | //cv.imshow("live-temp", face_img); 152 | var input_image = preprocessAge(face_img); 153 | face_img.delete(); 154 | 155 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]); 156 | input_tensor.data.set(input_image.data); 157 | const feeds = {"input": input_tensor}; 158 | 159 | const output_tensor = await session.run(feeds); 160 | const outputLayers = ["prob_stage_1", "prob_stage_2", "prob_stage_3", "stage1_delta_k", "stage2_delta_k", "stage3_delta_k", 161 | "index_offset_stage1", "index_offset_stage2", "index_offset_stage3"]; 162 | 163 | const outputFeat = []; 164 | for (let i = 0; i < outputLayers.length; i++) { 165 | const result = output_tensor[outputLayers[i]]; 166 | // console.log(outputLayers[i], ": ", result.size); 167 | for (let j = 0; j < result.size; j++) 168 | outputFeat.push(result.data[j]); 169 | } 170 | 171 | let age = mergeAge(outputFeat, 3, 3, 3, 1, 1); 172 | console.log("output age: ", age); 173 | result.push([x1, y1, x2, y2, age]); 174 | } 175 | 176 | img.delete(); 177 | return result; 178 | } 179 | 180 | export {loadAgeModel, predictAge} -------------------------------------------------------------------------------- /lib/fr_detect.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | import {cv} from "./load_opencv"; 5 | import {download} from "./download"; 6 | 7 | async function loadDetectionModel() { 8 | var detect_session = null; 9 | await InferenceSession.create("../model/fr_detect.onnx", {executionProviders: ['wasm']}) 10 | .then((session) => { 11 | detect_session = session; 12 | const input_tensor = new Tensor("float32", new Float32Array(320 * 240 * 3), [1, 3, 240, 320]); 13 | for (let i = 0; i < 320 * 240 * 3; i++) { 14 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 15 | } 16 | const feeds = {"input": input_tensor}; 17 | const output_tensor = detect_session.run(feeds) 18 | console.log("initialize the detection session.") 19 | }) 20 | return detect_session 21 | } 22 | 23 | async function loadDetectionModelPath(model_path) { 24 | const arr_buf = await download(model_path); 25 | const detection_session = await InferenceSession.create(arr_buf); 26 | return detection_session 27 | } 28 | 29 | function preprocessDetection(image) { 30 | var rows = image.rows, 31 | cols = image.cols; 32 | 33 | var img_data = ndarray(new Float32Array(rows * cols * 3), [rows, cols, 3]); 34 | 35 | for (var y = 0; y < rows; y++) 36 | for (var x = 0; x < cols; x++) { 37 | let pixel = image.ucharPtr(y, x); 38 | for (var c = 0; c < 3; c++) { 39 | var pixel_value = 0 40 | if (c === 0) // R 41 | pixel_value = (pixel[c] - 127) / 128.0; 42 | if (c === 1) // G 43 | pixel_value = (pixel[c] - 127) / 128.0; 44 | if (c === 2) // B 45 | pixel_value = (pixel[c] - 127) / 128.0; 46 | 47 | img_data.set(y, x, c, pixel_value) 48 | } 49 | } 50 | 51 | var preprocesed = ndarray(new Float32Array(3 * rows * cols), [1, 3, rows, cols]) 52 | 53 | // Transpose 54 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 55 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 56 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 57 | 58 | return preprocesed 59 | } 60 | 61 | async function detectFaceImage(session, img) { 62 | const onnx_config = { 63 | min_sizes: [[10, 16, 24], [32, 48], [64, 96], [128, 192, 256]], 64 | steps: [8, 16, 32, 64], 65 | variance: [0.1, 0.2], 66 | clip: false, 67 | confidence_threshold: 0.65, 68 | top_k: 750, 69 | nms_threshold: 0.4, 70 | }; 71 | 72 | var dsize = new cv.Size(320, 240); 73 | var resize_image = new cv.Mat(); 74 | cv.resize(img, resize_image, dsize); 75 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2RGB); 76 | 77 | const image = preprocessDetection(resize_image); 78 | 79 | var resize_param = {cols: img.cols / 320, rows: img.rows / 240}; 80 | 81 | const input_tensor = new Tensor("float32", new Float32Array(320 * 240 * 3), [1, 3, 240, 320]); 82 | input_tensor.data.set(image.data); 83 | 84 | const feeds = {"input": input_tensor}; 85 | const output_tensor = await session.run(feeds); 86 | 87 | const loc = output_tensor['boxes']; 88 | const conf = output_tensor['scores']; 89 | 90 | // const landmarks = output_tensor['585'] 91 | const total_result = conf.size / 2; 92 | 93 | const scale = [320, 240, 320, 240]; 94 | const scale1 = [800, 800, 800, 800, 800, 800, 800, 800, 800, 800]; 95 | 96 | const priors = definePriorBox([320, 240], onnx_config); 97 | 98 | const boxes_arr = decodeBBox(loc, priors, onnx_config); 99 | 100 | const scores_arr = ndarray(conf.data, [total_result, 2]).pick(null, 1); 101 | var landms_arr = null;//decode_landmark(landmarks, priors, onnx_config.variance); 102 | 103 | var box = ndarray(loc.data, [4420, 4]); 104 | var boxes_before = scaleMultiplyBBox(box, scale); 105 | var landms_before = null;//scale_multiply_landms(landms_arr, scale1); 106 | 107 | var [bbox_screen, scores_screen, landms_screen] = screenScore(boxes_before, scores_arr, landms_before, onnx_config.confidence_threshold); 108 | 109 | var [bbox_sorted, scores_sorted, landms_sorted] = sortScore(bbox_screen, scores_screen, landms_screen, onnx_config.top_k); 110 | 111 | var [bbox_small, score_result, landms_small, result_size] = cpuNMS(bbox_sorted, scores_sorted, landms_sorted, onnx_config.nms_threshold); 112 | var [bbox_result, landms_result] = scaleResult(bbox_small, landms_small, resize_param, img.cols, img.rows); 113 | 114 | var output = { 115 | bbox: bbox_result, 116 | landmark: landms_result, 117 | conf: score_result, 118 | size: result_size 119 | } 120 | 121 | resize_image.delete(); 122 | img.delete(); 123 | return output 124 | } 125 | 126 | async function detectFace(session, canvas_id) { 127 | 128 | var img = cv.imread(canvas_id); 129 | var output = await detectFaceImage(session, img); 130 | 131 | return output 132 | } 133 | 134 | async function detectFaceBase64(session, base64Image) { 135 | let image = new Image() 136 | image.src = base64Image 137 | await new Promise(r => { 138 | image.onload = r 139 | }) 140 | 141 | var img = cv.imread(image); 142 | var output = await detectFaceImage(session, img); 143 | 144 | return output 145 | } 146 | 147 | function product(x, y) { 148 | var size_x = x.length, 149 | size_y = y.length; 150 | var result = []; 151 | 152 | for (var i = 0; i < size_x; i++) 153 | for (var j = 0; j < size_y; j++) 154 | result.push([x[i], y[j]]); 155 | 156 | return result; 157 | } 158 | 159 | function range(num) { 160 | var result = []; 161 | for (var i = 0; i < num; i++) 162 | result.push(i); 163 | 164 | return result; 165 | } 166 | 167 | function definePriorBox(image_size, onnx_config) { 168 | var min_sizes = onnx_config.min_sizes, 169 | steps = onnx_config.steps, 170 | clip = onnx_config.clip, 171 | name = "s", 172 | feature_maps = steps.map((step) => [Math.ceil(image_size[0] / step), Math.ceil(image_size[1] / step)]); 173 | 174 | var anchors = []; 175 | 176 | feature_maps.forEach((f, k) => { 177 | var min_size = min_sizes[k]; 178 | product(range(f[0]), range(f[1])).forEach(([i, j]) => { 179 | min_size.forEach((m_size) => { 180 | var s_kx = m_size / image_size[0], 181 | s_ky = m_size / image_size[1]; 182 | var dense_cx = [j + 0.5].map((x) => x * steps[k] / image_size[0]), 183 | dense_cy = [i + 0.5].map((y) => y * steps[k] / image_size[1]); 184 | product(dense_cy, dense_cx).forEach(([cy, cx]) => { 185 | anchors.push(cx); 186 | anchors.push(cy); 187 | anchors.push(s_kx); 188 | anchors.push(s_ky); 189 | }) 190 | }); 191 | }); 192 | }); 193 | 194 | var output = ndarray(new Float32Array(anchors), [anchors.length / 4, 4]); 195 | 196 | if (clip) 197 | output = ndarray.ops.min(1, ops.max(output, 0)); 198 | 199 | return output; 200 | } 201 | 202 | function decodeBBox(bbox, priors, onnx_config) { 203 | var variances = onnx_config.variance 204 | var loc = ndarray(bbox.data, [4420, 4]); 205 | // console.log(bbox, priors); 206 | var before_prior = priors.hi(null, 2), 207 | after_prior = priors.lo(null, 2); 208 | 209 | var before_loc = loc.hi(null, 2), 210 | after_loc = loc.lo(null, 2); 211 | 212 | var before_result = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 213 | var before_temp = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 214 | var before_temp2 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 215 | 216 | var after_result = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 217 | var after_temp = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 218 | var after_temp2 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 219 | var after_temp3 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 220 | var after_temp4 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]); 221 | 222 | var boxes = ndarray(new Float32Array(before_loc.shape[0] * 4), [before_loc.shape[0], 4]); 223 | 224 | // Before 225 | ops.mul(before_temp, before_loc, after_prior); 226 | ops.muls(before_temp2, before_temp, variances[0]); 227 | ops.add(before_result, before_temp2, before_prior); 228 | 229 | // After 230 | ops.muls(after_temp, after_loc, variances[1]); 231 | ops.exp(after_temp2, after_temp); 232 | ops.mul(after_temp3, after_temp2, after_prior); 233 | 234 | for (var index = 0; index < 4; index++) 235 | ops.assign(after_result.pick(null, index), after_temp3.pick(null, index)); 236 | 237 | ops.divs(after_temp4, after_temp3, -2); 238 | ops.addeq(before_result, after_temp4); 239 | 240 | ops.addeq(after_result, before_result); 241 | 242 | ops.assign(boxes.pick(null, 0), before_result.pick(null, 0)); 243 | ops.assign(boxes.pick(null, 1), before_result.pick(null, 1)); 244 | ops.assign(boxes.pick(null, 2), after_result.pick(null, 0)); 245 | ops.assign(boxes.pick(null, 3), after_result.pick(null, 1)); 246 | 247 | return boxes; 248 | } 249 | 250 | function scaleMultiplyBBox(boxes_arr, scale) { 251 | var total_result = boxes_arr.shape[0]; 252 | var boxes_before = ndarray(new Float32Array(total_result * 4), [total_result, 4]); 253 | 254 | for (var index = 0; index < scale.length; index++) { 255 | let temp = boxes_arr.pick(null, index), 256 | before_result = ndarray(new Float32Array(total_result), [total_result]); 257 | ops.muls(before_result, temp, scale[index]); 258 | ops.assign(boxes_before.pick(null, index), before_result); 259 | } 260 | 261 | return boxes_before; 262 | } 263 | 264 | function scaleMultiplyLandmarks(landms_arr, scale1) { 265 | var total_result = landms_arr.shape[0]; 266 | var landms_before = ndarray(new Float32Array(total_result * 10), [total_result, 10]); 267 | 268 | for (var index = 0; index < scale1.length; index++) { 269 | let temp = landms_arr.pick(null, index), 270 | before_landms_result = ndarray(new Float32Array(total_result), [total_result]); 271 | ops.muls(before_landms_result, temp, scale1[index]); 272 | ops.assign(landms_before.pick(null, index), before_landms_result); 273 | } 274 | 275 | return landms_before; 276 | } 277 | 278 | function screenScore(bbox, scores, landms, threshold) { 279 | var total_size = scores.shape[0]; 280 | var index_arr = []; 281 | 282 | for (var index = 0; index < total_size; index++) { 283 | var score_temp = scores.get(index); 284 | 285 | if (score_temp >= threshold) { 286 | index_arr.push(index); 287 | } 288 | } 289 | 290 | var result_bbox = ndarray(new Float32Array(index_arr.length * 4), [index_arr.length, 4]); 291 | var result_scores = ndarray(new Float32Array(index_arr.length), [index_arr.length]); 292 | var result_landms = null;//ndarray(new Float32Array(index_arr.length * 10), [index_arr.length, 10]); 293 | 294 | index_arr.forEach((index, i) => { 295 | ops.assign(result_bbox.pick(i, null), bbox.pick(index, null)); 296 | //ops.assign(result_landms.pick(i, null), landms.pick(index, null)); 297 | ops.assign(result_scores.pick(i), scores.pick(index)); 298 | }); 299 | 300 | return [result_bbox, result_scores, result_landms]; 301 | } 302 | 303 | function sortScore(bbox, scores, landms, top_k) { 304 | var total_size = scores.shape[0]; 305 | var index_sort = new Array(total_size * 2); 306 | 307 | for (var index = 0; index < total_size; index++) { 308 | var temp = scores.get(index); 309 | index_sort[index] = [index, temp]; 310 | } 311 | 312 | index_sort.sort((a, b) => { 313 | if (a[1] < b[1]) return 1; 314 | if (a[1] > b[1]) return -1; 315 | 316 | return 0; 317 | }); 318 | 319 | var max_size = (total_size > top_k) ? top_k : total_size; 320 | 321 | var result_bbox = ndarray(new Float32Array(max_size * 4), [max_size, 4]); 322 | var result_scores = ndarray(new Float32Array(max_size), [max_size]); 323 | var result_landms = null;//ndarray(new Float32Array(max_size * 10), [max_size, 10]); 324 | 325 | for (var idx = 0; idx < max_size; idx++) { 326 | result_scores.set(idx, index_sort[idx][1]); 327 | ops.assign(result_bbox.pick(idx, null), bbox.pick(index_sort[idx][0], null)); 328 | //ops.assign(result_landms.pick(idx, null), landms.pick(index_sort[idx][0], null)); 329 | } 330 | 331 | return [result_bbox, result_scores, result_landms]; 332 | } 333 | 334 | function cpuNMS(bbox, scores, landms, thresh) { 335 | var {max, min} = Math; 336 | var size = bbox.shape[0]; 337 | var foundLocations = []; 338 | var pick = []; 339 | 340 | for (var i = 0; i < size; i++) { 341 | var x1 = bbox.get(i, 0), 342 | y1 = bbox.get(i, 1), 343 | x2 = bbox.get(i, 2), 344 | y2 = bbox.get(i, 3); 345 | 346 | var width = x2 - x1, 347 | height = y2 - y1; 348 | 349 | if (width > 0 && height > 0) { 350 | var area = width * height; 351 | foundLocations.push({x1, y1, x2, y2, width, height, area, index: i}); 352 | } 353 | } 354 | 355 | foundLocations.sort((b1, b2) => { 356 | return b1.y2 - b2.y2; 357 | }); 358 | 359 | while (foundLocations.length > 0) { 360 | var last = foundLocations[0] //[foundLocations.length - 1]; 361 | var suppress = [last]; 362 | pick.push(last.index) //foundLocations.length - 1); 363 | 364 | for (let i = 1; i < foundLocations.length; i++) { 365 | const box = foundLocations[i]; 366 | const xx1 = max(box.x1, last.x1); 367 | const yy1 = max(box.y1, last.y1); 368 | const xx2 = min(box.x2, last.x2); 369 | const yy2 = min(box.y2, last.y2); 370 | const w = max(0, xx2 - xx1 + 1); 371 | const h = max(0, yy2 - yy1 + 1); 372 | const overlap = (w * h) / box.area; 373 | 374 | if (overlap >= thresh) 375 | suppress.push(foundLocations[i]); 376 | } 377 | 378 | foundLocations = foundLocations.filter((box) => { 379 | return !suppress.find((supp) => { 380 | return supp === box; 381 | }) 382 | }); 383 | } 384 | 385 | var result_bbox = ndarray(new Float32Array(pick.length * 4), [pick.length, 4]); 386 | var result_scores = ndarray(new Float32Array(pick.length), [pick.length]); 387 | var result_landms = null;//ndarray(new Float32Array(pick.length * 10), [pick.length, 10]); 388 | 389 | // console.log("Pick index: ", pick); 390 | 391 | pick.forEach((pick_index, i) => { 392 | ops.assign(result_bbox.pick(i, null), bbox.pick(pick_index, null)); 393 | ops.assign(result_scores.pick(i), scores.pick(pick_index)); 394 | //ops.assign(result_landms.pick(i, null), landms.pick(pick_index, null)); 395 | }); 396 | 397 | return [result_bbox, result_scores, result_landms, pick.length]; 398 | } 399 | 400 | function scaleResult(bbox, landmark, resize_param, width, height) { 401 | var size = bbox.shape[0]; 402 | var result_bbox = ndarray(new Float32Array(size * 4), [size, 4]); 403 | var result_landms = null;//ndarray(new Float32Array(size * 10), [size, 10]); 404 | 405 | for (let i = 0; i < size; i++) { 406 | let x1 = bbox.get(i, 0) * resize_param.cols, 407 | y1 = bbox.get(i, 1) * resize_param.rows, 408 | x2 = bbox.get(i, 2) * resize_param.cols, 409 | y2 = bbox.get(i, 3) * resize_param.rows; 410 | 411 | const f_size = (y2 - y1); 412 | const ct_x = (x1 + x2) / 2, 413 | ct_y = (y1 + y2) / 2; 414 | 415 | x1 = (ct_x - f_size / 2) < 0 ? 0 : (ct_x - f_size / 2); 416 | y1 = (ct_y - f_size / 2) < 0 ? 0 : (ct_y - f_size / 2); 417 | x2 = (ct_x + f_size / 2) > width - 1 ? width - 1 : (ct_x + f_size / 2); 418 | y2 = (ct_y + f_size / 2) > height - 1 ? height - 1 : (ct_y + f_size / 2); 419 | 420 | result_bbox.set(i, 0, x1); 421 | result_bbox.set(i, 1, y1); 422 | result_bbox.set(i, 2, x2); 423 | result_bbox.set(i, 3, y2); 424 | } 425 | 426 | /* 427 | for (var j = 0; j < 5; j++) { 428 | let x = landmark.pick(null, j * 2), 429 | y = landmark.pick(null, j * 2 + 1); 430 | 431 | ops.divseq(x, resize_param.cols); // X 432 | ops.divseq(y, resize_param.rows); // Y 433 | 434 | ops.assign(result_landms.pick(null, j * 2), x); 435 | ops.assign(result_landms.pick(null, j * 2 + 1), y); 436 | } 437 | */ 438 | return [result_bbox, result_landms]; 439 | } 440 | 441 | function getBestFace(bbox) { 442 | var face_count = bbox.shape[0], 443 | bbox_size = bbox.shape[1]; 444 | 445 | var idx = -1, max_size = 0; 446 | for (let i = 0; i < face_count; i++) { 447 | var x1 = parseInt(bbox.data[i * bbox_size]), 448 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 449 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 450 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 451 | width = Math.abs(x2 - x1), 452 | height = Math.abs(y2 - y1); 453 | if (width * height > max_size) 454 | idx = i; 455 | } 456 | return idx 457 | } 458 | 459 | export {loadDetectionModel, loadDetectionModelPath, detectFace, detectFaceBase64} -------------------------------------------------------------------------------- /lib/fr_expression.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | import {softmax} from "./fr_pose"; 5 | 6 | async function loadExpressionModel() { 7 | var expression_session = null 8 | await InferenceSession.create("../model/fr_expression.onnx", {executionProviders: ['wasm']}) 9 | .then((session) => { 10 | expression_session = session 11 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]); 12 | for (let i = 0; i < 224 * 224 * 3; i++) { 13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 14 | } 15 | const feeds = {"input": input_tensor}; 16 | const output_tensor = expression_session.run(feeds) 17 | console.log("initialize the expression session.") 18 | }) 19 | return expression_session 20 | } 21 | 22 | function alignExpressionImage(image, bbox) { 23 | var src_h = image.rows, 24 | src_w = image.cols; 25 | 26 | var x = bbox[0] 27 | var y = bbox[1] 28 | var box_w = bbox[2] 29 | var box_h = bbox[3] 30 | 31 | var rect = new cv.Rect(x, y, Math.min(parseInt(box_w * 1.2), src_w - 1), Math.min(parseInt(box_h * 1.2), src_h - 1)) 32 | 33 | var face_image = new cv.Mat() 34 | face_image = image.roi(rect) 35 | 36 | var dsize = new cv.Size(224, 224); 37 | var resize_image = new cv.Mat(); 38 | cv.resize(face_image, resize_image, dsize); 39 | 40 | face_image.delete() 41 | return resize_image 42 | } 43 | 44 | function preprocessExpression(img) { 45 | var cols = img.cols; 46 | var rows = img.rows; 47 | var channels = 3; 48 | 49 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 50 | 51 | for (var y = 0; y < rows; y++) 52 | for (var x = 0; x < cols; x++) { 53 | let pixel = img.ucharPtr(y, x); 54 | for (var c = 0; c < channels; c++) { 55 | var pixel_value = pixel[c] / 255.0; 56 | img_data.set(y, x, c, pixel_value) 57 | } 58 | } 59 | 60 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols]) 61 | 62 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 63 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 64 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 65 | 66 | return preprocesed 67 | } 68 | 69 | async function predictExpression(session, canvas_id, bbox) { 70 | var img = cv.imread(canvas_id) 71 | 72 | var face_count = bbox.shape[0], 73 | bbox_size = bbox.shape[1]; 74 | 75 | const result = []; 76 | for (let i = 0; i < face_count; i++) { 77 | var x1 = parseInt(bbox.data[i * bbox_size]), 78 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 79 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 80 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 81 | width = Math.abs(x2 - x1), 82 | height = Math.abs(y2 - y1); 83 | 84 | var face_img = alignExpressionImage(img, [x1, y1, width, height]); 85 | //cv.imshow("live-temp", face_img); 86 | var input_image = preprocessExpression(face_img); 87 | face_img.delete(); 88 | 89 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]); 90 | input_tensor.data.set(input_image.data); 91 | const feeds = {"input": input_tensor}; 92 | 93 | const output_tensor = await session.run(feeds); 94 | const expression_arr = softmax(output_tensor['output'].data); 95 | 96 | var max_idx = null, max_val = 0; 97 | for (let i = 0; i < expression_arr.length; i++) 98 | if (max_val < expression_arr[i]) { 99 | max_idx = i; 100 | max_val = expression_arr[i]; 101 | } 102 | result.push([x1, y1, x2, y2, max_idx]); 103 | } 104 | img.delete(); 105 | return result; 106 | } 107 | 108 | export {loadExpressionModel, predictExpression} -------------------------------------------------------------------------------- /lib/fr_eye.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | import {softmax} from "./fr_pose"; 5 | 6 | async function loadEyeModel() { 7 | var eye_session = null 8 | await InferenceSession.create("../model/fr_eye.onnx", {executionProviders: ['wasm']}) 9 | .then((session) => { 10 | eye_session = session 11 | const input_tensor = new Tensor("float32", new Float32Array(24 * 24 * 1), [1, 1, 24, 24]); 12 | for (let i = 0; i < 24 * 24; i++) { 13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 14 | } 15 | const feeds = {"input": input_tensor}; 16 | const output_tensor = eye_session.run(feeds) 17 | console.log("initialize the eye session.") 18 | }) 19 | return eye_session 20 | } 21 | 22 | function getEyeBBox(landmark, size) { 23 | var height = size[0], width = size[1]; 24 | const padding_rate = 1.6; 25 | var left_eye_center_x = parseInt((landmark[74] + landmark[76] + landmark[80] + landmark[82]) / 4); 26 | var left_eye_center_y = parseInt((landmark[75] + landmark[77] + landmark[81] + landmark[83]) / 4); 27 | var left_eye_size = parseInt((landmark[78] - landmark[72]) * padding_rate); 28 | var left_corner_x = parseInt(left_eye_center_x - left_eye_size / 2); 29 | if (left_corner_x < 0) 30 | left_corner_x = 0; 31 | 32 | var left_corner_y = parseInt(left_eye_center_y - left_eye_size / 2); 33 | if (left_corner_y < 0) 34 | left_corner_y = 0; 35 | 36 | if (left_corner_x + left_eye_size >= width) 37 | left_eye_size = width - left_corner_x - 1 38 | 39 | if (left_corner_y + left_eye_size >= height) 40 | left_eye_size = height - left_corner_y - 1 41 | 42 | var right_eye_center_x = parseInt((landmark[86] + landmark[88] + landmark[92] + landmark[94]) / 4); 43 | var right_eye_center_y = parseInt((landmark[87] + landmark[89] + landmark[93] + landmark[95]) / 4); 44 | var right_eye_size = parseInt((landmark[90] - landmark[84]) * padding_rate); 45 | var right_corner_x = parseInt(right_eye_center_x - right_eye_size / 2); 46 | if (right_corner_x < 0) 47 | right_corner_x = 0 48 | var right_corner_y = parseInt(right_eye_center_y - right_eye_size / 2); 49 | if (right_corner_y < 0) 50 | right_corner_y = 0 51 | if (right_corner_x + right_eye_size >= width) 52 | right_eye_size = width - right_corner_x - 1 53 | if (right_corner_y + right_eye_size >= height) 54 | right_eye_size = height - right_corner_y - 1 55 | 56 | return [left_corner_x, left_corner_y, left_eye_size, left_eye_size, 57 | right_corner_x, right_corner_y, right_eye_size, right_eye_size] 58 | } 59 | 60 | function alignEyeImage(image, landmark) { 61 | var src_h = image.rows, 62 | src_w = image.cols; 63 | 64 | var eye_bbox = getEyeBBox(landmark, [src_h, src_w]) 65 | var rect = new cv.Rect(eye_bbox[0], eye_bbox[1], eye_bbox[2], eye_bbox[3]) 66 | 67 | var eye_image = new cv.Mat() 68 | eye_image = image.roi(rect) 69 | 70 | var dsize = new cv.Size(24, 24); 71 | var left_eye = new cv.Mat(); 72 | cv.resize(eye_image, left_eye, dsize); 73 | cv.cvtColor(left_eye, left_eye, cv.COLOR_BGR2GRAY) 74 | 75 | // right eye 76 | rect = new cv.Rect(eye_bbox[4], eye_bbox[5], eye_bbox[6], eye_bbox[7]) 77 | eye_image = image.roi(rect) 78 | var right_eye = new cv.Mat(); 79 | cv.resize(eye_image, right_eye, dsize); 80 | cv.cvtColor(right_eye, right_eye, cv.COLOR_BGR2GRAY) 81 | 82 | eye_image.delete() 83 | return [left_eye, right_eye] 84 | } 85 | 86 | function preprocessEye(imgs) { 87 | var cols = imgs[0].cols; 88 | var rows = imgs[0].rows; 89 | var channels = 1; 90 | 91 | var img_data1 = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 92 | var img_data2 = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 93 | 94 | for (var y = 0; y < rows; y++) 95 | for (var x = 0; x < cols; x++) { 96 | let pixel1 = imgs[0].ucharPtr(y, x); 97 | let pixel2 = imgs[1].ucharPtr(y, x); 98 | 99 | for (var c = 0; c < channels; c++) { 100 | var pixel_value1 = pixel1[c] / 255.0; 101 | var pixel_value2 = pixel2[c] / 255.0; 102 | 103 | img_data1.set(y, x, c, pixel_value1) 104 | img_data2.set(y, x, c, pixel_value2) 105 | } 106 | } 107 | 108 | var preprocesed1 = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols]) 109 | ops.assign(preprocesed1.pick(0, 0, null, null), img_data1.pick(null, null, 0)); 110 | var preprocesed2 = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols]) 111 | ops.assign(preprocesed2.pick(0, 0, null, null), img_data2.pick(null, null, 0)); 112 | 113 | return [preprocesed1, preprocesed2] 114 | } 115 | 116 | async function predictEye(session, canvas_id, landmarks) { 117 | var img = cv.imread(canvas_id) 118 | 119 | const result = []; 120 | for (let i = 0; i < landmarks.length; i++) { 121 | var face_imgs = alignEyeImage(img, landmarks[i]); 122 | // cv.imshow("live-temp", face_imgs[1]); 123 | var input_images = preprocessEye(face_imgs); 124 | face_imgs[0].delete(); 125 | face_imgs[1].delete(); 126 | 127 | const input_tensor1 = new Tensor("float32", new Float32Array(24 * 24), [1, 1, 24, 24]); 128 | input_tensor1.data.set(input_images[0].data); 129 | const feeds1 = {"input": input_tensor1}; 130 | 131 | const output_tensor1 = await session.run(feeds1); 132 | const left_res = softmax(output_tensor1['output'].data); 133 | 134 | const input_tensor2 = new Tensor("float32", new Float32Array(24 * 24), [1, 1, 24, 24]); 135 | input_tensor2.data.set(input_images[1].data); 136 | const feeds2 = {"input": input_tensor2}; 137 | 138 | const output_tensor2 = await session.run(feeds2); 139 | const right_res = softmax(output_tensor2['output'].data); 140 | result.push([left_res[0] > left_res[1], right_res[0] > right_res[1]]); 141 | } 142 | img.delete(); 143 | return result; 144 | } 145 | 146 | export {loadEyeModel, predictEye} -------------------------------------------------------------------------------- /lib/fr_feature.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | 5 | const REFERENCE_FACIAL_POINTS = [ 6 | [38.29459953, 51.69630051], 7 | [73.53179932, 51.50139999], 8 | [56.02519989, 71.73660278], 9 | [41.54930115, 92.3655014], 10 | [70.72990036, 92.20410156] 11 | ] 12 | 13 | async function loadFeatureModel() { 14 | var feature_session = null; 15 | await InferenceSession.create("../model/fr_feature.onnx", {executionProviders: ['wasm']}) 16 | .then((session) => { 17 | feature_session = session 18 | const input_tensor = new Tensor("float32", new Float32Array(112 * 112 * 3), [1, 3, 112, 112]); 19 | for (let i = 0; i < 112 * 112 * 3; i++) { 20 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 21 | } 22 | const feeds = {"input": input_tensor}; 23 | const output_tensor = feature_session.run(feeds) 24 | console.log("initialize the feature session.") 25 | }) 26 | return feature_session; 27 | } 28 | 29 | function convert68pts5pts(landmark) { 30 | var left_eye_x = (landmark[74] + landmark[76] + landmark[80] + landmark[82]) / 4, 31 | left_eye_y = (landmark[75] + landmark[77] + landmark[81] + landmark[83]) / 4, 32 | 33 | right_eye_x = (landmark[86] + landmark[88] + landmark[92] + landmark[94]) / 4, 34 | right_eye_y = (landmark[87] + landmark[89] + landmark[93] + landmark[95]) / 4, 35 | 36 | nose_x = landmark[60], nose_y = landmark[61], 37 | 38 | left_mouse_x = (landmark[96] + landmark[120]) / 2, 39 | left_mouse_y = (landmark[97] + landmark[121]) / 2, 40 | 41 | right_mouse_x = (landmark[108] + landmark[128]) / 2, 42 | right_mouse_y = (landmark[109] + landmark[129]) / 2; 43 | return [[left_eye_x, left_eye_y], [right_eye_x, right_eye_y], [nose_x, nose_y], [left_mouse_x, left_mouse_y], 44 | [right_mouse_x, right_mouse_y]] 45 | } 46 | 47 | function getReferenceFacialPoints() { 48 | let ref5pts = REFERENCE_FACIAL_POINTS; 49 | 50 | return ref5pts; 51 | } 52 | 53 | function warpAndCropFace(src, 54 | face_pts, 55 | ref_pts=null, 56 | crop_size=[112, 112]) { 57 | 58 | let srcTri = cv.matFromArray(3, 1, cv.CV_32FC2, [face_pts[0][0], face_pts[0][1], face_pts[1][0], face_pts[1][1], 59 | face_pts[2][0], face_pts[2][1]]); 60 | let dstTri = cv.matFromArray(3, 1, cv.CV_32FC2, [ref_pts[0][0], ref_pts[0][1], ref_pts[1][0], ref_pts[1][1], 61 | ref_pts[2][0], ref_pts[2][1]]); 62 | 63 | let tfm = cv.getAffineTransform(srcTri, dstTri); 64 | 65 | let dsize = new cv.Size(crop_size[0], crop_size[1]); 66 | let dst = new cv.Mat(); 67 | cv.warpAffine(src, dst, tfm, dsize); 68 | 69 | return dst; 70 | } 71 | 72 | function alignFeatureImage(image, landmark) { 73 | let facePoints = convert68pts5pts(landmark); 74 | let refPoints = getReferenceFacialPoints(); 75 | let alignImg = warpAndCropFace(image, facePoints, refPoints); 76 | return alignImg; 77 | } 78 | 79 | function preprocessFeature(image) { 80 | var rows = image.rows, 81 | cols = image.cols; 82 | 83 | var img_data = ndarray(new Float32Array(rows * cols * 3), [rows, cols, 3]); 84 | 85 | for (var y = 0; y < rows; y++) 86 | for (var x = 0; x < cols; x++) { 87 | let pixel = image.ucharPtr(y, x); 88 | for (var c = 0; c < 3; c++) { 89 | var pixel_value = 0 90 | if (c === 0) // R 91 | pixel_value = (pixel[c] - 127) / 128.0; 92 | if (c === 1) // G 93 | pixel_value = (pixel[c] - 127) / 128.0; 94 | if (c === 2) // B 95 | pixel_value = (pixel[c] - 127) / 128.0; 96 | 97 | img_data.set(y, x, c, pixel_value) 98 | } 99 | } 100 | 101 | var preprocesed = ndarray(new Float32Array(3 * rows * cols), [1, 3, rows, cols]) 102 | 103 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 104 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 105 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 106 | 107 | return preprocesed 108 | } 109 | 110 | async function extractFeatureImage(session, img, landmarks) { 111 | const result = []; 112 | for (let i = 0; i < landmarks.length; i++) { 113 | 114 | var face_img = alignFeatureImage(img, landmarks[i]); 115 | //cv.imshow("live-temp", face_img); 116 | var input_image = preprocessFeature(face_img); 117 | face_img.delete(); 118 | 119 | const input_tensor = new Tensor("float32", new Float32Array(112 * 112 * 3), [1, 3, 112, 112]); 120 | input_tensor.data.set(input_image.data); 121 | const feeds = {"input": input_tensor}; 122 | 123 | const output_tensor = await session.run(feeds); 124 | // console.log("Feature result: ", output_tensor); 125 | 126 | result.push(output_tensor); 127 | } 128 | img.delete(); 129 | return result; 130 | } 131 | 132 | async function extractFeature(session, canvas_id, landmarks) { 133 | var img = cv.imread(canvas_id); 134 | 135 | var result = await extractFeatureImage(session, img, landmarks); 136 | return result; 137 | } 138 | 139 | async function extractFeatureBase64(session, base64Image, landmarks) { 140 | let image = new Image() 141 | image.src = base64Image; 142 | await new Promise(r => { 143 | image.onload = r 144 | }) 145 | 146 | var img = cv.imread(image); 147 | 148 | var result = await extractFeatureImage(session, img, landmarks); 149 | return result; 150 | } 151 | 152 | function matchFeature(feature1, feature2) { 153 | const vectorSize = feature1.length; 154 | 155 | let meanFeat = []; 156 | let feature1Sum = 0 157 | let feature2Sum = 0 158 | 159 | for (let i = 0; i < vectorSize; i++) { 160 | let meanVal = (feature1[i] + feature2[i]) / 2; 161 | feature1[i] -= meanVal; 162 | feature2[i] -= meanVal; 163 | 164 | feature1Sum += feature1[i] * feature1[i]; 165 | feature2Sum += feature2[i] * feature2[i]; 166 | } 167 | 168 | let score = 0; 169 | for (let i = 0; i < vectorSize; i++) { 170 | feature1[i] = feature1[i] / Math.sqrt(feature1Sum); 171 | feature2[i] = feature2[i] / Math.sqrt(feature2Sum); 172 | 173 | score += feature1[i] * feature2[i]; 174 | } 175 | 176 | return score 177 | } 178 | 179 | export {loadFeatureModel, extractFeature, extractFeatureBase64, matchFeature} -------------------------------------------------------------------------------- /lib/fr_gender.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | 5 | async function loadGenderModel() { 6 | var feature_session = null; 7 | await InferenceSession.create("../model/fr_gender.onnx", {executionProviders: ['wasm']}) 8 | .then((session) => { 9 | feature_session = session 10 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]); 11 | for (let i = 0; i < 64 * 64 * 3; i++) { 12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 13 | } 14 | const feeds = {"input": input_tensor}; 15 | const output_tensor = feature_session.run(feeds) 16 | console.log("initialize the gender session.") 17 | }) 18 | return feature_session; 19 | } 20 | 21 | function alignGenderImage(image, bbox, scale_value) { 22 | var src_h = image.rows, 23 | src_w = image.cols; 24 | 25 | var x = bbox[0] 26 | var y = bbox[1] 27 | var box_w = bbox[2] 28 | var box_h = bbox[3] 29 | 30 | var scale = Math.min((src_h - 1) / box_h, Math.min((src_w - 1) / box_w, scale_value)) 31 | 32 | var new_width = box_w * scale 33 | var new_height = box_h * scale 34 | var center_x = box_w / 2 + x, 35 | center_y = box_h / 2 + y 36 | 37 | var left_top_x = center_x - new_width / 2 38 | var left_top_y = center_y - new_height / 2 39 | var right_bottom_x = center_x + new_width / 2 40 | var right_bottom_y = center_y + new_height / 2 41 | 42 | if (left_top_x < 0) { 43 | right_bottom_x -= left_top_x 44 | left_top_x = 0 45 | } 46 | 47 | if (left_top_y < 0) { 48 | right_bottom_y -= left_top_y 49 | left_top_y = 0 50 | } 51 | 52 | if (right_bottom_x > src_w - 1) { 53 | left_top_x -= right_bottom_x - src_w + 1 54 | right_bottom_x = src_w - 1 55 | } 56 | 57 | if (right_bottom_y > src_h - 1) { 58 | left_top_y -= right_bottom_y - src_h + 1 59 | right_bottom_y = src_h - 1 60 | } 61 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0), 62 | Math.min(parseInt(right_bottom_x - left_top_x), src_w - 1), Math.min(parseInt(right_bottom_y - left_top_y), src_h - 1)) 63 | 64 | var face_image = new cv.Mat() 65 | face_image = image.roi(rect) 66 | 67 | var dsize = new cv.Size(64, 64); 68 | var resize_image = new cv.Mat(); 69 | cv.resize(face_image, resize_image, dsize); 70 | 71 | face_image.delete() 72 | return resize_image 73 | } 74 | 75 | function mergeGender(x, s1, s2, s3, lambda_local, lambda_d) { 76 | let a = 0; 77 | let b = 0; 78 | let c = 0; 79 | 80 | const V = 1; 81 | 82 | for (let i = 0; i < s1; i++) 83 | a = a + (i + lambda_local * x[12 + i]) * x[i]; 84 | // console.log("a = ", a) 85 | 86 | a = a / (s1 * (1 + lambda_d * x[9])); 87 | 88 | for (let i = 0; i < s2; i++) 89 | b = b + (i + lambda_local * x[15 + i]) * x[3 + i]; 90 | //console.log("b = ", b) 91 | 92 | b = b / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])); 93 | 94 | for (let i = 0; i < s3; i++) 95 | c = c + (i + lambda_local * x[18 + i]) * x[6 + i]; 96 | //console.log("c = ", c) 97 | 98 | c = c / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])) / (s3 * (1 + lambda_d * x[11])); 99 | return (a + b + c) * V; 100 | } 101 | 102 | function preprocessGender(img) { 103 | var cols = img.cols; 104 | var rows = img.rows; 105 | var channels = 3; 106 | 107 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 108 | 109 | for (var y = 0; y < rows; y++) 110 | for (var x = 0; x < cols; x++) { 111 | let pixel = img.ucharPtr(y, x); 112 | // if(x == 0 && y == 0) 113 | // console.log(pixel); 114 | for (var c = 0; c < channels; c++) { 115 | var pixel_value = 0 116 | if (c === 0) // R 117 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229 118 | if (c === 1) // G 119 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224 120 | if (c === 2) // B 121 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225 122 | 123 | img_data.set(y, x, c, pixel_value) 124 | } 125 | } 126 | 127 | var preprocesed = ndarray(new Float32Array(3 * 64 * 64), [1, 3, 64, 64]) 128 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 129 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 130 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 131 | 132 | return preprocesed 133 | } 134 | 135 | async function predictGender(session, canvas_id, bbox) { 136 | var img = cv.imread(canvas_id); 137 | 138 | var face_size = bbox.shape[0]; 139 | var bbox_size = bbox.shape[1]; 140 | 141 | const result = []; 142 | for (let i = 0; i < face_size; i++) { 143 | var x1 = parseInt(bbox.data[i * bbox_size]), 144 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 145 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 146 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 147 | width = Math.abs(x2 - x1), 148 | height = Math.abs(y2 - y1); 149 | 150 | var face_img = alignGenderImage(img, [x1, y1, width, height], 1.4); 151 | //cv.imshow("live-temp", face_img); 152 | var input_image = preprocessGender(face_img); 153 | face_img.delete(); 154 | 155 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]); 156 | input_tensor.data.set(input_image.data); 157 | const feeds = {"input": input_tensor}; 158 | 159 | const output_tensor = await session.run(feeds); 160 | 161 | const outputLayers = ["prob_stage_1", "prob_stage_2", "prob_stage_3", "stage1_delta_k", "stage2_delta_k", "stage3_delta_k", 162 | "index_offset_stage1", "index_offset_stage2", "index_offset_stage3"]; 163 | 164 | const outputFeat = []; 165 | for (let i = 0; i < outputLayers.length; i++) { 166 | const result = output_tensor[outputLayers[i]]; 167 | // console.log(outputLayers[i], ": ", result.size); 168 | for (let j = 0; j < result.size; j++) 169 | outputFeat.push(result.data[j]); 170 | } 171 | 172 | // console.log("final result: ", outputFeat); 173 | let gender = mergeGender(outputFeat, 3, 3, 3, 1, 1); 174 | console.log("output gender: ", gender); 175 | result.push([x1, y1, x2, y2, gender]); 176 | } 177 | img.delete(); 178 | return result; 179 | } 180 | 181 | export {loadGenderModel, predictGender} -------------------------------------------------------------------------------- /lib/fr_landmark.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | 5 | 6 | async function loadLandmarkModel() { 7 | var landmark_session = null 8 | await InferenceSession.create("../model/fr_landmark.onnx", {executionProviders: ['wasm']}) 9 | .then((session) => { 10 | landmark_session = session 11 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64), [1, 1, 64, 64]); 12 | for (let i = 0; i < 64 * 64; i++) { 13 | input_tensor.data[i] = Math.random(); 14 | } 15 | const feeds = {"input": input_tensor}; 16 | const output_tensor = landmark_session.run(feeds) 17 | console.log("initialize the landmark session.") 18 | }) 19 | return landmark_session 20 | } 21 | 22 | function decodeLandmark(landmark, priors, variances) { 23 | 24 | var landms = ndarray(landmark.data, [26250, 10]); 25 | var before_prior = priors.hi(null, 2), 26 | after_prior = priors.lo(null, 2); 27 | var result = ndarray(new Float32Array(landms.shape[0] * landms.shape[1]), landms.shape); 28 | var priortemp = ndarray(new Float32Array(after_prior.shape[0] * 2), [after_prior.shape[0], 2]); 29 | var half_size = parseInt(Math.floor(landms.shape[1] / 2)); 30 | 31 | ops.muls(priortemp, after_prior, variances[0]); 32 | 33 | for (var index = 0; index < half_size; index++) { 34 | let temp = ndarray(new Float32Array(landms.shape[0] * 2), [landms.shape[0], 2]); 35 | let temp2 = ndarray(new Float32Array(landms.shape[0] * 2), [landms.shape[0], 2]); 36 | let preslice = landms.hi(null, (index + 1) * 2).lo(null, index * 2); 37 | ops.mul(temp, preslice, priortemp); 38 | ops.add(temp2, temp, before_prior); 39 | ops.assign(result.pick(null, index * 2), temp2.pick(null, 0)); 40 | ops.assign(result.pick(null, index * 2 + 1), temp2.pick(null, 1)); 41 | } 42 | 43 | return result; 44 | } 45 | 46 | function alignLandmarkImage(image, bbox, scale_value) { 47 | var src_h = image.rows, 48 | src_w = image.cols; 49 | 50 | var x = bbox[0] 51 | var y = bbox[1] 52 | var box_w = bbox[2] 53 | var box_h = bbox[3] 54 | 55 | var scale = Math.min((src_h-1)/box_h, Math.min((src_w-1)/box_w, scale_value)) 56 | 57 | var new_width = box_w * scale 58 | var new_height = box_h * scale 59 | var center_x = box_w/2+x, 60 | center_y = box_h/2+y 61 | 62 | var left_top_x = center_x-new_width/2 63 | var left_top_y = center_y-new_height/2 64 | var right_bottom_x = center_x+new_width/2 65 | var right_bottom_y = center_y+new_height/2 66 | 67 | if (left_top_x < 0) { 68 | right_bottom_x -= left_top_x 69 | left_top_x = 0 70 | } 71 | 72 | if (left_top_y < 0) { 73 | right_bottom_y -= left_top_y 74 | left_top_y = 0 75 | } 76 | 77 | if (right_bottom_x > src_w-1) { 78 | left_top_x -= right_bottom_x-src_w+1 79 | right_bottom_x = src_w-1 80 | } 81 | 82 | if (right_bottom_y > src_h-1) { 83 | left_top_y -= right_bottom_y-src_h+1 84 | right_bottom_y = src_h-1 85 | } 86 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0), 87 | Math.min(parseInt(right_bottom_x - left_top_x), src_w-1), Math.min(parseInt(right_bottom_y - left_top_y), src_h-1)) 88 | 89 | var face_image = new cv.Mat() 90 | face_image = image.roi(rect) 91 | 92 | var dsize = new cv.Size(64, 64); 93 | var resize_image = new cv.Mat(); 94 | cv.resize(face_image, resize_image, dsize); 95 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2GRAY) 96 | 97 | face_image.delete() 98 | return resize_image 99 | } 100 | 101 | function preprocessLandmark(img) { 102 | var cols = img.cols; 103 | var rows = img.rows; 104 | var channels = 1; 105 | 106 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 107 | 108 | for (var y = 0; y < rows; y++) 109 | for (var x = 0; x < cols; x++) { 110 | let pixel = img.ucharPtr(y, x); 111 | for (var c = 0; c < channels; c++) { 112 | var pixel_value = pixel[c] / 256.0; 113 | img_data.set(y, x, c, pixel_value) 114 | } 115 | } 116 | 117 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols]) 118 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 119 | 120 | return preprocesed; 121 | } 122 | 123 | async function predictLandmarkImage(session, img, bbox) { 124 | var face_size = bbox.shape[0]; 125 | var bbox_size = bbox.shape[1]; 126 | 127 | const landmarks = []; 128 | 129 | for (let i = 0; i < face_size; i++) { 130 | var x1 = parseInt(bbox.data[i * bbox_size]), 131 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 132 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 133 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 134 | width = Math.abs(x2 - x1), 135 | height = Math.abs(y2 - y1); 136 | 137 | var face_img = alignLandmarkImage(img, [x1, y1, width, height], 1.0); 138 | // cv.imshow("live-temp", face_img); 139 | var input_image = preprocessLandmark(face_img); 140 | face_img.delete(); 141 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64), [1, 1, 64, 64]); 142 | 143 | input_tensor.data.set(input_image.data); 144 | 145 | const feeds = {"input": input_tensor}; 146 | 147 | const output_tensor = await session.run(feeds); 148 | var landmark_arr = output_tensor['output'].data; 149 | 150 | for (let i = 0; i < landmark_arr.length; i++) { 151 | if (i % 2 === 0) 152 | landmark_arr[i] = parseInt(landmark_arr[i] * width + x1); 153 | else 154 | landmark_arr[i] = parseInt(landmark_arr[i] * height + y1); 155 | } 156 | landmarks.push(landmark_arr); 157 | // console.log("Landmark result: ", landmark_arr[0], landmark_arr[1], landmark_arr[74], landmark_arr[75], landmark_arr[76], landmark_arr[77]); 158 | } 159 | img.delete(); 160 | return landmarks 161 | } 162 | 163 | async function predictLandmark(session, canvas_id, bbox) { 164 | 165 | var img = cv.imread(canvas_id); 166 | var landmarks = await predictLandmarkImage(session, img, bbox); 167 | 168 | return landmarks; 169 | } 170 | 171 | async function predictLandmarkBase64(session, base64Image, bbox) { 172 | let image = new Image() 173 | image.src = base64Image 174 | await new Promise(r => { 175 | image.onload = r 176 | }) 177 | 178 | var img = cv.imread(image); 179 | var landmarks = await predictLandmarkImage(session, img, bbox); 180 | 181 | return landmarks; 182 | } 183 | 184 | export {loadLandmarkModel, predictLandmark, predictLandmarkBase64} -------------------------------------------------------------------------------- /lib/fr_liveness.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | import {softmax} from "./fr_pose"; 5 | 6 | async function loadLivenessModel() { 7 | var live_session = null 8 | await InferenceSession.create("../model/fr_liveness.onnx", {executionProviders: ['wasm']}) 9 | .then((session) => { 10 | live_session = session 11 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]); 12 | for (let i = 0; i < 128 * 128 * 3; i++) { 13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 14 | } 15 | const feeds = {"input": input_tensor}; 16 | const output_tensor = live_session.run(feeds) 17 | console.log("initialize the live session.") 18 | }) 19 | return live_session; 20 | } 21 | 22 | function alignLivenessImage(image, bbox, scale_value) { 23 | var src_h = image.rows, 24 | src_w = image.cols; 25 | 26 | var x = bbox[0] 27 | var y = bbox[1] 28 | var box_w = bbox[2] 29 | var box_h = bbox[3] 30 | 31 | var scale = Math.min((src_h-1)/box_h, Math.min((src_w-1)/box_w, scale_value)) 32 | 33 | var new_width = box_w * scale 34 | var new_height = box_h * scale 35 | var center_x = box_w/2+x, 36 | center_y = box_h/2+y 37 | 38 | var left_top_x = center_x-new_width/2 39 | var left_top_y = center_y-new_height/2 40 | var right_bottom_x = center_x+new_width/2 41 | var right_bottom_y = center_y+new_height/2 42 | 43 | if (left_top_x < 0) { 44 | right_bottom_x -= left_top_x 45 | left_top_x = 0 46 | } 47 | 48 | if (left_top_y < 0) { 49 | right_bottom_y -= left_top_y 50 | left_top_y = 0 51 | } 52 | 53 | if (right_bottom_x > src_w-1) { 54 | left_top_x -= right_bottom_x-src_w+1 55 | right_bottom_x = src_w-1 56 | } 57 | 58 | if (right_bottom_y > src_h-1) { 59 | left_top_y -= right_bottom_y-src_h+1 60 | right_bottom_y = src_h-1 61 | } 62 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0), 63 | Math.min(parseInt(right_bottom_x - left_top_x), src_w-1), Math.min(parseInt(right_bottom_y - left_top_y), src_h-1)) 64 | 65 | var face_image = new cv.Mat() 66 | face_image = image.roi(rect) 67 | 68 | var dsize = new cv.Size(128, 128); 69 | var resize_image = new cv.Mat(); 70 | cv.resize(face_image, resize_image, dsize); 71 | 72 | face_image.delete() 73 | return resize_image 74 | } 75 | 76 | function preprocessLiveness(img) { 77 | var cols = img.cols; 78 | var rows = img.rows; 79 | var channels = 3; 80 | 81 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 82 | 83 | for (var y = 0; y < rows; y++) 84 | for (var x = 0; x < cols; x++) { 85 | let pixel = img.ucharPtr(y, x); 86 | for (var c = 0; c < channels; c++) { 87 | var pixel_value = 0 88 | if (c === 0) // R 89 | pixel_value = pixel[c]; 90 | if (c === 1) // G 91 | pixel_value = pixel[c]; 92 | if (c === 2) // B 93 | pixel_value = pixel[c]; 94 | 95 | img_data.set(y, x, c, pixel_value) 96 | } 97 | } 98 | 99 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols]) 100 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 101 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 102 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 103 | 104 | return preprocesed 105 | } 106 | 107 | async function predictLiveness(session, canvas_id, bbox) { 108 | var img = cv.imread(canvas_id) 109 | 110 | var face_size = bbox.shape[0]; 111 | var bbox_size = bbox.shape[1]; 112 | 113 | const result = []; 114 | for (let i = 0; i < face_size; i++) { 115 | var x1 = parseInt(bbox.data[i * bbox_size]), 116 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 117 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 118 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 119 | width = Math.abs(x2 - x1), 120 | height = Math.abs(y2 - y1); 121 | 122 | var face_img = alignLivenessImage(img, [x1, y1, width, height], 2.7); 123 | //cv.imshow("live-temp", face_img); 124 | var input_image = preprocessLiveness(face_img); 125 | face_img.delete(); 126 | 127 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]); 128 | input_tensor.data.set(input_image.data); 129 | const feeds = {"input": input_tensor}; 130 | 131 | const output_tensor = await session.run(feeds); 132 | const score_arr = softmax(output_tensor['output'].data); 133 | console.log("Liveness result: ", score_arr); 134 | 135 | result.push([x1, y1, x2, y2, score_arr[0]]); 136 | } 137 | img.delete(); 138 | return result; 139 | 140 | } 141 | 142 | async function predictLivenessBase64(session, base64Image) { 143 | let image = new Image() 144 | image.src = base64Image 145 | await new Promise(r => { 146 | image.onload = r 147 | }) 148 | 149 | var img = cv.imread(image) 150 | 151 | var face_size = bbox.shape[0]; 152 | var bbox_size = bbox.shape[1]; 153 | 154 | const result = []; 155 | for (let i = 0; i < face_size; i++) { 156 | var x1 = parseInt(bbox.data[i * bbox_size]), 157 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 158 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 159 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 160 | width = Math.abs(x2 - x1), 161 | height = Math.abs(y2 - y1); 162 | 163 | var face_img = alignLivenessImage(img, [x1, y1, width, height], 2.7); 164 | //cv.imshow("live-temp", face_img); 165 | var input_image = preprocessLiveness(face_img); 166 | face_img.delete(); 167 | 168 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]); 169 | input_tensor.data.set(input_image.data); 170 | const feeds = {"input": input_tensor}; 171 | 172 | const output_tensor = await session.run(feeds); 173 | const score_arr = softmax(output_tensor['output'].data); 174 | console.log("Liveness result: ", score_arr); 175 | 176 | result.push([x1, y1, x2, y2, score_arr[0]]); 177 | } 178 | img.delete(); 179 | return result; 180 | 181 | } 182 | 183 | export {loadLivenessModel, predictLiveness} 184 | -------------------------------------------------------------------------------- /lib/fr_pose.js: -------------------------------------------------------------------------------- 1 | import {InferenceSession, Tensor} from "onnxruntime-web"; 2 | import ndarray from "ndarray"; 3 | import ops from "ndarray-ops"; 4 | 5 | async function loadPoseModel() { 6 | var pose_session = null 7 | await InferenceSession.create("../model/fr_pose.onnx", {executionProviders: ['wasm']}) 8 | .then((session) => { 9 | pose_session = session 10 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]); 11 | for (let i = 0; i < 224 * 224 * 3; i++) { 12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0; 13 | } 14 | const feeds = {"input": input_tensor}; 15 | const output_tensor = pose_session.run(feeds) 16 | console.log("initialize the pose session.") 17 | }) 18 | return pose_session 19 | } 20 | 21 | function preprocessPose(img) { 22 | var cols = img.cols; 23 | var rows = img.rows; 24 | var channels = 3; 25 | 26 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]); 27 | 28 | for (var y = 0; y < rows; y++) 29 | for (var x = 0; x < cols; x++) { 30 | let pixel = img.ucharPtr(y, x); 31 | // if(x == 0 && y == 0) 32 | // console.log(pixel); 33 | for (var c = 0; c < channels; c++) { 34 | var pixel_value = 0 35 | if (c === 0) // R 36 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229 37 | if (c === 1) // G 38 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224 39 | if (c === 2) // B 40 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225 41 | 42 | img_data.set(y, x, c, pixel_value) 43 | } 44 | } 45 | 46 | var preprocesed = ndarray(new Float32Array(3 * 224 * 224), [1, 3, 224, 224]) 47 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0)); 48 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1)); 49 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2)); 50 | 51 | return preprocesed 52 | } 53 | 54 | function softmax(arr) { 55 | return arr.map(function(value, index) { 56 | return Math.exp(value) / arr.map( function(y /*value*/){ return Math.exp(y) } ).reduce( function(a,b){ return a+b }) 57 | }) 58 | } 59 | 60 | async function predictPose(session, canvas_id, bbox) { 61 | var face_count = bbox.shape[0], 62 | bbox_size = bbox.shape[1]; 63 | 64 | var img = cv.imread(canvas_id); 65 | const result = []; 66 | 67 | for (let i = 0; i < face_count; i++) { 68 | var x1 = parseInt(bbox.data[i * bbox_size]), 69 | y1 = parseInt(bbox.data[i * bbox_size + 1]), 70 | x2 = parseInt(bbox.data[i * bbox_size + 2]), 71 | y2 = parseInt(bbox.data[i * bbox_size + 3]), 72 | width = Math.abs(x2 - x1), 73 | height = Math.abs(y2 - y1); 74 | 75 | var x11 = parseInt(x1 - width/4), 76 | y11 = parseInt(y1 - height/4), 77 | x22 = parseInt(x2 + width/4), 78 | y22 = parseInt(y2 + height/4); 79 | 80 | var rect = new cv.Rect(Math.max(x11, 0), Math.max(y11, 0), Math.min(x22 - x11, img.cols), Math.min(y22 - y11, img.rows)); 81 | var face_image = new cv.Mat(); 82 | 83 | face_image = img.roi(rect); 84 | 85 | var dsize = new cv.Size(224, 224); 86 | var resize_image = new cv.Mat(); 87 | cv.resize(face_image, resize_image, dsize); 88 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2RGB); 89 | 90 | // cv.imshow("live-temp", resize_image) 91 | 92 | var input_image = preprocessPose(resize_image); 93 | 94 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]); 95 | input_tensor.data.set(input_image.data); 96 | const feeds = {"input": input_tensor}; 97 | 98 | const output_tensor = await session.run(feeds); 99 | 100 | var arr = Array.apply(null, Array(66)); 101 | const index_arr = arr.map(function (x, i) { return i }) 102 | 103 | const yaw_arr = softmax(output_tensor['output'].data); 104 | const pitch_arr = softmax(output_tensor['617'].data); 105 | const roll_arr = softmax(output_tensor['618'].data); 106 | 107 | const yaw = yaw_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99; 108 | const pitch = pitch_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99; 109 | const roll = roll_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99; 110 | //console.log("Pose results: ", yaw, pitch, roll) 111 | result.push([x1, y1, x2, y2, yaw.toFixed(2), pitch.toFixed(2), roll.toFixed(2)]); 112 | 113 | resize_image.delete(); 114 | face_image.delete(); 115 | } 116 | img.delete(); 117 | return result; 118 | } 119 | 120 | export {loadPoseModel, softmax, predictPose} -------------------------------------------------------------------------------- /lib/load_opencv.js: -------------------------------------------------------------------------------- 1 | var cv = null; 2 | 3 | async function load_opencv() { 4 | if (!window.WebAssembly) { 5 | console.log("Your web browser doesn't support WebAssembly.") 6 | return 7 | } 8 | 9 | console.log("loading OpenCv.js") 10 | const script = document.createElement("script") 11 | script.type = "text/javascript" 12 | script.async = "async" 13 | script.src = "../js/opencv.js" 14 | document.body.appendChild(script) 15 | script.onload = () => { 16 | console.log("OpenCV.js is loaded.") 17 | } 18 | 19 | window.Module = { 20 | wasmBinaryFile: `../js/opencv_js.wasm`, // for wasm mode 21 | preRun: () => { 22 | console.log('preRun function on loading opencv') 23 | }, 24 | _main: () => { 25 | console.log('OpenCV.js is ready.') 26 | cv = window.cv 27 | } 28 | } 29 | return 30 | } 31 | 32 | export {load_opencv, cv} -------------------------------------------------------------------------------- /model/fr_age.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_age.onnx -------------------------------------------------------------------------------- /model/fr_detect.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_detect.onnx -------------------------------------------------------------------------------- /model/fr_expression.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_expression.onnx -------------------------------------------------------------------------------- /model/fr_eye.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_eye.onnx -------------------------------------------------------------------------------- /model/fr_feature.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_feature.onnx -------------------------------------------------------------------------------- /model/fr_gender.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_gender.onnx -------------------------------------------------------------------------------- /model/fr_landmark.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_landmark.onnx -------------------------------------------------------------------------------- /model/fr_liveness.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_liveness.onnx -------------------------------------------------------------------------------- /model/fr_pose.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_pose.onnx -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kby-face", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "jest", 8 | "build": "webpack" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/kby-ai/FaceRecognition-Javascript.git" 13 | }, 14 | "keywords": [], 15 | "license": "MIT", 16 | "bugs": { 17 | "url": "https://github.com/kby-ai/FaceRecognition-Javascript/issues" 18 | }, 19 | "homepage": "https://github.com/kby-ai/FaceRecognition-Javascript#readme", 20 | "dependencies": { 21 | "babel-core": "^6.26.3", 22 | "babel-jest": "^27.4.2", 23 | "babel-plugin-add-module-exports": "^1.0.4", 24 | "babel-preset-env": "^1.7.0", 25 | "clean-webpack-plugin": "^4.0.0", 26 | "eslint": "^8.3.0", 27 | "jest": "^27.4.3", 28 | "terser-webpack-plugin": "^5.2.5", 29 | "ndarray": "^1.0.19", 30 | "ndarray-ops": "^1.2.2", 31 | "onnxruntime-web": "^1.14.0" 32 | }, 33 | "devDependencies": { 34 | "@babel/core": "^7.18.10", 35 | "@babel/preset-env": "^7.18.10", 36 | "babel-loader": "^8.2.5", 37 | "copy-webpack-plugin": "^8.1.1", 38 | "webpack": "^5.64.4", 39 | "webpack-cli": "^4.9.1" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | // webpack.config.js 2 | const path = require('path'); 3 | const webpack = require('webpack'); 4 | const { CleanWebpackPlugin } = require('clean-webpack-plugin'); 5 | const TerserPlugin = require('terser-webpack-plugin'); 6 | const CopyPlugin = require("copy-webpack-plugin"); 7 | 8 | 9 | module.exports = { 10 | mode: 'production', 11 | devtool: false, 12 | entry: './index.js', 13 | target: ['web'], 14 | output: { 15 | path: path.resolve(__dirname, './dist'), 16 | filename: 'facerecognition-sdk.js', 17 | // globalObject: 'this', 18 | library: { 19 | type: 'umd' 20 | } 21 | }, 22 | 23 | plugins: [ 24 | new CleanWebpackPlugin(), 25 | // new webpack.SourceMapDevToolPlugin({ 26 | // filename: 'facerecognition-sdk.js.map' 27 | // }), 28 | new CopyPlugin({ 29 | // Use copy plugin to copy *.wasm to output folder. 30 | patterns: [{ from: 'node_modules/onnxruntime-web/dist/*.wasm', to: '[name][ext]' }] 31 | }) 32 | ], 33 | module: { 34 | rules: [ 35 | { 36 | test: /\.(js)$/, 37 | exclude: /node_modules/, 38 | use: "babel-loader", 39 | }, 40 | ], 41 | }, 42 | } 43 | --------------------------------------------------------------------------------