├── .gitignore
├── LICENSE
├── README.md
├── add.test.js
├── index.js
├── js
├── opencv.js
├── opencv_js.js
└── opencv_js.wasm
├── lib
├── download.js
├── fr_age.js
├── fr_detect.js
├── fr_expression.js
├── fr_eye.js
├── fr_feature.js
├── fr_gender.js
├── fr_landmark.js
├── fr_liveness.js
├── fr_pose.js
└── load_opencv.js
├── model
├── fr_age.onnx
├── fr_detect.onnx
├── fr_expression.onnx
├── fr_eye.onnx
├── fr_feature.onnx
├── fr_gender.onnx
├── fr_landmark.onnx
├── fr_liveness.onnx
└── fr_pose.onnx
├── package-lock.json
├── package.json
└── webpack.config.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | lerna-debug.log*
8 |
9 | # Diagnostic reports (https://nodejs.org/api/report.html)
10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
11 |
12 | # Runtime data
13 | pids
14 | *.pid
15 | *.seed
16 | *.pid.lock
17 |
18 | # Directory for instrumented libs generated by jscoverage/JSCover
19 | lib-cov
20 |
21 | # Coverage directory used by tools like istanbul
22 | coverage
23 | *.lcov
24 |
25 | # nyc test coverage
26 | .nyc_output
27 |
28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
29 | .grunt
30 |
31 | # Bower dependency directory (https://bower.io/)
32 | bower_components
33 |
34 | # node-waf configuration
35 | .lock-wscript
36 |
37 | # Compiled binary addons (https://nodejs.org/api/addons.html)
38 | build/Release
39 |
40 | # Dependency directories
41 | node_modules/
42 | jspm_packages/
43 |
44 | # TypeScript v1 declaration files
45 | typings/
46 |
47 | # TypeScript cache
48 | *.tsbuildinfo
49 |
50 | # Optional npm cache directory
51 | .npm
52 |
53 | # Optional eslint cache
54 | .eslintcache
55 |
56 | # Microbundle cache
57 | .rpt2_cache/
58 | .rts2_cache_cjs/
59 | .rts2_cache_es/
60 | .rts2_cache_umd/
61 |
62 | # Optional REPL history
63 | .node_repl_history
64 |
65 | # Output of 'npm pack'
66 | *.tgz
67 |
68 | # Yarn Integrity file
69 | .yarn-integrity
70 |
71 | # dotenv environment variables file
72 | .env
73 | .env.test
74 |
75 | # parcel-bundler cache (https://parceljs.org/)
76 | .cache
77 |
78 | # Next.js build output
79 | .next
80 |
81 | # Nuxt.js build / generate output
82 | .nuxt
83 |
84 | # Gatsby files
85 | .cache/
86 | # Comment in the public line in if your project uses Gatsby and *not* Next.js
87 | # https://nextjs.org/blog/next-9-1#public-directory-support
88 | # public
89 |
90 | # vuepress build output
91 | .vuepress/dist
92 |
93 | # Serverless directories
94 | .serverless/
95 |
96 | # FuseBox cache
97 | .fusebox/
98 |
99 | # DynamoDB Local files
100 | .dynamodb/
101 |
102 | # TernJS port file
103 | .tern-port
104 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 gitguanqi
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |

3 |
4 |
5 | # Face Recognition SDK Javascript - Fully On Premise
6 | ## Overview
7 | Experience the epitome of speed and fairness with our `face recognition model` **Top-ranked on NIST FRVT**, coupled with an advanced **iBeta level 2 liveness detection** engine that effectively safeguards against **printed photos, video replay, 3D masks, and deepfake threats**, ensuring top-tier security.
8 |
This is `on-premise SDK` which means everything is processed on the browser and **NO** data leaves the device
9 |
10 |
11 | ## Installation
12 |
13 | ```bash
14 | npm install faceplugin
15 | ```
16 |
17 | ## Table of Contents
18 |
19 | * **[Face Detection](#face-detection)**
20 | * **[Face Landmark Extraction](#face-landmark-extraction)**
21 | * **[Face Liveness Detection](#face-expression-detection)**
22 | * **[Face Expression Detection](#face-expression-detection)**
23 | * **[Face Pose Estimation](#face-pose-estimation)**
24 | * **[Eye Closeness Detection](#eye-closeness-detection)**
25 | * **[Gender Detection](#gender-detection)**
26 | * **[Age Detection](#age-detection)**
27 | * **[Face Feature Embedding](#face-recognition)**
28 |
29 | ## Examples
30 |
31 | https://github.com/kby-ai/FaceRecognition-Javascript/assets/125717930/551b6964-0fef-4483-85a7-76792c0f3b56
32 |
33 | * [Vue.js Demo](https://github.com/Faceplugin-ltd/FacePlugin-FaceRecognition-Vue)
34 | * [React.js Demo](https://github.com/Faceplugin-ltd/FacePlugin-FaceRecognition-React)
35 |
36 |
37 |
38 |
39 |
40 | ## List of our Products
41 |
42 | * **[Face Recognition with Liveness Detection-Android (Java, Kotlin)](https://github.com/Faceplugin-ltd/FaceRecognition-Android)**
43 | * **[Face Recognition with Liveness Detection-iOS (Objective C, Swift)](https://github.com/Faceplugin-ltd/FaceRecognition-iOS)**
44 | * **[Face Recognition with Liveness Detection-React Native](https://github.com/Faceplugin-ltd/FaceRecognition-React-Native)**
45 | * **[Face Recognition with Liveness Detection-Flutter](https://github.com/Faceplugin-ltd/FaceRecognition-Flutter)**
46 | * **[Face Recognition with Liveness Detection-Ionic Cordova](https://github.com/Faceplugin-ltd/FaceRecognition-Ionic-Cordova)**
47 | * **[Face Recognition with Liveness Detection-.Net MAUI](https://github.com/Faceplugin-ltd/FaceRecognition-.Net)**
48 | * **[Face Recognition with Liveness Detection-.Net WPF](https://github.com/Faceplugin-ltd/FaceRecognition-WPF-.Net)**
49 | * **[Face Recognition with Liveness Detection-Javascript](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript)**
50 | * **[Face Recognition with LivenessDetection-React](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-React)**
51 | * **[Face Recognition with LivenessDetection-Vue](https://github.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Vue)**
52 | * **[Face Liveness Detection-Android (Java, Kotlin)](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Android)**
53 | * **[Face Liveness Detection-iOS (Objective C, Swift)](https://github.com/Faceplugin-ltd/FaceLivenessDetection-iOS)**
54 | * **[Face Liveness Detection-Linux](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Linux)**
55 | * **[Face Liveness Detection-Docker](https://github.com/Faceplugin-ltd/FaceLivenessDetection-Docker)**
56 | * **[Open Source Face Recognition SDK](https://github.com/Faceplugin-ltd/Open-Source-Face-Recognition-SDK)**
57 | * **[Face Recognition SDK](https://github.com/Faceplugin-ltd/Face-Recognition-SDK)**
58 | * **[Liveness Detection SDK](https://github.com/Faceplugin-ltd/Face-Liveness-Detection-SDK)**
59 | * **[Palm Recognition SDK](https://github.com/Faceplugin-ltd/Palm-Recognition)**
60 | * **[ID Card Recognition](https://github.com/Faceplugin-ltd/ID-Card-Recognition)**
61 | * **[ID Document Liveness Detection](https://github.com/Faceplugin-ltd/ID-Document-Liveness-Detection)**
62 |
63 | ## Documentation
64 |
65 | Here are some useful documentation
66 |
67 |
68 | ### Face Detection
69 | Load detection model
70 | ```
71 | loadDetectionModel()
72 | ```
73 | Detect face in the image
74 | ```
75 | detectFace(session, canvas_id)
76 | ```
77 |
78 |
79 | ### Face Landmark Extraction
80 | Load landmark extraction model
81 | ```
82 | loadLandmarkModel()
83 | ```
84 | Extract face landmark in the image using detection result
85 | ```
86 | predictLandmark(session, canvas_id, bbox)
87 | ```
88 |
89 |
90 | ### Face Liveness Detection
91 | Load liveness detection model
92 | ```
93 | loadLivenessModel()
94 | ```
95 | Detect face liveness in the image using detection result. (Anti-spoofing)
96 | ```
97 | predictLiveness(session, canvas_id, bbox)
98 | ```
99 |
100 |
101 | ### Face Expression Detection
102 | Load expression detection model
103 | ```
104 | loadExpressionModel()
105 | ```
106 | Detect face expression
107 | ```
108 | predictExpression(session, canvas_id, bbox)
109 | ```
110 |
111 |
112 | ### Face Pose Estimation
113 | Load pose estimation model
114 | ```
115 | loadPoseModel()
116 | ```
117 | Predict facial pose
118 | ```
119 | predictPose(session, canvas_id, bbox, question)
120 | ```
121 |
122 |
123 | ### Eye Closeness Detection
124 | Load eye closeness model
125 | ```
126 | loadEyeModel()
127 | ```
128 | Predict eye closeness
129 | ```
130 | predictEye(session, canvas_id, landmark)
131 | ```
132 |
133 |
134 | ### Gender Detection
135 | Load gender detection model
136 | ```
137 | loadGenderModel()
138 | ```
139 | Predict gender using face image
140 | ```
141 | predictGender(session, canvas_id, landmark)
142 | ```
143 |
144 |
145 | ### Age Detection
146 | Load age detection model
147 | ```
148 | loadAgeModel()
149 | ```
150 | Predict age using face image
151 | ```
152 | predictAge(session, canvas_id, landmark)
153 | ```
154 |
155 |
156 | ### Face Recognition
157 | Load feature extraction model
158 | ```
159 | loadFeatureModel()
160 | ```
161 | Extract face feature vector in 512 dimension
162 | ```
163 | extractFeature(session, canvas_id, landmarks)
164 | ```
165 |
166 | ## Contact
167 | If you want to get better model, please contact us
168 |
169 |
174 |
175 |
176 |
--------------------------------------------------------------------------------
/add.test.js:
--------------------------------------------------------------------------------
1 | const add = require('./index.js').add;
2 |
3 | test('adds 1 + 2 to equal 3', () => {
4 | expect(add(1,2)).toBe(3);
5 | })
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | export * from "./lib/fr_detect";
2 | export * from "./lib/fr_expression";
3 | export * from "./lib/fr_eye";
4 | export * from "./lib/fr_landmark";
5 | export * from "./lib/fr_liveness";
6 | export * from "./lib/fr_pose";
7 | export * from "./lib/fr_age";
8 | export * from "./lib/fr_gender";
9 | export * from "./lib/fr_feature";
10 | export * from "./lib/load_opencv";
--------------------------------------------------------------------------------
/js/opencv_js.wasm:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/js/opencv_js.wasm
--------------------------------------------------------------------------------
/lib/download.js:
--------------------------------------------------------------------------------
1 | export const download = (url, logger = null) => {
2 | return new Promise((resolve, reject) => {
3 | const request = new XMLHttpRequest();
4 | request.open("GET", url, true);
5 | request.responseType = "arraybuffer";
6 | if (logger) {
7 | const [log, setState] = logger;
8 | request.onprogress = (e) => {
9 | const progress = (e.loaded / e.total) * 100;
10 | setState({ text: log, progress: progress.toFixed(2) });
11 | };
12 | }
13 | request.onload = function () {
14 | if (this.status >= 200 && this.status < 300) {
15 | resolve(request.response);
16 | } else {
17 | reject({
18 | status: this.status,
19 | statusText: request.statusText,
20 | });
21 | }
22 | resolve(request.response);
23 | };
24 | request.onerror = function () {
25 | reject({
26 | status: this.status,
27 | statusText: request.statusText,
28 | });
29 | };
30 | request.send();
31 | });
32 | };
33 |
--------------------------------------------------------------------------------
/lib/fr_age.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 |
5 | async function loadAgeModel() {
6 | var feature_session = null;
7 | await InferenceSession.create("../model/fr_age.onnx", {executionProviders: ['wasm']})
8 | .then((session) => {
9 | feature_session = session
10 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]);
11 | for (let i = 0; i < 64 * 64 * 3; i++) {
12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
13 | }
14 | const feeds = {"input": input_tensor};
15 | const output_tensor = feature_session.run(feeds)
16 | console.log("initialize the age session.")
17 | })
18 | return feature_session;
19 | }
20 |
21 | function alignAgeImage(image, bbox, scale_value) {
22 | var src_h = image.rows,
23 | src_w = image.cols;
24 |
25 | var x = bbox[0]
26 | var y = bbox[1]
27 | var box_w = bbox[2]
28 | var box_h = bbox[3]
29 |
30 | var scale = Math.min((src_h - 1) / box_h, Math.min((src_w - 1) / box_w, scale_value))
31 |
32 | var new_width = box_w * scale
33 | var new_height = box_h * scale
34 | var center_x = box_w / 2 + x,
35 | center_y = box_h / 2 + y
36 |
37 | var left_top_x = center_x - new_width / 2
38 | var left_top_y = center_y - new_height / 2
39 | var right_bottom_x = center_x + new_width / 2
40 | var right_bottom_y = center_y + new_height / 2
41 |
42 | if (left_top_x < 0) {
43 | right_bottom_x -= left_top_x
44 | left_top_x = 0
45 | }
46 |
47 | if (left_top_y < 0) {
48 | right_bottom_y -= left_top_y
49 | left_top_y = 0
50 | }
51 |
52 | if (right_bottom_x > src_w - 1) {
53 | left_top_x -= right_bottom_x - src_w + 1
54 | right_bottom_x = src_w - 1
55 | }
56 |
57 | if (right_bottom_y > src_h - 1) {
58 | left_top_y -= right_bottom_y - src_h + 1
59 | right_bottom_y = src_h - 1
60 | }
61 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0),
62 | Math.min(parseInt(right_bottom_x - left_top_x), src_w - 1), Math.min(parseInt(right_bottom_y - left_top_y), src_h - 1))
63 |
64 | var face_image = new cv.Mat()
65 | face_image = image.roi(rect)
66 |
67 | var dsize = new cv.Size(64, 64);
68 | var resize_image = new cv.Mat();
69 | cv.resize(face_image, resize_image, dsize);
70 |
71 | face_image.delete()
72 | return resize_image
73 | }
74 |
75 | function mergeAge(x, s1, s2, s3, lambda_local, lambda_d) {
76 | let a = 0;
77 | let b = 0;
78 | let c = 0;
79 |
80 | const V = 101;
81 |
82 | for (let i = 0; i < s1; i++)
83 | a = a + (i + lambda_local * x[12 + i]) * x[i];
84 | // console.log("a = ", a)
85 |
86 | a = a / (s1 * (1 + lambda_d * x[9]));
87 |
88 | for (let i = 0; i < s2; i++)
89 | b = b + (i + lambda_local * x[15 + i]) * x[3 + i];
90 | //console.log("b = ", b)
91 |
92 | b = b / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10]));
93 |
94 | for (let i = 0; i < s3; i++)
95 | c = c + (i + lambda_local * x[18 + i]) * x[6 + i];
96 | //console.log("c = ", c)
97 |
98 | c = c / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])) / (s3 * (1 + lambda_d * x[11]));
99 | return (a + b + c) * V;
100 | }
101 |
102 | function preprocessAge(img) {
103 | var cols = img.cols;
104 | var rows = img.rows;
105 | var channels = 3;
106 |
107 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
108 |
109 | for (var y = 0; y < rows; y++)
110 | for (var x = 0; x < cols; x++) {
111 | let pixel = img.ucharPtr(y, x);
112 | // if(x == 0 && y == 0)
113 | // console.log(pixel);
114 | for (var c = 0; c < channels; c++) {
115 | var pixel_value = 0
116 | if (c === 0) // R
117 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229
118 | if (c === 1) // G
119 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224
120 | if (c === 2) // B
121 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225
122 |
123 | img_data.set(y, x, c, pixel_value)
124 | }
125 | }
126 |
127 | var preprocesed = ndarray(new Float32Array(3 * 64 * 64), [1, 3, 64, 64])
128 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
129 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
130 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
131 |
132 | return preprocesed
133 | }
134 |
135 | async function predictAge(session, canvas_id, bbox) {
136 | var img = cv.imread(canvas_id);
137 |
138 | var face_size = bbox.shape[0];
139 | var bbox_size = bbox.shape[1];
140 |
141 | const result = [];
142 | for (let i = 0; i < face_size; i++) {
143 | var x1 = parseInt(bbox.data[i * bbox_size]),
144 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
145 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
146 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
147 | width = Math.abs(x2 - x1),
148 | height = Math.abs(y2 - y1);
149 |
150 | var face_img = alignAgeImage(img, [x1, y1, width, height], 1.4);
151 | //cv.imshow("live-temp", face_img);
152 | var input_image = preprocessAge(face_img);
153 | face_img.delete();
154 |
155 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]);
156 | input_tensor.data.set(input_image.data);
157 | const feeds = {"input": input_tensor};
158 |
159 | const output_tensor = await session.run(feeds);
160 | const outputLayers = ["prob_stage_1", "prob_stage_2", "prob_stage_3", "stage1_delta_k", "stage2_delta_k", "stage3_delta_k",
161 | "index_offset_stage1", "index_offset_stage2", "index_offset_stage3"];
162 |
163 | const outputFeat = [];
164 | for (let i = 0; i < outputLayers.length; i++) {
165 | const result = output_tensor[outputLayers[i]];
166 | // console.log(outputLayers[i], ": ", result.size);
167 | for (let j = 0; j < result.size; j++)
168 | outputFeat.push(result.data[j]);
169 | }
170 |
171 | let age = mergeAge(outputFeat, 3, 3, 3, 1, 1);
172 | console.log("output age: ", age);
173 | result.push([x1, y1, x2, y2, age]);
174 | }
175 |
176 | img.delete();
177 | return result;
178 | }
179 |
180 | export {loadAgeModel, predictAge}
--------------------------------------------------------------------------------
/lib/fr_detect.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 | import {cv} from "./load_opencv";
5 | import {download} from "./download";
6 |
7 | async function loadDetectionModel() {
8 | var detect_session = null;
9 | await InferenceSession.create("../model/fr_detect.onnx", {executionProviders: ['wasm']})
10 | .then((session) => {
11 | detect_session = session;
12 | const input_tensor = new Tensor("float32", new Float32Array(320 * 240 * 3), [1, 3, 240, 320]);
13 | for (let i = 0; i < 320 * 240 * 3; i++) {
14 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
15 | }
16 | const feeds = {"input": input_tensor};
17 | const output_tensor = detect_session.run(feeds)
18 | console.log("initialize the detection session.")
19 | })
20 | return detect_session
21 | }
22 |
23 | async function loadDetectionModelPath(model_path) {
24 | const arr_buf = await download(model_path);
25 | const detection_session = await InferenceSession.create(arr_buf);
26 | return detection_session
27 | }
28 |
29 | function preprocessDetection(image) {
30 | var rows = image.rows,
31 | cols = image.cols;
32 |
33 | var img_data = ndarray(new Float32Array(rows * cols * 3), [rows, cols, 3]);
34 |
35 | for (var y = 0; y < rows; y++)
36 | for (var x = 0; x < cols; x++) {
37 | let pixel = image.ucharPtr(y, x);
38 | for (var c = 0; c < 3; c++) {
39 | var pixel_value = 0
40 | if (c === 0) // R
41 | pixel_value = (pixel[c] - 127) / 128.0;
42 | if (c === 1) // G
43 | pixel_value = (pixel[c] - 127) / 128.0;
44 | if (c === 2) // B
45 | pixel_value = (pixel[c] - 127) / 128.0;
46 |
47 | img_data.set(y, x, c, pixel_value)
48 | }
49 | }
50 |
51 | var preprocesed = ndarray(new Float32Array(3 * rows * cols), [1, 3, rows, cols])
52 |
53 | // Transpose
54 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
55 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
56 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
57 |
58 | return preprocesed
59 | }
60 |
61 | async function detectFaceImage(session, img) {
62 | const onnx_config = {
63 | min_sizes: [[10, 16, 24], [32, 48], [64, 96], [128, 192, 256]],
64 | steps: [8, 16, 32, 64],
65 | variance: [0.1, 0.2],
66 | clip: false,
67 | confidence_threshold: 0.65,
68 | top_k: 750,
69 | nms_threshold: 0.4,
70 | };
71 |
72 | var dsize = new cv.Size(320, 240);
73 | var resize_image = new cv.Mat();
74 | cv.resize(img, resize_image, dsize);
75 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2RGB);
76 |
77 | const image = preprocessDetection(resize_image);
78 |
79 | var resize_param = {cols: img.cols / 320, rows: img.rows / 240};
80 |
81 | const input_tensor = new Tensor("float32", new Float32Array(320 * 240 * 3), [1, 3, 240, 320]);
82 | input_tensor.data.set(image.data);
83 |
84 | const feeds = {"input": input_tensor};
85 | const output_tensor = await session.run(feeds);
86 |
87 | const loc = output_tensor['boxes'];
88 | const conf = output_tensor['scores'];
89 |
90 | // const landmarks = output_tensor['585']
91 | const total_result = conf.size / 2;
92 |
93 | const scale = [320, 240, 320, 240];
94 | const scale1 = [800, 800, 800, 800, 800, 800, 800, 800, 800, 800];
95 |
96 | const priors = definePriorBox([320, 240], onnx_config);
97 |
98 | const boxes_arr = decodeBBox(loc, priors, onnx_config);
99 |
100 | const scores_arr = ndarray(conf.data, [total_result, 2]).pick(null, 1);
101 | var landms_arr = null;//decode_landmark(landmarks, priors, onnx_config.variance);
102 |
103 | var box = ndarray(loc.data, [4420, 4]);
104 | var boxes_before = scaleMultiplyBBox(box, scale);
105 | var landms_before = null;//scale_multiply_landms(landms_arr, scale1);
106 |
107 | var [bbox_screen, scores_screen, landms_screen] = screenScore(boxes_before, scores_arr, landms_before, onnx_config.confidence_threshold);
108 |
109 | var [bbox_sorted, scores_sorted, landms_sorted] = sortScore(bbox_screen, scores_screen, landms_screen, onnx_config.top_k);
110 |
111 | var [bbox_small, score_result, landms_small, result_size] = cpuNMS(bbox_sorted, scores_sorted, landms_sorted, onnx_config.nms_threshold);
112 | var [bbox_result, landms_result] = scaleResult(bbox_small, landms_small, resize_param, img.cols, img.rows);
113 |
114 | var output = {
115 | bbox: bbox_result,
116 | landmark: landms_result,
117 | conf: score_result,
118 | size: result_size
119 | }
120 |
121 | resize_image.delete();
122 | img.delete();
123 | return output
124 | }
125 |
126 | async function detectFace(session, canvas_id) {
127 |
128 | var img = cv.imread(canvas_id);
129 | var output = await detectFaceImage(session, img);
130 |
131 | return output
132 | }
133 |
134 | async function detectFaceBase64(session, base64Image) {
135 | let image = new Image()
136 | image.src = base64Image
137 | await new Promise(r => {
138 | image.onload = r
139 | })
140 |
141 | var img = cv.imread(image);
142 | var output = await detectFaceImage(session, img);
143 |
144 | return output
145 | }
146 |
147 | function product(x, y) {
148 | var size_x = x.length,
149 | size_y = y.length;
150 | var result = [];
151 |
152 | for (var i = 0; i < size_x; i++)
153 | for (var j = 0; j < size_y; j++)
154 | result.push([x[i], y[j]]);
155 |
156 | return result;
157 | }
158 |
159 | function range(num) {
160 | var result = [];
161 | for (var i = 0; i < num; i++)
162 | result.push(i);
163 |
164 | return result;
165 | }
166 |
167 | function definePriorBox(image_size, onnx_config) {
168 | var min_sizes = onnx_config.min_sizes,
169 | steps = onnx_config.steps,
170 | clip = onnx_config.clip,
171 | name = "s",
172 | feature_maps = steps.map((step) => [Math.ceil(image_size[0] / step), Math.ceil(image_size[1] / step)]);
173 |
174 | var anchors = [];
175 |
176 | feature_maps.forEach((f, k) => {
177 | var min_size = min_sizes[k];
178 | product(range(f[0]), range(f[1])).forEach(([i, j]) => {
179 | min_size.forEach((m_size) => {
180 | var s_kx = m_size / image_size[0],
181 | s_ky = m_size / image_size[1];
182 | var dense_cx = [j + 0.5].map((x) => x * steps[k] / image_size[0]),
183 | dense_cy = [i + 0.5].map((y) => y * steps[k] / image_size[1]);
184 | product(dense_cy, dense_cx).forEach(([cy, cx]) => {
185 | anchors.push(cx);
186 | anchors.push(cy);
187 | anchors.push(s_kx);
188 | anchors.push(s_ky);
189 | })
190 | });
191 | });
192 | });
193 |
194 | var output = ndarray(new Float32Array(anchors), [anchors.length / 4, 4]);
195 |
196 | if (clip)
197 | output = ndarray.ops.min(1, ops.max(output, 0));
198 |
199 | return output;
200 | }
201 |
202 | function decodeBBox(bbox, priors, onnx_config) {
203 | var variances = onnx_config.variance
204 | var loc = ndarray(bbox.data, [4420, 4]);
205 | // console.log(bbox, priors);
206 | var before_prior = priors.hi(null, 2),
207 | after_prior = priors.lo(null, 2);
208 |
209 | var before_loc = loc.hi(null, 2),
210 | after_loc = loc.lo(null, 2);
211 |
212 | var before_result = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
213 | var before_temp = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
214 | var before_temp2 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
215 |
216 | var after_result = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
217 | var after_temp = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
218 | var after_temp2 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
219 | var after_temp3 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
220 | var after_temp4 = ndarray(new Float32Array(before_loc.shape[0] * before_loc.shape[1]), [before_loc.shape[0], 2]);
221 |
222 | var boxes = ndarray(new Float32Array(before_loc.shape[0] * 4), [before_loc.shape[0], 4]);
223 |
224 | // Before
225 | ops.mul(before_temp, before_loc, after_prior);
226 | ops.muls(before_temp2, before_temp, variances[0]);
227 | ops.add(before_result, before_temp2, before_prior);
228 |
229 | // After
230 | ops.muls(after_temp, after_loc, variances[1]);
231 | ops.exp(after_temp2, after_temp);
232 | ops.mul(after_temp3, after_temp2, after_prior);
233 |
234 | for (var index = 0; index < 4; index++)
235 | ops.assign(after_result.pick(null, index), after_temp3.pick(null, index));
236 |
237 | ops.divs(after_temp4, after_temp3, -2);
238 | ops.addeq(before_result, after_temp4);
239 |
240 | ops.addeq(after_result, before_result);
241 |
242 | ops.assign(boxes.pick(null, 0), before_result.pick(null, 0));
243 | ops.assign(boxes.pick(null, 1), before_result.pick(null, 1));
244 | ops.assign(boxes.pick(null, 2), after_result.pick(null, 0));
245 | ops.assign(boxes.pick(null, 3), after_result.pick(null, 1));
246 |
247 | return boxes;
248 | }
249 |
250 | function scaleMultiplyBBox(boxes_arr, scale) {
251 | var total_result = boxes_arr.shape[0];
252 | var boxes_before = ndarray(new Float32Array(total_result * 4), [total_result, 4]);
253 |
254 | for (var index = 0; index < scale.length; index++) {
255 | let temp = boxes_arr.pick(null, index),
256 | before_result = ndarray(new Float32Array(total_result), [total_result]);
257 | ops.muls(before_result, temp, scale[index]);
258 | ops.assign(boxes_before.pick(null, index), before_result);
259 | }
260 |
261 | return boxes_before;
262 | }
263 |
264 | function scaleMultiplyLandmarks(landms_arr, scale1) {
265 | var total_result = landms_arr.shape[0];
266 | var landms_before = ndarray(new Float32Array(total_result * 10), [total_result, 10]);
267 |
268 | for (var index = 0; index < scale1.length; index++) {
269 | let temp = landms_arr.pick(null, index),
270 | before_landms_result = ndarray(new Float32Array(total_result), [total_result]);
271 | ops.muls(before_landms_result, temp, scale1[index]);
272 | ops.assign(landms_before.pick(null, index), before_landms_result);
273 | }
274 |
275 | return landms_before;
276 | }
277 |
278 | function screenScore(bbox, scores, landms, threshold) {
279 | var total_size = scores.shape[0];
280 | var index_arr = [];
281 |
282 | for (var index = 0; index < total_size; index++) {
283 | var score_temp = scores.get(index);
284 |
285 | if (score_temp >= threshold) {
286 | index_arr.push(index);
287 | }
288 | }
289 |
290 | var result_bbox = ndarray(new Float32Array(index_arr.length * 4), [index_arr.length, 4]);
291 | var result_scores = ndarray(new Float32Array(index_arr.length), [index_arr.length]);
292 | var result_landms = null;//ndarray(new Float32Array(index_arr.length * 10), [index_arr.length, 10]);
293 |
294 | index_arr.forEach((index, i) => {
295 | ops.assign(result_bbox.pick(i, null), bbox.pick(index, null));
296 | //ops.assign(result_landms.pick(i, null), landms.pick(index, null));
297 | ops.assign(result_scores.pick(i), scores.pick(index));
298 | });
299 |
300 | return [result_bbox, result_scores, result_landms];
301 | }
302 |
303 | function sortScore(bbox, scores, landms, top_k) {
304 | var total_size = scores.shape[0];
305 | var index_sort = new Array(total_size * 2);
306 |
307 | for (var index = 0; index < total_size; index++) {
308 | var temp = scores.get(index);
309 | index_sort[index] = [index, temp];
310 | }
311 |
312 | index_sort.sort((a, b) => {
313 | if (a[1] < b[1]) return 1;
314 | if (a[1] > b[1]) return -1;
315 |
316 | return 0;
317 | });
318 |
319 | var max_size = (total_size > top_k) ? top_k : total_size;
320 |
321 | var result_bbox = ndarray(new Float32Array(max_size * 4), [max_size, 4]);
322 | var result_scores = ndarray(new Float32Array(max_size), [max_size]);
323 | var result_landms = null;//ndarray(new Float32Array(max_size * 10), [max_size, 10]);
324 |
325 | for (var idx = 0; idx < max_size; idx++) {
326 | result_scores.set(idx, index_sort[idx][1]);
327 | ops.assign(result_bbox.pick(idx, null), bbox.pick(index_sort[idx][0], null));
328 | //ops.assign(result_landms.pick(idx, null), landms.pick(index_sort[idx][0], null));
329 | }
330 |
331 | return [result_bbox, result_scores, result_landms];
332 | }
333 |
334 | function cpuNMS(bbox, scores, landms, thresh) {
335 | var {max, min} = Math;
336 | var size = bbox.shape[0];
337 | var foundLocations = [];
338 | var pick = [];
339 |
340 | for (var i = 0; i < size; i++) {
341 | var x1 = bbox.get(i, 0),
342 | y1 = bbox.get(i, 1),
343 | x2 = bbox.get(i, 2),
344 | y2 = bbox.get(i, 3);
345 |
346 | var width = x2 - x1,
347 | height = y2 - y1;
348 |
349 | if (width > 0 && height > 0) {
350 | var area = width * height;
351 | foundLocations.push({x1, y1, x2, y2, width, height, area, index: i});
352 | }
353 | }
354 |
355 | foundLocations.sort((b1, b2) => {
356 | return b1.y2 - b2.y2;
357 | });
358 |
359 | while (foundLocations.length > 0) {
360 | var last = foundLocations[0] //[foundLocations.length - 1];
361 | var suppress = [last];
362 | pick.push(last.index) //foundLocations.length - 1);
363 |
364 | for (let i = 1; i < foundLocations.length; i++) {
365 | const box = foundLocations[i];
366 | const xx1 = max(box.x1, last.x1);
367 | const yy1 = max(box.y1, last.y1);
368 | const xx2 = min(box.x2, last.x2);
369 | const yy2 = min(box.y2, last.y2);
370 | const w = max(0, xx2 - xx1 + 1);
371 | const h = max(0, yy2 - yy1 + 1);
372 | const overlap = (w * h) / box.area;
373 |
374 | if (overlap >= thresh)
375 | suppress.push(foundLocations[i]);
376 | }
377 |
378 | foundLocations = foundLocations.filter((box) => {
379 | return !suppress.find((supp) => {
380 | return supp === box;
381 | })
382 | });
383 | }
384 |
385 | var result_bbox = ndarray(new Float32Array(pick.length * 4), [pick.length, 4]);
386 | var result_scores = ndarray(new Float32Array(pick.length), [pick.length]);
387 | var result_landms = null;//ndarray(new Float32Array(pick.length * 10), [pick.length, 10]);
388 |
389 | // console.log("Pick index: ", pick);
390 |
391 | pick.forEach((pick_index, i) => {
392 | ops.assign(result_bbox.pick(i, null), bbox.pick(pick_index, null));
393 | ops.assign(result_scores.pick(i), scores.pick(pick_index));
394 | //ops.assign(result_landms.pick(i, null), landms.pick(pick_index, null));
395 | });
396 |
397 | return [result_bbox, result_scores, result_landms, pick.length];
398 | }
399 |
400 | function scaleResult(bbox, landmark, resize_param, width, height) {
401 | var size = bbox.shape[0];
402 | var result_bbox = ndarray(new Float32Array(size * 4), [size, 4]);
403 | var result_landms = null;//ndarray(new Float32Array(size * 10), [size, 10]);
404 |
405 | for (let i = 0; i < size; i++) {
406 | let x1 = bbox.get(i, 0) * resize_param.cols,
407 | y1 = bbox.get(i, 1) * resize_param.rows,
408 | x2 = bbox.get(i, 2) * resize_param.cols,
409 | y2 = bbox.get(i, 3) * resize_param.rows;
410 |
411 | const f_size = (y2 - y1);
412 | const ct_x = (x1 + x2) / 2,
413 | ct_y = (y1 + y2) / 2;
414 |
415 | x1 = (ct_x - f_size / 2) < 0 ? 0 : (ct_x - f_size / 2);
416 | y1 = (ct_y - f_size / 2) < 0 ? 0 : (ct_y - f_size / 2);
417 | x2 = (ct_x + f_size / 2) > width - 1 ? width - 1 : (ct_x + f_size / 2);
418 | y2 = (ct_y + f_size / 2) > height - 1 ? height - 1 : (ct_y + f_size / 2);
419 |
420 | result_bbox.set(i, 0, x1);
421 | result_bbox.set(i, 1, y1);
422 | result_bbox.set(i, 2, x2);
423 | result_bbox.set(i, 3, y2);
424 | }
425 |
426 | /*
427 | for (var j = 0; j < 5; j++) {
428 | let x = landmark.pick(null, j * 2),
429 | y = landmark.pick(null, j * 2 + 1);
430 |
431 | ops.divseq(x, resize_param.cols); // X
432 | ops.divseq(y, resize_param.rows); // Y
433 |
434 | ops.assign(result_landms.pick(null, j * 2), x);
435 | ops.assign(result_landms.pick(null, j * 2 + 1), y);
436 | }
437 | */
438 | return [result_bbox, result_landms];
439 | }
440 |
441 | function getBestFace(bbox) {
442 | var face_count = bbox.shape[0],
443 | bbox_size = bbox.shape[1];
444 |
445 | var idx = -1, max_size = 0;
446 | for (let i = 0; i < face_count; i++) {
447 | var x1 = parseInt(bbox.data[i * bbox_size]),
448 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
449 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
450 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
451 | width = Math.abs(x2 - x1),
452 | height = Math.abs(y2 - y1);
453 | if (width * height > max_size)
454 | idx = i;
455 | }
456 | return idx
457 | }
458 |
459 | export {loadDetectionModel, loadDetectionModelPath, detectFace, detectFaceBase64}
--------------------------------------------------------------------------------
/lib/fr_expression.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 | import {softmax} from "./fr_pose";
5 |
6 | async function loadExpressionModel() {
7 | var expression_session = null
8 | await InferenceSession.create("../model/fr_expression.onnx", {executionProviders: ['wasm']})
9 | .then((session) => {
10 | expression_session = session
11 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]);
12 | for (let i = 0; i < 224 * 224 * 3; i++) {
13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
14 | }
15 | const feeds = {"input": input_tensor};
16 | const output_tensor = expression_session.run(feeds)
17 | console.log("initialize the expression session.")
18 | })
19 | return expression_session
20 | }
21 |
22 | function alignExpressionImage(image, bbox) {
23 | var src_h = image.rows,
24 | src_w = image.cols;
25 |
26 | var x = bbox[0]
27 | var y = bbox[1]
28 | var box_w = bbox[2]
29 | var box_h = bbox[3]
30 |
31 | var rect = new cv.Rect(x, y, Math.min(parseInt(box_w * 1.2), src_w - 1), Math.min(parseInt(box_h * 1.2), src_h - 1))
32 |
33 | var face_image = new cv.Mat()
34 | face_image = image.roi(rect)
35 |
36 | var dsize = new cv.Size(224, 224);
37 | var resize_image = new cv.Mat();
38 | cv.resize(face_image, resize_image, dsize);
39 |
40 | face_image.delete()
41 | return resize_image
42 | }
43 |
44 | function preprocessExpression(img) {
45 | var cols = img.cols;
46 | var rows = img.rows;
47 | var channels = 3;
48 |
49 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
50 |
51 | for (var y = 0; y < rows; y++)
52 | for (var x = 0; x < cols; x++) {
53 | let pixel = img.ucharPtr(y, x);
54 | for (var c = 0; c < channels; c++) {
55 | var pixel_value = pixel[c] / 255.0;
56 | img_data.set(y, x, c, pixel_value)
57 | }
58 | }
59 |
60 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols])
61 |
62 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
63 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
64 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
65 |
66 | return preprocesed
67 | }
68 |
69 | async function predictExpression(session, canvas_id, bbox) {
70 | var img = cv.imread(canvas_id)
71 |
72 | var face_count = bbox.shape[0],
73 | bbox_size = bbox.shape[1];
74 |
75 | const result = [];
76 | for (let i = 0; i < face_count; i++) {
77 | var x1 = parseInt(bbox.data[i * bbox_size]),
78 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
79 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
80 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
81 | width = Math.abs(x2 - x1),
82 | height = Math.abs(y2 - y1);
83 |
84 | var face_img = alignExpressionImage(img, [x1, y1, width, height]);
85 | //cv.imshow("live-temp", face_img);
86 | var input_image = preprocessExpression(face_img);
87 | face_img.delete();
88 |
89 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]);
90 | input_tensor.data.set(input_image.data);
91 | const feeds = {"input": input_tensor};
92 |
93 | const output_tensor = await session.run(feeds);
94 | const expression_arr = softmax(output_tensor['output'].data);
95 |
96 | var max_idx = null, max_val = 0;
97 | for (let i = 0; i < expression_arr.length; i++)
98 | if (max_val < expression_arr[i]) {
99 | max_idx = i;
100 | max_val = expression_arr[i];
101 | }
102 | result.push([x1, y1, x2, y2, max_idx]);
103 | }
104 | img.delete();
105 | return result;
106 | }
107 |
108 | export {loadExpressionModel, predictExpression}
--------------------------------------------------------------------------------
/lib/fr_eye.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 | import {softmax} from "./fr_pose";
5 |
6 | async function loadEyeModel() {
7 | var eye_session = null
8 | await InferenceSession.create("../model/fr_eye.onnx", {executionProviders: ['wasm']})
9 | .then((session) => {
10 | eye_session = session
11 | const input_tensor = new Tensor("float32", new Float32Array(24 * 24 * 1), [1, 1, 24, 24]);
12 | for (let i = 0; i < 24 * 24; i++) {
13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
14 | }
15 | const feeds = {"input": input_tensor};
16 | const output_tensor = eye_session.run(feeds)
17 | console.log("initialize the eye session.")
18 | })
19 | return eye_session
20 | }
21 |
22 | function getEyeBBox(landmark, size) {
23 | var height = size[0], width = size[1];
24 | const padding_rate = 1.6;
25 | var left_eye_center_x = parseInt((landmark[74] + landmark[76] + landmark[80] + landmark[82]) / 4);
26 | var left_eye_center_y = parseInt((landmark[75] + landmark[77] + landmark[81] + landmark[83]) / 4);
27 | var left_eye_size = parseInt((landmark[78] - landmark[72]) * padding_rate);
28 | var left_corner_x = parseInt(left_eye_center_x - left_eye_size / 2);
29 | if (left_corner_x < 0)
30 | left_corner_x = 0;
31 |
32 | var left_corner_y = parseInt(left_eye_center_y - left_eye_size / 2);
33 | if (left_corner_y < 0)
34 | left_corner_y = 0;
35 |
36 | if (left_corner_x + left_eye_size >= width)
37 | left_eye_size = width - left_corner_x - 1
38 |
39 | if (left_corner_y + left_eye_size >= height)
40 | left_eye_size = height - left_corner_y - 1
41 |
42 | var right_eye_center_x = parseInt((landmark[86] + landmark[88] + landmark[92] + landmark[94]) / 4);
43 | var right_eye_center_y = parseInt((landmark[87] + landmark[89] + landmark[93] + landmark[95]) / 4);
44 | var right_eye_size = parseInt((landmark[90] - landmark[84]) * padding_rate);
45 | var right_corner_x = parseInt(right_eye_center_x - right_eye_size / 2);
46 | if (right_corner_x < 0)
47 | right_corner_x = 0
48 | var right_corner_y = parseInt(right_eye_center_y - right_eye_size / 2);
49 | if (right_corner_y < 0)
50 | right_corner_y = 0
51 | if (right_corner_x + right_eye_size >= width)
52 | right_eye_size = width - right_corner_x - 1
53 | if (right_corner_y + right_eye_size >= height)
54 | right_eye_size = height - right_corner_y - 1
55 |
56 | return [left_corner_x, left_corner_y, left_eye_size, left_eye_size,
57 | right_corner_x, right_corner_y, right_eye_size, right_eye_size]
58 | }
59 |
60 | function alignEyeImage(image, landmark) {
61 | var src_h = image.rows,
62 | src_w = image.cols;
63 |
64 | var eye_bbox = getEyeBBox(landmark, [src_h, src_w])
65 | var rect = new cv.Rect(eye_bbox[0], eye_bbox[1], eye_bbox[2], eye_bbox[3])
66 |
67 | var eye_image = new cv.Mat()
68 | eye_image = image.roi(rect)
69 |
70 | var dsize = new cv.Size(24, 24);
71 | var left_eye = new cv.Mat();
72 | cv.resize(eye_image, left_eye, dsize);
73 | cv.cvtColor(left_eye, left_eye, cv.COLOR_BGR2GRAY)
74 |
75 | // right eye
76 | rect = new cv.Rect(eye_bbox[4], eye_bbox[5], eye_bbox[6], eye_bbox[7])
77 | eye_image = image.roi(rect)
78 | var right_eye = new cv.Mat();
79 | cv.resize(eye_image, right_eye, dsize);
80 | cv.cvtColor(right_eye, right_eye, cv.COLOR_BGR2GRAY)
81 |
82 | eye_image.delete()
83 | return [left_eye, right_eye]
84 | }
85 |
86 | function preprocessEye(imgs) {
87 | var cols = imgs[0].cols;
88 | var rows = imgs[0].rows;
89 | var channels = 1;
90 |
91 | var img_data1 = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
92 | var img_data2 = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
93 |
94 | for (var y = 0; y < rows; y++)
95 | for (var x = 0; x < cols; x++) {
96 | let pixel1 = imgs[0].ucharPtr(y, x);
97 | let pixel2 = imgs[1].ucharPtr(y, x);
98 |
99 | for (var c = 0; c < channels; c++) {
100 | var pixel_value1 = pixel1[c] / 255.0;
101 | var pixel_value2 = pixel2[c] / 255.0;
102 |
103 | img_data1.set(y, x, c, pixel_value1)
104 | img_data2.set(y, x, c, pixel_value2)
105 | }
106 | }
107 |
108 | var preprocesed1 = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols])
109 | ops.assign(preprocesed1.pick(0, 0, null, null), img_data1.pick(null, null, 0));
110 | var preprocesed2 = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols])
111 | ops.assign(preprocesed2.pick(0, 0, null, null), img_data2.pick(null, null, 0));
112 |
113 | return [preprocesed1, preprocesed2]
114 | }
115 |
116 | async function predictEye(session, canvas_id, landmarks) {
117 | var img = cv.imread(canvas_id)
118 |
119 | const result = [];
120 | for (let i = 0; i < landmarks.length; i++) {
121 | var face_imgs = alignEyeImage(img, landmarks[i]);
122 | // cv.imshow("live-temp", face_imgs[1]);
123 | var input_images = preprocessEye(face_imgs);
124 | face_imgs[0].delete();
125 | face_imgs[1].delete();
126 |
127 | const input_tensor1 = new Tensor("float32", new Float32Array(24 * 24), [1, 1, 24, 24]);
128 | input_tensor1.data.set(input_images[0].data);
129 | const feeds1 = {"input": input_tensor1};
130 |
131 | const output_tensor1 = await session.run(feeds1);
132 | const left_res = softmax(output_tensor1['output'].data);
133 |
134 | const input_tensor2 = new Tensor("float32", new Float32Array(24 * 24), [1, 1, 24, 24]);
135 | input_tensor2.data.set(input_images[1].data);
136 | const feeds2 = {"input": input_tensor2};
137 |
138 | const output_tensor2 = await session.run(feeds2);
139 | const right_res = softmax(output_tensor2['output'].data);
140 | result.push([left_res[0] > left_res[1], right_res[0] > right_res[1]]);
141 | }
142 | img.delete();
143 | return result;
144 | }
145 |
146 | export {loadEyeModel, predictEye}
--------------------------------------------------------------------------------
/lib/fr_feature.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 |
5 | const REFERENCE_FACIAL_POINTS = [
6 | [38.29459953, 51.69630051],
7 | [73.53179932, 51.50139999],
8 | [56.02519989, 71.73660278],
9 | [41.54930115, 92.3655014],
10 | [70.72990036, 92.20410156]
11 | ]
12 |
13 | async function loadFeatureModel() {
14 | var feature_session = null;
15 | await InferenceSession.create("../model/fr_feature.onnx", {executionProviders: ['wasm']})
16 | .then((session) => {
17 | feature_session = session
18 | const input_tensor = new Tensor("float32", new Float32Array(112 * 112 * 3), [1, 3, 112, 112]);
19 | for (let i = 0; i < 112 * 112 * 3; i++) {
20 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
21 | }
22 | const feeds = {"input": input_tensor};
23 | const output_tensor = feature_session.run(feeds)
24 | console.log("initialize the feature session.")
25 | })
26 | return feature_session;
27 | }
28 |
29 | function convert68pts5pts(landmark) {
30 | var left_eye_x = (landmark[74] + landmark[76] + landmark[80] + landmark[82]) / 4,
31 | left_eye_y = (landmark[75] + landmark[77] + landmark[81] + landmark[83]) / 4,
32 |
33 | right_eye_x = (landmark[86] + landmark[88] + landmark[92] + landmark[94]) / 4,
34 | right_eye_y = (landmark[87] + landmark[89] + landmark[93] + landmark[95]) / 4,
35 |
36 | nose_x = landmark[60], nose_y = landmark[61],
37 |
38 | left_mouse_x = (landmark[96] + landmark[120]) / 2,
39 | left_mouse_y = (landmark[97] + landmark[121]) / 2,
40 |
41 | right_mouse_x = (landmark[108] + landmark[128]) / 2,
42 | right_mouse_y = (landmark[109] + landmark[129]) / 2;
43 | return [[left_eye_x, left_eye_y], [right_eye_x, right_eye_y], [nose_x, nose_y], [left_mouse_x, left_mouse_y],
44 | [right_mouse_x, right_mouse_y]]
45 | }
46 |
47 | function getReferenceFacialPoints() {
48 | let ref5pts = REFERENCE_FACIAL_POINTS;
49 |
50 | return ref5pts;
51 | }
52 |
53 | function warpAndCropFace(src,
54 | face_pts,
55 | ref_pts=null,
56 | crop_size=[112, 112]) {
57 |
58 | let srcTri = cv.matFromArray(3, 1, cv.CV_32FC2, [face_pts[0][0], face_pts[0][1], face_pts[1][0], face_pts[1][1],
59 | face_pts[2][0], face_pts[2][1]]);
60 | let dstTri = cv.matFromArray(3, 1, cv.CV_32FC2, [ref_pts[0][0], ref_pts[0][1], ref_pts[1][0], ref_pts[1][1],
61 | ref_pts[2][0], ref_pts[2][1]]);
62 |
63 | let tfm = cv.getAffineTransform(srcTri, dstTri);
64 |
65 | let dsize = new cv.Size(crop_size[0], crop_size[1]);
66 | let dst = new cv.Mat();
67 | cv.warpAffine(src, dst, tfm, dsize);
68 |
69 | return dst;
70 | }
71 |
72 | function alignFeatureImage(image, landmark) {
73 | let facePoints = convert68pts5pts(landmark);
74 | let refPoints = getReferenceFacialPoints();
75 | let alignImg = warpAndCropFace(image, facePoints, refPoints);
76 | return alignImg;
77 | }
78 |
79 | function preprocessFeature(image) {
80 | var rows = image.rows,
81 | cols = image.cols;
82 |
83 | var img_data = ndarray(new Float32Array(rows * cols * 3), [rows, cols, 3]);
84 |
85 | for (var y = 0; y < rows; y++)
86 | for (var x = 0; x < cols; x++) {
87 | let pixel = image.ucharPtr(y, x);
88 | for (var c = 0; c < 3; c++) {
89 | var pixel_value = 0
90 | if (c === 0) // R
91 | pixel_value = (pixel[c] - 127) / 128.0;
92 | if (c === 1) // G
93 | pixel_value = (pixel[c] - 127) / 128.0;
94 | if (c === 2) // B
95 | pixel_value = (pixel[c] - 127) / 128.0;
96 |
97 | img_data.set(y, x, c, pixel_value)
98 | }
99 | }
100 |
101 | var preprocesed = ndarray(new Float32Array(3 * rows * cols), [1, 3, rows, cols])
102 |
103 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
104 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
105 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
106 |
107 | return preprocesed
108 | }
109 |
110 | async function extractFeatureImage(session, img, landmarks) {
111 | const result = [];
112 | for (let i = 0; i < landmarks.length; i++) {
113 |
114 | var face_img = alignFeatureImage(img, landmarks[i]);
115 | //cv.imshow("live-temp", face_img);
116 | var input_image = preprocessFeature(face_img);
117 | face_img.delete();
118 |
119 | const input_tensor = new Tensor("float32", new Float32Array(112 * 112 * 3), [1, 3, 112, 112]);
120 | input_tensor.data.set(input_image.data);
121 | const feeds = {"input": input_tensor};
122 |
123 | const output_tensor = await session.run(feeds);
124 | // console.log("Feature result: ", output_tensor);
125 |
126 | result.push(output_tensor);
127 | }
128 | img.delete();
129 | return result;
130 | }
131 |
132 | async function extractFeature(session, canvas_id, landmarks) {
133 | var img = cv.imread(canvas_id);
134 |
135 | var result = await extractFeatureImage(session, img, landmarks);
136 | return result;
137 | }
138 |
139 | async function extractFeatureBase64(session, base64Image, landmarks) {
140 | let image = new Image()
141 | image.src = base64Image;
142 | await new Promise(r => {
143 | image.onload = r
144 | })
145 |
146 | var img = cv.imread(image);
147 |
148 | var result = await extractFeatureImage(session, img, landmarks);
149 | return result;
150 | }
151 |
152 | function matchFeature(feature1, feature2) {
153 | const vectorSize = feature1.length;
154 |
155 | let meanFeat = [];
156 | let feature1Sum = 0
157 | let feature2Sum = 0
158 |
159 | for (let i = 0; i < vectorSize; i++) {
160 | let meanVal = (feature1[i] + feature2[i]) / 2;
161 | feature1[i] -= meanVal;
162 | feature2[i] -= meanVal;
163 |
164 | feature1Sum += feature1[i] * feature1[i];
165 | feature2Sum += feature2[i] * feature2[i];
166 | }
167 |
168 | let score = 0;
169 | for (let i = 0; i < vectorSize; i++) {
170 | feature1[i] = feature1[i] / Math.sqrt(feature1Sum);
171 | feature2[i] = feature2[i] / Math.sqrt(feature2Sum);
172 |
173 | score += feature1[i] * feature2[i];
174 | }
175 |
176 | return score
177 | }
178 |
179 | export {loadFeatureModel, extractFeature, extractFeatureBase64, matchFeature}
--------------------------------------------------------------------------------
/lib/fr_gender.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 |
5 | async function loadGenderModel() {
6 | var feature_session = null;
7 | await InferenceSession.create("../model/fr_gender.onnx", {executionProviders: ['wasm']})
8 | .then((session) => {
9 | feature_session = session
10 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]);
11 | for (let i = 0; i < 64 * 64 * 3; i++) {
12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
13 | }
14 | const feeds = {"input": input_tensor};
15 | const output_tensor = feature_session.run(feeds)
16 | console.log("initialize the gender session.")
17 | })
18 | return feature_session;
19 | }
20 |
21 | function alignGenderImage(image, bbox, scale_value) {
22 | var src_h = image.rows,
23 | src_w = image.cols;
24 |
25 | var x = bbox[0]
26 | var y = bbox[1]
27 | var box_w = bbox[2]
28 | var box_h = bbox[3]
29 |
30 | var scale = Math.min((src_h - 1) / box_h, Math.min((src_w - 1) / box_w, scale_value))
31 |
32 | var new_width = box_w * scale
33 | var new_height = box_h * scale
34 | var center_x = box_w / 2 + x,
35 | center_y = box_h / 2 + y
36 |
37 | var left_top_x = center_x - new_width / 2
38 | var left_top_y = center_y - new_height / 2
39 | var right_bottom_x = center_x + new_width / 2
40 | var right_bottom_y = center_y + new_height / 2
41 |
42 | if (left_top_x < 0) {
43 | right_bottom_x -= left_top_x
44 | left_top_x = 0
45 | }
46 |
47 | if (left_top_y < 0) {
48 | right_bottom_y -= left_top_y
49 | left_top_y = 0
50 | }
51 |
52 | if (right_bottom_x > src_w - 1) {
53 | left_top_x -= right_bottom_x - src_w + 1
54 | right_bottom_x = src_w - 1
55 | }
56 |
57 | if (right_bottom_y > src_h - 1) {
58 | left_top_y -= right_bottom_y - src_h + 1
59 | right_bottom_y = src_h - 1
60 | }
61 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0),
62 | Math.min(parseInt(right_bottom_x - left_top_x), src_w - 1), Math.min(parseInt(right_bottom_y - left_top_y), src_h - 1))
63 |
64 | var face_image = new cv.Mat()
65 | face_image = image.roi(rect)
66 |
67 | var dsize = new cv.Size(64, 64);
68 | var resize_image = new cv.Mat();
69 | cv.resize(face_image, resize_image, dsize);
70 |
71 | face_image.delete()
72 | return resize_image
73 | }
74 |
75 | function mergeGender(x, s1, s2, s3, lambda_local, lambda_d) {
76 | let a = 0;
77 | let b = 0;
78 | let c = 0;
79 |
80 | const V = 1;
81 |
82 | for (let i = 0; i < s1; i++)
83 | a = a + (i + lambda_local * x[12 + i]) * x[i];
84 | // console.log("a = ", a)
85 |
86 | a = a / (s1 * (1 + lambda_d * x[9]));
87 |
88 | for (let i = 0; i < s2; i++)
89 | b = b + (i + lambda_local * x[15 + i]) * x[3 + i];
90 | //console.log("b = ", b)
91 |
92 | b = b / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10]));
93 |
94 | for (let i = 0; i < s3; i++)
95 | c = c + (i + lambda_local * x[18 + i]) * x[6 + i];
96 | //console.log("c = ", c)
97 |
98 | c = c / (s1 * (1 + lambda_d * x[9])) / (s2 * (1 + lambda_d * x[10])) / (s3 * (1 + lambda_d * x[11]));
99 | return (a + b + c) * V;
100 | }
101 |
102 | function preprocessGender(img) {
103 | var cols = img.cols;
104 | var rows = img.rows;
105 | var channels = 3;
106 |
107 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
108 |
109 | for (var y = 0; y < rows; y++)
110 | for (var x = 0; x < cols; x++) {
111 | let pixel = img.ucharPtr(y, x);
112 | // if(x == 0 && y == 0)
113 | // console.log(pixel);
114 | for (var c = 0; c < channels; c++) {
115 | var pixel_value = 0
116 | if (c === 0) // R
117 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229
118 | if (c === 1) // G
119 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224
120 | if (c === 2) // B
121 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225
122 |
123 | img_data.set(y, x, c, pixel_value)
124 | }
125 | }
126 |
127 | var preprocesed = ndarray(new Float32Array(3 * 64 * 64), [1, 3, 64, 64])
128 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
129 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
130 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
131 |
132 | return preprocesed
133 | }
134 |
135 | async function predictGender(session, canvas_id, bbox) {
136 | var img = cv.imread(canvas_id);
137 |
138 | var face_size = bbox.shape[0];
139 | var bbox_size = bbox.shape[1];
140 |
141 | const result = [];
142 | for (let i = 0; i < face_size; i++) {
143 | var x1 = parseInt(bbox.data[i * bbox_size]),
144 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
145 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
146 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
147 | width = Math.abs(x2 - x1),
148 | height = Math.abs(y2 - y1);
149 |
150 | var face_img = alignGenderImage(img, [x1, y1, width, height], 1.4);
151 | //cv.imshow("live-temp", face_img);
152 | var input_image = preprocessGender(face_img);
153 | face_img.delete();
154 |
155 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64 * 3), [1, 3, 64, 64]);
156 | input_tensor.data.set(input_image.data);
157 | const feeds = {"input": input_tensor};
158 |
159 | const output_tensor = await session.run(feeds);
160 |
161 | const outputLayers = ["prob_stage_1", "prob_stage_2", "prob_stage_3", "stage1_delta_k", "stage2_delta_k", "stage3_delta_k",
162 | "index_offset_stage1", "index_offset_stage2", "index_offset_stage3"];
163 |
164 | const outputFeat = [];
165 | for (let i = 0; i < outputLayers.length; i++) {
166 | const result = output_tensor[outputLayers[i]];
167 | // console.log(outputLayers[i], ": ", result.size);
168 | for (let j = 0; j < result.size; j++)
169 | outputFeat.push(result.data[j]);
170 | }
171 |
172 | // console.log("final result: ", outputFeat);
173 | let gender = mergeGender(outputFeat, 3, 3, 3, 1, 1);
174 | console.log("output gender: ", gender);
175 | result.push([x1, y1, x2, y2, gender]);
176 | }
177 | img.delete();
178 | return result;
179 | }
180 |
181 | export {loadGenderModel, predictGender}
--------------------------------------------------------------------------------
/lib/fr_landmark.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 |
5 |
6 | async function loadLandmarkModel() {
7 | var landmark_session = null
8 | await InferenceSession.create("../model/fr_landmark.onnx", {executionProviders: ['wasm']})
9 | .then((session) => {
10 | landmark_session = session
11 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64), [1, 1, 64, 64]);
12 | for (let i = 0; i < 64 * 64; i++) {
13 | input_tensor.data[i] = Math.random();
14 | }
15 | const feeds = {"input": input_tensor};
16 | const output_tensor = landmark_session.run(feeds)
17 | console.log("initialize the landmark session.")
18 | })
19 | return landmark_session
20 | }
21 |
22 | function decodeLandmark(landmark, priors, variances) {
23 |
24 | var landms = ndarray(landmark.data, [26250, 10]);
25 | var before_prior = priors.hi(null, 2),
26 | after_prior = priors.lo(null, 2);
27 | var result = ndarray(new Float32Array(landms.shape[0] * landms.shape[1]), landms.shape);
28 | var priortemp = ndarray(new Float32Array(after_prior.shape[0] * 2), [after_prior.shape[0], 2]);
29 | var half_size = parseInt(Math.floor(landms.shape[1] / 2));
30 |
31 | ops.muls(priortemp, after_prior, variances[0]);
32 |
33 | for (var index = 0; index < half_size; index++) {
34 | let temp = ndarray(new Float32Array(landms.shape[0] * 2), [landms.shape[0], 2]);
35 | let temp2 = ndarray(new Float32Array(landms.shape[0] * 2), [landms.shape[0], 2]);
36 | let preslice = landms.hi(null, (index + 1) * 2).lo(null, index * 2);
37 | ops.mul(temp, preslice, priortemp);
38 | ops.add(temp2, temp, before_prior);
39 | ops.assign(result.pick(null, index * 2), temp2.pick(null, 0));
40 | ops.assign(result.pick(null, index * 2 + 1), temp2.pick(null, 1));
41 | }
42 |
43 | return result;
44 | }
45 |
46 | function alignLandmarkImage(image, bbox, scale_value) {
47 | var src_h = image.rows,
48 | src_w = image.cols;
49 |
50 | var x = bbox[0]
51 | var y = bbox[1]
52 | var box_w = bbox[2]
53 | var box_h = bbox[3]
54 |
55 | var scale = Math.min((src_h-1)/box_h, Math.min((src_w-1)/box_w, scale_value))
56 |
57 | var new_width = box_w * scale
58 | var new_height = box_h * scale
59 | var center_x = box_w/2+x,
60 | center_y = box_h/2+y
61 |
62 | var left_top_x = center_x-new_width/2
63 | var left_top_y = center_y-new_height/2
64 | var right_bottom_x = center_x+new_width/2
65 | var right_bottom_y = center_y+new_height/2
66 |
67 | if (left_top_x < 0) {
68 | right_bottom_x -= left_top_x
69 | left_top_x = 0
70 | }
71 |
72 | if (left_top_y < 0) {
73 | right_bottom_y -= left_top_y
74 | left_top_y = 0
75 | }
76 |
77 | if (right_bottom_x > src_w-1) {
78 | left_top_x -= right_bottom_x-src_w+1
79 | right_bottom_x = src_w-1
80 | }
81 |
82 | if (right_bottom_y > src_h-1) {
83 | left_top_y -= right_bottom_y-src_h+1
84 | right_bottom_y = src_h-1
85 | }
86 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0),
87 | Math.min(parseInt(right_bottom_x - left_top_x), src_w-1), Math.min(parseInt(right_bottom_y - left_top_y), src_h-1))
88 |
89 | var face_image = new cv.Mat()
90 | face_image = image.roi(rect)
91 |
92 | var dsize = new cv.Size(64, 64);
93 | var resize_image = new cv.Mat();
94 | cv.resize(face_image, resize_image, dsize);
95 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2GRAY)
96 |
97 | face_image.delete()
98 | return resize_image
99 | }
100 |
101 | function preprocessLandmark(img) {
102 | var cols = img.cols;
103 | var rows = img.rows;
104 | var channels = 1;
105 |
106 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
107 |
108 | for (var y = 0; y < rows; y++)
109 | for (var x = 0; x < cols; x++) {
110 | let pixel = img.ucharPtr(y, x);
111 | for (var c = 0; c < channels; c++) {
112 | var pixel_value = pixel[c] / 256.0;
113 | img_data.set(y, x, c, pixel_value)
114 | }
115 | }
116 |
117 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols])
118 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
119 |
120 | return preprocesed;
121 | }
122 |
123 | async function predictLandmarkImage(session, img, bbox) {
124 | var face_size = bbox.shape[0];
125 | var bbox_size = bbox.shape[1];
126 |
127 | const landmarks = [];
128 |
129 | for (let i = 0; i < face_size; i++) {
130 | var x1 = parseInt(bbox.data[i * bbox_size]),
131 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
132 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
133 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
134 | width = Math.abs(x2 - x1),
135 | height = Math.abs(y2 - y1);
136 |
137 | var face_img = alignLandmarkImage(img, [x1, y1, width, height], 1.0);
138 | // cv.imshow("live-temp", face_img);
139 | var input_image = preprocessLandmark(face_img);
140 | face_img.delete();
141 | const input_tensor = new Tensor("float32", new Float32Array(64 * 64), [1, 1, 64, 64]);
142 |
143 | input_tensor.data.set(input_image.data);
144 |
145 | const feeds = {"input": input_tensor};
146 |
147 | const output_tensor = await session.run(feeds);
148 | var landmark_arr = output_tensor['output'].data;
149 |
150 | for (let i = 0; i < landmark_arr.length; i++) {
151 | if (i % 2 === 0)
152 | landmark_arr[i] = parseInt(landmark_arr[i] * width + x1);
153 | else
154 | landmark_arr[i] = parseInt(landmark_arr[i] * height + y1);
155 | }
156 | landmarks.push(landmark_arr);
157 | // console.log("Landmark result: ", landmark_arr[0], landmark_arr[1], landmark_arr[74], landmark_arr[75], landmark_arr[76], landmark_arr[77]);
158 | }
159 | img.delete();
160 | return landmarks
161 | }
162 |
163 | async function predictLandmark(session, canvas_id, bbox) {
164 |
165 | var img = cv.imread(canvas_id);
166 | var landmarks = await predictLandmarkImage(session, img, bbox);
167 |
168 | return landmarks;
169 | }
170 |
171 | async function predictLandmarkBase64(session, base64Image, bbox) {
172 | let image = new Image()
173 | image.src = base64Image
174 | await new Promise(r => {
175 | image.onload = r
176 | })
177 |
178 | var img = cv.imread(image);
179 | var landmarks = await predictLandmarkImage(session, img, bbox);
180 |
181 | return landmarks;
182 | }
183 |
184 | export {loadLandmarkModel, predictLandmark, predictLandmarkBase64}
--------------------------------------------------------------------------------
/lib/fr_liveness.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 | import {softmax} from "./fr_pose";
5 |
6 | async function loadLivenessModel() {
7 | var live_session = null
8 | await InferenceSession.create("../model/fr_liveness.onnx", {executionProviders: ['wasm']})
9 | .then((session) => {
10 | live_session = session
11 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]);
12 | for (let i = 0; i < 128 * 128 * 3; i++) {
13 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
14 | }
15 | const feeds = {"input": input_tensor};
16 | const output_tensor = live_session.run(feeds)
17 | console.log("initialize the live session.")
18 | })
19 | return live_session;
20 | }
21 |
22 | function alignLivenessImage(image, bbox, scale_value) {
23 | var src_h = image.rows,
24 | src_w = image.cols;
25 |
26 | var x = bbox[0]
27 | var y = bbox[1]
28 | var box_w = bbox[2]
29 | var box_h = bbox[3]
30 |
31 | var scale = Math.min((src_h-1)/box_h, Math.min((src_w-1)/box_w, scale_value))
32 |
33 | var new_width = box_w * scale
34 | var new_height = box_h * scale
35 | var center_x = box_w/2+x,
36 | center_y = box_h/2+y
37 |
38 | var left_top_x = center_x-new_width/2
39 | var left_top_y = center_y-new_height/2
40 | var right_bottom_x = center_x+new_width/2
41 | var right_bottom_y = center_y+new_height/2
42 |
43 | if (left_top_x < 0) {
44 | right_bottom_x -= left_top_x
45 | left_top_x = 0
46 | }
47 |
48 | if (left_top_y < 0) {
49 | right_bottom_y -= left_top_y
50 | left_top_y = 0
51 | }
52 |
53 | if (right_bottom_x > src_w-1) {
54 | left_top_x -= right_bottom_x-src_w+1
55 | right_bottom_x = src_w-1
56 | }
57 |
58 | if (right_bottom_y > src_h-1) {
59 | left_top_y -= right_bottom_y-src_h+1
60 | right_bottom_y = src_h-1
61 | }
62 | var rect = new cv.Rect(Math.max(parseInt(left_top_x), 0), Math.max(parseInt(left_top_y), 0),
63 | Math.min(parseInt(right_bottom_x - left_top_x), src_w-1), Math.min(parseInt(right_bottom_y - left_top_y), src_h-1))
64 |
65 | var face_image = new cv.Mat()
66 | face_image = image.roi(rect)
67 |
68 | var dsize = new cv.Size(128, 128);
69 | var resize_image = new cv.Mat();
70 | cv.resize(face_image, resize_image, dsize);
71 |
72 | face_image.delete()
73 | return resize_image
74 | }
75 |
76 | function preprocessLiveness(img) {
77 | var cols = img.cols;
78 | var rows = img.rows;
79 | var channels = 3;
80 |
81 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
82 |
83 | for (var y = 0; y < rows; y++)
84 | for (var x = 0; x < cols; x++) {
85 | let pixel = img.ucharPtr(y, x);
86 | for (var c = 0; c < channels; c++) {
87 | var pixel_value = 0
88 | if (c === 0) // R
89 | pixel_value = pixel[c];
90 | if (c === 1) // G
91 | pixel_value = pixel[c];
92 | if (c === 2) // B
93 | pixel_value = pixel[c];
94 |
95 | img_data.set(y, x, c, pixel_value)
96 | }
97 | }
98 |
99 | var preprocesed = ndarray(new Float32Array(channels * cols * rows), [1, channels, rows, cols])
100 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
101 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
102 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
103 |
104 | return preprocesed
105 | }
106 |
107 | async function predictLiveness(session, canvas_id, bbox) {
108 | var img = cv.imread(canvas_id)
109 |
110 | var face_size = bbox.shape[0];
111 | var bbox_size = bbox.shape[1];
112 |
113 | const result = [];
114 | for (let i = 0; i < face_size; i++) {
115 | var x1 = parseInt(bbox.data[i * bbox_size]),
116 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
117 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
118 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
119 | width = Math.abs(x2 - x1),
120 | height = Math.abs(y2 - y1);
121 |
122 | var face_img = alignLivenessImage(img, [x1, y1, width, height], 2.7);
123 | //cv.imshow("live-temp", face_img);
124 | var input_image = preprocessLiveness(face_img);
125 | face_img.delete();
126 |
127 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]);
128 | input_tensor.data.set(input_image.data);
129 | const feeds = {"input": input_tensor};
130 |
131 | const output_tensor = await session.run(feeds);
132 | const score_arr = softmax(output_tensor['output'].data);
133 | console.log("Liveness result: ", score_arr);
134 |
135 | result.push([x1, y1, x2, y2, score_arr[0]]);
136 | }
137 | img.delete();
138 | return result;
139 |
140 | }
141 |
142 | async function predictLivenessBase64(session, base64Image) {
143 | let image = new Image()
144 | image.src = base64Image
145 | await new Promise(r => {
146 | image.onload = r
147 | })
148 |
149 | var img = cv.imread(image)
150 |
151 | var face_size = bbox.shape[0];
152 | var bbox_size = bbox.shape[1];
153 |
154 | const result = [];
155 | for (let i = 0; i < face_size; i++) {
156 | var x1 = parseInt(bbox.data[i * bbox_size]),
157 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
158 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
159 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
160 | width = Math.abs(x2 - x1),
161 | height = Math.abs(y2 - y1);
162 |
163 | var face_img = alignLivenessImage(img, [x1, y1, width, height], 2.7);
164 | //cv.imshow("live-temp", face_img);
165 | var input_image = preprocessLiveness(face_img);
166 | face_img.delete();
167 |
168 | const input_tensor = new Tensor("float32", new Float32Array(128 * 128 * 3), [1, 3, 128, 128]);
169 | input_tensor.data.set(input_image.data);
170 | const feeds = {"input": input_tensor};
171 |
172 | const output_tensor = await session.run(feeds);
173 | const score_arr = softmax(output_tensor['output'].data);
174 | console.log("Liveness result: ", score_arr);
175 |
176 | result.push([x1, y1, x2, y2, score_arr[0]]);
177 | }
178 | img.delete();
179 | return result;
180 |
181 | }
182 |
183 | export {loadLivenessModel, predictLiveness}
184 |
--------------------------------------------------------------------------------
/lib/fr_pose.js:
--------------------------------------------------------------------------------
1 | import {InferenceSession, Tensor} from "onnxruntime-web";
2 | import ndarray from "ndarray";
3 | import ops from "ndarray-ops";
4 |
5 | async function loadPoseModel() {
6 | var pose_session = null
7 | await InferenceSession.create("../model/fr_pose.onnx", {executionProviders: ['wasm']})
8 | .then((session) => {
9 | pose_session = session
10 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]);
11 | for (let i = 0; i < 224 * 224 * 3; i++) {
12 | input_tensor.data[i] = Math.random() * 2.0 - 1.0;
13 | }
14 | const feeds = {"input": input_tensor};
15 | const output_tensor = pose_session.run(feeds)
16 | console.log("initialize the pose session.")
17 | })
18 | return pose_session
19 | }
20 |
21 | function preprocessPose(img) {
22 | var cols = img.cols;
23 | var rows = img.rows;
24 | var channels = 3;
25 |
26 | var img_data = ndarray(new Float32Array(rows * cols * channels), [rows, cols, channels]);
27 |
28 | for (var y = 0; y < rows; y++)
29 | for (var x = 0; x < cols; x++) {
30 | let pixel = img.ucharPtr(y, x);
31 | // if(x == 0 && y == 0)
32 | // console.log(pixel);
33 | for (var c = 0; c < channels; c++) {
34 | var pixel_value = 0
35 | if (c === 0) // R
36 | pixel_value = (pixel[c] / 255.0 - 0.485) / 0.229
37 | if (c === 1) // G
38 | pixel_value = (pixel[c] / 255.0 - 0.456) / 0.224
39 | if (c === 2) // B
40 | pixel_value = (pixel[c] / 255.0 - 0.406) / 0.225
41 |
42 | img_data.set(y, x, c, pixel_value)
43 | }
44 | }
45 |
46 | var preprocesed = ndarray(new Float32Array(3 * 224 * 224), [1, 3, 224, 224])
47 | ops.assign(preprocesed.pick(0, 0, null, null), img_data.pick(null, null, 0));
48 | ops.assign(preprocesed.pick(0, 1, null, null), img_data.pick(null, null, 1));
49 | ops.assign(preprocesed.pick(0, 2, null, null), img_data.pick(null, null, 2));
50 |
51 | return preprocesed
52 | }
53 |
54 | function softmax(arr) {
55 | return arr.map(function(value, index) {
56 | return Math.exp(value) / arr.map( function(y /*value*/){ return Math.exp(y) } ).reduce( function(a,b){ return a+b })
57 | })
58 | }
59 |
60 | async function predictPose(session, canvas_id, bbox) {
61 | var face_count = bbox.shape[0],
62 | bbox_size = bbox.shape[1];
63 |
64 | var img = cv.imread(canvas_id);
65 | const result = [];
66 |
67 | for (let i = 0; i < face_count; i++) {
68 | var x1 = parseInt(bbox.data[i * bbox_size]),
69 | y1 = parseInt(bbox.data[i * bbox_size + 1]),
70 | x2 = parseInt(bbox.data[i * bbox_size + 2]),
71 | y2 = parseInt(bbox.data[i * bbox_size + 3]),
72 | width = Math.abs(x2 - x1),
73 | height = Math.abs(y2 - y1);
74 |
75 | var x11 = parseInt(x1 - width/4),
76 | y11 = parseInt(y1 - height/4),
77 | x22 = parseInt(x2 + width/4),
78 | y22 = parseInt(y2 + height/4);
79 |
80 | var rect = new cv.Rect(Math.max(x11, 0), Math.max(y11, 0), Math.min(x22 - x11, img.cols), Math.min(y22 - y11, img.rows));
81 | var face_image = new cv.Mat();
82 |
83 | face_image = img.roi(rect);
84 |
85 | var dsize = new cv.Size(224, 224);
86 | var resize_image = new cv.Mat();
87 | cv.resize(face_image, resize_image, dsize);
88 | cv.cvtColor(resize_image, resize_image, cv.COLOR_BGR2RGB);
89 |
90 | // cv.imshow("live-temp", resize_image)
91 |
92 | var input_image = preprocessPose(resize_image);
93 |
94 | const input_tensor = new Tensor("float32", new Float32Array(224 * 224 * 3), [1, 3, 224, 224]);
95 | input_tensor.data.set(input_image.data);
96 | const feeds = {"input": input_tensor};
97 |
98 | const output_tensor = await session.run(feeds);
99 |
100 | var arr = Array.apply(null, Array(66));
101 | const index_arr = arr.map(function (x, i) { return i })
102 |
103 | const yaw_arr = softmax(output_tensor['output'].data);
104 | const pitch_arr = softmax(output_tensor['617'].data);
105 | const roll_arr = softmax(output_tensor['618'].data);
106 |
107 | const yaw = yaw_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99;
108 | const pitch = pitch_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99;
109 | const roll = roll_arr.reduce(function (r, a, i){return r + a * index_arr[i]}, 0) * 3 - 99;
110 | //console.log("Pose results: ", yaw, pitch, roll)
111 | result.push([x1, y1, x2, y2, yaw.toFixed(2), pitch.toFixed(2), roll.toFixed(2)]);
112 |
113 | resize_image.delete();
114 | face_image.delete();
115 | }
116 | img.delete();
117 | return result;
118 | }
119 |
120 | export {loadPoseModel, softmax, predictPose}
--------------------------------------------------------------------------------
/lib/load_opencv.js:
--------------------------------------------------------------------------------
1 | var cv = null;
2 |
3 | async function load_opencv() {
4 | if (!window.WebAssembly) {
5 | console.log("Your web browser doesn't support WebAssembly.")
6 | return
7 | }
8 |
9 | console.log("loading OpenCv.js")
10 | const script = document.createElement("script")
11 | script.type = "text/javascript"
12 | script.async = "async"
13 | script.src = "../js/opencv.js"
14 | document.body.appendChild(script)
15 | script.onload = () => {
16 | console.log("OpenCV.js is loaded.")
17 | }
18 |
19 | window.Module = {
20 | wasmBinaryFile: `../js/opencv_js.wasm`, // for wasm mode
21 | preRun: () => {
22 | console.log('preRun function on loading opencv')
23 | },
24 | _main: () => {
25 | console.log('OpenCV.js is ready.')
26 | cv = window.cv
27 | }
28 | }
29 | return
30 | }
31 |
32 | export {load_opencv, cv}
--------------------------------------------------------------------------------
/model/fr_age.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_age.onnx
--------------------------------------------------------------------------------
/model/fr_detect.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_detect.onnx
--------------------------------------------------------------------------------
/model/fr_expression.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_expression.onnx
--------------------------------------------------------------------------------
/model/fr_eye.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_eye.onnx
--------------------------------------------------------------------------------
/model/fr_feature.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_feature.onnx
--------------------------------------------------------------------------------
/model/fr_gender.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_gender.onnx
--------------------------------------------------------------------------------
/model/fr_landmark.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_landmark.onnx
--------------------------------------------------------------------------------
/model/fr_liveness.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_liveness.onnx
--------------------------------------------------------------------------------
/model/fr_pose.onnx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Faceplugin-ltd/FaceRecognition-LivenessDetection-Javascript/469ecf9e773e14b0c9834143f8fbc504b2cec5e0/model/fr_pose.onnx
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "kby-face",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "jest",
8 | "build": "webpack"
9 | },
10 | "repository": {
11 | "type": "git",
12 | "url": "git+https://github.com/kby-ai/FaceRecognition-Javascript.git"
13 | },
14 | "keywords": [],
15 | "license": "MIT",
16 | "bugs": {
17 | "url": "https://github.com/kby-ai/FaceRecognition-Javascript/issues"
18 | },
19 | "homepage": "https://github.com/kby-ai/FaceRecognition-Javascript#readme",
20 | "dependencies": {
21 | "babel-core": "^6.26.3",
22 | "babel-jest": "^27.4.2",
23 | "babel-plugin-add-module-exports": "^1.0.4",
24 | "babel-preset-env": "^1.7.0",
25 | "clean-webpack-plugin": "^4.0.0",
26 | "eslint": "^8.3.0",
27 | "jest": "^27.4.3",
28 | "terser-webpack-plugin": "^5.2.5",
29 | "ndarray": "^1.0.19",
30 | "ndarray-ops": "^1.2.2",
31 | "onnxruntime-web": "^1.14.0"
32 | },
33 | "devDependencies": {
34 | "@babel/core": "^7.18.10",
35 | "@babel/preset-env": "^7.18.10",
36 | "babel-loader": "^8.2.5",
37 | "copy-webpack-plugin": "^8.1.1",
38 | "webpack": "^5.64.4",
39 | "webpack-cli": "^4.9.1"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
1 | // webpack.config.js
2 | const path = require('path');
3 | const webpack = require('webpack');
4 | const { CleanWebpackPlugin } = require('clean-webpack-plugin');
5 | const TerserPlugin = require('terser-webpack-plugin');
6 | const CopyPlugin = require("copy-webpack-plugin");
7 |
8 |
9 | module.exports = {
10 | mode: 'production',
11 | devtool: false,
12 | entry: './index.js',
13 | target: ['web'],
14 | output: {
15 | path: path.resolve(__dirname, './dist'),
16 | filename: 'facerecognition-sdk.js',
17 | // globalObject: 'this',
18 | library: {
19 | type: 'umd'
20 | }
21 | },
22 |
23 | plugins: [
24 | new CleanWebpackPlugin(),
25 | // new webpack.SourceMapDevToolPlugin({
26 | // filename: 'facerecognition-sdk.js.map'
27 | // }),
28 | new CopyPlugin({
29 | // Use copy plugin to copy *.wasm to output folder.
30 | patterns: [{ from: 'node_modules/onnxruntime-web/dist/*.wasm', to: '[name][ext]' }]
31 | })
32 | ],
33 | module: {
34 | rules: [
35 | {
36 | test: /\.(js)$/,
37 | exclude: /node_modules/,
38 | use: "babel-loader",
39 | },
40 | ],
41 | },
42 | }
43 |
--------------------------------------------------------------------------------