├── .github └── FUNDING.yml ├── .gitignore ├── .npmignore ├── KnownIssues.md ├── LICENSE ├── README.md ├── custom_tfjs ├── custom_tfjs.js └── custom_tfjs_core.js ├── custom_tfjs_config.json ├── examples ├── face-tracking │ ├── assets │ │ ├── canonical_face_model_uv_visualization.png │ │ ├── earring │ │ │ ├── license.txt │ │ │ ├── scene.bin │ │ │ ├── scene.gltf │ │ │ ├── textures │ │ │ │ ├── PEARL_DROP_EARRINGSG_baseColor.png │ │ │ │ ├── PEARL_DROP_EARRINGSG_metallicRoughness.png │ │ │ │ └── PEARL_DROP_EARRINGSG_normal.png │ │ │ └── thumbnail.png │ │ ├── face1.jpeg │ │ ├── glasses │ │ │ ├── license.txt │ │ │ ├── scene.bin │ │ │ ├── scene.gltf │ │ │ └── thumbnail.png │ │ ├── glasses2 │ │ │ ├── scene.bin │ │ │ ├── scene.gltf │ │ │ └── thumbnail.png │ │ ├── hat │ │ │ ├── license.txt │ │ │ ├── scene.bin │ │ │ ├── scene.gltf │ │ │ └── thumbnail.png │ │ ├── hat2 │ │ │ ├── license.txt │ │ │ ├── scene.bin │ │ │ ├── scene.gltf │ │ │ ├── textures │ │ │ │ ├── 21_-_Default_baseColor.png │ │ │ │ └── 21_-_Default_normal.png │ │ │ └── thumbnail.png │ │ └── sparkar │ │ │ ├── face-mask.png │ │ │ ├── face-mesk-license.txt │ │ │ ├── headOccluder.glb │ │ │ └── licenses.txt │ ├── debug.html │ ├── example1.html │ ├── example2.html │ ├── example3.html │ ├── three-blendshapes.html │ ├── three-css.html │ ├── three-facemesh.html │ └── three.html ├── image-tracking │ ├── assets │ │ ├── band-example │ │ │ ├── band.mind │ │ │ ├── bear-color.png │ │ │ ├── bear.png │ │ │ ├── bear │ │ │ │ ├── scene.bin │ │ │ │ ├── scene.gltf │ │ │ │ └── textures │ │ │ │ │ ├── Cello_paint_baseColor.png │ │ │ │ │ └── bear_paint_baseColor.png │ │ │ ├── new-raccoon.mind │ │ │ ├── pictarize.mind │ │ │ ├── raccoon-card.mind │ │ │ ├── raccoon-color.png │ │ │ ├── raccoon-highres.mind │ │ │ ├── raccoon.mind │ │ │ ├── raccoon.png │ │ │ └── raccoon │ │ │ │ ├── scene.bin │ │ │ │ ├── scene.gltf │ │ │ │ └── textures │ │ │ │ ├── drum_defaultCol_baseColor.png │ │ │ │ ├── drum_paint_baseColor.png │ │ │ │ └── raccoon_paint_baseColor.png │ │ ├── card-example │ │ │ ├── card.mind │ │ │ ├── card.png │ │ │ ├── softmind │ │ │ │ ├── scene.bin │ │ │ │ ├── scene.gltf │ │ │ │ └── textures │ │ │ │ │ ├── material_baseColor.png │ │ │ │ │ ├── material_emissive.png │ │ │ │ │ ├── material_metallicRoughness.png │ │ │ │ │ └── material_normal.png │ │ │ └── test-card.png │ │ └── pictarize-example │ │ │ ├── pictarize.jpg │ │ │ └── pictarize.mind │ ├── compile.html │ ├── debug.html │ ├── example1.html │ ├── example2.html │ ├── example3.html │ ├── example4.html │ └── three.html └── nodejs │ └── createImageTargetLibrary.js ├── index.html ├── index.js ├── package-lock.json ├── package.json ├── src ├── face-target │ ├── aframe.js │ ├── controller.js │ ├── face-geometry │ │ ├── canonical-face-model.obj │ │ ├── estimator.js │ │ ├── face-data-generator.js │ │ ├── face-data.js │ │ └── face-geometry.js │ ├── face-mesh-helper.js │ ├── index.js │ └── three.js ├── image-target │ ├── aframe.js │ ├── compiler-base.js │ ├── compiler.js │ ├── compiler.worker.js │ ├── controller.js │ ├── controller.worker.js │ ├── detector │ │ ├── crop-detector.js │ │ ├── detector.js │ │ ├── freak.js │ │ └── kernels │ │ │ ├── cpu │ │ │ ├── binomialFilter.js │ │ │ ├── buildExtremas.js │ │ │ ├── computeExtremaAngles.js │ │ │ ├── computeExtremaFreak.js │ │ │ ├── computeFreakDescriptors.js │ │ │ ├── computeLocalization.js │ │ │ ├── computeOrientationHistograms.js │ │ │ ├── downsampleBilinear.js │ │ │ ├── extremaReduction.js │ │ │ ├── fakeShader.js │ │ │ ├── index.js │ │ │ ├── prune.js │ │ │ ├── smoothHistograms.js │ │ │ └── upsampleBilinear.js │ │ │ ├── index.js │ │ │ └── webgl │ │ │ ├── binomialFilter.js │ │ │ ├── buildExtremas.js │ │ │ ├── computeExtremaAngles.js │ │ │ ├── computeExtremaFreak.js │ │ │ ├── computeFreakDescriptors.js │ │ │ ├── computeLocalization.js │ │ │ ├── computeOrientationHistograms.js │ │ │ ├── downsampleBilinear.js │ │ │ ├── extremaReduction.js │ │ │ ├── index.js │ │ │ ├── smoothHistograms.js │ │ │ └── upsampleBilinear.js │ ├── estimation │ │ ├── esimate-experiment.js │ │ ├── estimate.js │ │ ├── estimator.js │ │ ├── refine-estimate-experiment.js │ │ ├── refine-estimate.js │ │ └── utils.js │ ├── image-list.js │ ├── index.js │ ├── input-loader.js │ ├── matching │ │ ├── hamming-distance.js │ │ ├── hierarchical-clustering.js │ │ ├── hough.js │ │ ├── matcher.js │ │ ├── matching.js │ │ └── ransacHomography.js │ ├── offline-compiler.js │ ├── three.js │ ├── tracker │ │ ├── extract-utils.js │ │ ├── extract.js │ │ └── tracker.js │ └── utils │ │ ├── cumsum.js │ │ ├── geometry.js │ │ ├── homography.js │ │ ├── images.js │ │ └── randomizer.js ├── libs │ ├── one-euro-filter.js │ ├── opencv-helper.js │ ├── opencv-wasm.js │ └── opencv.js └── ui │ ├── compatibility.html │ ├── loading.html │ ├── scanning.html │ ├── ui.js │ └── ui.scss ├── testing ├── common.js ├── detection.html ├── detection.js ├── glsl │ ├── applyFilter.js │ ├── applyFilter2.js │ ├── glsl.html │ ├── loadImageToTensor.js │ ├── loadImageToTensor2.js │ ├── packKernel.js │ └── randomImage.js ├── homography.html ├── homography.js ├── matching-all.html ├── matching-all.js ├── matching.html ├── matching.js ├── mobile-console.css ├── mobile-console.js ├── speed.html ├── speed.js ├── tracking.html └── tracking.js ├── vite.config.dev.js └── vite.config.prod.js /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: hiukim 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | tests 3 | *.swp 4 | dist-dev 5 | dist/ 6 | *.DS_Store 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | testing/ 2 | examples/ 3 | dist-dev/ 4 | custom_tfjs_config.json 5 | custom_tfjs/ 6 | vite.config.* 7 | mind-ar-*.tgz -------------------------------------------------------------------------------- /KnownIssues.md: -------------------------------------------------------------------------------- 1 | Mac (node v18) failed to install node-canvas 2 | 3 | > brew install pkg-config cairo pango libpng jpeg giflib librsvg pixman 4 | 5 | Ref: https://github.com/Automattic/node-canvas/issues/2025 6 | 7 | 8 | Import MindAR in ESBuild 9 | 10 | If you are using esbuild and running into require errors, you can try importing the classes directly 11 | 12 | Example: 13 | 14 | `import {MindARThree} from 'mind-ar/src/image-target/three.js'` 15 | 16 | You'll need to use the esbuild-plugin-inline-worker 17 | 18 | Ref issue: https://github.com/hiukim/mind-ar-js/issues/360 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 hiukim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /custom_tfjs/custom_tfjs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2022 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | // This file is autogenerated. 19 | 20 | 21 | import {registerKernel} from '@tensorflow/tfjs-core/dist/base'; 22 | import '@tensorflow/tfjs-core/dist/base_side_effects'; 23 | export * from '@tensorflow/tfjs-core/dist/base'; 24 | 25 | //backend = webgl 26 | export * from '@tensorflow/tfjs-backend-webgl/dist/base'; 27 | import {reshapeConfig as Reshape_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Reshape'; 28 | registerKernel(Reshape_webgl); 29 | import {subConfig as Sub_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Sub'; 30 | registerKernel(Sub_webgl); 31 | import {castConfig as Cast_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Cast'; 32 | registerKernel(Cast_webgl); 33 | import {packConfig as Pack_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Pack'; 34 | registerKernel(Pack_webgl); -------------------------------------------------------------------------------- /custom_tfjs/custom_tfjs_core.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2022 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | // This file is autogenerated. 19 | 20 | 21 | import {registerKernel} from '@tensorflow/tfjs-core/dist/base'; 22 | import '@tensorflow/tfjs-core/dist/base_side_effects'; 23 | export * from '@tensorflow/tfjs-core/dist/base'; -------------------------------------------------------------------------------- /custom_tfjs_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "kernels": ["Reshape", 3 | "Sub", 4 | "Cast", 5 | "Pack" 6 | ], 7 | "backends": [ 8 | "webgl" 9 | ], 10 | "models": [ 11 | 12 | ], 13 | "outputPath": "./custom_tfjs", 14 | "forwardModeOnly": true 15 | } -------------------------------------------------------------------------------- /examples/face-tracking/assets/canonical_face_model_uv_visualization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/canonical_face_model_uv_visualization.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/license.txt: -------------------------------------------------------------------------------- 1 | Model Information: 2 | * title: Earring Jade 3 | * source: https://sketchfab.com/3d-models/earring-jade-c91e552728fb4b149d52193fd3e03ac5 4 | * author: Softmind Game Factory (https://sketchfab.com/softmind) 5 | 6 | Model License: 7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) 8 | * requirements: Author must be credited. Commercial use is allowed. 9 | 10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it: 11 | This work is based on "Earring Jade" (https://sketchfab.com/3d-models/earring-jade-c91e552728fb4b149d52193fd3e03ac5) by Softmind Game Factory (https://sketchfab.com/softmind) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/scene.bin -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_baseColor.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_metallicRoughness.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_metallicRoughness.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_normal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_normal.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/earring/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/thumbnail.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/face1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/face1.jpeg -------------------------------------------------------------------------------- /examples/face-tracking/assets/glasses/license.txt: -------------------------------------------------------------------------------- 1 | Model Information: 2 | * title: Thug Life | Cool glasses | Stylise goggles 3 | * source: https://sketchfab.com/3d-models/thug-life-cool-glasses-stylise-goggles-75945f055a8c47098fa8aa4b6a698b07 4 | * author: MR EXPERT (https://sketchfab.com/Rahul.Kumar5) 5 | 6 | Model License: 7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) 8 | * requirements: Author must be credited. Commercial use is allowed. 9 | 10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it: 11 | This work is based on "Thug Life | Cool glasses | Stylise goggles" (https://sketchfab.com/3d-models/thug-life-cool-glasses-stylise-goggles-75945f055a8c47098fa8aa4b6a698b07) by MR EXPERT (https://sketchfab.com/Rahul.Kumar5) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) -------------------------------------------------------------------------------- /examples/face-tracking/assets/glasses/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses/scene.bin -------------------------------------------------------------------------------- /examples/face-tracking/assets/glasses/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses/thumbnail.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/glasses2/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses2/scene.bin -------------------------------------------------------------------------------- /examples/face-tracking/assets/glasses2/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses2/thumbnail.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat/license.txt: -------------------------------------------------------------------------------- 1 | Model Information: 2 | * title: Clown Hat 3 | * source: https://sketchfab.com/3d-models/clown-hat-0f5ef2e6205c4b438c593a979a3ccdae 4 | * author: PatelDev (https://sketchfab.com/PatelDev) 5 | 6 | Model License: 7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) 8 | * requirements: Author must be credited. Commercial use is allowed. 9 | 10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it: 11 | This work is based on "Clown Hat" (https://sketchfab.com/3d-models/clown-hat-0f5ef2e6205c4b438c593a979a3ccdae) by PatelDev (https://sketchfab.com/PatelDev) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/) -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat/scene.bin -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat/thumbnail.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat2/license.txt: -------------------------------------------------------------------------------- 1 | credit: https://sketchfab.com/3d-models/marios-cap-super-mario-bros-82bcc48237ec4ff98ce770de60913d26 2 | -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat2/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/scene.bin -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat2/textures/21_-_Default_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/textures/21_-_Default_baseColor.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat2/textures/21_-_Default_normal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/textures/21_-_Default_normal.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/hat2/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/thumbnail.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/sparkar/face-mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/sparkar/face-mask.png -------------------------------------------------------------------------------- /examples/face-tracking/assets/sparkar/face-mesk-license.txt: -------------------------------------------------------------------------------- 1 | LICENSE 2 | Copyright (c) 2018-present, Facebook, Inc. All rights reserved. 3 | 4 | You are hereby granted a non-exclusive, worldwide, royalty-free license to use, 5 | copy, modify, and distribute this software in source code or binary form for use 6 | in connection with the web services and APIs provided by Facebook. 7 | 8 | As with any software that integrates with the Facebook platform, your use of 9 | this software is subject to the Facebook Developer Principles and Policies 10 | [http://developers.facebook.com/policy/]. This copyright notice shall be 11 | included in all copies or substantial portions of the software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /examples/face-tracking/assets/sparkar/headOccluder.glb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/sparkar/headOccluder.glb -------------------------------------------------------------------------------- /examples/face-tracking/assets/sparkar/licenses.txt: -------------------------------------------------------------------------------- 1 | Model Information: 2 | * title: Head Occluder 3 | * source: https://sparkar.facebook.com/ar-studio/learn/articles/people-tracking/face-reference-assets 4 | * author: Facebook 5 | 6 | This model is from Facebook SparkAR, please check out licensing agreement with them if you want to use it. 7 | -------------------------------------------------------------------------------- /examples/face-tracking/debug.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /examples/face-tracking/example1.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 24 | 25 | 26 | 27 |
28 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /examples/face-tracking/example2.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 36 | 37 | 66 | 67 | 68 | 69 |
70 |
71 | 72 | 73 | 74 | 75 | 76 |
77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 |
119 | 120 | 121 | -------------------------------------------------------------------------------- /examples/face-tracking/example3.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 51 | 52 | 76 | 77 | 78 | 79 |
80 |
81 | 82 | 83 | 84 |
85 |
86 | 87 |
88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 |
96 | 97 | 98 | -------------------------------------------------------------------------------- /examples/face-tracking/three-blendshapes.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 48 | 49 | 75 | 76 | 77 | 78 |
79 |
80 | 81 |
82 |
83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /examples/face-tracking/three-css.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 24 | 25 | 51 | 52 | 53 |
54 |
55 | 56 |
57 |
58 | This is a CSS div 59 |
60 | 61 |
62 |
63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /examples/face-tracking/three-facemesh.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 30 | 31 | 43 | 44 | 45 | 46 |
47 |
48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /examples/face-tracking/three.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 86 | 87 | 106 | 107 | 108 | 109 |
110 | 111 | 112 | 113 | 114 | Front Camera 115 | 118 | Back Camera 119 | 122 |
123 | 124 |
125 |
126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/band.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/band.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/bear-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear-color.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/bear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/bear/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/scene.bin -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/bear/textures/Cello_paint_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/textures/Cello_paint_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/bear/textures/bear_paint_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/textures/bear_paint_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/new-raccoon.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/new-raccoon.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/pictarize.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/pictarize.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon-card.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-card.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-color.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon-highres.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-highres.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/scene.bin -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon/textures/drum_defaultCol_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/drum_defaultCol_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon/textures/drum_paint_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/drum_paint_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/band-example/raccoon/textures/raccoon_paint_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/raccoon_paint_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/card.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/card.mind -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/card.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/card.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/softmind/scene.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/scene.bin -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/softmind/textures/material_baseColor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_baseColor.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/softmind/textures/material_emissive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_emissive.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/softmind/textures/material_metallicRoughness.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_metallicRoughness.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/softmind/textures/material_normal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_normal.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/card-example/test-card.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/test-card.png -------------------------------------------------------------------------------- /examples/image-tracking/assets/pictarize-example/pictarize.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/pictarize-example/pictarize.jpg -------------------------------------------------------------------------------- /examples/image-tracking/assets/pictarize-example/pictarize.mind: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/pictarize-example/pictarize.mind -------------------------------------------------------------------------------- /examples/image-tracking/debug.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /examples/image-tracking/example1.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 22 | 23 | 24 | 25 |
26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 40 | 41 | 42 |
43 | 44 | 45 | -------------------------------------------------------------------------------- /examples/image-tracking/example2.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 30 | 31 | 57 | 58 | 59 |
60 |
61 | 62 | 63 | 64 | 65 |
66 |
67 | 68 |
69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 |
85 | 86 | 87 | -------------------------------------------------------------------------------- /examples/image-tracking/example4.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 22 | 23 | 24 | 25 |
26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 40 | 41 | 42 |
43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /examples/image-tracking/three.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 92 | 93 | 112 | 113 | 114 | 115 |
116 | 117 | 118 | 119 | 120 | Front Camera 121 | 124 | Back Camera 125 | 128 |
129 | 130 |
131 |
132 | 133 | 134 | 135 | -------------------------------------------------------------------------------- /examples/nodejs/createImageTargetLibrary.js: -------------------------------------------------------------------------------- 1 | import { OfflineCompiler } from '../../src/image-target/offline-compiler.js'; 2 | 3 | import { writeFile } from 'fs/promises' 4 | import { loadImage } from 'canvas'; 5 | 6 | //canvas.loadImage treats path as relative from where nodejs was executed, not relative to the script's location 7 | const imagePaths = ['examples/image-tracking/assets/card-example/card.png']; 8 | 9 | async function run() { 10 | //load all images 11 | const images = await Promise.all(imagePaths.map(value => loadImage(value))); 12 | const compiler = new OfflineCompiler(); 13 | await compiler.compileImageTargets(images, console.log); 14 | const buffer = compiler.exportData(); 15 | await writeFile('targets.mind', buffer); 16 | } 17 | 18 | 19 | 20 | 21 | run(); -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |

Examples

8 |

Image-Tracking

9 | 18 |

Face-Tracking

19 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = {}; 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mind-ar", 3 | "version": "1.2.5", 4 | "description": "web augmented reality framework", 5 | "main": "index.js", 6 | "scripts": { 7 | "dev": "vite --config vite.config.dev.js --host", 8 | "watch": "vite build --watch --config vite.config.dev.js", 9 | "build-dev": "vite build --config vite.config.dev.js", 10 | "build": "vite build --config vite.config.prod.js" 11 | }, 12 | "engines": { 13 | "node": ">=14.0.0" 14 | }, 15 | "type": "module", 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/hiukim/mind-ar-js" 19 | }, 20 | "keywords": [ 21 | "web AR", 22 | "aframe", 23 | "web augmented reality", 24 | "image tracking", 25 | "face tracking", 26 | "tensorflowjs", 27 | "mediapipe" 28 | ], 29 | "author": "hiukim", 30 | "license": "MIT", 31 | "dependencies": { 32 | "@mediapipe/tasks-vision": "^0.10.9", 33 | "@msgpack/msgpack": "^2.8.0", 34 | "@tensorflow/tfjs": "^4.16.0", 35 | "@vitejs/plugin-basic-ssl": "^1.1.0", 36 | "canvas": "^2.10.2", 37 | "mathjs": "^11.6.0", 38 | "ml-matrix": "^6.10.4", 39 | "svd-js": "^1.1.1", 40 | "tinyqueue": "^2.0.3" 41 | }, 42 | "peerDependencies": { 43 | "three": ">=0.136.0" 44 | }, 45 | "files": [ 46 | "dist/", 47 | "src/" 48 | ], 49 | "devDependencies": { 50 | "vite": "^5.0.11" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/face-target/face-geometry/face-data-generator.js: -------------------------------------------------------------------------------- 1 | // canonical-face-model.obj source: 2 | // https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data 3 | // 4 | // this script parse the model data. To run: node face-data-generator.js 5 | 6 | import * as fs from "fs"; 7 | const text = fs.readFileSync("./canonical-face-model.obj", "utf8"); 8 | const textByLine = text.split("\n") 9 | 10 | const positions = []; 11 | const uvs = []; 12 | const faces = []; 13 | const uvIndexes = []; 14 | 15 | textByLine.forEach((line) => { 16 | const ss = line.split(" "); 17 | if (ss[0] === 'f') { 18 | for (let i = 1; i <= 3; i++) { 19 | const [index, uvIndex] = ss[i].split("/"); 20 | uvIndexes[parseInt(uvIndex)-1] = parseInt(index)-1; 21 | } 22 | } 23 | }); 24 | 25 | let uvCount = 0; 26 | textByLine.forEach((line) => { 27 | const ss = line.split(" "); 28 | if (ss[0] === 'v') { 29 | positions.push([ parseFloat(ss[1]),parseFloat(ss[2]),parseFloat(ss[3]) ]); 30 | } else if (ss[0] === 'vt') { 31 | uvs[uvIndexes[uvCount++]] = [ parseFloat(ss[1]),parseFloat(ss[2]) ]; 32 | } else if (ss[0] === 'f') { 33 | faces.push(parseInt(ss[1].split("/")[0]-1),parseInt(ss[2].split("/")[0]-1),parseInt(ss[3].split("/")[0])-1); 34 | } 35 | }); 36 | 37 | // important landmarks for computing transformation 38 | // pairs of [positionIndex, weight] 39 | const landmarkBasis = [[4, 0.070909939706326], [6, 0.032100144773722], [10, 0.008446550928056], [33, 0.058724168688059], [54, 0.007667080033571], [67, 0.009078059345484], [117, 0.009791937656701], [119, 0.014565368182957], [121, 0.018591361120343], [127, 0.005197994410992], [129, 0.120625205338001], [132, 0.005560018587857], [133, 0.05328618362546], [136, 0.066890455782413], [143, 0.014816547743976], [147, 0.014262833632529], [198, 0.025462191551924], [205, 0.047252278774977], [263, 0.058724168688059], [284, 0.007667080033571], [297, 0.009078059345484], [346, 0.009791937656701], [348, 0.014565368182957], [350, 0.018591361120343], [356, 0.005197994410992], [358, 0.120625205338001], [361, 0.005560018587857], [362, 0.05328618362546], [365, 0.066890455782413], [372, 0.014816547743976], [376, 0.014262833632529], [420, 0.025462191551924], [425, 0.047252278774977]] 40 | 41 | let output = ""; 42 | output += "//This is a generated file\n"; 43 | output += "const positions=" + JSON.stringify(positions) + ";\n"; 44 | output += "const uvs=" + JSON.stringify(uvs) + ";\n"; 45 | output += "const faces=" + JSON.stringify(faces) + ";\n"; 46 | output += "const landmarkBasis=" + JSON.stringify(landmarkBasis) + ";\n"; 47 | 48 | output += ` 49 | export { 50 | positions, uvs, faces, landmarkBasis 51 | } 52 | ` 53 | 54 | fs.writeFileSync("./face-data.js", output); 55 | -------------------------------------------------------------------------------- /src/face-target/face-geometry/face-geometry.js: -------------------------------------------------------------------------------- 1 | import {uvs, faces} from "./face-data.js"; 2 | 3 | const nLandmarks = uvs.length; 4 | 5 | const createThreeFaceGeometry = (THREE) => { 6 | class FaceGeometry extends THREE.BufferGeometry { 7 | constructor(options = {}) { 8 | super(); 9 | 10 | this.positions = new Float32Array(nLandmarks * 3); 11 | this.uvs = new Float32Array(nLandmarks * 2); 12 | this.setAttribute("position", new THREE.BufferAttribute(this.positions, 3)); 13 | this.setAttribute("uv", new THREE.BufferAttribute(this.uvs, 2)); 14 | this.setUvs(); 15 | this.setIndex(faces); 16 | } 17 | 18 | setUvs() { 19 | for (let j = 0; j < nLandmarks; j++) { 20 | this.uvs[j * 2] = uvs[j][0]; 21 | this.uvs[j * 2 + 1] = uvs[j][1]; 22 | } 23 | this.getAttribute("uv").needsUpdate = true; 24 | } 25 | 26 | updatePositions(landmarks) { 27 | for (let i = 0; i < nLandmarks; i++) { 28 | this.positions[i*3+0] = landmarks[i][0]; 29 | this.positions[i*3+1] = landmarks[i][1]; 30 | this.positions[i*3+2] = landmarks[i][2]; 31 | } 32 | this.attributes.position.needsUpdate = true; 33 | this.computeVertexNormals(); 34 | } 35 | } 36 | return new FaceGeometry(); 37 | } 38 | 39 | export { 40 | createThreeFaceGeometry 41 | } 42 | -------------------------------------------------------------------------------- /src/face-target/face-mesh-helper.js: -------------------------------------------------------------------------------- 1 | import * as vision from "@mediapipe/tasks-vision"; 2 | //import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3"; 3 | 4 | class FaceMeshHelper { 5 | constructor() { 6 | } 7 | 8 | async init() { 9 | const filesetResolver = await vision.FilesetResolver.forVisionTasks( 10 | "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.9/wasm" 11 | //"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" 12 | ); 13 | this.faceLandmarker = await vision.FaceLandmarker.createFromOptions(filesetResolver, { 14 | baseOptions: { 15 | modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`, 16 | delegate: "GPU" 17 | }, 18 | outputFaceBlendshapes: true, 19 | // outputFacialTransformationMatrixes: true, 20 | runningMode: "IMAGE", 21 | numFaces: 1 22 | }); 23 | } 24 | 25 | async detect(input) { 26 | const faceLandmarkerResult = this.faceLandmarker.detect(input); 27 | return faceLandmarkerResult; 28 | } 29 | } 30 | 31 | export { 32 | FaceMeshHelper 33 | } 34 | -------------------------------------------------------------------------------- /src/face-target/index.js: -------------------------------------------------------------------------------- 1 | import {Controller} from './controller.js'; 2 | import {UI} from '../ui/ui.js'; 3 | 4 | const e = { 5 | Controller, 6 | UI 7 | } 8 | 9 | if (!window.MINDAR) { 10 | window.MINDAR = {}; 11 | } 12 | 13 | window.MINDAR.FACE = e; 14 | 15 | 16 | export {Controller,UI} -------------------------------------------------------------------------------- /src/image-target/compiler.js: -------------------------------------------------------------------------------- 1 | import {CompilerBase} from './compiler-base.js' 2 | import CompilerWorker from "./compiler.worker.js?worker&inline"; 3 | 4 | export class Compiler extends CompilerBase { 5 | createProcessCanvas(img) { 6 | const processCanvas = document.createElement('canvas'); 7 | processCanvas.width = img.width; 8 | processCanvas.height = img.height; 9 | return processCanvas; 10 | } 11 | 12 | compileTrack({progressCallback, targetImages, basePercent}) { 13 | return new Promise((resolve, reject) => { 14 | const worker = new CompilerWorker(); 15 | worker.onmessage = (e) => { 16 | if (e.data.type === 'progress') { 17 | progressCallback(basePercent + e.data.percent * basePercent/100); 18 | } else if (e.data.type === 'compileDone') { 19 | resolve(e.data.list); 20 | } 21 | }; 22 | worker.postMessage({ type: 'compile', targetImages }); 23 | }); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/image-target/compiler.worker.js: -------------------------------------------------------------------------------- 1 | import { extractTrackingFeatures } from './tracker/extract-utils.js'; 2 | import { buildTrackingImageList } from './image-list.js'; 3 | 4 | onmessage = (msg) => { 5 | const { data } = msg; 6 | if (data.type === 'compile') { 7 | //console.log("worker compile..."); 8 | const { targetImages } = data; 9 | const percentPerImage = 100.0 / targetImages.length; 10 | let percent = 0.0; 11 | const list = []; 12 | for (let i = 0; i < targetImages.length; i++) { 13 | const targetImage = targetImages[i]; 14 | const imageList = buildTrackingImageList(targetImage); 15 | const percentPerAction = percentPerImage / imageList.length; 16 | 17 | //console.log("compiling tracking...", i); 18 | const trackingData = extractTrackingFeatures(imageList, (index) => { 19 | //console.log("done tracking", i, index); 20 | percent += percentPerAction 21 | postMessage({ type: 'progress', percent }); 22 | }); 23 | list.push(trackingData); 24 | } 25 | postMessage({ 26 | type: 'compileDone', 27 | list, 28 | }); 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /src/image-target/controller.worker.js: -------------------------------------------------------------------------------- 1 | import { Matcher } from './matching/matcher.js'; 2 | import { Estimator } from './estimation/estimator.js'; 3 | 4 | let projectionTransform = null; 5 | let matchingDataList = null; 6 | let debugMode = false; 7 | let matcher = null; 8 | let estimator = null; 9 | 10 | onmessage = (msg) => { 11 | const { data } = msg; 12 | 13 | switch (data.type) { 14 | case "setup": 15 | projectionTransform = data.projectionTransform; 16 | matchingDataList = data.matchingDataList; 17 | debugMode = data.debugMode; 18 | matcher = new Matcher(data.inputWidth, data.inputHeight, debugMode); 19 | estimator = new Estimator(data.projectionTransform); 20 | break; 21 | 22 | case "match": 23 | const interestedTargetIndexes = data.targetIndexes; 24 | 25 | let matchedTargetIndex = -1; 26 | let matchedModelViewTransform = null; 27 | let matchedDebugExtra = null; 28 | 29 | for (let i = 0; i < interestedTargetIndexes.length; i++) { 30 | const matchingIndex = interestedTargetIndexes[i]; 31 | 32 | const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(matchingDataList[matchingIndex], data.featurePoints); 33 | matchedDebugExtra = debugExtra; 34 | 35 | if (keyframeIndex !== -1) { 36 | const modelViewTransform = estimator.estimate({ screenCoords, worldCoords }); 37 | 38 | if (modelViewTransform) { 39 | matchedTargetIndex = matchingIndex; 40 | matchedModelViewTransform = modelViewTransform; 41 | } 42 | break; 43 | } 44 | } 45 | 46 | postMessage({ 47 | type: 'matchDone', 48 | targetIndex: matchedTargetIndex, 49 | modelViewTransform: matchedModelViewTransform, 50 | debugExtra: matchedDebugExtra 51 | }); 52 | break; 53 | 54 | case 'trackUpdate': 55 | const { modelViewTransform, worldCoords, screenCoords } = data; 56 | const finalModelViewTransform = estimator.refineEstimate({ initialModelViewTransform: modelViewTransform, worldCoords, screenCoords }); 57 | postMessage({ 58 | type: 'trackUpdateDone', 59 | modelViewTransform: finalModelViewTransform, 60 | }); 61 | break; 62 | 63 | case "dispose": 64 | close(); 65 | break; 66 | 67 | default: 68 | throw new Error(`Invalid message type '${data.type}'`); 69 | } 70 | }; 71 | 72 | -------------------------------------------------------------------------------- /src/image-target/detector/crop-detector.js: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs'; 2 | import {Detector} from './detector.js'; 3 | import {buildModelViewProjectionTransform, computeScreenCoordiate} from '../estimation/utils.js'; 4 | 5 | class CropDetector { 6 | constructor(width, height, debugMode=false) { 7 | this.debugMode = debugMode; 8 | this.width = width; 9 | this.height = height; 10 | 11 | // nearest power of 2, min dimensions 12 | let minDimension = Math.min(width, height) / 2; 13 | let cropSize = Math.pow( 2, Math.round( Math.log( minDimension ) / Math.log( 2 ) ) ); 14 | this.cropSize = cropSize; 15 | 16 | this.detector = new Detector(cropSize, cropSize, debugMode); 17 | 18 | this.kernelCaches = {}; 19 | this.lastRandomIndex = 4; 20 | } 21 | 22 | detect(inputImageT) { // crop center 23 | const startY = Math.floor(this.height / 2 - this.cropSize / 2); 24 | const startX = Math.floor(this.width / 2 - this.cropSize / 2); 25 | const result = this._detect(inputImageT, startX, startY); 26 | 27 | if (this.debugMode) { 28 | result.debugExtra.crop = {startX, startY, cropSize: this.cropSize}; 29 | } 30 | return result; 31 | } 32 | 33 | detectMoving(inputImageT) { // loop a few locations around center 34 | const dx = this.lastRandomIndex % 3; 35 | const dy = Math.floor(this.lastRandomIndex / 3); 36 | 37 | let startY = Math.floor(this.height / 2 - this.cropSize + dy * this.cropSize / 2); 38 | let startX = Math.floor(this.width / 2 - this.cropSize + dx * this.cropSize / 2); 39 | 40 | if (startX < 0) startX = 0; 41 | if (startY < 0) startY = 0; 42 | if (startX >= this.width - this.cropSize) startX = this.width - this.cropSize - 1; 43 | if (startY >= this.height - this.cropSize) startY = this.height - this.cropSize - 1; 44 | 45 | this.lastRandomIndex = (this.lastRandomIndex + 1) % 9; 46 | 47 | const result = this._detect(inputImageT, startX, startY); 48 | return result; 49 | } 50 | 51 | _detect(inputImageT, startX, startY) { 52 | const cropInputImageT = inputImageT.slice([startY, startX], [this.cropSize, this.cropSize]); 53 | const {featurePoints, debugExtra} = this.detector.detect(cropInputImageT); 54 | featurePoints.forEach((p) => { 55 | p.x += startX; 56 | p.y += startY; 57 | }); 58 | if (this.debugMode) { 59 | debugExtra.projectedImage = cropInputImageT.arraySync(); 60 | } 61 | cropInputImageT.dispose(); 62 | return {featurePoints: featurePoints, debugExtra}; 63 | } 64 | } 65 | 66 | export { 67 | CropDetector 68 | }; 69 | -------------------------------------------------------------------------------- /src/image-target/detector/freak.js: -------------------------------------------------------------------------------- 1 | // 37 points = 6 rings x 6 points per ring + 1 center 2 | const FREAK_RINGS = [ 3 | // ring 5 4 | { 5 | sigma: 0.550000, 6 | points: [ 7 | [-1.000000, 0.000000], 8 | [-0.500000, -0.866025], 9 | [0.500000, -0.866025], 10 | [1.000000, -0.000000], 11 | [0.500000, 0.866025], 12 | [-0.500000, 0.866025] 13 | ] 14 | }, 15 | // ring 4 16 | { 17 | sigma: 0.475000, 18 | points: [ 19 | [0.000000, 0.930969], 20 | [-0.806243, 0.465485], 21 | [-0.806243, -0.465485], 22 | [-0.000000, -0.930969], 23 | [0.806243, -0.465485], 24 | [0.806243, 0.465485] 25 | ] 26 | }, 27 | // ring 3 28 | { 29 | sigma: 0.400000, 30 | points: [ 31 | [0.847306, -0.000000], 32 | [0.423653, 0.733789], 33 | [-0.423653, 0.733789], 34 | [-0.847306, 0.000000], 35 | [-0.423653, -0.733789], 36 | [0.423653, -0.733789] 37 | ] 38 | }, 39 | // ring 2 40 | { 41 | sigma: 0.325000, 42 | points: [ 43 | [-0.000000, -0.741094], 44 | [0.641806, -0.370547], 45 | [0.641806, 0.370547], 46 | [0.000000, 0.741094], 47 | [-0.641806, 0.370547], 48 | [-0.641806, -0.370547] 49 | ] 50 | }, 51 | // ring 1 52 | { 53 | sigma: 0.250000, 54 | points: [ 55 | [-0.595502, 0.000000], 56 | [-0.297751, -0.515720], 57 | [0.297751, -0.515720], 58 | [0.595502, -0.000000], 59 | [0.297751, 0.515720], 60 | [-0.297751, 0.515720] 61 | ] 62 | }, 63 | // ring 0 64 | { 65 | sigma: 0.175000, 66 | points: [ 67 | [0.000000, 0.362783], 68 | [-0.314179, 0.181391], 69 | [-0.314179, -0.181391], 70 | [-0.000000, -0.362783], 71 | [0.314179, -0.181391], 72 | [0.314179, 0.181391] 73 | ] 74 | }, 75 | // center 76 | { 77 | sigma: 0.100000, 78 | points: [ 79 | [0, 0] 80 | ] 81 | } 82 | ]; 83 | 84 | const FREAKPOINTS = []; 85 | for (let r = 0; r < FREAK_RINGS.length; r++) { 86 | const sigma = FREAK_RINGS[r].sigma; 87 | for (let i = 0; i < FREAK_RINGS[r].points.length; i++) { 88 | const point = FREAK_RINGS[r].points[i]; 89 | FREAKPOINTS.push([sigma, point[0], point[1]]); 90 | } 91 | } 92 | 93 | export { 94 | FREAKPOINTS 95 | }; 96 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/binomialFilter.js: -------------------------------------------------------------------------------- 1 | import * as FakeShader from './fakeShader.js'; 2 | 3 | 4 | function GetKernels(image) { 5 | const imageWidth = image.shape[1]; 6 | const key = 'w' + imageWidth; 7 | 8 | const imageHeight = image.shape[0]; 9 | const kernel1 = { 10 | variableNames: ['p'], 11 | outputShape: [imageHeight, imageWidth], 12 | userCode: function () { 13 | const coords = this.getOutputCoords(); 14 | 15 | let sum = this.getP(coords[0], coords[1] - 2); 16 | sum += this.getP(coords[0], coords[1] - 1) * 4.; 17 | sum += this.getP(coords[0], coords[1]) * 6.; 18 | sum += this.getP(coords[0], coords[1] + 1) * 4.; 19 | sum += this.getP(coords[0], coords[1] + 2); 20 | this.setOutput(sum); 21 | } 22 | 23 | }; 24 | const kernel2 = { 25 | variableNames: ['p'], 26 | outputShape: [imageHeight, imageWidth], 27 | userCode: function () { 28 | const coords = this.getOutputCoords(); 29 | 30 | let sum = this.getP(coords[0] - 2, coords[1]); 31 | sum += this.getP(coords[0] - 1, coords[1]) * 4.; 32 | sum += this.getP(coords[0], coords[1]) * 6.; 33 | sum += this.getP(coords[0] + 1, coords[1]) * 4.; 34 | sum += this.getP(coords[0] + 2, coords[1]); 35 | sum /= 256.; 36 | this.setOutput(sum); 37 | } 38 | 39 | }; 40 | return [kernel1, kernel2]; 41 | //} 42 | 43 | } 44 | 45 | 46 | 47 | 48 | export const binomialFilter = (args) => {//{inputs: UnaryInputs, backend: MathBackendCPU} 49 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 50 | const image = args.inputs.image; 51 | /** @type {MathBackendCPU} */ 52 | const backend = args.backend; 53 | 54 | const [kernel1, kernel2] = GetKernels(image); 55 | 56 | const result1 = FakeShader.runCode(backend, kernel1, [image], image.dtype); 57 | return FakeShader.runCode(backend, kernel2, [result1], image.dtype); 58 | } 59 | 60 | 61 | 62 | 63 | export const binomialFilterConfig = {//: KernelConfig 64 | kernelName: "BinomialFilter", 65 | backendName: 'cpu', 66 | kernelFunc: binomialFilter,// as {} as KernelFunc, 67 | }; 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/buildExtremas.js: -------------------------------------------------------------------------------- 1 | 2 | import * as FakeShader from './fakeShader.js'; 3 | import {engine} from '@tensorflow/tfjs' 4 | const FREAK_EXPANSION_FACTOR = 7.0; 5 | 6 | const LAPLACIAN_THRESHOLD = 3.0; 7 | const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD; 8 | 9 | const EDGE_THRESHOLD = 4.0; 10 | const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD + 1) * (EDGE_THRESHOLD + 1) / EDGE_THRESHOLD); 11 | 12 | function GetProgram(image) { 13 | const imageWidth = image.shape[1]; 14 | const imageHeight = image.shape[0]; 15 | const kernel = { 16 | variableNames: ['image0', 'image1', 'image2'], 17 | outputShape: [imageHeight, imageWidth], 18 | userCode: 19 | function () { 20 | const coords = this.getOutputCoords(); 21 | 22 | const y = coords[0]; 23 | const x = coords[1]; 24 | 25 | const value = this.getImage1(y, x); 26 | 27 | // Step 1: find local maxima/minima 28 | if (value * value < LAPLACIAN_SQR_THRESHOLD) { 29 | this.setOutput(0.0); 30 | return; 31 | } 32 | if (y < FREAK_EXPANSION_FACTOR || y > imageHeight - 1 - FREAK_EXPANSION_FACTOR) { 33 | this.setOutput(0.0); 34 | return; 35 | } 36 | if (x < FREAK_EXPANSION_FACTOR || x > imageWidth - 1 - FREAK_EXPANSION_FACTOR) { 37 | this.setOutput(0.0); 38 | return; 39 | } 40 | 41 | let isMax = true; 42 | let isMin = true; 43 | for (let dy = -1; dy <= 1; dy++) { 44 | for (let dx = -1; dx <= 1; dx++) { 45 | const value0 = this.getImage0(y + dy, x + dx); 46 | const value1 = this.getImage1(y + dy, x + dx); 47 | const value2 = this.getImage2(y + dy, x + dx); 48 | 49 | if (value < value0 || value < value1 || value < value2) { 50 | isMax = false; 51 | } 52 | if (value > value0 || value > value1 || value > value2) { 53 | isMin = false; 54 | } 55 | } 56 | } 57 | 58 | if (!isMax && !isMin) { 59 | this.setOutput(0.0); 60 | return; 61 | } 62 | 63 | // compute edge score and reject based on threshold 64 | const dxx = this.getImage1(y, x + 1) + this.getImage1(y, x - 1) - 2. * this.getImage1(y, x); 65 | const dyy = this.getImage1(y + 1, x) + this.getImage1(y - 1, x) - 2. * this.getImage1(y, x); 66 | const dxy = 0.25 * (this.getImage1(y - 1, x - 1) + this.getImage1(y + 1, x + 1) - this.getImage1(y - 1, x + 1) - this.getImage1(y + 1, x - 1)); 67 | 68 | const det = (dxx * dyy) - (dxy * dxy); 69 | 70 | if (Math.abs(det) < 0.0001) { // determinant undefined. no solution 71 | this.setOutput(0.0); 72 | return; 73 | } 74 | 75 | const edgeScore = (dxx + dyy) * (dxx + dyy) / det; 76 | 77 | if (Math.abs(edgeScore) >= EDGE_HESSIAN_THRESHOLD) { 78 | this.setOutput(0.0); 79 | return; 80 | } 81 | this.setOutput(this.getImage1(y, x)); 82 | } 83 | 84 | }; 85 | 86 | 87 | return kernel; 88 | } 89 | 90 | 91 | export const buildExtremas = (args) => { 92 | let { image0, image1, image2 } = args.inputs; 93 | /** @type {MathBackendCPU} */ 94 | const backend = args.backend; 95 | 96 | image0 = engine().runKernel('DownsampleBilinear', { image: image0 }); 97 | image2 = engine().runKernel('UpsampleBilinear', { image: image2, targetImage: image1 }); 98 | const program=GetProgram(image1); 99 | return FakeShader.runCode(backend,program,[image0,image1,image2],image1.dtype); 100 | } 101 | 102 | export const buildExtremasConfig = {//: KernelConfig 103 | kernelName: "BuildExtremas", 104 | backendName: 'cpu', 105 | kernelFunc: buildExtremas,// as {} as KernelFunc, 106 | }; 107 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/computeExtremaAngles.js: -------------------------------------------------------------------------------- 1 | const ORIENTATION_NUM_BINS = 36; 2 | 3 | 4 | export function computeExtremaAnglesImpl(histogram) { 5 | const resultValues = new Float32Array(histogram.height); 6 | 7 | function getHistogram(featureIndex, prev) { 8 | return histogram.values[featureIndex * histogram.width + prev] 9 | }; 10 | function setOutput(featureIndex, an) { 11 | 12 | resultValues[featureIndex] = an; 13 | } 14 | 15 | function imod(x, y) { 16 | return Math.trunc(x - y * Math.floor(x / y)); 17 | } 18 | 19 | for (let featureIndex = 0; featureIndex < histogram.height; featureIndex++) { 20 | let maxIndex = 0; 21 | for (let i = 1; i < ORIENTATION_NUM_BINS; i++) { 22 | if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) { 23 | maxIndex = i; 24 | } 25 | } 26 | let prev = imod(maxIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS); 27 | let next = imod(maxIndex + 1, ORIENTATION_NUM_BINS); 28 | /** 29 | * Fit a quatratic to 3 points. The system of equations is: 30 | * 31 | * y0 = A*x0^2 + B*x0 + C 32 | * y1 = A*x1^2 + B*x1 + C 33 | * y2 = A*x2^2 + B*x2 + C 34 | * 35 | * This system of equations is solved for A,B,C. 36 | */ 37 | const p10 = maxIndex - 1; 38 | const p11 = getHistogram(featureIndex, prev); 39 | const p20 = maxIndex; 40 | const p21 = getHistogram(featureIndex, maxIndex); 41 | const p30 = maxIndex + 1; 42 | const p31 = getHistogram(featureIndex, next); 43 | 44 | const d1 = (p30 - p20) * (p30 - p10); 45 | const d2 = (p10 - p20) * (p30 - p10); 46 | const d3 = p10 - p20; 47 | 48 | // If any of the denominators are zero then, just use maxIndex. 49 | let fbin = maxIndex; 50 | if (Math.abs(d1) > 0.00001 && Math.abs(d2) > 0.00001 && Math.abs(d3) > 0.00001) { 51 | const a = p10 * p10; 52 | const b = p20 * p20; 53 | 54 | // Solve for the coefficients A,B,C 55 | let A = ((p31 - p21) / d1) - ((p11 - p21) / d2); 56 | if(Number.isNaN(A)) A=0; 57 | const B = ((p11 - p21) + (A * (b - a))) / d3; 58 | //const C = p11 - (A * a) - (B * p10); 59 | fbin = -B / (2.0 * A); 60 | if(Number.isNaN(fbin)) fbin=0;//console.warn(`computeExtremaAngles::NaN! fbin=${fbin} maxIndex=${maxIndex} A=${A} B=${B} p31=${p31} p21=${p21} p11=${p11}`); 61 | } 62 | 63 | const an = 2.0 * Math.PI * (fbin + 0.5) / ORIENTATION_NUM_BINS - Math.PI; 64 | 65 | setOutput(featureIndex,an); 66 | } 67 | return resultValues; 68 | } 69 | 70 | export const computeExtremaAngles = (args) => { 71 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 72 | const { histograms } = args.inputs; 73 | /** @type {import('@tensorflow/tfjs-backend-cpu').MathBackendCPU} */ 74 | const backend = args.backend; 75 | 76 | /** @type {TypedArray} */ 77 | const histogramValues ={values: backend.data.get(histograms.dataId).values,width:histograms.shape[1],height:histograms.shape[0]}; 78 | 79 | const resultValues = computeExtremaAnglesImpl(histogramValues); 80 | 81 | return backend.makeOutput(resultValues, [histograms.shape[0]], histograms.dtype); 82 | } 83 | 84 | 85 | export const computeExtremaAnglesConfig = {//: KernelConfig 86 | kernelName: "ComputeExtremaAngles", 87 | backendName: 'cpu', 88 | kernelFunc: computeExtremaAngles,// as {} as KernelFunc, 89 | }; 90 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/computeExtremaFreak.js: -------------------------------------------------------------------------------- 1 | import { FREAKPOINTS } from '../../freak.js'; 2 | import * as FakeShader from './fakeShader.js'; 3 | const FREAK_EXPANSION_FACTOR = 7.0; 4 | 5 | function GetProgram(prunedExtremasHeight, pyramidImagesLength) { 6 | 7 | const imageVariableNames = []; 8 | for (let i = 1; i < pyramidImagesLength; i++) { 9 | imageVariableNames.push('image' + i); 10 | } 11 | 12 | 13 | const kernel = { 14 | variableNames: [...imageVariableNames, 'extrema', 'angles', 'freakPoints'], 15 | outputShape: [prunedExtremasHeight, FREAKPOINTS.length], 16 | userCode: function () { 17 | const getPixel=(octave, y, x)=> { 18 | const key = 'getImage' + octave; 19 | if (octave < 1 || octave >= pyramidImagesLength) 20 | return 0.0; 21 | return this[key](y, x); 22 | } 23 | const coords = this.getOutputCoords(); 24 | const featureIndex = coords[0]; 25 | const freakIndex = coords[1]; 26 | 27 | //const freakSigma = this.getFreakPoints(freakIndex, 0); 28 | const freakX = this.getFreakPoints(freakIndex, 1); 29 | const freakY = this.getFreakPoints(freakIndex, 2); 30 | 31 | const octave = this.int(this.getExtrema(featureIndex, 1)); 32 | const inputY = this.getExtrema(featureIndex, 2); 33 | const inputX = this.getExtrema(featureIndex, 3); 34 | const inputAngle = this.getAngles(featureIndex); 35 | const cos = FREAK_EXPANSION_FACTOR * Math.cos(inputAngle); 36 | const sin = FREAK_EXPANSION_FACTOR * Math.sin(inputAngle); 37 | 38 | const yp = inputY + freakX * sin + freakY * cos; 39 | const xp = inputX + freakX * cos + freakY * -sin; 40 | 41 | const x0 = this.int(Math.floor(xp)); 42 | const x1 = x0 + 1; 43 | const y0 = this.int(Math.floor(yp)); 44 | const y1 = y0 + 1; 45 | 46 | const f1 = getPixel(octave, y0, x0); 47 | const f2 = getPixel(octave, y0, x1); 48 | const f3 = getPixel(octave, y1, x0); 49 | const f4 = getPixel(octave, y1, x1); 50 | 51 | /* const x1f = float(x1); 52 | const y1f = float(y1); 53 | const x0f = float(x0); 54 | const y0f = float(y0); */ 55 | 56 | // ratio for interpolation between four neighbouring points 57 | const value = (x1 - xp) * (y1 - yp) * f1 58 | + (xp - x0) * (y1 - yp) * f2 59 | + (x1 - xp) * (yp - y0) * f3 60 | + (xp - x0) * (yp - y0) * f4; 61 | 62 | this.setOutput(value); 63 | } 64 | 65 | } 66 | 67 | return kernel; 68 | 69 | } 70 | 71 | 72 | 73 | export const computeExtremaFreak = (args) => { 74 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 75 | const { gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT, pyramidImagesLength } = args.inputs; 76 | /** @type {MathBackendCPU} */ 77 | const backend = args.backend; 78 | const prog = GetProgram(prunedExtremas.shape[0], pyramidImagesLength); 79 | return FakeShader.runCode(backend, prog, [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT], 'float32'); 80 | } 81 | 82 | 83 | 84 | export const computeExtremaFreakConfig = {//: KernelConfig 85 | kernelName: "ComputeExtremaFreak", 86 | backendName: 'cpu', 87 | kernelFunc: computeExtremaFreak,// as {} as KernelFunc, 88 | }; 89 | 90 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/computeFreakDescriptors.js: -------------------------------------------------------------------------------- 1 | 2 | import { FREAKPOINTS } from '../../freak.js'; 3 | 4 | const FREAK_CONPARISON_COUNT = (FREAKPOINTS.length - 1) * (FREAKPOINTS.length) / 2; 5 | const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8); 6 | 7 | 8 | function computeFreakDescriptorImpl(extremaFreaks,positionT) { 9 | const resultValues = new Float32Array(extremaFreaks.height* descriptorCount); 10 | function getP(y, x) { 11 | return positionT.values[y*positionT.width+x]; 12 | } 13 | function getFreak(y, x) { 14 | return extremaFreaks.values[y*extremaFreaks.width+x]; 15 | } 16 | function setOutput(y,x,o){ 17 | resultValues[y*descriptorCount+x]=o; 18 | } 19 | 20 | for (let featureIndex = 0; featureIndex < extremaFreaks.height; featureIndex++) { 21 | for (let _descIndex = 0; _descIndex < descriptorCount; _descIndex++) { 22 | const descIndex = _descIndex * 8; 23 | let sum = 0; 24 | for (let i = 0; i < 8; i++) { 25 | if (descIndex + i >= FREAK_CONPARISON_COUNT) { 26 | continue; 27 | } 28 | 29 | const p1 = Math.trunc(getP(descIndex + i, 0)); 30 | const p2 = Math.trunc(getP(descIndex + i, 1)); 31 | 32 | const v1 = getFreak(featureIndex, p1); 33 | const v2 = getFreak(featureIndex, p2); 34 | 35 | if (v1 < v2 + 0.01) { 36 | sum += Math.trunc(Math.pow(2.0, (7.0 - i))); 37 | } 38 | } 39 | setOutput(featureIndex,_descIndex, sum); 40 | } 41 | } 42 | 43 | 44 | 45 | return resultValues; 46 | } 47 | 48 | export const computeFreakDescriptor = (args) => { 49 | const { extremaFreaks, positionT } = args.inputs; 50 | const { backend } = args; 51 | 52 | const freaksData ={values:backend.data.get(extremaFreaks.dataId).values,height:extremaFreaks.shape[0], width:extremaFreaks.shape[1]}; 53 | const positionData ={values:backend.data.get(positionT.dataId).values,width:positionT.shape[1]}; 54 | //backend.runWebGLProgram(program,[extremaFreaks, positionT],'int32'); 55 | 56 | const resultValues=computeFreakDescriptorImpl(freaksData,positionData); 57 | return backend.makeOutput(resultValues, [extremaFreaks.shape[0], descriptorCount], 'int32'); 58 | } 59 | 60 | export const computeFreakDescriptorConfig = {//: KernelConfig 61 | kernelName: "ComputeFreakDescriptors", 62 | backendName: 'cpu', 63 | kernelFunc: computeFreakDescriptor,// as {} as KernelFunc, 64 | }; 65 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/computeLocalization.js: -------------------------------------------------------------------------------- 1 | import {tensor} from '@tensorflow/tfjs' 2 | import * as FakeShader from './fakeShader.js'; 3 | 4 | function GetProgram(numDogPyramidImages, extremasListLength) { 5 | 6 | const dogVariableNames = []; 7 | 8 | for (let i = 1; i < numDogPyramidImages; i++) { 9 | dogVariableNames.push('image' + i); 10 | } 11 | 12 | 13 | const program = { 14 | variableNames: [...dogVariableNames, 'extrema'], 15 | outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema 16 | userCode: function () { 17 | const getPixel = (octave, y, x) => { 18 | const k = 'getImage' + octave 19 | if (!this.hasOwnProperty(k)) { 20 | throw new Error(`ComputeLocalization:: ${k} does not exist`); 21 | } 22 | return this[k](y, x); 23 | } 24 | const coords = this.getOutputCoords(); 25 | const featureIndex = coords[0]; 26 | const score = this.getExtrema(featureIndex, 0); 27 | if (score == 0.0) { 28 | return; 29 | } 30 | 31 | const dy = coords[1] - 1; 32 | const dx = coords[2] - 1; 33 | const octave = this.int(this.getExtrema(featureIndex, 1)); 34 | const y = this.int(this.getExtrema(featureIndex, 2)); 35 | const x = this.int(this.getExtrema(featureIndex, 3)); 36 | this.setOutput(getPixel(octave, y + dy, x + dx)); 37 | } 38 | 39 | }; 40 | //} 41 | return program; 42 | } 43 | 44 | 45 | 46 | const int = Math.trunc; 47 | function clamp(n, min, max) { 48 | return Math.min(Math.max(min, n), max - 1); 49 | } 50 | 51 | 52 | 53 | 54 | 55 | 56 | export const computeLocalization = (args) => { 57 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 58 | const { prunedExtremasList, dogPyramidImagesT } = args.inputs; 59 | /** @type {MathBackendCPU} */ 60 | const backend = args.backend; 61 | 62 | const program = GetProgram(dogPyramidImagesT.length, prunedExtremasList.length); 63 | const prunedExtremasT = tensor(prunedExtremasList, [prunedExtremasList.length, prunedExtremasList[0].length], 'int32'); 64 | return FakeShader.runCode(backend, program, [...dogPyramidImagesT.slice(1), prunedExtremasT], dogPyramidImagesT[0].dtype); 65 | } 66 | 67 | export const computeLocalizationConfig = {//: KernelConfig 68 | kernelName: "ComputeLocalization", 69 | backendName: 'cpu', 70 | kernelFunc: computeLocalization,// as {} as KernelFunc, 71 | }; 72 | 73 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/downsampleBilinear.js: -------------------------------------------------------------------------------- 1 | import * as FakeShader from './fakeShader.js'; 2 | 3 | export const downsampleBilinear = (args) => { 4 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 5 | const x = args.inputs.image; 6 | /** @type {MathBackendCPU} */ 7 | const backend = args.backend; 8 | 9 | const kernel = { 10 | variableNames: ['p'], 11 | outputShape: [Math.floor(x.shape[0] / 2), Math.floor(x.shape[1] / 2)], 12 | userCode: 13 | function () { 14 | const coords = this.getOutputCoords(); 15 | const y = coords[0] * 2; 16 | const x = coords[1] * 2; 17 | let sum = new Float32Array(1); 18 | sum[0] = Math.fround(this.getP(y, x) * 0.25); 19 | sum[0] += Math.fround(this.getP(y + 1, x) * 0.25); 20 | sum[0] += Math.fround(this.getP(y, x + 1) * 0.25); 21 | sum[0] += Math.fround(this.getP(y + 1, x + 1) * 0.25); 22 | 23 | this.setOutput(sum[0]); 24 | } 25 | } 26 | return FakeShader.runCode(backend, kernel, [x], x.dtype); 27 | } 28 | 29 | export const downsampleBilinearConfig = {//: KernelConfig 30 | kernelName: "DownsampleBilinear", 31 | backendName: 'cpu', 32 | kernelFunc: downsampleBilinear,// as {} as KernelFunc, 33 | }; 34 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/extremaReduction.js: -------------------------------------------------------------------------------- 1 | import * as FakeShader from './fakeShader.js'; 2 | function GetProgram(outHeight,outWidth) { 3 | const kernel = { 4 | variableNames: ['extrema'], 5 | outputShape: [outHeight, outWidth], 6 | userCode: function () { 7 | const coords = this.getOutputCoords(); 8 | const y = coords[0] * 2; 9 | const x = coords[1] * 2; 10 | 11 | let location = 0.0; 12 | let values = this.getExtrema(y, x); 13 | 14 | if (this.getExtrema(y + 1, x) != 0.0) { 15 | location = 1.0; 16 | values = this.getExtrema(y + 1, x); 17 | } 18 | else if (this.getExtrema(y, x + 1) != 0.0) { 19 | location = 2.0; 20 | values = this.getExtrema(y, x + 1); 21 | } 22 | else if (this.getExtrema(y + 1, x + 1) != 0.0) { 23 | location = 3.0; 24 | values = this.getExtrema(y + 1, x + 1); 25 | } 26 | 27 | if (values < 0.0) { 28 | this.setOutput(location * -1000.0 + values); 29 | } else { 30 | this.setOutput(location * 1000.0 + values); 31 | } 32 | } 33 | 34 | } 35 | return kernel; 36 | } 37 | 38 | 39 | export const extremaReduction = (args) => { 40 | const { extremasResultT } = args.inputs; 41 | /** @type {MathBackendCPU} */ 42 | const backend = args.backend; 43 | const extremaHeight = extremasResultT.shape[0]; 44 | const extremaWidth = extremasResultT.shape[1]; 45 | const outHeight = Math.floor(extremaHeight / 2.0); 46 | const outWidth = Math.floor(extremaWidth / 2.0); 47 | const program=GetProgram(outHeight,outWidth); 48 | 49 | return FakeShader.runCode(backend,program,[extremasResultT],extremasResultT.dtype); 50 | } 51 | 52 | export const extremaReductionConfig = {//: KernelConfig 53 | kernelName: "ExtremaReduction", 54 | backendName: 'cpu', 55 | kernelFunc: extremaReduction,// as {} as KernelFunc, 56 | }; 57 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/fakeShader.js: -------------------------------------------------------------------------------- 1 | import {zeros,map, flatten as mathjsflatten} from 'mathjs' 2 | /** 3 | * @typedef {Object} Kernel 4 | * @property {string[]} variableNames 5 | * @property {number[]} outputShape 6 | * @property {Function} userCode 7 | */ 8 | 9 | /** 10 | * 11 | * @param {MathBackendCPU} backend 12 | * @param {Kernel} kernel 13 | * @param {Array<.TensorInfo>} inputs 14 | * @param {DataType} dtype 15 | * @returns {Tensor} 16 | */ 17 | function runCode(backend, kernel, inputs, dtype) { 18 | const inputData = inputs.map((value) => { return backend.data.get(value.dataId).values; }); 19 | 20 | //create getter functions for every variable name, clamping the input. 21 | const tempData = {} 22 | kernel.variableNames.forEach((name, index) => { 23 | const funName=`get${capFirstLetter(name)}`; 24 | //console.log("Making function:",funName,inputs[index].shape); 25 | tempData[funName] = function (...args) { 26 | const inputIndex=index; 27 | for (let i = 0; i < args.length; i++) { 28 | args[i] = clampInt(args[i], 0, inputs[inputIndex].shape[i] ); 29 | } 30 | return inputData[index][flatten(args, inputs[inputIndex].shape)]; 31 | } 32 | }); 33 | tempData.int=Math.trunc; 34 | tempData.atan=Math.atan2; 35 | //create an empty matrix to map the output size, because i'm lazy and want to use Matrix.map(...) 36 | //const temp = new Matrix(); 37 | //console.log("Creating output shape:",kernel.outputShape); 38 | const temp=zeros(kernel.outputShape);//reshape([0,0,0],kernel.outputShape); 39 | const output = map(temp,(value, index,matrix) => { 40 | 41 | tempData.getOutputCoords = () => { return index; } 42 | let out; 43 | 44 | tempData.setOutput = (newValue) => { out = Number.isNaN(newValue) ? 0 : Math.fround(newValue); } 45 | //bind the method calls and run the code 46 | kernel.userCode.bind(tempData)(); 47 | return out; 48 | }) 49 | 50 | //output.flat() 51 | //convert the output from a matrix into a tensor 52 | 53 | return backend.makeOutput(mathjsflatten(output), kernel.outputShape, dtype); 54 | } 55 | 56 | /** 57 | * 58 | * @param {string} word 59 | * @returns 60 | */ 61 | function capFirstLetter(word) { 62 | return word[0].toUpperCase() + word.substring(1); 63 | } 64 | 65 | function clampInt(n, min, max) { 66 | return Math.min(Math.max(n, min), max - 1); 67 | } 68 | /** 69 | * 70 | * @param {number[]} input 71 | * @param {number[]} max 72 | * @returns 73 | */ 74 | function flatten(input, max) { 75 | return input.reduce((prev, current, index) => { 76 | for (let i = index + 1; i < max.length; i++) { 77 | current *= max[i]; 78 | } 79 | return prev + current; 80 | },0); 81 | } 82 | 83 | export { runCode }; -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/index.js: -------------------------------------------------------------------------------- 1 | import { registerKernel } from '@tensorflow/tfjs'; 2 | import { binomialFilterConfig } from './binomialFilter.js'; 3 | import { buildExtremasConfig } from './buildExtremas.js'; 4 | import { computeExtremaAnglesConfig } from './computeExtremaAngles.js'; 5 | import { computeExtremaFreakConfig } from './computeExtremaFreak.js'; 6 | import { computeFreakDescriptorConfig } from './computeFreakDescriptors.js'; 7 | import { computeLocalizationConfig } from './computeLocalization.js'; 8 | import { computeOrientationHistogramsConfig } from './computeOrientationHistograms.js'; 9 | import { downsampleBilinearConfig } from './downsampleBilinear.js'; 10 | import { extremaReductionConfig } from './extremaReduction.js'; 11 | import { smoothHistogramsConfig } from './smoothHistograms.js'; 12 | import { upsampleBilinearConfig } from './upsampleBilinear.js'; 13 | 14 | //export function Register(){ 15 | registerKernel(binomialFilterConfig); 16 | registerKernel(buildExtremasConfig); 17 | registerKernel(computeExtremaAnglesConfig); 18 | registerKernel(computeExtremaFreakConfig); 19 | registerKernel(computeFreakDescriptorConfig); 20 | registerKernel(computeLocalizationConfig); 21 | registerKernel(computeOrientationHistogramsConfig); 22 | registerKernel(downsampleBilinearConfig); 23 | registerKernel(extremaReductionConfig); 24 | registerKernel(smoothHistogramsConfig); 25 | registerKernel(upsampleBilinearConfig); 26 | //} 27 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/prune.js: -------------------------------------------------------------------------------- 1 | import * as FakeShader from './fakeShader.js'; 2 | /* 3 | const kernel = { 4 | variableNames: ['extrema'], 5 | outputShape: [Math.floor(extremaHeight/2), Math.floor(extremaWidth/2)], 6 | userCode: ` 7 | void main() { 8 | ivec2 coords = getOutputCoords(); 9 | int y = coords[0] * 2; 10 | int x = coords[1] * 2; 11 | 12 | float location = 0.0; 13 | float values = getExtrema(y, x); 14 | 15 | if (getExtrema(y+1, x) != 0.0) { 16 | location = 1.0; 17 | values = getExtrema(y+1, x); 18 | } 19 | else if (getExtrema(y, x+1) != 0.0) { 20 | location = 2.0; 21 | values = getExtrema(y, x+1); 22 | } 23 | else if (getExtrema(y+1, x+1) != 0.0) { 24 | location = 3.0; 25 | values = getExtrema(y+1, x+1); 26 | } 27 | 28 | if (values < 0.0) { 29 | setOutput(location * -1000.0 + values); 30 | } else { 31 | setOutput(location * 1000.0 + values); 32 | } 33 | } 34 | ` 35 | } 36 | 37 | */ 38 | function clamp(n, min, max) { 39 | return Math.min(Math.max(min, n), max-1); 40 | } 41 | 42 | const pruneImpl=(vals,width,height)=>{ 43 | const w=Math.floor(width/2); 44 | const h=Math.floor(height/2); 45 | const resultValues = new Float32Array(w*h); 46 | function getExtrema(x,y){ 47 | x=clamp(x,0,width); 48 | y=clamp(y,0,height); 49 | return vals[y*width+x]; 50 | } 51 | function setOutput(x,y,o){ 52 | resultValues[y*width+x]=o; 53 | } 54 | 55 | for(let x=0;x{ 89 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 90 | const x = args.inputs.x; 91 | /** @type {MathBackendCPU} */ 92 | const cpuBackend = args.backend; 93 | const imageHeight = x.shape[0]; 94 | const imageWidth = x.shape[1]; 95 | /** @type {TypedArray} */ 96 | const values = cpuBackend.data.get(x.dataId).values; 97 | 98 | const resultValues = pruneImpl(values,imageWidth,imageHeight); 99 | 100 | return cpuBackend.makeOutput(resultValues, [Math.floor(imageHeight/2), Math.floor(imageWidth/2)], 'float32'); 101 | } 102 | 103 | const pruneConfig = {//: KernelConfig 104 | kernelName: "Prune", 105 | backendName: 'cpu', 106 | kernelFunc: prune,// as {} as KernelFunc, 107 | }; 108 | 109 | module.exports={ 110 | pruneConfig, 111 | prune, 112 | pruneImpl 113 | } -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/smoothHistograms.js: -------------------------------------------------------------------------------- 1 | 2 | const ORIENTATION_NUM_BINS = 36; 3 | const ORIENTATION_SMOOTHING_ITERATIONS = 5; 4 | 5 | 6 | function smoothHistogramsImpl(histograms) { 7 | const resultValues = new Float32Array(histograms.height * ORIENTATION_NUM_BINS); 8 | function getHistogram(y, x) { 9 | return histograms.values[y * histograms.width + x]; 10 | } 11 | function setOutput(y, x, o) { 12 | resultValues[y * ORIENTATION_NUM_BINS + x] = o; 13 | } 14 | function imod(x,y){ 15 | return Math.trunc(x - y * Math.floor(x/y)); 16 | } 17 | for (let featureIndex = 0; featureIndex < histograms.height; featureIndex++) { 18 | for (let binIndex = 0; binIndex < ORIENTATION_NUM_BINS; binIndex++) { 19 | const prevBin = imod(binIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS); 20 | const nextBin = imod(binIndex + 1, ORIENTATION_NUM_BINS); 21 | const result = 0.274068619061197 * getHistogram(featureIndex, prevBin) + 0.451862761877606 * getHistogram(featureIndex, binIndex) + 0.274068619061197 * getHistogram(featureIndex, nextBin); 22 | 23 | setOutput(featureIndex, binIndex, result); 24 | } 25 | } 26 | 27 | return resultValues; 28 | } 29 | 30 | export const smoothHistograms = (args) => { 31 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 32 | const { histograms } = args.inputs; 33 | /** @type {MathBackendCPU} */ 34 | const backend = args.backend; 35 | const histogramsData = { values: backend.data.get(histograms.dataId).values, height: histograms.shape[0], width: histograms.shape[1] }; 36 | //const program = GetProgram(histograms); 37 | for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) { 38 | histogramsData.values = smoothHistogramsImpl(histogramsData);//backend.runWebGLProgram(program, [histograms], histograms.dtype);//this._compileAndRun(program, [histograms]); 39 | } 40 | return backend.makeOutput(histogramsData.values,[histograms.shape[0],ORIENTATION_NUM_BINS],histograms.dtype); 41 | 42 | } 43 | 44 | 45 | 46 | export const smoothHistogramsConfig = {//: KernelConfig 47 | kernelName: "SmoothHistograms", 48 | backendName: 'cpu', 49 | kernelFunc: smoothHistograms,// as {} as KernelFunc, 50 | }; 51 | 52 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/cpu/upsampleBilinear.js: -------------------------------------------------------------------------------- 1 | import * as FakeShader from './fakeShader.js'; 2 | function getProgram(targetImage) { 3 | const kernel = { 4 | variableNames: ['p'], 5 | outputShape: [targetImage.shape[0], targetImage.shape[1]], 6 | userCode: function () { 7 | const coords = this.getOutputCoords(); 8 | const j = coords[0]; 9 | const i = coords[1]; 10 | 11 | const sj = Math.fround(0.5 * j) - 0.25; 12 | const si = Math.fround(0.5 * i) - 0.25; 13 | 14 | const sj0 = Math.floor(sj); 15 | const sj1 = Math.ceil(sj); 16 | const si0 = Math.floor(si); 17 | const si1 = Math.ceil(si); 18 | 19 | const sj0I = this.int(sj0); 20 | const sj1I = this.int(sj1); 21 | const si0I = this.int(si0); 22 | const si1I = this.int(si1); 23 | 24 | let sum = 0.0; 25 | sum += this.getP(sj0I, si0I) * Math.fround((si1 - si) * (sj1 - sj)); 26 | sum += this.getP(sj1I, si0I) * Math.fround((si1 - si) * (sj - sj0)); 27 | sum += this.getP(sj0I, si1I) * Math.fround((si - si0) * (sj1 - sj)); 28 | sum += this.getP(sj1I, si1I) * Math.fround((si - si0) * (sj - sj0)); 29 | this.setOutput(sum); 30 | } 31 | 32 | }; 33 | return kernel; 34 | } 35 | 36 | export const upsampleBilinear = (args) => { 37 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 38 | const { image, targetImage } = args.inputs; 39 | 40 | /** @type {MathBackendCPU} */ 41 | const cpuBackend = args.backend; 42 | 43 | const program = getProgram(targetImage); 44 | return FakeShader.runCode(cpuBackend,program,[image],image.dtype); 45 | 46 | } 47 | 48 | export const upsampleBilinearConfig = {//: KernelConfig 49 | kernelName: "UpsampleBilinear", 50 | backendName: 'cpu', 51 | kernelFunc: upsampleBilinear,// as {} as KernelFunc, 52 | }; 53 | 54 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/index.js: -------------------------------------------------------------------------------- 1 | import './webgl/index.js'; 2 | import './cpu/index.js'; 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/binomialFilter.js: -------------------------------------------------------------------------------- 1 | const cache={}; 2 | 3 | /** 4 | * 5 | * @param {string} key 6 | * @param {TensorInfo} image 7 | * @returns {[GPGPUProgram,GPGPUProgram]} 8 | */ 9 | function GetKernels(image){ 10 | const imageWidth = image.shape[1]; 11 | const imageHeight = image.shape[0]; 12 | const key = 'w' + imageWidth + "h" + imageHeight; 13 | if(!cache.hasOwnProperty(key)){ 14 | const kernel1 = { 15 | variableNames: ['p'], 16 | outputShape: [imageHeight, imageWidth], 17 | userCode: ` 18 | void main() { 19 | ivec2 coords = getOutputCoords(); 20 | 21 | float sum = getP(coords[0], coords[1]-2); 22 | sum += getP(coords[0], coords[1]-1) * 4.; 23 | sum += getP(coords[0], coords[1]) * 6.; 24 | sum += getP(coords[0], coords[1]+1) * 4.; 25 | sum += getP(coords[0], coords[1]+2); 26 | setOutput(sum); 27 | } 28 | ` 29 | }; 30 | const kernel2 = { 31 | variableNames: ['p'], 32 | outputShape: [imageHeight, imageWidth], 33 | userCode: ` 34 | void main() { 35 | ivec2 coords = getOutputCoords(); 36 | 37 | float sum = getP(coords[0]-2, coords[1]); 38 | sum += getP(coords[0]-1, coords[1]) * 4.; 39 | sum += getP(coords[0], coords[1]) * 6.; 40 | sum += getP(coords[0]+1, coords[1]) * 4.; 41 | sum += getP(coords[0]+2, coords[1]); 42 | sum /= 256.; 43 | setOutput(sum); 44 | } 45 | ` 46 | }; 47 | cache[key]=[kernel1,kernel2]; 48 | } 49 | return cache[key]; 50 | } 51 | 52 | export const binomialFilter = (args) =>{//{inputs: UnaryInputs, backend: MathBackendCPU} 53 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 54 | const image = args.inputs.image; 55 | /** @type {MathBackendWebGL} */ 56 | const backend = args.backend; 57 | 58 | const[kernel1,kernel2]=GetKernels(image); 59 | 60 | const result1=backend.runWebGLProgram(kernel1,[image],image.dtype); 61 | const result2=backend.runWebGLProgram(kernel2,[result1],image.dtype); 62 | backend.disposeIntermediateTensorInfo(result1); 63 | return result2; 64 | } 65 | 66 | 67 | 68 | 69 | export const binomialFilterConfig = {//: KernelConfig 70 | kernelName: "BinomialFilter", 71 | backendName: 'webgl', 72 | kernelFunc: binomialFilter,// as {} as KernelFunc, 73 | }; 74 | 75 | 76 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/buildExtremas.js: -------------------------------------------------------------------------------- 1 | import {engine} from '@tensorflow/tfjs'; 2 | 3 | const FREAK_EXPANSION_FACTOR = 7.0; 4 | 5 | const LAPLACIAN_THRESHOLD = 3.0; 6 | const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD; 7 | 8 | const EDGE_THRESHOLD = 4.0; 9 | const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD+1) * (EDGE_THRESHOLD+1) / EDGE_THRESHOLD); 10 | 11 | 12 | const cache={}; 13 | function GetProgram(image){ 14 | const imageWidth = image.shape[1]; 15 | const imageHeight = image.shape[0]; 16 | const kernelKey = 'w' + imageWidth + "h" + imageHeight; 17 | if(!cache.hasOwnProperty(kernelKey)){ 18 | const kernel = { 19 | variableNames: ['image0', 'image1', 'image2'], 20 | outputShape: [imageHeight, imageWidth], 21 | userCode: ` 22 | void main() { 23 | ivec2 coords = getOutputCoords(); 24 | 25 | int y = coords[0]; 26 | int x = coords[1]; 27 | 28 | float value = getImage1(y, x); 29 | 30 | // Step 1: find local maxima/minima 31 | if (value * value < ${LAPLACIAN_SQR_THRESHOLD}.) { 32 | setOutput(0.); 33 | return; 34 | } 35 | if (y < ${FREAK_EXPANSION_FACTOR} || y > ${imageHeight - 1 - FREAK_EXPANSION_FACTOR}) { 36 | setOutput(0.); 37 | return; 38 | } 39 | if (x < ${FREAK_EXPANSION_FACTOR} || x > ${imageWidth - 1 - FREAK_EXPANSION_FACTOR}) { 40 | setOutput(0.); 41 | return; 42 | } 43 | 44 | bool isMax = true; 45 | bool isMin = true; 46 | for (int dy = -1; dy <= 1; dy++) { 47 | for (int dx = -1; dx <= 1; dx++) { 48 | float value0 = getImage0(y+dy, x+dx); 49 | float value1 = getImage1(y+dy, x+dx); 50 | float value2 = getImage2(y+dy, x+dx); 51 | 52 | if (value < value0 || value < value1 || value < value2) { 53 | isMax = false; 54 | } 55 | if (value > value0 || value > value1 || value > value2) { 56 | isMin = false; 57 | } 58 | } 59 | } 60 | 61 | if (!isMax && !isMin) { 62 | setOutput(0.); 63 | return; 64 | } 65 | 66 | // compute edge score and reject based on threshold 67 | float dxx = getImage1(y, x+1) + getImage1(y, x-1) - 2. * getImage1(y, x); 68 | float dyy = getImage1(y+1, x) + getImage1(y-1, x) - 2. * getImage1(y, x); 69 | float dxy = 0.25 * (getImage1(y-1,x-1) + getImage1(y+1,x+1) - getImage1(y-1,x+1) - getImage1(y+1,x-1)); 70 | 71 | float det = (dxx * dyy) - (dxy * dxy); 72 | 73 | if (abs(det) < 0.0001) { // determinant undefined. no solution 74 | setOutput(0.); 75 | return; 76 | } 77 | 78 | float edgeScore = (dxx + dyy) * (dxx + dyy) / det; 79 | 80 | if (abs(edgeScore) >= ${EDGE_HESSIAN_THRESHOLD} ) { 81 | setOutput(0.); 82 | return; 83 | } 84 | setOutput(getImage1(y,x)); 85 | } 86 | ` 87 | }; 88 | cache[kernelKey]=kernel; 89 | } 90 | return cache[kernelKey]; 91 | } 92 | 93 | export const buildExtremas=(args)=>{ 94 | let {image0,image1,image2}=args.inputs; 95 | /** @type {MathBackendWebGL} */ 96 | const backend = args.backend; 97 | 98 | const program=GetProgram(image1); 99 | 100 | image0=engine().runKernel('DownsampleBilinear',{image:image0}); 101 | image2=engine().runKernel('UpsampleBilinear',{image:image2,targetImage:image1}); 102 | return backend.runWebGLProgram(program,[image0,image1,image2],image1.dtype); 103 | } 104 | 105 | export const buildExtremasConfig = {//: KernelConfig 106 | kernelName: "BuildExtremas", 107 | backendName: 'webgl', 108 | kernelFunc: buildExtremas,// as {} as KernelFunc, 109 | }; 110 | 111 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/computeExtremaAngles.js: -------------------------------------------------------------------------------- 1 | 2 | const ORIENTATION_NUM_BINS = 36; 3 | 4 | const cache={}; 5 | function GetProgram(histograms){ 6 | const key=histograms.shape[0]; 7 | if(!cache.hasOwnProperty(key)){ 8 | const kernel = { 9 | variableNames: ['histogram'], 10 | outputShape: [histograms.shape[0]], 11 | userCode: ` 12 | void main() { 13 | int featureIndex = getOutputCoords(); 14 | 15 | int maxIndex = 0; 16 | for (int i = 1; i < ${ORIENTATION_NUM_BINS}; i++) { 17 | if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) { 18 | maxIndex = i; 19 | } 20 | } 21 | 22 | int prev = imod(maxIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS}); 23 | int next = imod(maxIndex + 1, ${ORIENTATION_NUM_BINS}); 24 | 25 | /** 26 | * Fit a quatratic to 3 points. The system of equations is: 27 | * 28 | * y0 = A*x0^2 + B*x0 + C 29 | * y1 = A*x1^2 + B*x1 + C 30 | * y2 = A*x2^2 + B*x2 + C 31 | * 32 | * This system of equations is solved for A,B,C. 33 | */ 34 | float p10 = float(maxIndex - 1); 35 | float p11 = getHistogram(featureIndex, prev); 36 | float p20 = float(maxIndex); 37 | float p21 = getHistogram(featureIndex, maxIndex); 38 | float p30 = float(maxIndex + 1); 39 | float p31 = getHistogram(featureIndex, next); 40 | 41 | float d1 = (p30-p20)*(p30-p10); 42 | float d2 = (p10-p20)*(p30-p10); 43 | float d3 = p10-p20; 44 | 45 | // If any of the denominators are zero then, just use maxIndex. 46 | float fbin = float(maxIndex); 47 | if ( abs(d1) > 0.00001 && abs(d2) > 0.00001 && abs(d3) > 0.00001) { 48 | float a = p10*p10; 49 | float b = p20*p20; 50 | 51 | // Solve for the coefficients A,B,C 52 | float A = ((p31-p21)/d1)-((p11-p21)/d2); 53 | float B = ((p11-p21)+(A*(b-a)))/d3; 54 | float C = p11-(A*a)-(B*p10); 55 | fbin = -B / (2. * A); 56 | } 57 | 58 | float an = 2.0 *${Math.PI} * (fbin + 0.5) / ${ORIENTATION_NUM_BINS}. - ${Math.PI}; 59 | setOutput(an); 60 | } 61 | ` 62 | } 63 | cache[key]=kernel; 64 | } 65 | return cache[key]; 66 | } 67 | 68 | export const computeExtremaAngles=(args)=>{ 69 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 70 | const {histograms} = args.inputs; 71 | /** @type {MathBackendWebGL} */ 72 | const backend = args.backend; 73 | 74 | const program = GetProgram(histograms); 75 | return backend.runWebGLProgram(program,[histograms],histograms.dtype); 76 | } 77 | 78 | 79 | export const computeExtremaAnglesConfig = {//: KernelConfig 80 | kernelName: "ComputeExtremaAngles", 81 | backendName: 'webgl', 82 | kernelFunc: computeExtremaAngles,// as {} as KernelFunc, 83 | }; 84 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/computeExtremaFreak.js: -------------------------------------------------------------------------------- 1 | import { FREAKPOINTS } from '../../freak.js' 2 | 3 | const FREAK_EXPANSION_FACTOR = 7.0; 4 | const cache = {}; 5 | function GetProgram(imageCount, prunedExtremas) { 6 | const key = `${imageCount}|${prunedExtremas.shape[0]}`; 7 | if (!cache.hasOwnProperty(key)) { 8 | const imageVariableNames = []; 9 | for (let i = 1; i < imageCount; i++) { 10 | imageVariableNames.push('image' + i); 11 | } 12 | 13 | let pixelsSubCodes = `float getPixel(int octave, int y, int x) {`; 14 | for (let i = 1; i < imageCount; i++) { 15 | pixelsSubCodes += ` 16 | if (octave == ${i}) { 17 | return getImage${i}(y, x); 18 | } 19 | ` 20 | } 21 | pixelsSubCodes += `}`; 22 | 23 | const kernel = { 24 | variableNames: [...imageVariableNames, 'extrema', 'angles', 'freakPoints'], 25 | outputShape: [prunedExtremas.shape[0], FREAKPOINTS.length], 26 | userCode: ` 27 | ${pixelsSubCodes} 28 | void main() { 29 | ivec2 coords = getOutputCoords(); 30 | int featureIndex = coords[0]; 31 | int freakIndex = coords[1]; 32 | 33 | float freakSigma = getFreakPoints(freakIndex, 0); 34 | float freakX = getFreakPoints(freakIndex, 1); 35 | float freakY = getFreakPoints(freakIndex, 2); 36 | 37 | int octave = int(getExtrema(featureIndex, 1)); 38 | float inputY = getExtrema(featureIndex, 2); 39 | float inputX = getExtrema(featureIndex, 3); 40 | float inputAngle = getAngles(featureIndex); 41 | float cos = ${FREAK_EXPANSION_FACTOR}. * cos(inputAngle); 42 | float sin = ${FREAK_EXPANSION_FACTOR}. * sin(inputAngle); 43 | 44 | float yp = inputY + freakX * sin + freakY * cos; 45 | float xp = inputX + freakX * cos + freakY * -sin; 46 | 47 | int x0 = int(floor(xp)); 48 | int x1 = x0 + 1; 49 | int y0 = int(floor(yp)); 50 | int y1 = y0 + 1; 51 | 52 | float f1 = getPixel(octave, y0, x0); 53 | float f2 = getPixel(octave, y0, x1); 54 | float f3 = getPixel(octave, y1, x0); 55 | float f4 = getPixel(octave, y1, x1); 56 | 57 | float x1f = float(x1); 58 | float y1f = float(y1); 59 | float x0f = float(x0); 60 | float y0f = float(y0); 61 | 62 | // ratio for interpolation between four neighbouring points 63 | float value = (x1f - xp) * (y1f - yp) * f1 64 | + (xp - x0f) * (y1f - yp) * f2 65 | + (x1f - xp) * (yp - y0f) * f3 66 | + (xp - x0f) * (yp - y0f) * f4; 67 | 68 | setOutput(value); 69 | } 70 | ` 71 | } 72 | 73 | cache[key] = kernel; 74 | } 75 | 76 | return cache[key]; 77 | } 78 | 79 | export const computeExtremaFreak = (args) => { 80 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 81 | const { gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT,pyramidImagesLength } = args.inputs; 82 | /** @type {MathBackendWebGL} */ 83 | const backend = args.backend; 84 | 85 | const program = GetProgram(pyramidImagesLength, prunedExtremas); 86 | 87 | return backend.runWebGLProgram(program, [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT], 'float32'); 88 | } 89 | 90 | export const computeExtremaFreakConfig = {//: KernelConfig 91 | kernelName: "ComputeExtremaFreak", 92 | backendName: 'webgl', 93 | kernelFunc: computeExtremaFreak,// as {} as KernelFunc, 94 | }; 95 | 96 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/computeFreakDescriptors.js: -------------------------------------------------------------------------------- 1 | 2 | import { FREAKPOINTS } from '../../freak.js'; 3 | 4 | const FREAK_CONPARISON_COUNT = (FREAKPOINTS.length - 1) * (FREAKPOINTS.length) / 2; 5 | const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8); 6 | 7 | const cache={}; 8 | function GetProgram(extremaFreaks){ 9 | const key=`${extremaFreaks.shape[0]}`; 10 | if (!cache.hasOwnProperty(key)) { 11 | const kernel = { 12 | variableNames: ['freak', 'p'], 13 | outputShape: [extremaFreaks.shape[0], descriptorCount], 14 | userCode: ` 15 | void main() { 16 | ivec2 coords = getOutputCoords(); 17 | int featureIndex = coords[0]; 18 | int descIndex = coords[1] * 8; 19 | 20 | int sum = 0; 21 | for (int i = 0; i < 8; i++) { 22 | if (descIndex + i >= ${FREAK_CONPARISON_COUNT}) { 23 | continue; 24 | } 25 | 26 | int p1 = int(getP(descIndex + i, 0)); 27 | int p2 = int(getP(descIndex + i, 1)); 28 | 29 | float v1 = getFreak(featureIndex, p1); 30 | float v2 = getFreak(featureIndex, p2); 31 | 32 | if (v1 < v2 + 0.01) { 33 | sum += int(pow(2.0, float(7 - i))); 34 | } 35 | } 36 | setOutput(float(sum)); 37 | } 38 | ` 39 | } 40 | cache[key]=kernel; 41 | } 42 | return cache[key]; 43 | } 44 | 45 | export const computeFreakDescriptor=(args)=>{ 46 | const {extremaFreaks, positionT} =args.inputs; 47 | const {backend} = args; 48 | const program = GetProgram(extremaFreaks); 49 | return backend.runWebGLProgram(program,[extremaFreaks, positionT],'int32'); 50 | } 51 | 52 | export const computeFreakDescriptorConfig = {//: KernelConfig 53 | kernelName: "ComputeFreakDescriptors", 54 | backendName: 'webgl', 55 | kernelFunc: computeFreakDescriptor,// as {} as KernelFunc, 56 | }; 57 | 58 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/computeLocalization.js: -------------------------------------------------------------------------------- 1 | import {tensor} from '@tensorflow/tfjs' 2 | 3 | const cache={}; 4 | function GetProgram(numDogPyramidImages,extremasListLength){ 5 | const kernelKey=`${numDogPyramidImages}|${extremasListLength}`; 6 | if(!cache.hasOwnProperty(kernelKey)){ 7 | const dogVariableNames = []; 8 | let dogSubCodes = `float getPixel(int octave, int y, int x) {`; 9 | for (let i = 1; i < numDogPyramidImages; i++) { // extrema starts from second octave 10 | dogVariableNames.push('image' + i); 11 | dogSubCodes += ` 12 | if (octave == ${i}) { 13 | return getImage${i}(y, x); 14 | } 15 | `; 16 | } 17 | dogSubCodes += `}`; 18 | 19 | cache[kernelKey] = { 20 | variableNames: [...dogVariableNames, 'extrema'], 21 | outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema 22 | userCode: ` 23 | ${dogSubCodes} 24 | 25 | void main() { 26 | ivec3 coords = getOutputCoords(); 27 | int featureIndex = coords[0]; 28 | float score = getExtrema(featureIndex, 0); 29 | if (score == 0.0) { 30 | return; 31 | } 32 | 33 | int dy = coords[1]-1; 34 | int dx = coords[2]-1; 35 | int octave = int(getExtrema(featureIndex, 1)); 36 | int y = int(getExtrema(featureIndex, 2)); 37 | int x = int(getExtrema(featureIndex, 3)); 38 | setOutput(getPixel(octave, y+dy, x+dx)); 39 | } 40 | ` 41 | }; 42 | } 43 | return cache[kernelKey]; 44 | } 45 | 46 | export const computeLocalization=(args)=>{ 47 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 48 | const {prunedExtremasList, dogPyramidImagesT} = args.inputs; 49 | /** @type {MathBackendWebGL} */ 50 | const backend = args.backend; 51 | const program = GetProgram(dogPyramidImagesT.length,prunedExtremasList.length); 52 | const prunedExtremasT = tensor(prunedExtremasList, [prunedExtremasList.length, prunedExtremasList[0].length], 'int32'); 53 | return backend.runWebGLProgram(program, [...dogPyramidImagesT.slice(1), prunedExtremasT],dogPyramidImagesT[0].dtype); 54 | 55 | } 56 | 57 | export const computeLocalizationConfig = {//: KernelConfig 58 | kernelName: "ComputeLocalization", 59 | backendName: 'webgl', 60 | kernelFunc: computeLocalization,// as {} as KernelFunc, 61 | }; 62 | 63 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/downsampleBilinear.js: -------------------------------------------------------------------------------- 1 | const cache={}; 2 | /** 3 | * 4 | * @param {TensorInfo} image 5 | * @returns {GPGPUProgram} 6 | */ 7 | function GetProgram(image){ 8 | const imageWidth = image.shape[1]; 9 | const imageHeight = image.shape[0]; 10 | const kernelKey = 'w' + imageWidth + "h" + imageHeight; 11 | if(!cache.hasOwnProperty(kernelKey)){ 12 | const kernel = { 13 | variableNames: ['p'], 14 | outputShape: [Math.floor(imageHeight/2), Math.floor(imageWidth/2)], 15 | userCode: ` 16 | void main() { 17 | ivec2 coords = getOutputCoords(); 18 | int y = coords[0] * 2; 19 | int x = coords[1] * 2; 20 | 21 | float sum = getP(y, x) * 0.25; 22 | sum += getP(y+1,x) * 0.25; 23 | sum += getP(y, x+1) * 0.25; 24 | sum += getP(y+1,x+1) * 0.25; 25 | setOutput(sum); 26 | } 27 | ` 28 | }; 29 | cache[kernelKey]=kernel; 30 | } 31 | return cache[kernelKey]; 32 | } 33 | 34 | 35 | export const downsampleBilinear =(args)=>{ 36 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 37 | const image = args.inputs.image; 38 | /** @type {MathBackendWebGL} */ 39 | const backend = args.backend; 40 | 41 | const program=GetProgram(image); 42 | 43 | return backend.runWebGLProgram(program,[image],image.dtype); 44 | } 45 | 46 | export const downsampleBilinearConfig = {//: KernelConfig 47 | kernelName: "DownsampleBilinear", 48 | backendName: 'webgl', 49 | kernelFunc: downsampleBilinear,// as {} as KernelFunc, 50 | }; 51 | 52 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/extremaReduction.js: -------------------------------------------------------------------------------- 1 | export const extremaReduction=(args)=>{ 2 | /** @type {import('@tensorflow/tfjs').TensorInfo[]} */ 3 | const {extremasResultT} = args.inputs; 4 | /** @type {MathBackendWebGL} */ 5 | const backend = args.backend; 6 | const extremaHeight = extremasResultT.shape[0]; 7 | const extremaWidth = extremasResultT.shape[1]; 8 | const kernel = { 9 | variableNames: ['extrema'], 10 | outputShape: [Math.floor(extremaHeight/2), Math.floor(extremaWidth/2)], 11 | userCode: ` 12 | void main() { 13 | ivec2 coords = getOutputCoords(); 14 | int y = coords[0] * 2; 15 | int x = coords[1] * 2; 16 | 17 | float location = 0.0; 18 | float values = getExtrema(y, x); 19 | 20 | if (getExtrema(y+1, x) != 0.0) { 21 | location = 1.0; 22 | values = getExtrema(y+1, x); 23 | } 24 | else if (getExtrema(y, x+1) != 0.0) { 25 | location = 2.0; 26 | values = getExtrema(y, x+1); 27 | } 28 | else if (getExtrema(y+1, x+1) != 0.0) { 29 | location = 3.0; 30 | values = getExtrema(y+1, x+1); 31 | } 32 | 33 | if (values < 0.0) { 34 | setOutput(location * -1000.0 + values); 35 | } else { 36 | setOutput(location * 1000.0 + values); 37 | } 38 | } 39 | ` 40 | } 41 | 42 | return backend.runWebGLProgram(kernel,[extremasResultT],extremasResultT.dtype); 43 | } 44 | 45 | export const extremaReductionConfig = {//: KernelConfig 46 | kernelName: "ExtremaReduction", 47 | backendName: 'webgl', 48 | kernelFunc: extremaReduction,// as {} as KernelFunc, 49 | }; 50 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/index.js: -------------------------------------------------------------------------------- 1 | import { registerKernel } from '@tensorflow/tfjs'; 2 | import { binomialFilterConfig } from './binomialFilter.js'; 3 | import { buildExtremasConfig } from './buildExtremas.js'; 4 | import { computeExtremaAnglesConfig } from './computeExtremaAngles.js'; 5 | import { computeExtremaFreakConfig } from './computeExtremaFreak.js' 6 | import { computeFreakDescriptorConfig } from './computeFreakDescriptors.js' 7 | import { computeLocalizationConfig } from './computeLocalization.js' 8 | import { computeOrientationHistogramsConfig } from './computeOrientationHistograms.js' 9 | import { downsampleBilinearConfig } from './downsampleBilinear.js'; 10 | import { extremaReductionConfig } from './extremaReduction.js'; 11 | import { smoothHistogramsConfig } from './smoothHistograms.js'; 12 | import { upsampleBilinearConfig } from './upsampleBilinear.js'; 13 | 14 | //export function Register(){ 15 | registerKernel(binomialFilterConfig); 16 | registerKernel(buildExtremasConfig); 17 | registerKernel(computeExtremaAnglesConfig); 18 | registerKernel(computeExtremaFreakConfig); 19 | registerKernel(computeFreakDescriptorConfig); 20 | registerKernel(computeLocalizationConfig); 21 | registerKernel(computeOrientationHistogramsConfig); 22 | registerKernel(downsampleBilinearConfig); 23 | registerKernel(extremaReductionConfig); 24 | registerKernel(smoothHistogramsConfig); 25 | registerKernel(upsampleBilinearConfig); 26 | //} 27 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/smoothHistograms.js: -------------------------------------------------------------------------------- 1 | 2 | const ORIENTATION_NUM_BINS = 36; 3 | const ORIENTATION_SMOOTHING_ITERATIONS = 5; 4 | 5 | const cache={}; 6 | function GetProgram(histograms){ 7 | const kernelKey=`h${histograms.shape[0]}`; 8 | if(!cache.hasOwnProperty(kernelKey)){ 9 | const kernel = { 10 | variableNames: ['histogram'], 11 | outputShape: [histograms.shape[0], ORIENTATION_NUM_BINS], 12 | userCode: ` 13 | void main() { 14 | ivec2 coords = getOutputCoords(); 15 | 16 | int featureIndex = coords[0]; 17 | int binIndex = coords[1]; 18 | 19 | int prevBin = imod(binIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS}); 20 | int nextBin = imod(binIndex + 1, ${ORIENTATION_NUM_BINS}); 21 | float result = 0.274068619061197 * getHistogram(featureIndex, prevBin) + 0.451862761877606 * getHistogram(featureIndex, binIndex) + 0.274068619061197 * getHistogram(featureIndex, nextBin); 22 | 23 | setOutput(result); 24 | } 25 | ` 26 | }; 27 | cache[kernelKey]=kernel; 28 | } 29 | return cache[kernelKey]; 30 | } 31 | 32 | export const smoothHistograms=(args)=>{ 33 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 34 | let {histograms} = args.inputs; 35 | /** @type {MathBackendWebGL} */ 36 | const backend = args.backend; 37 | 38 | const program = GetProgram(histograms); 39 | for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) { 40 | const _histograms = histograms; 41 | histograms = backend.runWebGLProgram(program,[histograms],histograms.dtype);//this._compileAndRun(program, [histograms]); 42 | if (i > 0) { 43 | backend.disposeIntermediateTensorInfo(_histograms); 44 | } 45 | } 46 | return histograms; 47 | 48 | } 49 | 50 | 51 | 52 | export const smoothHistogramsConfig = {//: KernelConfig 53 | kernelName: "SmoothHistograms", 54 | backendName: 'webgl', 55 | kernelFunc: smoothHistograms,// as {} as KernelFunc, 56 | }; 57 | 58 | -------------------------------------------------------------------------------- /src/image-target/detector/kernels/webgl/upsampleBilinear.js: -------------------------------------------------------------------------------- 1 | import {MathBackendWebGL} from '@tensorflow/tfjs-backend-webgl'; 2 | 3 | const cache={}; 4 | function GetProgram(image,targetImage){ 5 | const targetImageWidth = targetImage.shape[1]; 6 | const targetImageHeight = targetImage.shape[0]; 7 | const kernelKey = 'w' + targetImageWidth + "h" + targetImageHeight; 8 | if(!cache.hasOwnProperty(kernelKey)){ 9 | const kernel = { 10 | variableNames: ['p'], 11 | outputShape: [targetImageHeight, targetImageWidth], 12 | userCode: ` 13 | void main() { 14 | ivec2 coords = getOutputCoords(); 15 | int j = coords[0]; 16 | int i = coords[1]; 17 | 18 | float sj = 0.5 * float(j) - 0.25; 19 | float si = 0.5 * float(i) - 0.25; 20 | 21 | float sj0 = floor(sj); 22 | float sj1 = ceil(sj); 23 | float si0 = floor(si); 24 | float si1 = ceil(si); 25 | 26 | int sj0I = int(sj0); 27 | int sj1I = int(sj1); 28 | int si0I = int(si0); 29 | int si1I = int(si1); 30 | 31 | float sum = 0.0; 32 | sum += getP(sj0I, si0I) * (si1 - si) * (sj1 - sj); 33 | sum += getP(sj1I, si0I) * (si1 - si) * (sj - sj0); 34 | sum += getP(sj0I, si1I) * (si - si0) * (sj1 - sj); 35 | sum += getP(sj1I, si1I) * (si - si0) * (sj - sj0); 36 | setOutput(sum); 37 | } 38 | ` 39 | }; 40 | cache[kernelKey]=kernel; 41 | } 42 | 43 | return cache[kernelKey]; 44 | } 45 | 46 | export const upsampleBilinear =(args)=>{ 47 | /** @type {import('@tensorflow/tfjs').TensorInfo} */ 48 | const {image,targetImage} = args.inputs; 49 | 50 | 51 | /** @type {MathBackendWebGL} */ 52 | const backend = args.backend; 53 | 54 | const program = GetProgram(image,targetImage); 55 | return backend.runWebGLProgram(program,[image],image.dtype); 56 | 57 | 58 | } 59 | 60 | export const upsampleBilinearConfig = {//: KernelConfig 61 | kernelName: "UpsampleBilinear", 62 | backendName: 'webgl', 63 | kernelFunc: upsampleBilinear,// as {} as KernelFunc, 64 | }; 65 | -------------------------------------------------------------------------------- /src/image-target/estimation/estimate.js: -------------------------------------------------------------------------------- 1 | import {Matrix, inverse} from 'ml-matrix'; 2 | import {solveHomography} from '../utils/homography.js'; 3 | 4 | // build world matrix with list of matching worldCoords|screenCoords 5 | // 6 | // Step 1. estimate homography with list of pairs 7 | // Ref: https://www.uio.no/studier/emner/matnat/its/TEK5030/v19/lect/lecture_4_3-estimating-homographies-from-feature-correspondences.pdf (Basic homography estimation from points) 8 | // 9 | // Step 2. decompose homography into rotation and translation matrixes (i.e. world matrix) 10 | // Ref: can anyone provide reference? 11 | const estimate = ({screenCoords, worldCoords, projectionTransform}) => { 12 | const Harray = solveHomography(worldCoords.map((p) => [p.x, p.y]), screenCoords.map((p) => [p.x, p.y])); 13 | const H = new Matrix([ 14 | [Harray[0], Harray[1], Harray[2]], 15 | [Harray[3], Harray[4], Harray[5]], 16 | [Harray[6], Harray[7], Harray[8]], 17 | ]); 18 | 19 | const K = new Matrix(projectionTransform); 20 | const KInv = inverse(K); 21 | 22 | const _KInvH = KInv.mmul(H); 23 | const KInvH = _KInvH.to1DArray(); 24 | 25 | const norm1 = Math.sqrt( KInvH[0] * KInvH[0] + KInvH[3] * KInvH[3] + KInvH[6] * KInvH[6]); 26 | const norm2 = Math.sqrt( KInvH[1] * KInvH[1] + KInvH[4] * KInvH[4] + KInvH[7] * KInvH[7]); 27 | const tnorm = (norm1 + norm2) / 2; 28 | 29 | const rotate = []; 30 | rotate[0] = KInvH[0] / norm1; 31 | rotate[3] = KInvH[3] / norm1; 32 | rotate[6] = KInvH[6] / norm1; 33 | 34 | rotate[1] = KInvH[1] / norm2; 35 | rotate[4] = KInvH[4] / norm2; 36 | rotate[7] = KInvH[7] / norm2; 37 | 38 | rotate[2] = rotate[3] * rotate[7] - rotate[6] * rotate[4]; 39 | rotate[5] = rotate[6] * rotate[1] - rotate[0] * rotate[7]; 40 | rotate[8] = rotate[0] * rotate[4] - rotate[1] * rotate[3]; 41 | 42 | const norm3 = Math.sqrt(rotate[2] * rotate[2] + rotate[5] * rotate[5] + rotate[8] * rotate[8]); 43 | rotate[2] /= norm3; 44 | rotate[5] /= norm3; 45 | rotate[8] /= norm3; 46 | 47 | // TODO: artoolkit has check_rotation() that somehow switch the rotate vector. not sure what that does. Can anyone advice? 48 | // https://github.com/artoolkitx/artoolkit5/blob/5bf0b671ff16ead527b9b892e6aeb1a2771f97be/lib/SRC/ARICP/icpUtil.c#L215 49 | 50 | const tran = [] 51 | tran[0] = KInvH[2] / tnorm; 52 | tran[1] = KInvH[5] / tnorm; 53 | tran[2] = KInvH[8] / tnorm; 54 | 55 | let initialModelViewTransform = [ 56 | [rotate[0], rotate[1], rotate[2], tran[0]], 57 | [rotate[3], rotate[4], rotate[5], tran[1]], 58 | [rotate[6], rotate[7], rotate[8], tran[2]] 59 | ]; 60 | 61 | return initialModelViewTransform; 62 | }; 63 | 64 | export { 65 | estimate 66 | } 67 | -------------------------------------------------------------------------------- /src/image-target/estimation/estimator.js: -------------------------------------------------------------------------------- 1 | import {estimate} from './estimate.js'; 2 | import {refineEstimate} from './refine-estimate.js'; 3 | 4 | class Estimator { 5 | constructor(projectionTransform) { 6 | this.projectionTransform = projectionTransform; 7 | } 8 | 9 | // Solve homography between screen points and world points using Direct Linear Transformation 10 | // then decompose homography into rotation and translation matrix (i.e. modelViewTransform) 11 | estimate({screenCoords, worldCoords}) { 12 | const modelViewTransform = estimate({screenCoords, worldCoords, projectionTransform: this.projectionTransform}); 13 | return modelViewTransform; 14 | } 15 | 16 | // Given an initial guess of the modelViewTransform and new pairs of screen-world coordinates, 17 | // use Iterative Closest Point to refine the transformation 18 | //refineEstimate({initialModelViewTransform, screenCoords, worldCoords}) { 19 | refineEstimate({initialModelViewTransform, worldCoords, screenCoords}) { 20 | const updatedModelViewTransform = refineEstimate({initialModelViewTransform, worldCoords, screenCoords, projectionTransform: this.projectionTransform}); 21 | return updatedModelViewTransform; 22 | } 23 | } 24 | 25 | export { 26 | Estimator, 27 | } 28 | -------------------------------------------------------------------------------- /src/image-target/estimation/utils.js: -------------------------------------------------------------------------------- 1 | const buildModelViewProjectionTransform = (projectionTransform, modelViewTransform) => { 2 | // assume the projectTransform has the following format: 3 | // [[fx, 0, cx], 4 | // [0, fy, cy] 5 | // [0, 0, 1]] 6 | const modelViewProjectionTransform = [ 7 | [ 8 | projectionTransform[0][0] * modelViewTransform[0][0] + projectionTransform[0][2] * modelViewTransform[2][0], 9 | projectionTransform[0][0] * modelViewTransform[0][1] + projectionTransform[0][2] * modelViewTransform[2][1], 10 | projectionTransform[0][0] * modelViewTransform[0][2] + projectionTransform[0][2] * modelViewTransform[2][2], 11 | projectionTransform[0][0] * modelViewTransform[0][3] + projectionTransform[0][2] * modelViewTransform[2][3], 12 | ], 13 | [ 14 | projectionTransform[1][1] * modelViewTransform[1][0] + projectionTransform[1][2] * modelViewTransform[2][0], 15 | projectionTransform[1][1] * modelViewTransform[1][1] + projectionTransform[1][2] * modelViewTransform[2][1], 16 | projectionTransform[1][1] * modelViewTransform[1][2] + projectionTransform[1][2] * modelViewTransform[2][2], 17 | projectionTransform[1][1] * modelViewTransform[1][3] + projectionTransform[1][2] * modelViewTransform[2][3], 18 | ], 19 | [ 20 | modelViewTransform[2][0], 21 | modelViewTransform[2][1], 22 | modelViewTransform[2][2], 23 | modelViewTransform[2][3], 24 | ] 25 | ]; 26 | return modelViewProjectionTransform; 27 | 28 | /* 29 | // this is the full computation if the projectTransform does not look like the expected format, but more computations 30 | // 31 | const modelViewProjectionTransform = [[],[],[]]; 32 | for (let j = 0; j < 3; j++ ) { 33 | for (let i = 0; i < 4; i++) { 34 | modelViewProjectionTransform[j][i] = projectionTransform[j][0] * modelViewTransform[0][i] 35 | + projectionTransform[j][1] * modelViewTransform[1][i] 36 | + projectionTransform[j][2] * modelViewTransform[2][i]; 37 | } 38 | } 39 | return modelViewProjectionTransform; 40 | */ 41 | } 42 | 43 | const applyModelViewProjectionTransform = (modelViewProjectionTransform, x, y, z) => { 44 | // assume z is zero 45 | const ux = modelViewProjectionTransform[0][0] * x + modelViewProjectionTransform[0][1] * y + modelViewProjectionTransform[0][3]; 46 | const uy = modelViewProjectionTransform[1][0] * x + modelViewProjectionTransform[1][1] * y + modelViewProjectionTransform[1][3]; 47 | const uz = modelViewProjectionTransform[2][0] * x + modelViewProjectionTransform[2][1] * y + modelViewProjectionTransform[2][3]; 48 | return {x: ux, y: uy, z: uz}; 49 | } 50 | 51 | const computeScreenCoordiate = (modelViewProjectionTransform, x, y, z) => { 52 | const {x: ux, y: uy, z: uz} = applyModelViewProjectionTransform(modelViewProjectionTransform, x, y, z); 53 | //if( Math.abs(uz) < 0.000001 ) return null; 54 | return {x: ux/uz, y: uy/uz}; 55 | } 56 | 57 | const screenToMarkerCoordinate = (modelViewProjectionTransform, sx, sy) => { 58 | const c11 = modelViewProjectionTransform[2][0] * sx - modelViewProjectionTransform[0][0]; 59 | const c12 = modelViewProjectionTransform[2][1] * sx - modelViewProjectionTransform[0][1]; 60 | const c21 = modelViewProjectionTransform[2][0] * sy - modelViewProjectionTransform[1][0]; 61 | const c22 = modelViewProjectionTransform[2][1] * sy - modelViewProjectionTransform[1][1]; 62 | const b1 = modelViewProjectionTransform[0][3] - modelViewProjectionTransform[2][3] * sx; 63 | const b2 = modelViewProjectionTransform[1][3] - modelViewProjectionTransform[2][3] * sy; 64 | 65 | const m = c11 * c22 - c12 * c21; 66 | return { 67 | x: (c22 * b1 - c12 * b2) / m, 68 | y: (c11 * b2 - c21 * b1) / m 69 | } 70 | } 71 | 72 | export { 73 | buildModelViewProjectionTransform, 74 | applyModelViewProjectionTransform, 75 | computeScreenCoordiate, 76 | } 77 | -------------------------------------------------------------------------------- /src/image-target/image-list.js: -------------------------------------------------------------------------------- 1 | import {resize} from "./utils/images.js"; 2 | 3 | const MIN_IMAGE_PIXEL_SIZE = 100; 4 | 5 | // Build a list of image {data, width, height, scale} with different scales 6 | const buildImageList = (inputImage) => { 7 | const minScale = MIN_IMAGE_PIXEL_SIZE / Math.min(inputImage.width, inputImage.height); 8 | 9 | const scaleList = []; 10 | let c = minScale; 11 | while (true) { 12 | scaleList.push(c); 13 | c *= Math.pow(2.0, 1.0/3.0); 14 | if (c >= 0.95) { 15 | c = 1; 16 | break; 17 | } 18 | } 19 | scaleList.push(c); 20 | scaleList.reverse(); 21 | 22 | const imageList = []; 23 | for (let i = 0; i < scaleList.length; i++) { 24 | const w = inputImage.width * scaleList[i]; 25 | const h = inputImage.height * scaleList[i]; 26 | imageList.push(Object.assign(resize({image: inputImage, ratio: scaleList[i]}), {scale: scaleList[i]})); 27 | } 28 | return imageList; 29 | } 30 | 31 | const buildTrackingImageList = (inputImage) => { 32 | const minDimension = Math.min(inputImage.width, inputImage.height); 33 | const scaleList = []; 34 | const imageList = []; 35 | scaleList.push( 256.0 / minDimension); 36 | scaleList.push( 128.0 / minDimension); 37 | for (let i = 0; i < scaleList.length; i++) { 38 | imageList.push(Object.assign(resize({image: inputImage, ratio: scaleList[i]}), {scale: scaleList[i]})); 39 | } 40 | return imageList; 41 | } 42 | 43 | export { 44 | buildImageList, 45 | buildTrackingImageList 46 | } 47 | -------------------------------------------------------------------------------- /src/image-target/index.js: -------------------------------------------------------------------------------- 1 | import {Controller} from './controller.js'; 2 | import {Compiler} from './compiler.js'; 3 | import {UI} from '../ui/ui.js'; 4 | 5 | export { 6 | Controller, 7 | Compiler, 8 | UI 9 | } 10 | 11 | if (!window.MINDAR) { 12 | window.MINDAR = {}; 13 | } 14 | 15 | window.MINDAR.IMAGE = { 16 | Controller, 17 | Compiler, 18 | UI 19 | }; 20 | -------------------------------------------------------------------------------- /src/image-target/input-loader.js: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs'; 2 | 3 | // More efficient implementation for tf.browser.fromPixels 4 | // original implementation: /node_modules/@tensorflow/tfjs-backend-webgl/src/kernels/FromPixels.ts 5 | // 6 | // This implementation return grey scale instead of RGBA in the orignal implementation 7 | 8 | class InputLoader { 9 | constructor(width, height) { 10 | this.width = width; 11 | this.height = height; 12 | this.texShape = [height, width]; 13 | 14 | const context = document.createElement('canvas').getContext('2d'); 15 | context.canvas.width = width; 16 | context.canvas.height = height; 17 | this.context = context; 18 | 19 | this.program = this.buildProgram(width, height); 20 | 21 | const backend = tf.backend(); 22 | //this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'int32'); 23 | this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'float32'); 24 | // warning!!! 25 | // usage type should be TextureUsage.PIXELS, but tfjs didn't export this enum type, so we hard-coded 2 here 26 | // i.e. backend.texData.get(tempPixelHandle.dataId).usage = TextureUsage.PIXELS; 27 | backend.texData.get(this.tempPixelHandle.dataId).usage = 2; 28 | } 29 | 30 | // old method 31 | _loadInput(input) { 32 | return tf.tidy(() => { 33 | let inputImage = tf.browser.fromPixels(input); 34 | inputImage = inputImage.mean(2); 35 | return inputImage; 36 | }); 37 | } 38 | 39 | // input is instance of HTMLVideoElement or HTMLImageElement 40 | loadInput(input) { 41 | const context = this.context; 42 | context.clearRect(0, 0, this.context.canvas.width, this.context.canvas.height); 43 | 44 | const isInputRotated = input.width === this.height && input.height === this.width; 45 | if (isInputRotated) { // rotate 90 degree and draw 46 | let x = this.context.canvas.width / 2; 47 | let y = this.context.canvas.height / 2; 48 | let angleInDegrees = 90; 49 | 50 | context.save(); // save the current context state 51 | context.translate(x, y); // move the context origin to the center of the image 52 | context.rotate(angleInDegrees * Math.PI / 180); // rotate the context 53 | 54 | // draw the image with its center at the origin 55 | context.drawImage(input, -input.width / 2, -input.height / 2); 56 | context.restore(); // restore the context to its original state 57 | } else { 58 | this.context.drawImage(input, 0, 0, input.width, input.height); 59 | } 60 | 61 | const backend = tf.backend(); 62 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(this.tempPixelHandle.dataId), this.context.canvas); 63 | 64 | //const res = backend.compileAndRun(this.program, [this.tempPixelHandle]); 65 | const res = this._compileAndRun(this.program, [this.tempPixelHandle]); 66 | //const res = this._runWebGLProgram(this.program, [this.tempPixelHandle], 'float32'); 67 | //backend.disposeData(tempPixelHandle.dataId); 68 | return res; 69 | } 70 | 71 | buildProgram(width, height) { 72 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D'; 73 | 74 | const program = { 75 | variableNames: ['A'], 76 | outputShape: this.texShape, 77 | userCode:` 78 | void main() { 79 | ivec2 coords = getOutputCoords(); 80 | int texR = coords[0]; 81 | int texC = coords[1]; 82 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0); 83 | 84 | vec4 values = ${textureMethod}(A, uv); 85 | setOutput((0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0); 86 | } 87 | ` 88 | } 89 | return program; 90 | } 91 | 92 | _compileAndRun(program, inputs) { 93 | const outInfo = tf.backend().compileAndRun(program, inputs); 94 | return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype); 95 | } 96 | 97 | _runWebGLProgram(program, inputs, outputType) { 98 | const outInfo = tf.backend().runWebGLProgram(program, inputs, outputType); 99 | return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype); 100 | } 101 | } 102 | 103 | export { 104 | InputLoader 105 | }; 106 | -------------------------------------------------------------------------------- /src/image-target/matching/hamming-distance.js: -------------------------------------------------------------------------------- 1 | // Fast computation on number of bit sets 2 | // Ref: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 3 | const compute = (options) => { 4 | const {v1, v2} = options; 5 | let d = 0; 6 | 7 | for (let i = 0; i < v1.length; i++) { 8 | let x = (v1[i] ^ v2[i]) >>> 0; 9 | d += bitCount(x); 10 | } 11 | return d; 12 | } 13 | 14 | const bitCount = (v) => { 15 | var c = v - ((v >> 1) & 0x55555555); 16 | c = ((c >> 2) & 0x33333333) + (c & 0x33333333); 17 | c = ((c >> 4) + c) & 0x0F0F0F0F; 18 | c = ((c >> 8) + c) & 0x00FF00FF; 19 | c = ((c >> 16) + c) & 0x0000FFFF; 20 | return c; 21 | } 22 | 23 | export { 24 | compute 25 | }; 26 | -------------------------------------------------------------------------------- /src/image-target/matching/hierarchical-clustering.js: -------------------------------------------------------------------------------- 1 | import {compute as hammingCompute} from './hamming-distance.js'; 2 | import {createRandomizer} from '../utils/randomizer.js'; 3 | 4 | const MIN_FEATURE_PER_NODE = 16; 5 | const NUM_ASSIGNMENT_HYPOTHESES = 128; 6 | const NUM_CENTERS = 8; 7 | 8 | 9 | const _computeKMedoids = (options) => { 10 | const {points, pointIndexes, randomizer} = options; 11 | 12 | const randomPointIndexes = []; 13 | for (let i = 0; i < pointIndexes.length; i++) { 14 | randomPointIndexes.push(i); 15 | } 16 | 17 | let bestSumD = Number.MAX_SAFE_INTEGER; 18 | let bestAssignmentIndex = -1; 19 | 20 | const assignments = []; 21 | for (let i = 0; i < NUM_ASSIGNMENT_HYPOTHESES; i++) { 22 | randomizer.arrayShuffle({arr: randomPointIndexes, sampleSize: NUM_CENTERS}); 23 | 24 | let sumD = 0; 25 | const assignment = []; 26 | for (let j = 0; j < pointIndexes.length; j++) { 27 | let bestD = Number.MAX_SAFE_INTEGER; 28 | for (let k = 0; k < NUM_CENTERS; k++) { 29 | const centerIndex = pointIndexes[randomPointIndexes[k]]; 30 | const d = hammingCompute({v1: points[pointIndexes[j]].descriptors, v2: points[centerIndex].descriptors}); 31 | if (d < bestD) { 32 | assignment[j] = randomPointIndexes[k]; 33 | bestD = d; 34 | } 35 | } 36 | sumD += bestD; 37 | } 38 | assignments.push(assignment); 39 | 40 | if (sumD < bestSumD) { 41 | bestSumD = sumD; 42 | bestAssignmentIndex = i; 43 | } 44 | } 45 | return assignments[bestAssignmentIndex]; 46 | } 47 | 48 | // kmedoids clustering of points, with hamming distance of FREAK descriptor 49 | // 50 | // node = { 51 | // isLeaf: bool, 52 | // children: [], list of children node 53 | // pointIndexes: [], list of int, point indexes 54 | // centerPointIndex: int 55 | // } 56 | const build = ({points}) => { 57 | const pointIndexes = []; 58 | for (let i = 0; i < points.length; i++) { 59 | pointIndexes.push(i); 60 | } 61 | 62 | const randomizer = createRandomizer(); 63 | 64 | const rootNode = _build({points: points, pointIndexes: pointIndexes, centerPointIndex: null, randomizer}); 65 | return {rootNode}; 66 | } 67 | 68 | // recursive build hierarchy clusters 69 | const _build = (options) => { 70 | const {points, pointIndexes, centerPointIndex, randomizer} = options; 71 | 72 | let isLeaf = false; 73 | 74 | if (pointIndexes.length <= NUM_CENTERS || pointIndexes.length <= MIN_FEATURE_PER_NODE) { 75 | isLeaf = true; 76 | } 77 | 78 | const clusters = {}; 79 | if (!isLeaf) { 80 | // compute clusters 81 | const assignment = _computeKMedoids({points, pointIndexes, randomizer}); 82 | 83 | for (let i = 0; i < assignment.length; i++) { 84 | if (clusters[pointIndexes[assignment[i]]] === undefined) { 85 | clusters[pointIndexes[assignment[i]]] = []; 86 | } 87 | clusters[pointIndexes[assignment[i]]].push(pointIndexes[i]); 88 | } 89 | } 90 | if (Object.keys(clusters).length === 1) { 91 | isLeaf = true; 92 | } 93 | 94 | const node = { 95 | centerPointIndex: centerPointIndex 96 | } 97 | 98 | if (isLeaf) { 99 | node.leaf = true; 100 | node.pointIndexes = []; 101 | for (let i = 0; i < pointIndexes.length; i++) { 102 | node.pointIndexes.push(pointIndexes[i]); 103 | } 104 | return node; 105 | } 106 | 107 | // recursive build children 108 | node.leaf = false; 109 | node.children = []; 110 | 111 | Object.keys(clusters).forEach((centerIndex) => { 112 | node.children.push(_build({points: points, pointIndexes: clusters[centerIndex], centerPointIndex: centerIndex, randomizer})); 113 | }); 114 | return node; 115 | } 116 | 117 | export { 118 | build, 119 | }; 120 | 121 | -------------------------------------------------------------------------------- /src/image-target/matching/matcher.js: -------------------------------------------------------------------------------- 1 | import {match} from './matching.js'; 2 | 3 | class Matcher { 4 | constructor(queryWidth, queryHeight, debugMode = false) { 5 | this.queryWidth = queryWidth; 6 | this.queryHeight = queryHeight; 7 | this.debugMode = debugMode; 8 | } 9 | 10 | matchDetection(keyframes, featurePoints) { 11 | let debugExtra = {frames: []}; 12 | 13 | let bestResult = null; 14 | for (let i = 0; i < keyframes.length; i++) { 15 | const {H, matches, debugExtra: frameDebugExtra} = match({keyframe: keyframes[i], querypoints: featurePoints, querywidth: this.queryWidth, queryheight: this.queryHeight, debugMode: this.debugMode}); 16 | debugExtra.frames.push(frameDebugExtra); 17 | 18 | if (H) { 19 | if (bestResult === null || bestResult.matches.length < matches.length) { 20 | bestResult = {keyframeIndex: i, H, matches}; 21 | } 22 | } 23 | } 24 | 25 | if (bestResult === null) { 26 | return {keyframeIndex: -1, debugExtra}; 27 | } 28 | 29 | const screenCoords = []; 30 | const worldCoords = []; 31 | const keyframe = keyframes[bestResult.keyframeIndex]; 32 | for (let i = 0; i < bestResult.matches.length; i++) { 33 | const querypoint = bestResult.matches[i].querypoint; 34 | const keypoint = bestResult.matches[i].keypoint; 35 | screenCoords.push({ 36 | x: querypoint.x, 37 | y: querypoint.y, 38 | }) 39 | worldCoords.push({ 40 | x: (keypoint.x + 0.5) / keyframe.scale, 41 | y: (keypoint.y + 0.5) / keyframe.scale, 42 | z: 0, 43 | }) 44 | } 45 | return {screenCoords, worldCoords, keyframeIndex: bestResult.keyframeIndex, debugExtra}; 46 | } 47 | } 48 | 49 | export { 50 | Matcher 51 | } 52 | -------------------------------------------------------------------------------- /src/image-target/offline-compiler.js: -------------------------------------------------------------------------------- 1 | import {CompilerBase} from './compiler-base.js' 2 | import { buildTrackingImageList } from './image-list.js'; 3 | import { extractTrackingFeatures } from './tracker/extract-utils.js'; 4 | import { createCanvas } from 'canvas' 5 | import './detector/kernels/cpu/index.js' 6 | 7 | export class OfflineCompiler extends CompilerBase { 8 | createProcessCanvas(img) { 9 | const processCanvas = createCanvas(img.width, img.height); 10 | return processCanvas; 11 | } 12 | 13 | compileTrack({progressCallback, targetImages, basePercent}) { 14 | return new Promise((resolve, reject) => { 15 | const percentPerImage = (100-basePercent) / targetImages.length; 16 | let percent = 0; 17 | const list = []; 18 | for (let i = 0; i < targetImages.length; i++) { 19 | const targetImage = targetImages[i]; 20 | const imageList = buildTrackingImageList(targetImage); 21 | const percentPerAction = percentPerImage / imageList.length; 22 | 23 | //console.log("compiling tracking...", i); 24 | const trackingData = extractTrackingFeatures(imageList, (index) => { 25 | //console.log("done tracking", i, index); 26 | percent += percentPerAction; 27 | progressCallback(basePercent+percent); 28 | }); 29 | list.push(trackingData); 30 | } 31 | resolve(list); 32 | }); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/image-target/tracker/extract-utils.js: -------------------------------------------------------------------------------- 1 | import { extract } from './extract.js'; 2 | 3 | export const extractTrackingFeatures = (imageList, doneCallback) => { 4 | const featureSets = []; 5 | for (let i = 0; i < imageList.length; i++) { 6 | const image = imageList[i]; 7 | const points = extract(image); 8 | 9 | const featureSet = { 10 | data: image.data, 11 | scale: image.scale, 12 | width: image.width, 13 | height: image.height, 14 | points, 15 | }; 16 | featureSets.push(featureSet); 17 | 18 | doneCallback(i); 19 | } 20 | return featureSets; 21 | } 22 | -------------------------------------------------------------------------------- /src/image-target/utils/cumsum.js: -------------------------------------------------------------------------------- 1 | // fast 2D submatrix sum using cumulative sum algorithm 2 | class Cumsum { 3 | constructor(data, width, height) { 4 | this.cumsum = []; 5 | for (let j = 0; j < height; j++) { 6 | this.cumsum.push([]); 7 | for (let i = 0; i < width; i++) { 8 | this.cumsum[j].push(0); 9 | } 10 | } 11 | 12 | this.cumsum[0][0] = data[0]; 13 | for (let i = 1; i < width; i++) { 14 | this.cumsum[0][i] = this.cumsum[0][i-1] + data[i]; 15 | } 16 | for (let j = 1; j < height; j++) { 17 | this.cumsum[j][0] = this.cumsum[j-1][0] + data[j*width]; 18 | } 19 | 20 | for (let j = 1; j < height; j++) { 21 | for (let i = 1; i < width; i++) { 22 | this.cumsum[j][i] = data[j*width+i] 23 | + this.cumsum[j-1][i] 24 | + this.cumsum[j][i-1] 25 | - this.cumsum[j-1][i-1]; 26 | } 27 | } 28 | } 29 | 30 | query(x1, y1, x2, y2) { 31 | let ret = this.cumsum[y2][x2]; 32 | if (y1 > 0) ret -= this.cumsum[y1-1][x2]; 33 | if (x1 > 0) ret -= this.cumsum[y2][x1-1]; 34 | if (x1 > 0 && y1 > 0) ret += this.cumsum[y1-1][x1-1]; 35 | return ret; 36 | } 37 | } 38 | 39 | export { 40 | Cumsum 41 | } 42 | -------------------------------------------------------------------------------- /src/image-target/utils/geometry.js: -------------------------------------------------------------------------------- 1 | // check which side point C on the line from A to B 2 | const linePointSide = (A, B, C) => { 3 | return ((B[0]-A[0])*(C[1]-A[1])-(B[1]-A[1])*(C[0]-A[0])); 4 | } 5 | 6 | // srcPoints, dstPoints: array of four elements [x, y] 7 | const checkFourPointsConsistent = (x1, x2, x3, x4, x1p, x2p, x3p, x4p) => { 8 | if ((linePointSide(x1, x2, x3) > 0) !== (linePointSide(x1p, x2p, x3p) > 0)) return false; 9 | if ((linePointSide(x2, x3, x4) > 0) !== (linePointSide(x2p, x3p, x4p) > 0)) return false; 10 | if ((linePointSide(x3, x4, x1) > 0) !== (linePointSide(x3p, x4p, x1p) > 0)) return false; 11 | if ((linePointSide(x4, x1, x2) > 0) !== (linePointSide(x4p, x1p, x2p) > 0)) return false; 12 | return true; 13 | } 14 | 15 | const checkThreePointsConsistent = (x1, x2, x3, x1p, x2p, x3p) => { 16 | if ((linePointSide(x1, x2, x3) > 0) !== (linePointSide(x1p, x2p, x3p) > 0)) return false; 17 | return true; 18 | } 19 | 20 | const determinant = (A) => { 21 | const C1 = A[4] * A[8] - A[5] * A[7]; 22 | const C2 = A[3] * A[8] - A[5] * A[6]; 23 | const C3 = A[3] * A[7] - A[4] * A[6]; 24 | return A[0] * C1 - A[1] * C2 + A[2] * C3; 25 | } 26 | 27 | const matrixInverse33 = (A, threshold) => { 28 | const det = determinant(A); 29 | if (Math.abs(det) <= threshold) return null; 30 | const oneOver = 1.0 / det; 31 | 32 | const B = [ 33 | (A[4] * A[8] - A[5] * A[7]) * oneOver, 34 | (A[2] * A[7] - A[1] * A[8]) * oneOver, 35 | (A[1] * A[5] - A[2] * A[4]) * oneOver, 36 | (A[5] * A[6] - A[3] * A[8]) * oneOver, 37 | (A[0] * A[8] - A[2] * A[6]) * oneOver, 38 | (A[2] * A[3] - A[0] * A[5]) * oneOver, 39 | (A[3] * A[7] - A[4] * A[6]) * oneOver, 40 | (A[1] * A[6] - A[0] * A[7]) * oneOver, 41 | (A[0] * A[4] - A[1] * A[3]) * oneOver, 42 | ]; 43 | return B; 44 | } 45 | 46 | const matrixMul33 = (A, B) => { 47 | const C = []; 48 | C[0] = A[0]*B[0] + A[1]*B[3] + A[2]*B[6]; 49 | C[1] = A[0]*B[1] + A[1]*B[4] + A[2]*B[7]; 50 | C[2] = A[0]*B[2] + A[1]*B[5] + A[2]*B[8]; 51 | C[3] = A[3]*B[0] + A[4]*B[3] + A[5]*B[6]; 52 | C[4] = A[3]*B[1] + A[4]*B[4] + A[5]*B[7]; 53 | C[5] = A[3]*B[2] + A[4]*B[5] + A[5]*B[8]; 54 | C[6] = A[6]*B[0] + A[7]*B[3] + A[8]*B[6]; 55 | C[7] = A[6]*B[1] + A[7]*B[4] + A[8]*B[7]; 56 | C[8] = A[6]*B[2] + A[7]*B[5] + A[8]*B[8]; 57 | return C; 58 | } 59 | 60 | const multiplyPointHomographyInhomogenous = (x, H) => { 61 | const w = H[6]*x[0] + H[7]*x[1] + H[8]; 62 | const xp = []; 63 | xp[0] = (H[0]*x[0] + H[1]*x[1] + H[2])/w; 64 | xp[1] = (H[3]*x[0] + H[4]*x[1] + H[5])/w; 65 | return xp; 66 | } 67 | 68 | const smallestTriangleArea = (x1, x2, x3, x4) => { 69 | const v12 = _vector(x2, x1); 70 | const v13 = _vector(x3, x1); 71 | const v14 = _vector(x4, x1); 72 | const v32 = _vector(x2, x3); 73 | const v34 = _vector(x4, x3); 74 | const a1 = _areaOfTriangle(v12, v13); 75 | const a2 = _areaOfTriangle(v13, v14); 76 | const a3 = _areaOfTriangle(v12, v14); 77 | const a4 = _areaOfTriangle(v32, v34); 78 | return Math.min(Math.min(Math.min(a1, a2), a3), a4); 79 | } 80 | 81 | // check if four points form a convex quadrilaternal. 82 | // all four combinations should have same sign 83 | const quadrilateralConvex = (x1, x2, x3, x4) => { 84 | const first = linePointSide(x1, x2, x3) <= 0; 85 | if ( (linePointSide(x2, x3, x4) <= 0) !== first) return false; 86 | if ( (linePointSide(x3, x4, x1) <= 0) !== first) return false; 87 | if ( (linePointSide(x4, x1, x2) <= 0) !== first) return false; 88 | 89 | //if (linePointSide(x1, x2, x3) <= 0) return false; 90 | //if (linePointSide(x2, x3, x4) <= 0) return false; 91 | //if (linePointSide(x3, x4, x1) <= 0) return false; 92 | //if (linePointSide(x4, x1, x2) <= 0) return false; 93 | return true; 94 | } 95 | 96 | const _vector = (a, b) => { 97 | return [ 98 | a[0] - b[0], 99 | a[1] - b[1] 100 | ] 101 | } 102 | 103 | const _areaOfTriangle = (u, v) => { 104 | const a = u[0]*v[1] - u[1]*v[0]; 105 | return Math.abs(a) * 0.5; 106 | } 107 | 108 | export { 109 | matrixInverse33, 110 | matrixMul33, 111 | quadrilateralConvex, 112 | smallestTriangleArea, 113 | multiplyPointHomographyInhomogenous, 114 | checkThreePointsConsistent, 115 | checkFourPointsConsistent, 116 | determinant 117 | } 118 | 119 | -------------------------------------------------------------------------------- /src/image-target/utils/homography.js: -------------------------------------------------------------------------------- 1 | import {Matrix, inverse} from 'ml-matrix'; 2 | 3 | const solveHomography = (srcPoints, dstPoints) => { 4 | const {normPoints: normSrcPoints, param: srcParam} = _normalizePoints(srcPoints); 5 | const {normPoints: normDstPoints, param: dstParam} = _normalizePoints(dstPoints); 6 | 7 | const num = normDstPoints.length; 8 | const AData = []; 9 | const BData = []; 10 | for (let j = 0; j < num; j++) { 11 | const row1 = [ 12 | normSrcPoints[j][0], 13 | normSrcPoints[j][1], 14 | 1, 15 | 0, 16 | 0, 17 | 0, 18 | -(normSrcPoints[j][0] * normDstPoints[j][0]), 19 | -(normSrcPoints[j][1] * normDstPoints[j][0]), 20 | ]; 21 | const row2 = [ 22 | 0, 23 | 0, 24 | 0, 25 | normSrcPoints[j][0], 26 | normSrcPoints[j][1], 27 | 1, 28 | -(normSrcPoints[j][0] * normDstPoints[j][1]), 29 | -(normSrcPoints[j][1] * normDstPoints[j][1]), 30 | ]; 31 | AData.push(row1); 32 | AData.push(row2); 33 | 34 | BData.push([normDstPoints[j][0]]); 35 | BData.push([normDstPoints[j][1]]); 36 | } 37 | 38 | try { 39 | const A = new Matrix(AData); 40 | const B = new Matrix(BData); 41 | const AT = A.transpose(); 42 | const ATA = AT.mmul(A); 43 | const ATB = AT.mmul(B); 44 | const ATAInv = inverse(ATA); 45 | const C = ATAInv.mmul(ATB).to1DArray(); 46 | const H = _denormalizeHomography(C, srcParam, dstParam); 47 | return H; 48 | } catch (e) { 49 | return null; 50 | } 51 | } 52 | 53 | // centroid at origin and avg distance from origin is sqrt(2) 54 | const _normalizePoints = (coords) => { 55 | //return {normalizedCoords: coords, param: {meanX: 0, meanY: 0, s: 1}}; // skip normalization 56 | 57 | let sumX = 0; 58 | let sumY = 0; 59 | for (let i = 0; i < coords.length; i++) { 60 | sumX += coords[i][0]; 61 | sumY += coords[i][1]; 62 | } 63 | let meanX = sumX / coords.length; 64 | let meanY = sumY / coords.length; 65 | 66 | let sumDiff = 0; 67 | for (let i = 0; i < coords.length; i++) { 68 | const diffX = coords[i][0] - meanX; 69 | const diffY = coords[i][1] - meanY; 70 | sumDiff += Math.sqrt(diffX * diffX + diffY * diffY); 71 | } 72 | let s = Math.sqrt(2) * coords.length / sumDiff; 73 | 74 | const normPoints = []; 75 | for (let i = 0; i < coords.length; i++) { 76 | normPoints.push([ 77 | (coords[i][0] - meanX) * s, 78 | (coords[i][1] - meanY) * s, 79 | ]); 80 | } 81 | return {normPoints, param: {meanX, meanY, s}}; 82 | } 83 | 84 | // Denormalize homography 85 | // where T is the normalization matrix, i.e. 86 | // 87 | // [1 0 -meanX] 88 | // T = [0 1 -meanY] 89 | // [0 0 1/s] 90 | // 91 | // [1 0 s*meanX] 92 | // inv(T) = [0 1 s*meanY] 93 | // [0 0 s] 94 | // 95 | // H = inv(Tdst) * Hn * Tsrc 96 | // 97 | // @param { 98 | // nH: normH, 99 | // srcParam: param of src transform, 100 | // dstParam: param of dst transform 101 | // } 102 | const _denormalizeHomography = (nH, srcParam, dstParam) => { 103 | /* 104 | Matrix version 105 | const normH = new Matrix([ 106 | [nH[0], nH[1], nH[2]], 107 | [nH[3], nH[4], nH[5]], 108 | [nH[6], nH[7], 1], 109 | ]); 110 | const Tsrc = new Matrix([ 111 | [1, 0, -srcParam.meanX], 112 | [0, 1, -srcParam.meanY], 113 | [0, 0, 1/srcParam.s], 114 | ]); 115 | 116 | const invTdst = new Matrix([ 117 | [1, 0, dstParam.s * dstParam.meanX], 118 | [0, 1, dstParam.s * dstParam.meanY], 119 | [0, 0, dstParam.s], 120 | ]); 121 | const H = invTdst.mmul(normH).mmul(Tsrc); 122 | */ 123 | 124 | // plain implementation of the above using Matrix 125 | const sMeanX = dstParam.s * dstParam.meanX; 126 | const sMeanY = dstParam.s * dstParam.meanY; 127 | 128 | const H = [ 129 | nH[0] + sMeanX * nH[6], 130 | nH[1] + sMeanX * nH[7], 131 | (nH[0] + sMeanX * nH[6]) * -srcParam.meanX + (nH[1] + sMeanX * nH[7]) * -srcParam.meanY + (nH[2] + sMeanX) / srcParam.s, 132 | nH[3] + sMeanY * nH[6], 133 | nH[4] + sMeanY * nH[7], 134 | (nH[3] + sMeanY * nH[6]) * -srcParam.meanX + (nH[4] + sMeanY * nH[7]) * -srcParam.meanY + (nH[5] + sMeanY) / srcParam.s, 135 | dstParam.s * nH[6], 136 | dstParam.s * nH[7], 137 | dstParam.s * nH[6] * -srcParam.meanX + dstParam.s * nH[7] * -srcParam.meanY + dstParam.s / srcParam.s, 138 | ]; 139 | 140 | // make H[8] === 1; 141 | for (let i = 0; i < 9; i++) { 142 | H[i] = H[i] / H[8]; 143 | } 144 | return H; 145 | } 146 | 147 | export { 148 | solveHomography 149 | } 150 | -------------------------------------------------------------------------------- /src/image-target/utils/images.js: -------------------------------------------------------------------------------- 1 | // simpler version of upsampling. better performance 2 | const _upsampleBilinear = ({image, padOneWidth, padOneHeight}) => { 3 | const {width, height, data} = image; 4 | const dstWidth = image.width * 2 + (padOneWidth?1:0); 5 | const dstHeight = image.height * 2 + (padOneHeight?1:0); 6 | const temp = new Float32Array(dstWidth * dstHeight); 7 | 8 | for (let i = 0; i < width; i++) { 9 | for (let j = 0; j < height; j++) { 10 | const v = 0.25 * data[j * width + i]; 11 | const ii = Math.floor(i/2); 12 | const jj = Math.floor(j/2); 13 | const pos = Math.floor(j/2) * dstWidth + Math.floor(i/2); 14 | temp[pos] += v; 15 | temp[pos+1] += v; 16 | temp[pos+dstWidth] += v; 17 | temp[pos+dstWidth+1] += v; 18 | } 19 | } 20 | return {data: temp, width: dstWidth, height: dstHeight}; 21 | } 22 | 23 | // artoolkit version. slower. is it necessary? 24 | const upsampleBilinear = ({image, padOneWidth, padOneHeight}) => { 25 | const {width, height, data} = image; 26 | 27 | const dstWidth = image.width * 2 + (padOneWidth?1:0); 28 | const dstHeight = image.height * 2 + (padOneHeight?1:0); 29 | 30 | const temp = new Float32Array(dstWidth * dstHeight); 31 | for (let i = 0; i < dstWidth; i++) { 32 | const si = 0.5 * i - 0.25; 33 | let si0 = Math.floor(si); 34 | let si1 = Math.ceil(si); 35 | if (si0 < 0) si0 = 0; // border 36 | if (si1 >= width) si1 = width - 1; // border 37 | 38 | for (let j = 0; j < dstHeight; j++) { 39 | const sj = 0.5 * j - 0.25; 40 | let sj0 = Math.floor(sj); 41 | let sj1 = Math.ceil(sj); 42 | if (sj0 < 0) sj0 = 0; // border 43 | if (sj1 >= height) sj1 = height - 1; //border 44 | 45 | const value = (si1 - si) * (sj1 - sj) * data[ sj0 * width + si0 ] + 46 | (si1 - si) * (sj - sj0) * data[ sj1 * width + si0 ] + 47 | (si - si0) * (sj1 - sj) * data[ sj0 * width + si1 ] + 48 | (si - si0) * (sj - sj0) * data[ sj1 * width + si1 ]; 49 | 50 | temp[j * dstWidth + i] = value; 51 | } 52 | } 53 | 54 | return {data: temp, width: dstWidth, height: dstHeight}; 55 | } 56 | 57 | const downsampleBilinear = ({image}) => { 58 | const {data, width, height} = image; 59 | 60 | const dstWidth = Math.floor(width / 2); 61 | const dstHeight = Math.floor(height / 2); 62 | 63 | const temp = new Float32Array(dstWidth * dstHeight); 64 | const offsets = [0, 1, width, width+1]; 65 | 66 | for (let j = 0; j < dstHeight; j++) { 67 | for (let i = 0; i < dstWidth; i++) { 68 | let srcPos = j*2 * width + i*2; 69 | let value = 0.0; 70 | for (let d = 0; d < offsets.length; d++) { 71 | value += data[srcPos+ offsets[d]]; 72 | } 73 | value *= 0.25; 74 | temp[j*dstWidth+i] = value; 75 | } 76 | } 77 | return {data: temp, width: dstWidth, height: dstHeight}; 78 | } 79 | 80 | const resize = ({image, ratio}) => { 81 | const width = Math.round(image.width * ratio); 82 | const height = Math.round(image.height * ratio); 83 | 84 | //const imageData = new Float32Array(width * height); 85 | const imageData = new Uint8Array(width * height); 86 | for (let i = 0; i < width; i++) { 87 | let si1 = Math.round(1.0 * i / ratio); 88 | let si2 = Math.round(1.0 * (i+1) / ratio) - 1; 89 | if (si2 >= image.width) si2 = image.width - 1; 90 | 91 | for (let j = 0; j < height; j++) { 92 | let sj1 = Math.round(1.0 * j / ratio); 93 | let sj2 = Math.round(1.0 * (j+1) / ratio) - 1; 94 | if (sj2 >= image.height) sj2 = image.height - 1; 95 | 96 | let sum = 0; 97 | let count = 0; 98 | for (let ii = si1; ii <= si2; ii++) { 99 | for (let jj = sj1; jj <= sj2; jj++) { 100 | sum += (1.0 * image.data[jj * image.width + ii]); 101 | count += 1; 102 | } 103 | } 104 | imageData[j * width + i] = Math.floor(sum / count); 105 | } 106 | } 107 | return {data: imageData, width: width, height: height}; 108 | } 109 | 110 | export { 111 | downsampleBilinear, 112 | upsampleBilinear, 113 | resize, 114 | } 115 | 116 | -------------------------------------------------------------------------------- /src/image-target/utils/randomizer.js: -------------------------------------------------------------------------------- 1 | const mRandSeed = 1234; 2 | 3 | const createRandomizer = () => { 4 | const randomizer = { 5 | seed: mRandSeed, 6 | 7 | arrayShuffle(options) { 8 | const {arr, sampleSize} = options; 9 | for (let i = 0; i < sampleSize; i++) { 10 | 11 | this.seed = (214013 * this.seed + 2531011) % (1 << 31); 12 | let k = (this.seed >> 16) & 0x7fff; 13 | k = k % arr.length; 14 | 15 | let tmp = arr[i]; 16 | arr[i] = arr[k]; 17 | arr[k] = tmp; 18 | } 19 | }, 20 | 21 | nextInt(maxValue) { 22 | this.seed = (214013 * this.seed + 2531011) % (1 << 31); 23 | let k = (this.seed >> 16) & 0x7fff; 24 | k = k % maxValue; 25 | return k; 26 | } 27 | } 28 | return randomizer; 29 | } 30 | export { 31 | createRandomizer 32 | } 33 | -------------------------------------------------------------------------------- /src/libs/one-euro-filter.js: -------------------------------------------------------------------------------- 1 | // Ref: https://jaantollander.com/post/noise-filtering-using-one-euro-filter/#mjx-eqn%3A1 2 | 3 | const smoothingFactor = (te, cutoff) => { 4 | const r = 2 * Math.PI * cutoff * te; 5 | return r / (r+1); 6 | } 7 | 8 | const exponentialSmoothing = (a, x, xPrev) => { 9 | return a * x + (1 - a) * xPrev; 10 | } 11 | 12 | class OneEuroFilter { 13 | constructor({minCutOff, beta}) { 14 | this.minCutOff = minCutOff; 15 | this.beta = beta; 16 | this.dCutOff = 0.001; // period in milliseconds, so default to 0.001 = 1Hz 17 | 18 | this.xPrev = null; 19 | this.dxPrev = null; 20 | this.tPrev = null; 21 | this.initialized = false; 22 | } 23 | 24 | reset() { 25 | this.initialized = false; 26 | } 27 | 28 | filter(t, x) { 29 | if (!this.initialized) { 30 | this.initialized = true; 31 | this.xPrev = x; 32 | this.dxPrev = x.map(() => 0); 33 | this.tPrev = t; 34 | return x; 35 | } 36 | 37 | const {xPrev, tPrev, dxPrev} = this; 38 | 39 | //console.log("filter", x, xPrev, x.map((xx, i) => x[i] - xPrev[i])); 40 | 41 | const te = t - tPrev; 42 | 43 | const ad = smoothingFactor(te, this.dCutOff); 44 | 45 | const dx = []; 46 | const dxHat = []; 47 | const xHat = []; 48 | for (let i = 0; i < x.length; i++) { 49 | // The filtered derivative of the signal. 50 | dx[i] = (x[i] - xPrev[i]) / te; 51 | dxHat[i] = exponentialSmoothing(ad, dx[i], dxPrev[i]); 52 | 53 | // The filtered signal 54 | const cutOff = this.minCutOff + this.beta * Math.abs(dxHat[i]); 55 | const a = smoothingFactor(te, cutOff); 56 | xHat[i] = exponentialSmoothing(a, x[i], xPrev[i]); 57 | } 58 | 59 | // update prev 60 | this.xPrev = xHat; 61 | this.dxPrev = dxHat; 62 | this.tPrev = t; 63 | 64 | return xHat; 65 | } 66 | } 67 | 68 | export { 69 | OneEuroFilter 70 | } 71 | -------------------------------------------------------------------------------- /src/libs/opencv-helper.js: -------------------------------------------------------------------------------- 1 | import {cv} from './opencv.js' 2 | let initialized = false; 3 | 4 | const _cv = {}; 5 | 6 | const waitResolves = []; 7 | 8 | export const waitCV = async() => { 9 | if (initialized) return true; 10 | return new Promise((resolve, reject) => { 11 | waitResolves.push(resolve); 12 | }); 13 | } 14 | 15 | cv().then((target) => { 16 | initialized = true; 17 | Object.assign(_cv, target); 18 | waitResolves.forEach((resolve) => { 19 | resolve(); 20 | }); 21 | }); 22 | 23 | export const opencv=_cv; 24 | //module.exports={waitCV,opencv} 25 | -------------------------------------------------------------------------------- /src/ui/compatibility.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Failed to launch :(

4 |

5 | Looks like your device/browser is not compatible. 6 |

7 | 8 |
9 |
10 |

11 | Please try the following recommended browsers: 12 |

13 |

14 | For Android device - Chrome 15 |

16 |

17 | For iOS device - Safari 18 |

19 |
20 |
21 | -------------------------------------------------------------------------------- /src/ui/loading.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | -------------------------------------------------------------------------------- /src/ui/scanning.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | -------------------------------------------------------------------------------- /src/ui/ui.js: -------------------------------------------------------------------------------- 1 | //import "./ui.scss"; 2 | import loadingHTML from './loading.html?raw'; 3 | import compatibilityHTML from './compatibility.html?raw'; 4 | import scanningHTML from './scanning.html?raw'; 5 | 6 | const css=`.mindar-ui-overlay{display:flex;align-items:center;justify-content:center;position:absolute;left:0;right:0;top:0;bottom:0;background:transparent;z-index:2}.mindar-ui-overlay.hidden{display:none}.mindar-ui-loading .loader{border:16px solid #222;border-top:16px solid white;opacity:.8;border-radius:50%;width:120px;height:120px;animation:spin 2s linear infinite}@keyframes spin{0%{transform:rotate(0)}to{transform:rotate(360deg)}}.mindar-ui-compatibility .content{background:black;color:#fff;opacity:.8;text-align:center;margin:20px;padding:20px;min-height:50vh}@media (min-aspect-ratio: 1/1){.mindar-ui-scanning .scanning{width:50vh;height:50vh}}@media (max-aspect-ratio: 1/1){.mindar-ui-scanning .scanning{width:80vw;height:80vw}}.mindar-ui-scanning .scanning .inner{position:relative;width:100%;height:100%;opacity:.8;background:linear-gradient(to right,white 10px,transparent 10px) 0 0,linear-gradient(to right,white 10px,transparent 10px) 0 100%,linear-gradient(to left,white 10px,transparent 10px) 100% 0,linear-gradient(to left,white 10px,transparent 10px) 100% 100%,linear-gradient(to bottom,white 10px,transparent 10px) 0 0,linear-gradient(to bottom,white 10px,transparent 10px) 100% 0,linear-gradient(to top,white 10px,transparent 10px) 0 100%,linear-gradient(to top,white 10px,transparent 10px) 100% 100%;background-repeat:no-repeat;background-size:40px 40px}.mindar-ui-scanning .scanning .inner .scanline{position:absolute;width:100%;height:10px;background:white;animation:move 2s linear infinite}@keyframes move{0%,to{top:0%}50%{top:calc(100% - 10px)}}`; 7 | 8 | export class UI { 9 | constructor({uiLoading, uiScanning, uiError}) { 10 | const cssBlock=document.createElement('style'); 11 | cssBlock.innerText=css; 12 | document.head.appendChild(cssBlock); 13 | if (uiLoading === 'yes') { 14 | this.loadingModal = this._loadHTML(loadingHTML); 15 | } else if (uiLoading !== 'no') { 16 | this.loadingModal = document.querySelector(uiLoading); 17 | } 18 | 19 | if (uiError === 'yes') { 20 | this.compatibilityModal = this._loadHTML(compatibilityHTML); 21 | } else if (uiError !== 'no') { 22 | this.compatibilityModal = document.querySelector(uiError); 23 | } 24 | 25 | if (uiScanning === 'yes') { 26 | this.scanningMask = this._loadHTML(scanningHTML); 27 | } else if (uiScanning !== 'no') { 28 | this.scanningMask = document.querySelector(uiScanning); 29 | } 30 | 31 | this.hideLoading(); 32 | this.hideCompatibility(); 33 | this.hideScanning(); 34 | } 35 | 36 | showLoading() { 37 | if (!this.loadingModal) return; 38 | this.loadingModal.classList.remove("hidden"); 39 | } 40 | hideLoading() { 41 | if (!this.loadingModal) return; 42 | this.loadingModal.classList.add("hidden"); 43 | } 44 | showCompatibility() { 45 | if (!this.compatibilityModal) return; 46 | this.compatibilityModal.classList.remove("hidden"); 47 | } 48 | hideCompatibility() { 49 | if (!this.compatibilityModal) return; 50 | this.compatibilityModal.classList.add("hidden"); 51 | } 52 | showScanning() { 53 | if (!this.scanningMask) return; 54 | this.scanningMask.classList.remove("hidden"); 55 | } 56 | hideScanning() { 57 | if (!this.scanningMask) return; 58 | this.scanningMask.classList.add("hidden"); 59 | } 60 | 61 | _loadHTML(html) { 62 | const e = document.createElement('template'); 63 | e.innerHTML = html.trim(); 64 | const rootNode = e.content.firstChild; 65 | document.getElementsByTagName('body')[0].appendChild(rootNode); 66 | return rootNode; 67 | } 68 | } 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /src/ui/ui.scss: -------------------------------------------------------------------------------- 1 | $scan-mask-border: 10px; 2 | $primary-color: white; 3 | 4 | .mindar-ui-overlay { 5 | display: flex; 6 | align-items: center; 7 | justify-content: center; 8 | position: absolute; 9 | left: 0; 10 | right: 0; 11 | top: 0; 12 | bottom: 0; 13 | background: transparent; 14 | z-index: 2; 15 | 16 | &.hidden { 17 | display: none; 18 | } 19 | } 20 | 21 | .mindar-ui-loading { 22 | .loader { 23 | border: 16px solid #222; // background color 24 | border-top: 16px solid $primary-color; // foreground color 25 | opacity: 0.8; 26 | border-radius: 50%; 27 | width: 120px; 28 | height: 120px; 29 | animation: spin 2s linear infinite; 30 | } 31 | 32 | @keyframes spin { 33 | 0% { transform: rotate(0deg); } 34 | 100% { transform: rotate(360deg); } 35 | } 36 | } 37 | 38 | .mindar-ui-compatibility { 39 | .content { 40 | background: black; 41 | color: white; 42 | opacity: 0.8; 43 | text-align: center; 44 | 45 | margin: 20px; 46 | padding: 20px; 47 | min-height: 50vh; 48 | } 49 | } 50 | 51 | .mindar-ui-scanning { 52 | .scanning { 53 | @media (min-aspect-ratio: "1/1") { 54 | width: 50vh; 55 | height: 50vh; 56 | } 57 | @media (max-aspect-ratio: "1/1") { 58 | width: 80vw; 59 | height: 80vw; 60 | } 61 | .inner { 62 | position: relative; 63 | width: 100%; 64 | height: 100%; 65 | opacity: 0.8; 66 | 67 | background: 68 | linear-gradient(to right, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 0, 69 | linear-gradient(to right, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 100%, 70 | linear-gradient(to left, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 0, 71 | linear-gradient(to left, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 100%, 72 | linear-gradient(to bottom, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 0, 73 | linear-gradient(to bottom, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 0, 74 | linear-gradient(to top, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 100%, 75 | linear-gradient(to top, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 100%; 76 | 77 | background-repeat: no-repeat; 78 | background-size: 40px 40px; 79 | 80 | .scanline { 81 | position: absolute; 82 | width: 100%; 83 | height: 10px; 84 | background: $primary-color; 85 | animation: move 2s linear infinite; 86 | 87 | @keyframes move { 88 | 0%, 100% { top: 0% } 89 | 50% { top: calc(100% - 10px) } 90 | } 91 | } 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /testing/common.js: -------------------------------------------------------------------------------- 1 | const utils = { 2 | loadImage: async (src) => { 3 | const img = new Image(); 4 | return new Promise((resolve, reject) => { 5 | let img = new Image() 6 | img.onload = () => resolve(img); 7 | img.onerror = reject; 8 | img.src = src; 9 | }) 10 | }, 11 | 12 | drawPoint: (ctx, color, centerX, centerY, radius=1) => { 13 | ctx.beginPath(); 14 | ctx.arc(centerX, centerY, radius, 0, 2 * Math.PI, false); 15 | ctx.fillStyle = color; 16 | //ctx.lineWidth = 1; 17 | ctx.strokeStyle = color; 18 | ctx.stroke(); 19 | }, 20 | 21 | drawRect: (ctx, color, centerX, centerY, width) => { 22 | ctx.beginPath(); 23 | ctx.strokeStyle = color; 24 | ctx.rect(centerX - width/2, centerY - width /2 , width, width); 25 | ctx.stroke(); 26 | }, 27 | 28 | drawLine: (ctx, color, fromX, fromY, toX, toY) => { 29 | ctx.beginPath(); 30 | ctx.lineWidth = 2; 31 | ctx.fillStyle = color; 32 | ctx.strokeStyle = color; 33 | ctx.moveTo(fromX, fromY); 34 | ctx.lineTo(toX, toY); 35 | ctx.stroke(); 36 | }, 37 | 38 | pixel2DToImageData: (pixels) => { 39 | const height = pixels.length; 40 | const width = pixels[0].length; 41 | const data = new Uint8ClampedArray(height * width * 4); 42 | for (let j = 0; j < height; j++) { 43 | for (let i = 0; i < width; i++) { 44 | const pos = j * width + i; 45 | data[pos*4 + 0] = pixels[j][i]; 46 | data[pos*4 + 1] = pixels[j][i]; 47 | data[pos*4 + 2] = pixels[j][i]; 48 | data[pos*4 + 3] = 255; 49 | } 50 | } 51 | const imageData = new ImageData(data, width, height); 52 | return imageData; 53 | }, 54 | 55 | pixel3DToImageData: (pixels) => { 56 | const height = pixels.length; 57 | const width = pixels[0].length; 58 | const data = new Uint8ClampedArray(height * width * 4); 59 | for (let j = 0; j < height; j++) { 60 | for (let i = 0; i < width; i++) { 61 | const pos = j * width + i; 62 | data[pos*4 + 0] = pixels[j][i][0]; 63 | data[pos*4 + 1] = pixels[j][i][1]; 64 | data[pos*4 + 2] = pixels[j][i][2]; 65 | data[pos*4 + 3] = 255; 66 | } 67 | } 68 | const imageData = new ImageData(data, width, height); 69 | return imageData; 70 | }, 71 | 72 | pixel1DToImageData: (pixels, width, height) => { 73 | const data = new Uint8ClampedArray(pixels.length * 4); 74 | for (let j = 0; j < pixels.length; j++) { 75 | data[j*4 + 0] = pixels[j]; 76 | data[j*4 + 1] = pixels[j]; 77 | data[j*4 + 2] = pixels[j]; 78 | data[j*4 + 3] = 255; 79 | } 80 | const imageData = new ImageData(data, width, height); 81 | return imageData; 82 | }, 83 | 84 | matrixInverse33: (A, threshold) => { 85 | const det = utils.determinant(A); 86 | if (Math.abs(det) <= threshold) return null; 87 | const oneOver = 1.0 / det; 88 | 89 | const B = [ 90 | (A[4] * A[8] - A[5] * A[7]) * oneOver, 91 | (A[2] * A[7] - A[1] * A[8]) * oneOver, 92 | (A[1] * A[5] - A[2] * A[4]) * oneOver, 93 | (A[5] * A[6] - A[3] * A[8]) * oneOver, 94 | (A[0] * A[8] - A[2] * A[6]) * oneOver, 95 | (A[2] * A[3] - A[0] * A[5]) * oneOver, 96 | (A[3] * A[7] - A[4] * A[6]) * oneOver, 97 | (A[1] * A[6] - A[0] * A[7]) * oneOver, 98 | (A[0] * A[4] - A[1] * A[3]) * oneOver, 99 | ]; 100 | return B; 101 | }, 102 | determinant: (A) => { 103 | const C1 = A[4] * A[8] - A[5] * A[7]; 104 | const C2 = A[3] * A[8] - A[5] * A[6]; 105 | const C3 = A[3] * A[7] - A[4] * A[6]; 106 | return A[0] * C1 - A[1] * C2 + A[2] * C3; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /testing/detection.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 19 | 20 | 25 | 26 | 27 | 28 |
29 |
30 | 31 | 32 | -------------------------------------------------------------------------------- /testing/glsl/applyFilter.js: -------------------------------------------------------------------------------- 1 | const applyFilter = (image) => { 2 | const imageHeight = image.shape[0]; 3 | const imageWidth = image.shape[1]; 4 | 5 | const imaxmin = ` 6 | int imax(int a, int b) { 7 | if (a > b) return a; 8 | return b; 9 | } 10 | int imin(int a, int b) { 11 | if (a < b) return a; 12 | return b; 13 | } 14 | 15 | ` 16 | const kernel1 = { 17 | variableNames: ['p'], 18 | outputShape: [imageHeight, imageWidth], 19 | userCode: ` 20 | ${imaxmin} 21 | void main() { 22 | ivec2 coords = getOutputCoords(); 23 | 24 | float sum = getP(coords[0], imax(0, coords[1]-2)); 25 | sum += getP(coords[0], imax(0, coords[1]-1)) * 4.; 26 | sum += getP(coords[0], coords[1]) * 6.; 27 | sum += getP(coords[0], imin(${imageWidth}-1, coords[1]+1)) * 4.; 28 | sum += getP(coords[0], imin(${imageWidth}-1, coords[1]+2)); 29 | setOutput(sum); 30 | 31 | } 32 | ` 33 | }; 34 | 35 | const kernel2 = { 36 | variableNames: ['p'], 37 | outputShape: [imageHeight, imageWidth], 38 | userCode: ` 39 | ${imaxmin} 40 | void main() { 41 | ivec2 coords = getOutputCoords(); 42 | 43 | float sum = getP(imax(coords[0]-2, 0), coords[1]); 44 | sum += getP(imax(coords[0]-1, 0), coords[1]) * 4.; 45 | sum += getP(coords[0], coords[1]) * 6.; 46 | sum += getP(imin(coords[0]+1, ${imageHeight}-1), coords[1]) * 4.; 47 | sum += getP(imin(coords[0]+2, ${imageHeight}-1), coords[1]); 48 | sum /= 256.; 49 | setOutput(sum); 50 | } 51 | ` 52 | }; 53 | 54 | return tf.tidy(() => { 55 | const result1 = runWebGLProgram(kernel1, [image], 'float32'); 56 | const result2 = runWebGLProgram(kernel2, [result1], 'float32'); 57 | return result2; 58 | }); 59 | } 60 | -------------------------------------------------------------------------------- /testing/glsl/applyFilter2.js: -------------------------------------------------------------------------------- 1 | const applyFilter2 = (image) => { 2 | const imageHeight = image.shape[0]; 3 | const imageWidth = image.shape[1]; 4 | 5 | const imaxmin = ` 6 | int imax(int a, int b) { 7 | if (a > b) return a; 8 | return b; 9 | } 10 | int imin(int a, int b) { 11 | if (a < b) return a; 12 | return b; 13 | } 14 | 15 | ` 16 | const kernel1 = { 17 | variableNames: ['p'], 18 | outputShape: [imageHeight, imageWidth], 19 | userCode: ` 20 | ${imaxmin} 21 | void main() { 22 | ivec2 coords = getOutputCoords(); 23 | float sum = getP(coords[0], coords[1]-1) * 0.25; 24 | sum += getP(coords[0], coords[1]) * 0.5; 25 | sum += getP(coords[0], coords[1]+1) * 0.25; 26 | setOutput(sum); 27 | } 28 | ` 29 | }; 30 | const kernel2 = { 31 | variableNames: ['p'], 32 | outputShape: [imageHeight, imageWidth], 33 | userCode: ` 34 | ${imaxmin} 35 | void main() { 36 | ivec2 coords = getOutputCoords(); 37 | float sum = getP(coords[0]-1, coords[1]) * 0.25; 38 | sum += getP(coords[0], coords[1]) * 0.5; 39 | sum += getP(coords[0]+1, coords[1]) * 0.25; 40 | setOutput(sum); 41 | } 42 | ` 43 | }; 44 | 45 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D'; 46 | 47 | const dummyKernel = { 48 | variableNames: ['p'], 49 | outputShape: [imageHeight, imageWidth], 50 | packedOutput: true, 51 | userCode: ` 52 | void main() { 53 | ivec2 coords = getOutputCoords(); 54 | 55 | vec4 values = ${textureMethod}(p, resultUV); 56 | vec4 v = vec4(getP(coords[0], coords[1]), 0, 0, 0); 57 | setOutput(v); 58 | } 59 | ` 60 | } 61 | 62 | return tf.tidy(() => { 63 | //const resultDummy = runWebGLProgram(dummyKernel, [image], 'int32'); 64 | //const result = compileAndRun(kernel1, [resultDummy]); 65 | //return result; 66 | 67 | const out1 = tf.backend().runWebGLProgram(kernel1, [image], 'float32'); 68 | const out2 = tf.backend().runWebGLProgram(kernel2, [out1], 'float32'); 69 | tf.backend().disposeIntermediateTensorInfo(out1); 70 | return tf.engine().makeTensorFromDataId(out2.dataId, out2.shape, out2.dtype); 71 | 72 | const result1 = compileAndRun(kernel1, [image]); 73 | return result1; 74 | const result2 = compileAndRun(kernel2, [result1]); 75 | return result2; 76 | }); 77 | } 78 | -------------------------------------------------------------------------------- /testing/glsl/loadImageToTensor.js: -------------------------------------------------------------------------------- 1 | const loadImageToTensor = (canvas) => { 2 | const width = canvas.width; 3 | const height = canvas.height; 4 | const backend = tf.backend(); 5 | 6 | const texShape = [height, width]; 7 | 8 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D'; 9 | const program = { 10 | variableNames: ['A'], 11 | outputShape: texShape, 12 | userCode:` 13 | void main() { 14 | ivec2 coords = getOutputCoords(); 15 | int texR = coords[0]; 16 | int texC = coords[1]; 17 | 18 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0); 19 | vec4 values = ${textureMethod}(A, uv); 20 | float res = (values.r + values.g + values.b) * 255.0 / 3.0; 21 | 22 | setOutput(res); 23 | } 24 | ` 25 | } 26 | 27 | if (!caches.loadImageToTensor) { 28 | const tempPixelHandle = backend.makeTensorInfo(texShape, 'int32'); 29 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(tempPixelHandle.dataId), canvas); 30 | backend.texData.get(tempPixelHandle.dataId).usage = 2; 31 | 32 | caches.loadImageToTensor = { 33 | tempPixelHandle 34 | } 35 | } 36 | 37 | const {tempPixelHandle} = caches.loadImageToTensor; 38 | 39 | const inputs = [tempPixelHandle]; 40 | 41 | const result = runWebGLProgram(program, inputs, 'int32'); 42 | return result; 43 | 44 | //const outInfo = tf.backend().compileAndRun(program, inputs); 45 | //const res = tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype); 46 | //return res; 47 | } 48 | -------------------------------------------------------------------------------- /testing/glsl/loadImageToTensor2.js: -------------------------------------------------------------------------------- 1 | const loadImageToTensor2 = (canvas) => { 2 | const width = canvas.width; 3 | const height = canvas.height; 4 | const backend = tf.backend(); 5 | 6 | const texShape = [height, width]; 7 | 8 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D'; 9 | const program = { 10 | variableNames: ['A'], 11 | outputShape: texShape, 12 | userCode:` 13 | void main() { 14 | vec4 values = ${textureMethod}(A, resultUV); 15 | float res = (0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0; 16 | setOutput(res); 17 | } 18 | ` 19 | } 20 | 21 | if (!caches.loadImageToTensor2) { 22 | const tempPixelHandle = backend.makeTensorInfo(texShape, 'int32'); 23 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(tempPixelHandle.dataId), canvas); 24 | backend.texData.get(tempPixelHandle.dataId).usage = 2; 25 | 26 | caches.loadImageToTensor2 = { 27 | tempPixelHandle 28 | } 29 | } 30 | 31 | const {tempPixelHandle} = caches.loadImageToTensor2; 32 | 33 | const inputs = [tempPixelHandle]; 34 | 35 | const outInfo = tf.backend().compileAndRun(program, inputs); 36 | const res = tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype); 37 | 38 | return res; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /testing/glsl/packKernel.js: -------------------------------------------------------------------------------- 1 | const packKernel = (image) => { 2 | //const imageHeight = image.shape[0]; 3 | const imageHeight = 10; 4 | const imageWidth = image.shape[1]; 5 | 6 | const kernel = { 7 | variableNames: ['p'], 8 | outputShape: [imageHeight], 9 | packedOutput: true, 10 | userCode: ` 11 | void main() { 12 | int coords = getOutputCoords(); 13 | //setOutput(float(coords)); 14 | setOutput(vec4(1,0,0,0)); 15 | } 16 | ` 17 | }; 18 | 19 | const dummyKernel = { 20 | variableNames: ['a'], 21 | outputShape: [imageHeight], 22 | packedInputs: true, 23 | userCode: ` 24 | void main() { 25 | int coords = getOutputCoords(); 26 | 27 | int texNumC = 10; 28 | int texNumR = 1; 29 | int texelIndex = coords; 30 | int texR = texelIndex / texNumC; 31 | int texC = texelIndex - texR * texNumC; 32 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR); 33 | 34 | vec4 values = texture(a, vec2(0.3, 0.1)); 35 | //vec4 values = texture(a, uv); 36 | //vec4 values = texture(a, resultUV); 37 | setOutput(values.r); 38 | 39 | //setOutput(getA(coords).r); 40 | //setOutput(getAAtOuttCoords().r); 41 | } 42 | 43 | ` 44 | } 45 | 46 | return tf.tidy(() => { 47 | //const resultDummy = runWebGLProgram(dummyKernel, [image], 'int32'); 48 | //const result = compileAndRun(kernel1, [resultDummy]); 49 | //return result; 50 | 51 | const out1 = tf.backend().runWebGLProgram(kernel, [image], 'int32'); 52 | const out2 = tf.backend().runWebGLProgram(dummyKernel, [out1], 'int32'); 53 | const result = tf.engine().makeTensorFromDataId(out1.dataId, out1.shape, out1.dtype); 54 | const result2 = tf.engine().makeTensorFromDataId(out2.dataId, out2.shape, out2.dtype); 55 | 56 | console.log("test pack", result, result.arraySync()); 57 | console.log("test pack", result2, result2.arraySync()); 58 | 59 | return result; 60 | }); 61 | } 62 | -------------------------------------------------------------------------------- /testing/glsl/randomImage.js: -------------------------------------------------------------------------------- 1 | const randomImage = (height, width) => { 2 | const mRandSeed = 1234; 3 | 4 | const createRandomizer = () => { 5 | const randomizer = { 6 | seed: mRandSeed, 7 | 8 | arrayShuffle(options) { 9 | const {arr, sampleSize} = options; 10 | for (let i = 0; i < sampleSize; i++) { 11 | 12 | this.seed = (214013 * this.seed + 2531011) % (1 << 31); 13 | let k = (this.seed >> 16) & 0x7fff; 14 | k = k % arr.length; 15 | 16 | let tmp = arr[i]; 17 | arr[i] = arr[k]; 18 | arr[k] = tmp; 19 | } 20 | }, 21 | 22 | nextInt(maxValue) { 23 | this.seed = (214013 * this.seed + 2531011) % (1 << 31); 24 | let k = (this.seed >> 16) & 0x7fff; 25 | k = k % maxValue; 26 | return k; 27 | } 28 | } 29 | return randomizer; 30 | } 31 | 32 | const randomizer = createRandomizer(); 33 | 34 | const tData = new Uint8ClampedArray(height * width * 4); 35 | for (let j = 0; j < height; j++) { 36 | for (let i = 0; i < width; i++) { 37 | const index = j * width + i; 38 | tData[index*4] = randomizer.nextInt(255); 39 | tData[index*4+1] = randomizer.nextInt(255); 40 | tData[index*4+2] = randomizer.nextInt(255); 41 | tData[index*4+3] = 255; 42 | } 43 | } 44 | const imageData = new ImageData(tData, width, height); 45 | const canvas = document.createElement('canvas'); 46 | canvas.width = width; 47 | canvas.height = height; 48 | const ctx = canvas.getContext('2d'); 49 | ctx.putImageData(imageData, 0, 0); 50 | 51 | console.log("tData", tData); 52 | 53 | return new Promise((resolve, reject) => { 54 | const img = document.createElement("img"); 55 | img.onload = () => { 56 | resolve(img); 57 | } 58 | img.src = canvas.toDataURL("image/png"); 59 | }); 60 | } 61 | -------------------------------------------------------------------------------- /testing/homography.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 21 | 22 | 28 | 29 | 30 | 31 |
32 | 33 | 34 | -------------------------------------------------------------------------------- /testing/matching-all.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 44 | 45 | 47 | 48 | 49 | 50 |
51 |
52 | 53 | 54 | -------------------------------------------------------------------------------- /testing/matching.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 29 | 30 | 35 | 36 | 37 | 38 |
39 |
40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /testing/mobile-console.css: -------------------------------------------------------------------------------- 1 | #js-mobile-console{ 2 | width: 100%; 3 | position: fixed; 4 | bottom: 10px; 5 | z-index: 10000; 6 | } 7 | 8 | #jsmc-log{ 9 | border-top: 1px solid #808080; 10 | max-height: 500px; 11 | overflow-y: auto; 12 | margin-right: 10px; 13 | font-size: 1em; 14 | } 15 | 16 | #jsmc-input-container{ 17 | position: relative; 18 | } 19 | 20 | #jsmc-input{ 21 | width: 100%; 22 | } 23 | 24 | #jsmc-button{ 25 | position: fixed; 26 | bottom: 31px; 27 | right: 48px; 28 | } 29 | 30 | .jsmc-log-el{ 31 | color: red; 32 | } 33 | 34 | .jsmc-log-error .jsmc-log-text{ 35 | color: red; 36 | } 37 | 38 | .jsmc-log-command .jsmc-log-text{ 39 | color: #009EE4; 40 | } 41 | 42 | .jsmc-log-text{ 43 | color: black; 44 | } 45 | 46 | .jsmc-log-target{ 47 | color: #7979EF; 48 | } 49 | 50 | .jsmc-log-target{ 51 | clear: both; 52 | } 53 | 54 | #jsmc-content{ 55 | background: white; 56 | clear: both; 57 | } 58 | 59 | #jsmc-collapse, 60 | #jsmc-commands, 61 | #jsmc-clear{ 62 | cursor: pointer; 63 | width: 25px; 64 | background: white; 65 | text-align: center; 66 | color: gray; 67 | border: 1px solid #808080; 68 | display: inline-block; 69 | } 70 | 71 | #jsmc-clear{ 72 | font-weight: bolder; 73 | height: 19px; 74 | } 75 | 76 | #jsmc-commands{ 77 | display: none; 78 | } 79 | 80 | #jsmc-commands-container{ 81 | background: #FFF; 82 | display: none; 83 | } 84 | 85 | .jsmc-command-wrapper{ 86 | border-bottom: 1px solid #808080; 87 | padding: 0px 14px; 88 | cursor: pointer; 89 | } 90 | 91 | .jsmc-command:last-child{ 92 | border-bottom: none; 93 | } 94 | -------------------------------------------------------------------------------- /testing/speed.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 21 | 22 | 28 | 29 | 30 | 31 |
32 | 33 | 34 | -------------------------------------------------------------------------------- /testing/speed.js: -------------------------------------------------------------------------------- 1 | const { useEffect, useMemo, useRef, useState, useCallback } = React; 2 | 3 | let queryImages = []; 4 | let controller; 5 | 6 | const Main = () => { 7 | const [numRun, setNumRun] = useState(10); 8 | const [ready, setReady] = useState(false); 9 | const [result, setResult] = useState(null); 10 | 11 | useEffect(() => { 12 | const init = async () => { 13 | //for (let i = 11; i <= 15; i+=1) { 14 | // queryImages.push(await utils.loadImage('../tests/video2/out' + i + '.png')); 15 | //} 16 | //queryImages.push(await utils.loadImage('../tests/video2/out01_540.png')); 17 | //queryImages.push(await utils.loadImage('../tests/video2/out01_540.png')); 18 | queryImages.push(await utils.loadImage('../tests/video2/out01.png')); 19 | queryImages.push(await utils.loadImage('../tests/video2/out02.png')); 20 | const queryImage0 = queryImages[0]; 21 | 22 | const inputWidth = queryImage0.width; 23 | const inputHeight = queryImage0.height; 24 | controller = new MINDAR.Controller({inputWidth, inputHeight, debugMode: false}); 25 | const {dimensions, matchingDataList, trackingDataList, imageListList} = await controller.addImageTargets('../examples/assets/card-example/card.mind'); 26 | controller.dummyRun(queryImage0); 27 | setReady(true); 28 | } 29 | init(); 30 | }, []); 31 | 32 | const start = useCallback(async () => { 33 | const targetIndex = 0; 34 | console.log("start", numRun); 35 | 36 | let timeSpent = { 37 | detect: [], 38 | match: [], 39 | track: [], 40 | trackUpdate: [], 41 | }; 42 | let _start = null; 43 | 44 | for (let i = 0; i < numRun; i++) { 45 | _start = new Date(); 46 | const {featurePoints} = await controller.detect(queryImages[0]); 47 | timeSpent['detect'].push(new Date() - _start); 48 | 49 | _start = new Date(); 50 | const {modelViewTransform: firstModelViewTransform, allMatchResults, debugExtra} = await controller.match(featurePoints, targetIndex); 51 | 52 | /* 53 | const sumTime = {}; 54 | for (let j = 0; j < debugExtras[0].length; j++) { 55 | Object.keys(debugExtras[0][j].time).forEach((k) => { 56 | if (!sumTime[k]) sumTime[k] = 0; 57 | sumTime[k] += debugExtras[0][j].time[k]; 58 | }); 59 | } 60 | const timeDiv = document.createElement("div"); 61 | timeDiv.innerHTML = JSON.stringify(sumTime); 62 | document.body.appendChild(timeDiv); 63 | */ 64 | 65 | timeSpent['match'].push(new Date() - _start); 66 | 67 | if (!firstModelViewTransform) { 68 | timeSpent['track'].push(0); 69 | timeSpent['trackUpdate'].push(0); 70 | continue; 71 | } 72 | 73 | _start = new Date(); 74 | const nKeyframes = 1; 75 | const trackResult = await controller.track(queryImages[1], firstModelViewTransform, targetIndex); 76 | timeSpent['track'].push(new Date() - _start); 77 | //console.log("trackResultsf", trackResult); 78 | 79 | _start = new Date(); 80 | const bestSelectedFeatures = trackResult; 81 | const newModelViewTransform = await controller.trackUpdate(firstModelViewTransform, bestSelectedFeatures); 82 | timeSpent['trackUpdate'].push(new Date() - _start); 83 | 84 | controller.showTFStats(); 85 | } 86 | 87 | const avg = {}; 88 | Object.keys(timeSpent).forEach((key) => { 89 | const sum = timeSpent[key].reduce((acc, t) => acc + t, 0.0); 90 | avg[key] = sum / timeSpent[key].length; 91 | }); 92 | 93 | setResult({timeSpent, avg}); 94 | }); 95 | 96 | console.log("result", result); 97 | 98 | return ( 99 |
100 |

Speed Test

101 | {!ready && Initializing...} 102 | {ready && ( 103 |
104 | setNumRun(e.target.value)}/> 105 | 106 |
107 | )} 108 | 109 | {result && ( 110 |
111 | 112 | 113 | 114 | 115 | {result.timeSpent['detect'].map((spent, index) => ( 116 | 117 | ))} 118 | 119 | 120 | 121 | 122 | {Object.keys(result.timeSpent).map((key, keyIndex) => ( 123 | 124 | 125 | {result.timeSpent[key].map((t, index) => ( 126 | 127 | ))} 128 | 129 | 130 | ))} 131 | 132 |
{index+1}avg
{key}{t}{result.avg[key]}
133 |
134 | )} 135 |
136 | ) 137 | } 138 | 139 | ReactDOM.render( 140 |
, 141 | document.getElementById('root') 142 | ); 143 | -------------------------------------------------------------------------------- /testing/tracking.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 37 | 38 | 39 | 40 |
41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /vite.config.dev.js: -------------------------------------------------------------------------------- 1 | import {defineConfig,build} from 'vite' 2 | import * as fs from 'fs/promises'; 3 | import * as path from 'path'; 4 | import basicSsl from '@vitejs/plugin-basic-ssl' 5 | 6 | const outDir = 'dist-dev' 7 | 8 | const moduleConfig={ 9 | mode: 'development', 10 | assetsInclude:'**/*.html', 11 | base:'./', 12 | plugins:[ 13 | basicSsl() 14 | ], 15 | build: { 16 | outDir: outDir, 17 | emptyOutDir:false, 18 | sourcemap:'inline' , 19 | lib: { 20 | fileName:"[name]", 21 | entry:'index.js', 22 | formats:['es'] 23 | }, 24 | rollupOptions:{ 25 | external:(id)=>(id==='three'||id.includes('three/examples/jsm/')||id.includes('three/addons/')), 26 | input:{ 27 | 'mindar-image': './src/image-target/index.js', 28 | 'mindar-image-three': './src/image-target/three.js', 29 | 'mindar-face': './src/face-target/index.js', 30 | 'mindar-face-three': './src/face-target/three.js', 31 | } 32 | }, 33 | }, 34 | resolve:{ 35 | alias:{ 36 | 'three/addons/':'three/examples/jsm/' 37 | } 38 | } 39 | }; 40 | const faceAframeConfig=defineConfig({ 41 | mode: 'development', 42 | build: { 43 | outDir: outDir, 44 | emptyOutDir:false, 45 | sourcemap:'inline' , 46 | minify: false, 47 | lib: { 48 | name:"MINDAR", 49 | fileName:"[name]", 50 | entry:'index.js', 51 | formats:['iife'] 52 | }, 53 | rollupOptions:{ 54 | input:{ 55 | 'mindar-face-aframe': './src/face-target/aframe.js', 56 | }, 57 | 58 | } 59 | } 60 | }) 61 | /** @type {import('vite').UserConfig} */ 62 | const imageAframeConfig=defineConfig({ 63 | mode: 'development', 64 | build: { 65 | outDir: outDir, 66 | emptyOutDir:false, 67 | sourcemap:'inline' , 68 | minify: false, 69 | lib: { 70 | name:"MINDAR", 71 | fileName:"[name]", 72 | entry:'index.js', 73 | formats:['iife'], 74 | 75 | }, 76 | rollupOptions:{ 77 | input:{ 78 | 'mindar-image-aframe': './src/image-target/aframe.js' 79 | } 80 | } 81 | } 82 | }) 83 | 84 | export default defineConfig(async ({ command, mode }) => { 85 | await fs.rm(outDir,{recursive:true,force:true}); 86 | if (command === 'build') { 87 | await build(imageAframeConfig); 88 | await build(faceAframeConfig); 89 | const files=await fs.readdir(outDir); 90 | //rename the aframe builds 91 | await Promise.all(files.map(async (filename)=>{ 92 | if(filename.includes(".iife.js")){ 93 | const newName=filename.replace(".iife.js",".js"); 94 | console.log(filename,"->",newName) 95 | await fs.rename(path.join(outDir,filename),path.join(outDir,newName)); 96 | } 97 | })); 98 | } 99 | return moduleConfig 100 | }) 101 | -------------------------------------------------------------------------------- /vite.config.prod.js: -------------------------------------------------------------------------------- 1 | import {defineConfig,build} from 'vite' 2 | import * as fs from 'fs/promises'; 3 | import * as path from 'path'; 4 | 5 | const outDir = 'dist' 6 | 7 | /** @type {import('vite').UserConfig} */ 8 | const moduleConfig= defineConfig({ 9 | mode: 'production', 10 | publicDir:false, 11 | base:'./', 12 | build: { 13 | outDir:outDir, 14 | emptyOutDir:false, 15 | copyPublicDir:false, 16 | lib: { 17 | fileName:"[name].prod", 18 | entry:'index.js', 19 | formats:["es"], 20 | }, 21 | rollupOptions:{ 22 | external:(id)=>(id==='three'||id.includes('three/examples/jsm/')||id.includes('three/addons/')), 23 | input:{ 24 | 'mindar-image': './src/image-target/index.js', 25 | 'mindar-image-three': './src/image-target/three.js', 26 | 'mindar-face': './src/face-target/index.js', 27 | 'mindar-face-three': './src/face-target/three.js' 28 | } 29 | } 30 | }, 31 | resolve:{ 32 | alias:{ 33 | 'three/addons/':'three/examples/jsm/' 34 | } 35 | } 36 | }); 37 | const faceAframeConfig=defineConfig({ 38 | mode: 'production', 39 | build: { 40 | outDir: outDir, 41 | emptyOutDir:false, 42 | lib: { 43 | name:"MINDAR", 44 | fileName:"[name].prod", 45 | entry:'index.js', 46 | formats:['iife'] 47 | }, 48 | rollupOptions:{ 49 | input:{ 50 | 'mindar-face-aframe': './src/face-target/aframe.js', 51 | }, 52 | 53 | } 54 | } 55 | }) 56 | /** @type {import('vite').UserConfig} */ 57 | const imageAframeConfig=defineConfig({ 58 | mode: 'production', 59 | build: { 60 | outDir: outDir, 61 | emptyOutDir:false, 62 | lib: { 63 | name:"MINDAR", 64 | fileName:"[name].prod", 65 | entry:'index.js', 66 | formats:['iife'], 67 | 68 | }, 69 | rollupOptions:{ 70 | input:{ 71 | 'mindar-image-aframe': './src/image-target/aframe.js' 72 | } 73 | } 74 | } 75 | }) 76 | 77 | export default defineConfig(async ({ command, mode }) => { 78 | await fs.rm(outDir,{recursive:true,force:true}); 79 | if (command === 'build') { 80 | await build(imageAframeConfig); 81 | await build(faceAframeConfig); 82 | const files=await fs.readdir(outDir); 83 | //rename the aframe builds 84 | await Promise.all(files.map(async (filename)=>{ 85 | if(filename.includes(".iife.js")){ 86 | const newName=filename.replace(".iife.js",".js"); 87 | console.log(filename,"->",newName) 88 | await fs.rename(path.join(outDir,filename),path.join(outDir,newName)); 89 | } 90 | })); 91 | } 92 | return moduleConfig 93 | }) 94 | --------------------------------------------------------------------------------