├── .github
└── FUNDING.yml
├── .gitignore
├── .npmignore
├── KnownIssues.md
├── LICENSE
├── README.md
├── custom_tfjs
├── custom_tfjs.js
└── custom_tfjs_core.js
├── custom_tfjs_config.json
├── examples
├── face-tracking
│ ├── assets
│ │ ├── canonical_face_model_uv_visualization.png
│ │ ├── earring
│ │ │ ├── license.txt
│ │ │ ├── scene.bin
│ │ │ ├── scene.gltf
│ │ │ ├── textures
│ │ │ │ ├── PEARL_DROP_EARRINGSG_baseColor.png
│ │ │ │ ├── PEARL_DROP_EARRINGSG_metallicRoughness.png
│ │ │ │ └── PEARL_DROP_EARRINGSG_normal.png
│ │ │ └── thumbnail.png
│ │ ├── face1.jpeg
│ │ ├── glasses
│ │ │ ├── license.txt
│ │ │ ├── scene.bin
│ │ │ ├── scene.gltf
│ │ │ └── thumbnail.png
│ │ ├── glasses2
│ │ │ ├── scene.bin
│ │ │ ├── scene.gltf
│ │ │ └── thumbnail.png
│ │ ├── hat
│ │ │ ├── license.txt
│ │ │ ├── scene.bin
│ │ │ ├── scene.gltf
│ │ │ └── thumbnail.png
│ │ ├── hat2
│ │ │ ├── license.txt
│ │ │ ├── scene.bin
│ │ │ ├── scene.gltf
│ │ │ ├── textures
│ │ │ │ ├── 21_-_Default_baseColor.png
│ │ │ │ └── 21_-_Default_normal.png
│ │ │ └── thumbnail.png
│ │ └── sparkar
│ │ │ ├── face-mask.png
│ │ │ ├── face-mesk-license.txt
│ │ │ ├── headOccluder.glb
│ │ │ └── licenses.txt
│ ├── debug.html
│ ├── example1.html
│ ├── example2.html
│ ├── example3.html
│ ├── three-blendshapes.html
│ ├── three-css.html
│ ├── three-facemesh.html
│ └── three.html
├── image-tracking
│ ├── assets
│ │ ├── band-example
│ │ │ ├── band.mind
│ │ │ ├── bear-color.png
│ │ │ ├── bear.png
│ │ │ ├── bear
│ │ │ │ ├── scene.bin
│ │ │ │ ├── scene.gltf
│ │ │ │ └── textures
│ │ │ │ │ ├── Cello_paint_baseColor.png
│ │ │ │ │ └── bear_paint_baseColor.png
│ │ │ ├── new-raccoon.mind
│ │ │ ├── pictarize.mind
│ │ │ ├── raccoon-card.mind
│ │ │ ├── raccoon-color.png
│ │ │ ├── raccoon-highres.mind
│ │ │ ├── raccoon.mind
│ │ │ ├── raccoon.png
│ │ │ └── raccoon
│ │ │ │ ├── scene.bin
│ │ │ │ ├── scene.gltf
│ │ │ │ └── textures
│ │ │ │ ├── drum_defaultCol_baseColor.png
│ │ │ │ ├── drum_paint_baseColor.png
│ │ │ │ └── raccoon_paint_baseColor.png
│ │ ├── card-example
│ │ │ ├── card.mind
│ │ │ ├── card.png
│ │ │ ├── softmind
│ │ │ │ ├── scene.bin
│ │ │ │ ├── scene.gltf
│ │ │ │ └── textures
│ │ │ │ │ ├── material_baseColor.png
│ │ │ │ │ ├── material_emissive.png
│ │ │ │ │ ├── material_metallicRoughness.png
│ │ │ │ │ └── material_normal.png
│ │ │ └── test-card.png
│ │ └── pictarize-example
│ │ │ ├── pictarize.jpg
│ │ │ └── pictarize.mind
│ ├── compile.html
│ ├── debug.html
│ ├── example1.html
│ ├── example2.html
│ ├── example3.html
│ ├── example4.html
│ └── three.html
└── nodejs
│ └── createImageTargetLibrary.js
├── index.html
├── index.js
├── package-lock.json
├── package.json
├── src
├── face-target
│ ├── aframe.js
│ ├── controller.js
│ ├── face-geometry
│ │ ├── canonical-face-model.obj
│ │ ├── estimator.js
│ │ ├── face-data-generator.js
│ │ ├── face-data.js
│ │ └── face-geometry.js
│ ├── face-mesh-helper.js
│ ├── index.js
│ └── three.js
├── image-target
│ ├── aframe.js
│ ├── compiler-base.js
│ ├── compiler.js
│ ├── compiler.worker.js
│ ├── controller.js
│ ├── controller.worker.js
│ ├── detector
│ │ ├── crop-detector.js
│ │ ├── detector.js
│ │ ├── freak.js
│ │ └── kernels
│ │ │ ├── cpu
│ │ │ ├── binomialFilter.js
│ │ │ ├── buildExtremas.js
│ │ │ ├── computeExtremaAngles.js
│ │ │ ├── computeExtremaFreak.js
│ │ │ ├── computeFreakDescriptors.js
│ │ │ ├── computeLocalization.js
│ │ │ ├── computeOrientationHistograms.js
│ │ │ ├── downsampleBilinear.js
│ │ │ ├── extremaReduction.js
│ │ │ ├── fakeShader.js
│ │ │ ├── index.js
│ │ │ ├── prune.js
│ │ │ ├── smoothHistograms.js
│ │ │ └── upsampleBilinear.js
│ │ │ ├── index.js
│ │ │ └── webgl
│ │ │ ├── binomialFilter.js
│ │ │ ├── buildExtremas.js
│ │ │ ├── computeExtremaAngles.js
│ │ │ ├── computeExtremaFreak.js
│ │ │ ├── computeFreakDescriptors.js
│ │ │ ├── computeLocalization.js
│ │ │ ├── computeOrientationHistograms.js
│ │ │ ├── downsampleBilinear.js
│ │ │ ├── extremaReduction.js
│ │ │ ├── index.js
│ │ │ ├── smoothHistograms.js
│ │ │ └── upsampleBilinear.js
│ ├── estimation
│ │ ├── esimate-experiment.js
│ │ ├── estimate.js
│ │ ├── estimator.js
│ │ ├── refine-estimate-experiment.js
│ │ ├── refine-estimate.js
│ │ └── utils.js
│ ├── image-list.js
│ ├── index.js
│ ├── input-loader.js
│ ├── matching
│ │ ├── hamming-distance.js
│ │ ├── hierarchical-clustering.js
│ │ ├── hough.js
│ │ ├── matcher.js
│ │ ├── matching.js
│ │ └── ransacHomography.js
│ ├── offline-compiler.js
│ ├── three.js
│ ├── tracker
│ │ ├── extract-utils.js
│ │ ├── extract.js
│ │ └── tracker.js
│ └── utils
│ │ ├── cumsum.js
│ │ ├── geometry.js
│ │ ├── homography.js
│ │ ├── images.js
│ │ └── randomizer.js
├── libs
│ ├── one-euro-filter.js
│ ├── opencv-helper.js
│ ├── opencv-wasm.js
│ └── opencv.js
└── ui
│ ├── compatibility.html
│ ├── loading.html
│ ├── scanning.html
│ ├── ui.js
│ └── ui.scss
├── testing
├── common.js
├── detection.html
├── detection.js
├── glsl
│ ├── applyFilter.js
│ ├── applyFilter2.js
│ ├── glsl.html
│ ├── loadImageToTensor.js
│ ├── loadImageToTensor2.js
│ ├── packKernel.js
│ └── randomImage.js
├── homography.html
├── homography.js
├── matching-all.html
├── matching-all.js
├── matching.html
├── matching.js
├── mobile-console.css
├── mobile-console.js
├── speed.html
├── speed.js
├── tracking.html
└── tracking.js
├── vite.config.dev.js
└── vite.config.prod.js
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | # These are supported funding model platforms
2 |
3 | github: hiukim
4 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | tests
3 | *.swp
4 | dist-dev
5 | dist/
6 | *.DS_Store
7 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | testing/
2 | examples/
3 | dist-dev/
4 | custom_tfjs_config.json
5 | custom_tfjs/
6 | vite.config.*
7 | mind-ar-*.tgz
--------------------------------------------------------------------------------
/KnownIssues.md:
--------------------------------------------------------------------------------
1 | Mac (node v18) failed to install node-canvas
2 |
3 | > brew install pkg-config cairo pango libpng jpeg giflib librsvg pixman
4 |
5 | Ref: https://github.com/Automattic/node-canvas/issues/2025
6 |
7 |
8 | Import MindAR in ESBuild
9 |
10 | If you are using esbuild and running into require errors, you can try importing the classes directly
11 |
12 | Example:
13 |
14 | `import {MindARThree} from 'mind-ar/src/image-target/three.js'`
15 |
16 | You'll need to use the esbuild-plugin-inline-worker
17 |
18 | Ref issue: https://github.com/hiukim/mind-ar-js/issues/360
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 hiukim
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/custom_tfjs/custom_tfjs.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @license
3 | * Copyright 2022 Google LLC. All Rights Reserved.
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | * =============================================================================
16 | */
17 |
18 | // This file is autogenerated.
19 |
20 |
21 | import {registerKernel} from '@tensorflow/tfjs-core/dist/base';
22 | import '@tensorflow/tfjs-core/dist/base_side_effects';
23 | export * from '@tensorflow/tfjs-core/dist/base';
24 |
25 | //backend = webgl
26 | export * from '@tensorflow/tfjs-backend-webgl/dist/base';
27 | import {reshapeConfig as Reshape_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Reshape';
28 | registerKernel(Reshape_webgl);
29 | import {subConfig as Sub_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Sub';
30 | registerKernel(Sub_webgl);
31 | import {castConfig as Cast_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Cast';
32 | registerKernel(Cast_webgl);
33 | import {packConfig as Pack_webgl} from '@tensorflow/tfjs-backend-webgl/dist/kernels/Pack';
34 | registerKernel(Pack_webgl);
--------------------------------------------------------------------------------
/custom_tfjs/custom_tfjs_core.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @license
3 | * Copyright 2022 Google LLC. All Rights Reserved.
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | * =============================================================================
16 | */
17 |
18 | // This file is autogenerated.
19 |
20 |
21 | import {registerKernel} from '@tensorflow/tfjs-core/dist/base';
22 | import '@tensorflow/tfjs-core/dist/base_side_effects';
23 | export * from '@tensorflow/tfjs-core/dist/base';
--------------------------------------------------------------------------------
/custom_tfjs_config.json:
--------------------------------------------------------------------------------
1 | {
2 | "kernels": ["Reshape",
3 | "Sub",
4 | "Cast",
5 | "Pack"
6 | ],
7 | "backends": [
8 | "webgl"
9 | ],
10 | "models": [
11 |
12 | ],
13 | "outputPath": "./custom_tfjs",
14 | "forwardModeOnly": true
15 | }
--------------------------------------------------------------------------------
/examples/face-tracking/assets/canonical_face_model_uv_visualization.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/canonical_face_model_uv_visualization.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/license.txt:
--------------------------------------------------------------------------------
1 | Model Information:
2 | * title: Earring Jade
3 | * source: https://sketchfab.com/3d-models/earring-jade-c91e552728fb4b149d52193fd3e03ac5
4 | * author: Softmind Game Factory (https://sketchfab.com/softmind)
5 |
6 | Model License:
7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
8 | * requirements: Author must be credited. Commercial use is allowed.
9 |
10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it:
11 | This work is based on "Earring Jade" (https://sketchfab.com/3d-models/earring-jade-c91e552728fb4b149d52193fd3e03ac5) by Softmind Game Factory (https://sketchfab.com/softmind) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/scene.bin
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_baseColor.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_metallicRoughness.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_metallicRoughness.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_normal.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/textures/PEARL_DROP_EARRINGSG_normal.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/earring/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/earring/thumbnail.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/face1.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/face1.jpeg
--------------------------------------------------------------------------------
/examples/face-tracking/assets/glasses/license.txt:
--------------------------------------------------------------------------------
1 | Model Information:
2 | * title: Thug Life | Cool glasses | Stylise goggles
3 | * source: https://sketchfab.com/3d-models/thug-life-cool-glasses-stylise-goggles-75945f055a8c47098fa8aa4b6a698b07
4 | * author: MR EXPERT (https://sketchfab.com/Rahul.Kumar5)
5 |
6 | Model License:
7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
8 | * requirements: Author must be credited. Commercial use is allowed.
9 |
10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it:
11 | This work is based on "Thug Life | Cool glasses | Stylise goggles" (https://sketchfab.com/3d-models/thug-life-cool-glasses-stylise-goggles-75945f055a8c47098fa8aa4b6a698b07) by MR EXPERT (https://sketchfab.com/Rahul.Kumar5) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
--------------------------------------------------------------------------------
/examples/face-tracking/assets/glasses/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses/scene.bin
--------------------------------------------------------------------------------
/examples/face-tracking/assets/glasses/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses/thumbnail.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/glasses2/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses2/scene.bin
--------------------------------------------------------------------------------
/examples/face-tracking/assets/glasses2/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/glasses2/thumbnail.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat/license.txt:
--------------------------------------------------------------------------------
1 | Model Information:
2 | * title: Clown Hat
3 | * source: https://sketchfab.com/3d-models/clown-hat-0f5ef2e6205c4b438c593a979a3ccdae
4 | * author: PatelDev (https://sketchfab.com/PatelDev)
5 |
6 | Model License:
7 | * license type: CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
8 | * requirements: Author must be credited. Commercial use is allowed.
9 |
10 | If you use this 3D model in your project be sure to copy paste this credit wherever you share it:
11 | This work is based on "Clown Hat" (https://sketchfab.com/3d-models/clown-hat-0f5ef2e6205c4b438c593a979a3ccdae) by PatelDev (https://sketchfab.com/PatelDev) licensed under CC-BY-4.0 (http://creativecommons.org/licenses/by/4.0/)
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat/scene.bin
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat/thumbnail.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat2/license.txt:
--------------------------------------------------------------------------------
1 | credit: https://sketchfab.com/3d-models/marios-cap-super-mario-bros-82bcc48237ec4ff98ce770de60913d26
2 |
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat2/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/scene.bin
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat2/textures/21_-_Default_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/textures/21_-_Default_baseColor.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat2/textures/21_-_Default_normal.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/textures/21_-_Default_normal.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/hat2/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/hat2/thumbnail.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/sparkar/face-mask.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/sparkar/face-mask.png
--------------------------------------------------------------------------------
/examples/face-tracking/assets/sparkar/face-mesk-license.txt:
--------------------------------------------------------------------------------
1 | LICENSE
2 | Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
3 |
4 | You are hereby granted a non-exclusive, worldwide, royalty-free license to use,
5 | copy, modify, and distribute this software in source code or binary form for use
6 | in connection with the web services and APIs provided by Facebook.
7 |
8 | As with any software that integrates with the Facebook platform, your use of
9 | this software is subject to the Facebook Developer Principles and Policies
10 | [http://developers.facebook.com/policy/]. This copyright notice shall be
11 | included in all copies or substantial portions of the software.
12 |
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/examples/face-tracking/assets/sparkar/headOccluder.glb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/face-tracking/assets/sparkar/headOccluder.glb
--------------------------------------------------------------------------------
/examples/face-tracking/assets/sparkar/licenses.txt:
--------------------------------------------------------------------------------
1 | Model Information:
2 | * title: Head Occluder
3 | * source: https://sparkar.facebook.com/ar-studio/learn/articles/people-tracking/face-reference-assets
4 | * author: Facebook
5 |
6 | This model is from Facebook SparkAR, please check out licensing agreement with them if you want to use it.
7 |
--------------------------------------------------------------------------------
/examples/face-tracking/debug.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
12 |
13 |
28 |
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/examples/face-tracking/example1.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
24 |
25 |
26 |
27 |
28 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/examples/face-tracking/example2.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
36 |
37 |
66 |
67 |
68 |
69 |
70 |
71 |

72 |

73 |

74 |

75 |

76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
--------------------------------------------------------------------------------
/examples/face-tracking/example3.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
51 |
52 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
--------------------------------------------------------------------------------
/examples/face-tracking/three-blendshapes.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
48 |
49 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
--------------------------------------------------------------------------------
/examples/face-tracking/three-css.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
24 |
25 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 | This is a CSS div
59 |
60 |

61 |
62 |
63 |
64 |
65 |
66 |
--------------------------------------------------------------------------------
/examples/face-tracking/three-facemesh.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
30 |
31 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/examples/face-tracking/three.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
86 |
87 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 | Front Camera
115 |
118 | Back Camera
119 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/band.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/band.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/bear-color.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear-color.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/bear.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/bear/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/scene.bin
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/bear/textures/Cello_paint_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/textures/Cello_paint_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/bear/textures/bear_paint_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/bear/textures/bear_paint_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/new-raccoon.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/new-raccoon.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/pictarize.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/pictarize.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon-card.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-card.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon-color.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-color.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon-highres.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon-highres.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/scene.bin
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon/textures/drum_defaultCol_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/drum_defaultCol_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon/textures/drum_paint_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/drum_paint_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/band-example/raccoon/textures/raccoon_paint_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/band-example/raccoon/textures/raccoon_paint_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/card.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/card.mind
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/card.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/card.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/softmind/scene.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/scene.bin
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/softmind/textures/material_baseColor.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_baseColor.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/softmind/textures/material_emissive.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_emissive.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/softmind/textures/material_metallicRoughness.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_metallicRoughness.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/softmind/textures/material_normal.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/softmind/textures/material_normal.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/card-example/test-card.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/card-example/test-card.png
--------------------------------------------------------------------------------
/examples/image-tracking/assets/pictarize-example/pictarize.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/pictarize-example/pictarize.jpg
--------------------------------------------------------------------------------
/examples/image-tracking/assets/pictarize-example/pictarize.mind:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/webarkit/mind-ar-js/fb5db87b1e5076ba86fde82101955da70b629a1c/examples/image-tracking/assets/pictarize-example/pictarize.mind
--------------------------------------------------------------------------------
/examples/image-tracking/debug.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
12 |
13 |
36 |
37 |
38 |
39 |
40 |
--------------------------------------------------------------------------------
/examples/image-tracking/example1.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
40 |
41 |
42 |
43 |
44 |
45 |
--------------------------------------------------------------------------------
/examples/image-tracking/example2.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
30 |
31 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
--------------------------------------------------------------------------------
/examples/image-tracking/example4.html:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
--------------------------------------------------------------------------------
/examples/image-tracking/three.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
92 |
93 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 | Front Camera
121 |
124 | Back Camera
125 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
--------------------------------------------------------------------------------
/examples/nodejs/createImageTargetLibrary.js:
--------------------------------------------------------------------------------
1 | import { OfflineCompiler } from '../../src/image-target/offline-compiler.js';
2 |
3 | import { writeFile } from 'fs/promises'
4 | import { loadImage } from 'canvas';
5 |
6 | //canvas.loadImage treats path as relative from where nodejs was executed, not relative to the script's location
7 | const imagePaths = ['examples/image-tracking/assets/card-example/card.png'];
8 |
9 | async function run() {
10 | //load all images
11 | const images = await Promise.all(imagePaths.map(value => loadImage(value)));
12 | const compiler = new OfflineCompiler();
13 | await compiler.compileImageTargets(images, console.log);
14 | const buffer = compiler.exportData();
15 | await writeFile('targets.mind', buffer);
16 | }
17 |
18 |
19 |
20 |
21 | run();
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Examples
8 | Image-Tracking
9 |
18 | Face-Tracking
19 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | module.exports = {};
2 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mind-ar",
3 | "version": "1.2.5",
4 | "description": "web augmented reality framework",
5 | "main": "index.js",
6 | "scripts": {
7 | "dev": "vite --config vite.config.dev.js --host",
8 | "watch": "vite build --watch --config vite.config.dev.js",
9 | "build-dev": "vite build --config vite.config.dev.js",
10 | "build": "vite build --config vite.config.prod.js"
11 | },
12 | "engines": {
13 | "node": ">=14.0.0"
14 | },
15 | "type": "module",
16 | "repository": {
17 | "type": "git",
18 | "url": "https://github.com/hiukim/mind-ar-js"
19 | },
20 | "keywords": [
21 | "web AR",
22 | "aframe",
23 | "web augmented reality",
24 | "image tracking",
25 | "face tracking",
26 | "tensorflowjs",
27 | "mediapipe"
28 | ],
29 | "author": "hiukim",
30 | "license": "MIT",
31 | "dependencies": {
32 | "@mediapipe/tasks-vision": "^0.10.9",
33 | "@msgpack/msgpack": "^2.8.0",
34 | "@tensorflow/tfjs": "^4.16.0",
35 | "@vitejs/plugin-basic-ssl": "^1.1.0",
36 | "canvas": "^2.10.2",
37 | "mathjs": "^11.6.0",
38 | "ml-matrix": "^6.10.4",
39 | "svd-js": "^1.1.1",
40 | "tinyqueue": "^2.0.3"
41 | },
42 | "peerDependencies": {
43 | "three": ">=0.136.0"
44 | },
45 | "files": [
46 | "dist/",
47 | "src/"
48 | ],
49 | "devDependencies": {
50 | "vite": "^5.0.11"
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/face-target/face-geometry/face-data-generator.js:
--------------------------------------------------------------------------------
1 | // canonical-face-model.obj source:
2 | // https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data
3 | //
4 | // this script parse the model data. To run: node face-data-generator.js
5 |
6 | import * as fs from "fs";
7 | const text = fs.readFileSync("./canonical-face-model.obj", "utf8");
8 | const textByLine = text.split("\n")
9 |
10 | const positions = [];
11 | const uvs = [];
12 | const faces = [];
13 | const uvIndexes = [];
14 |
15 | textByLine.forEach((line) => {
16 | const ss = line.split(" ");
17 | if (ss[0] === 'f') {
18 | for (let i = 1; i <= 3; i++) {
19 | const [index, uvIndex] = ss[i].split("/");
20 | uvIndexes[parseInt(uvIndex)-1] = parseInt(index)-1;
21 | }
22 | }
23 | });
24 |
25 | let uvCount = 0;
26 | textByLine.forEach((line) => {
27 | const ss = line.split(" ");
28 | if (ss[0] === 'v') {
29 | positions.push([ parseFloat(ss[1]),parseFloat(ss[2]),parseFloat(ss[3]) ]);
30 | } else if (ss[0] === 'vt') {
31 | uvs[uvIndexes[uvCount++]] = [ parseFloat(ss[1]),parseFloat(ss[2]) ];
32 | } else if (ss[0] === 'f') {
33 | faces.push(parseInt(ss[1].split("/")[0]-1),parseInt(ss[2].split("/")[0]-1),parseInt(ss[3].split("/")[0])-1);
34 | }
35 | });
36 |
37 | // important landmarks for computing transformation
38 | // pairs of [positionIndex, weight]
39 | const landmarkBasis = [[4, 0.070909939706326], [6, 0.032100144773722], [10, 0.008446550928056], [33, 0.058724168688059], [54, 0.007667080033571], [67, 0.009078059345484], [117, 0.009791937656701], [119, 0.014565368182957], [121, 0.018591361120343], [127, 0.005197994410992], [129, 0.120625205338001], [132, 0.005560018587857], [133, 0.05328618362546], [136, 0.066890455782413], [143, 0.014816547743976], [147, 0.014262833632529], [198, 0.025462191551924], [205, 0.047252278774977], [263, 0.058724168688059], [284, 0.007667080033571], [297, 0.009078059345484], [346, 0.009791937656701], [348, 0.014565368182957], [350, 0.018591361120343], [356, 0.005197994410992], [358, 0.120625205338001], [361, 0.005560018587857], [362, 0.05328618362546], [365, 0.066890455782413], [372, 0.014816547743976], [376, 0.014262833632529], [420, 0.025462191551924], [425, 0.047252278774977]]
40 |
41 | let output = "";
42 | output += "//This is a generated file\n";
43 | output += "const positions=" + JSON.stringify(positions) + ";\n";
44 | output += "const uvs=" + JSON.stringify(uvs) + ";\n";
45 | output += "const faces=" + JSON.stringify(faces) + ";\n";
46 | output += "const landmarkBasis=" + JSON.stringify(landmarkBasis) + ";\n";
47 |
48 | output += `
49 | export {
50 | positions, uvs, faces, landmarkBasis
51 | }
52 | `
53 |
54 | fs.writeFileSync("./face-data.js", output);
55 |
--------------------------------------------------------------------------------
/src/face-target/face-geometry/face-geometry.js:
--------------------------------------------------------------------------------
1 | import {uvs, faces} from "./face-data.js";
2 |
3 | const nLandmarks = uvs.length;
4 |
5 | const createThreeFaceGeometry = (THREE) => {
6 | class FaceGeometry extends THREE.BufferGeometry {
7 | constructor(options = {}) {
8 | super();
9 |
10 | this.positions = new Float32Array(nLandmarks * 3);
11 | this.uvs = new Float32Array(nLandmarks * 2);
12 | this.setAttribute("position", new THREE.BufferAttribute(this.positions, 3));
13 | this.setAttribute("uv", new THREE.BufferAttribute(this.uvs, 2));
14 | this.setUvs();
15 | this.setIndex(faces);
16 | }
17 |
18 | setUvs() {
19 | for (let j = 0; j < nLandmarks; j++) {
20 | this.uvs[j * 2] = uvs[j][0];
21 | this.uvs[j * 2 + 1] = uvs[j][1];
22 | }
23 | this.getAttribute("uv").needsUpdate = true;
24 | }
25 |
26 | updatePositions(landmarks) {
27 | for (let i = 0; i < nLandmarks; i++) {
28 | this.positions[i*3+0] = landmarks[i][0];
29 | this.positions[i*3+1] = landmarks[i][1];
30 | this.positions[i*3+2] = landmarks[i][2];
31 | }
32 | this.attributes.position.needsUpdate = true;
33 | this.computeVertexNormals();
34 | }
35 | }
36 | return new FaceGeometry();
37 | }
38 |
39 | export {
40 | createThreeFaceGeometry
41 | }
42 |
--------------------------------------------------------------------------------
/src/face-target/face-mesh-helper.js:
--------------------------------------------------------------------------------
1 | import * as vision from "@mediapipe/tasks-vision";
2 | //import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
3 |
4 | class FaceMeshHelper {
5 | constructor() {
6 | }
7 |
8 | async init() {
9 | const filesetResolver = await vision.FilesetResolver.forVisionTasks(
10 | "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.9/wasm"
11 | //"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
12 | );
13 | this.faceLandmarker = await vision.FaceLandmarker.createFromOptions(filesetResolver, {
14 | baseOptions: {
15 | modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
16 | delegate: "GPU"
17 | },
18 | outputFaceBlendshapes: true,
19 | // outputFacialTransformationMatrixes: true,
20 | runningMode: "IMAGE",
21 | numFaces: 1
22 | });
23 | }
24 |
25 | async detect(input) {
26 | const faceLandmarkerResult = this.faceLandmarker.detect(input);
27 | return faceLandmarkerResult;
28 | }
29 | }
30 |
31 | export {
32 | FaceMeshHelper
33 | }
34 |
--------------------------------------------------------------------------------
/src/face-target/index.js:
--------------------------------------------------------------------------------
1 | import {Controller} from './controller.js';
2 | import {UI} from '../ui/ui.js';
3 |
4 | const e = {
5 | Controller,
6 | UI
7 | }
8 |
9 | if (!window.MINDAR) {
10 | window.MINDAR = {};
11 | }
12 |
13 | window.MINDAR.FACE = e;
14 |
15 |
16 | export {Controller,UI}
--------------------------------------------------------------------------------
/src/image-target/compiler.js:
--------------------------------------------------------------------------------
1 | import {CompilerBase} from './compiler-base.js'
2 | import CompilerWorker from "./compiler.worker.js?worker&inline";
3 |
4 | export class Compiler extends CompilerBase {
5 | createProcessCanvas(img) {
6 | const processCanvas = document.createElement('canvas');
7 | processCanvas.width = img.width;
8 | processCanvas.height = img.height;
9 | return processCanvas;
10 | }
11 |
12 | compileTrack({progressCallback, targetImages, basePercent}) {
13 | return new Promise((resolve, reject) => {
14 | const worker = new CompilerWorker();
15 | worker.onmessage = (e) => {
16 | if (e.data.type === 'progress') {
17 | progressCallback(basePercent + e.data.percent * basePercent/100);
18 | } else if (e.data.type === 'compileDone') {
19 | resolve(e.data.list);
20 | }
21 | };
22 | worker.postMessage({ type: 'compile', targetImages });
23 | });
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/image-target/compiler.worker.js:
--------------------------------------------------------------------------------
1 | import { extractTrackingFeatures } from './tracker/extract-utils.js';
2 | import { buildTrackingImageList } from './image-list.js';
3 |
4 | onmessage = (msg) => {
5 | const { data } = msg;
6 | if (data.type === 'compile') {
7 | //console.log("worker compile...");
8 | const { targetImages } = data;
9 | const percentPerImage = 100.0 / targetImages.length;
10 | let percent = 0.0;
11 | const list = [];
12 | for (let i = 0; i < targetImages.length; i++) {
13 | const targetImage = targetImages[i];
14 | const imageList = buildTrackingImageList(targetImage);
15 | const percentPerAction = percentPerImage / imageList.length;
16 |
17 | //console.log("compiling tracking...", i);
18 | const trackingData = extractTrackingFeatures(imageList, (index) => {
19 | //console.log("done tracking", i, index);
20 | percent += percentPerAction
21 | postMessage({ type: 'progress', percent });
22 | });
23 | list.push(trackingData);
24 | }
25 | postMessage({
26 | type: 'compileDone',
27 | list,
28 | });
29 | }
30 | };
31 |
--------------------------------------------------------------------------------
/src/image-target/controller.worker.js:
--------------------------------------------------------------------------------
1 | import { Matcher } from './matching/matcher.js';
2 | import { Estimator } from './estimation/estimator.js';
3 |
4 | let projectionTransform = null;
5 | let matchingDataList = null;
6 | let debugMode = false;
7 | let matcher = null;
8 | let estimator = null;
9 |
10 | onmessage = (msg) => {
11 | const { data } = msg;
12 |
13 | switch (data.type) {
14 | case "setup":
15 | projectionTransform = data.projectionTransform;
16 | matchingDataList = data.matchingDataList;
17 | debugMode = data.debugMode;
18 | matcher = new Matcher(data.inputWidth, data.inputHeight, debugMode);
19 | estimator = new Estimator(data.projectionTransform);
20 | break;
21 |
22 | case "match":
23 | const interestedTargetIndexes = data.targetIndexes;
24 |
25 | let matchedTargetIndex = -1;
26 | let matchedModelViewTransform = null;
27 | let matchedDebugExtra = null;
28 |
29 | for (let i = 0; i < interestedTargetIndexes.length; i++) {
30 | const matchingIndex = interestedTargetIndexes[i];
31 |
32 | const { keyframeIndex, screenCoords, worldCoords, debugExtra } = matcher.matchDetection(matchingDataList[matchingIndex], data.featurePoints);
33 | matchedDebugExtra = debugExtra;
34 |
35 | if (keyframeIndex !== -1) {
36 | const modelViewTransform = estimator.estimate({ screenCoords, worldCoords });
37 |
38 | if (modelViewTransform) {
39 | matchedTargetIndex = matchingIndex;
40 | matchedModelViewTransform = modelViewTransform;
41 | }
42 | break;
43 | }
44 | }
45 |
46 | postMessage({
47 | type: 'matchDone',
48 | targetIndex: matchedTargetIndex,
49 | modelViewTransform: matchedModelViewTransform,
50 | debugExtra: matchedDebugExtra
51 | });
52 | break;
53 |
54 | case 'trackUpdate':
55 | const { modelViewTransform, worldCoords, screenCoords } = data;
56 | const finalModelViewTransform = estimator.refineEstimate({ initialModelViewTransform: modelViewTransform, worldCoords, screenCoords });
57 | postMessage({
58 | type: 'trackUpdateDone',
59 | modelViewTransform: finalModelViewTransform,
60 | });
61 | break;
62 |
63 | case "dispose":
64 | close();
65 | break;
66 |
67 | default:
68 | throw new Error(`Invalid message type '${data.type}'`);
69 | }
70 | };
71 |
72 |
--------------------------------------------------------------------------------
/src/image-target/detector/crop-detector.js:
--------------------------------------------------------------------------------
1 | import * as tf from '@tensorflow/tfjs';
2 | import {Detector} from './detector.js';
3 | import {buildModelViewProjectionTransform, computeScreenCoordiate} from '../estimation/utils.js';
4 |
5 | class CropDetector {
6 | constructor(width, height, debugMode=false) {
7 | this.debugMode = debugMode;
8 | this.width = width;
9 | this.height = height;
10 |
11 | // nearest power of 2, min dimensions
12 | let minDimension = Math.min(width, height) / 2;
13 | let cropSize = Math.pow( 2, Math.round( Math.log( minDimension ) / Math.log( 2 ) ) );
14 | this.cropSize = cropSize;
15 |
16 | this.detector = new Detector(cropSize, cropSize, debugMode);
17 |
18 | this.kernelCaches = {};
19 | this.lastRandomIndex = 4;
20 | }
21 |
22 | detect(inputImageT) { // crop center
23 | const startY = Math.floor(this.height / 2 - this.cropSize / 2);
24 | const startX = Math.floor(this.width / 2 - this.cropSize / 2);
25 | const result = this._detect(inputImageT, startX, startY);
26 |
27 | if (this.debugMode) {
28 | result.debugExtra.crop = {startX, startY, cropSize: this.cropSize};
29 | }
30 | return result;
31 | }
32 |
33 | detectMoving(inputImageT) { // loop a few locations around center
34 | const dx = this.lastRandomIndex % 3;
35 | const dy = Math.floor(this.lastRandomIndex / 3);
36 |
37 | let startY = Math.floor(this.height / 2 - this.cropSize + dy * this.cropSize / 2);
38 | let startX = Math.floor(this.width / 2 - this.cropSize + dx * this.cropSize / 2);
39 |
40 | if (startX < 0) startX = 0;
41 | if (startY < 0) startY = 0;
42 | if (startX >= this.width - this.cropSize) startX = this.width - this.cropSize - 1;
43 | if (startY >= this.height - this.cropSize) startY = this.height - this.cropSize - 1;
44 |
45 | this.lastRandomIndex = (this.lastRandomIndex + 1) % 9;
46 |
47 | const result = this._detect(inputImageT, startX, startY);
48 | return result;
49 | }
50 |
51 | _detect(inputImageT, startX, startY) {
52 | const cropInputImageT = inputImageT.slice([startY, startX], [this.cropSize, this.cropSize]);
53 | const {featurePoints, debugExtra} = this.detector.detect(cropInputImageT);
54 | featurePoints.forEach((p) => {
55 | p.x += startX;
56 | p.y += startY;
57 | });
58 | if (this.debugMode) {
59 | debugExtra.projectedImage = cropInputImageT.arraySync();
60 | }
61 | cropInputImageT.dispose();
62 | return {featurePoints: featurePoints, debugExtra};
63 | }
64 | }
65 |
66 | export {
67 | CropDetector
68 | };
69 |
--------------------------------------------------------------------------------
/src/image-target/detector/freak.js:
--------------------------------------------------------------------------------
1 | // 37 points = 6 rings x 6 points per ring + 1 center
2 | const FREAK_RINGS = [
3 | // ring 5
4 | {
5 | sigma: 0.550000,
6 | points: [
7 | [-1.000000, 0.000000],
8 | [-0.500000, -0.866025],
9 | [0.500000, -0.866025],
10 | [1.000000, -0.000000],
11 | [0.500000, 0.866025],
12 | [-0.500000, 0.866025]
13 | ]
14 | },
15 | // ring 4
16 | {
17 | sigma: 0.475000,
18 | points: [
19 | [0.000000, 0.930969],
20 | [-0.806243, 0.465485],
21 | [-0.806243, -0.465485],
22 | [-0.000000, -0.930969],
23 | [0.806243, -0.465485],
24 | [0.806243, 0.465485]
25 | ]
26 | },
27 | // ring 3
28 | {
29 | sigma: 0.400000,
30 | points: [
31 | [0.847306, -0.000000],
32 | [0.423653, 0.733789],
33 | [-0.423653, 0.733789],
34 | [-0.847306, 0.000000],
35 | [-0.423653, -0.733789],
36 | [0.423653, -0.733789]
37 | ]
38 | },
39 | // ring 2
40 | {
41 | sigma: 0.325000,
42 | points: [
43 | [-0.000000, -0.741094],
44 | [0.641806, -0.370547],
45 | [0.641806, 0.370547],
46 | [0.000000, 0.741094],
47 | [-0.641806, 0.370547],
48 | [-0.641806, -0.370547]
49 | ]
50 | },
51 | // ring 1
52 | {
53 | sigma: 0.250000,
54 | points: [
55 | [-0.595502, 0.000000],
56 | [-0.297751, -0.515720],
57 | [0.297751, -0.515720],
58 | [0.595502, -0.000000],
59 | [0.297751, 0.515720],
60 | [-0.297751, 0.515720]
61 | ]
62 | },
63 | // ring 0
64 | {
65 | sigma: 0.175000,
66 | points: [
67 | [0.000000, 0.362783],
68 | [-0.314179, 0.181391],
69 | [-0.314179, -0.181391],
70 | [-0.000000, -0.362783],
71 | [0.314179, -0.181391],
72 | [0.314179, 0.181391]
73 | ]
74 | },
75 | // center
76 | {
77 | sigma: 0.100000,
78 | points: [
79 | [0, 0]
80 | ]
81 | }
82 | ];
83 |
84 | const FREAKPOINTS = [];
85 | for (let r = 0; r < FREAK_RINGS.length; r++) {
86 | const sigma = FREAK_RINGS[r].sigma;
87 | for (let i = 0; i < FREAK_RINGS[r].points.length; i++) {
88 | const point = FREAK_RINGS[r].points[i];
89 | FREAKPOINTS.push([sigma, point[0], point[1]]);
90 | }
91 | }
92 |
93 | export {
94 | FREAKPOINTS
95 | };
96 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/binomialFilter.js:
--------------------------------------------------------------------------------
1 | import * as FakeShader from './fakeShader.js';
2 |
3 |
4 | function GetKernels(image) {
5 | const imageWidth = image.shape[1];
6 | const key = 'w' + imageWidth;
7 |
8 | const imageHeight = image.shape[0];
9 | const kernel1 = {
10 | variableNames: ['p'],
11 | outputShape: [imageHeight, imageWidth],
12 | userCode: function () {
13 | const coords = this.getOutputCoords();
14 |
15 | let sum = this.getP(coords[0], coords[1] - 2);
16 | sum += this.getP(coords[0], coords[1] - 1) * 4.;
17 | sum += this.getP(coords[0], coords[1]) * 6.;
18 | sum += this.getP(coords[0], coords[1] + 1) * 4.;
19 | sum += this.getP(coords[0], coords[1] + 2);
20 | this.setOutput(sum);
21 | }
22 |
23 | };
24 | const kernel2 = {
25 | variableNames: ['p'],
26 | outputShape: [imageHeight, imageWidth],
27 | userCode: function () {
28 | const coords = this.getOutputCoords();
29 |
30 | let sum = this.getP(coords[0] - 2, coords[1]);
31 | sum += this.getP(coords[0] - 1, coords[1]) * 4.;
32 | sum += this.getP(coords[0], coords[1]) * 6.;
33 | sum += this.getP(coords[0] + 1, coords[1]) * 4.;
34 | sum += this.getP(coords[0] + 2, coords[1]);
35 | sum /= 256.;
36 | this.setOutput(sum);
37 | }
38 |
39 | };
40 | return [kernel1, kernel2];
41 | //}
42 |
43 | }
44 |
45 |
46 |
47 |
48 | export const binomialFilter = (args) => {//{inputs: UnaryInputs, backend: MathBackendCPU}
49 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
50 | const image = args.inputs.image;
51 | /** @type {MathBackendCPU} */
52 | const backend = args.backend;
53 |
54 | const [kernel1, kernel2] = GetKernels(image);
55 |
56 | const result1 = FakeShader.runCode(backend, kernel1, [image], image.dtype);
57 | return FakeShader.runCode(backend, kernel2, [result1], image.dtype);
58 | }
59 |
60 |
61 |
62 |
63 | export const binomialFilterConfig = {//: KernelConfig
64 | kernelName: "BinomialFilter",
65 | backendName: 'cpu',
66 | kernelFunc: binomialFilter,// as {} as KernelFunc,
67 | };
68 |
69 |
70 |
71 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/buildExtremas.js:
--------------------------------------------------------------------------------
1 |
2 | import * as FakeShader from './fakeShader.js';
3 | import {engine} from '@tensorflow/tfjs'
4 | const FREAK_EXPANSION_FACTOR = 7.0;
5 |
6 | const LAPLACIAN_THRESHOLD = 3.0;
7 | const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD;
8 |
9 | const EDGE_THRESHOLD = 4.0;
10 | const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD + 1) * (EDGE_THRESHOLD + 1) / EDGE_THRESHOLD);
11 |
12 | function GetProgram(image) {
13 | const imageWidth = image.shape[1];
14 | const imageHeight = image.shape[0];
15 | const kernel = {
16 | variableNames: ['image0', 'image1', 'image2'],
17 | outputShape: [imageHeight, imageWidth],
18 | userCode:
19 | function () {
20 | const coords = this.getOutputCoords();
21 |
22 | const y = coords[0];
23 | const x = coords[1];
24 |
25 | const value = this.getImage1(y, x);
26 |
27 | // Step 1: find local maxima/minima
28 | if (value * value < LAPLACIAN_SQR_THRESHOLD) {
29 | this.setOutput(0.0);
30 | return;
31 | }
32 | if (y < FREAK_EXPANSION_FACTOR || y > imageHeight - 1 - FREAK_EXPANSION_FACTOR) {
33 | this.setOutput(0.0);
34 | return;
35 | }
36 | if (x < FREAK_EXPANSION_FACTOR || x > imageWidth - 1 - FREAK_EXPANSION_FACTOR) {
37 | this.setOutput(0.0);
38 | return;
39 | }
40 |
41 | let isMax = true;
42 | let isMin = true;
43 | for (let dy = -1; dy <= 1; dy++) {
44 | for (let dx = -1; dx <= 1; dx++) {
45 | const value0 = this.getImage0(y + dy, x + dx);
46 | const value1 = this.getImage1(y + dy, x + dx);
47 | const value2 = this.getImage2(y + dy, x + dx);
48 |
49 | if (value < value0 || value < value1 || value < value2) {
50 | isMax = false;
51 | }
52 | if (value > value0 || value > value1 || value > value2) {
53 | isMin = false;
54 | }
55 | }
56 | }
57 |
58 | if (!isMax && !isMin) {
59 | this.setOutput(0.0);
60 | return;
61 | }
62 |
63 | // compute edge score and reject based on threshold
64 | const dxx = this.getImage1(y, x + 1) + this.getImage1(y, x - 1) - 2. * this.getImage1(y, x);
65 | const dyy = this.getImage1(y + 1, x) + this.getImage1(y - 1, x) - 2. * this.getImage1(y, x);
66 | const dxy = 0.25 * (this.getImage1(y - 1, x - 1) + this.getImage1(y + 1, x + 1) - this.getImage1(y - 1, x + 1) - this.getImage1(y + 1, x - 1));
67 |
68 | const det = (dxx * dyy) - (dxy * dxy);
69 |
70 | if (Math.abs(det) < 0.0001) { // determinant undefined. no solution
71 | this.setOutput(0.0);
72 | return;
73 | }
74 |
75 | const edgeScore = (dxx + dyy) * (dxx + dyy) / det;
76 |
77 | if (Math.abs(edgeScore) >= EDGE_HESSIAN_THRESHOLD) {
78 | this.setOutput(0.0);
79 | return;
80 | }
81 | this.setOutput(this.getImage1(y, x));
82 | }
83 |
84 | };
85 |
86 |
87 | return kernel;
88 | }
89 |
90 |
91 | export const buildExtremas = (args) => {
92 | let { image0, image1, image2 } = args.inputs;
93 | /** @type {MathBackendCPU} */
94 | const backend = args.backend;
95 |
96 | image0 = engine().runKernel('DownsampleBilinear', { image: image0 });
97 | image2 = engine().runKernel('UpsampleBilinear', { image: image2, targetImage: image1 });
98 | const program=GetProgram(image1);
99 | return FakeShader.runCode(backend,program,[image0,image1,image2],image1.dtype);
100 | }
101 |
102 | export const buildExtremasConfig = {//: KernelConfig
103 | kernelName: "BuildExtremas",
104 | backendName: 'cpu',
105 | kernelFunc: buildExtremas,// as {} as KernelFunc,
106 | };
107 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/computeExtremaAngles.js:
--------------------------------------------------------------------------------
1 | const ORIENTATION_NUM_BINS = 36;
2 |
3 |
4 | export function computeExtremaAnglesImpl(histogram) {
5 | const resultValues = new Float32Array(histogram.height);
6 |
7 | function getHistogram(featureIndex, prev) {
8 | return histogram.values[featureIndex * histogram.width + prev]
9 | };
10 | function setOutput(featureIndex, an) {
11 |
12 | resultValues[featureIndex] = an;
13 | }
14 |
15 | function imod(x, y) {
16 | return Math.trunc(x - y * Math.floor(x / y));
17 | }
18 |
19 | for (let featureIndex = 0; featureIndex < histogram.height; featureIndex++) {
20 | let maxIndex = 0;
21 | for (let i = 1; i < ORIENTATION_NUM_BINS; i++) {
22 | if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) {
23 | maxIndex = i;
24 | }
25 | }
26 | let prev = imod(maxIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
27 | let next = imod(maxIndex + 1, ORIENTATION_NUM_BINS);
28 | /**
29 | * Fit a quatratic to 3 points. The system of equations is:
30 | *
31 | * y0 = A*x0^2 + B*x0 + C
32 | * y1 = A*x1^2 + B*x1 + C
33 | * y2 = A*x2^2 + B*x2 + C
34 | *
35 | * This system of equations is solved for A,B,C.
36 | */
37 | const p10 = maxIndex - 1;
38 | const p11 = getHistogram(featureIndex, prev);
39 | const p20 = maxIndex;
40 | const p21 = getHistogram(featureIndex, maxIndex);
41 | const p30 = maxIndex + 1;
42 | const p31 = getHistogram(featureIndex, next);
43 |
44 | const d1 = (p30 - p20) * (p30 - p10);
45 | const d2 = (p10 - p20) * (p30 - p10);
46 | const d3 = p10 - p20;
47 |
48 | // If any of the denominators are zero then, just use maxIndex.
49 | let fbin = maxIndex;
50 | if (Math.abs(d1) > 0.00001 && Math.abs(d2) > 0.00001 && Math.abs(d3) > 0.00001) {
51 | const a = p10 * p10;
52 | const b = p20 * p20;
53 |
54 | // Solve for the coefficients A,B,C
55 | let A = ((p31 - p21) / d1) - ((p11 - p21) / d2);
56 | if(Number.isNaN(A)) A=0;
57 | const B = ((p11 - p21) + (A * (b - a))) / d3;
58 | //const C = p11 - (A * a) - (B * p10);
59 | fbin = -B / (2.0 * A);
60 | if(Number.isNaN(fbin)) fbin=0;//console.warn(`computeExtremaAngles::NaN! fbin=${fbin} maxIndex=${maxIndex} A=${A} B=${B} p31=${p31} p21=${p21} p11=${p11}`);
61 | }
62 |
63 | const an = 2.0 * Math.PI * (fbin + 0.5) / ORIENTATION_NUM_BINS - Math.PI;
64 |
65 | setOutput(featureIndex,an);
66 | }
67 | return resultValues;
68 | }
69 |
70 | export const computeExtremaAngles = (args) => {
71 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
72 | const { histograms } = args.inputs;
73 | /** @type {import('@tensorflow/tfjs-backend-cpu').MathBackendCPU} */
74 | const backend = args.backend;
75 |
76 | /** @type {TypedArray} */
77 | const histogramValues ={values: backend.data.get(histograms.dataId).values,width:histograms.shape[1],height:histograms.shape[0]};
78 |
79 | const resultValues = computeExtremaAnglesImpl(histogramValues);
80 |
81 | return backend.makeOutput(resultValues, [histograms.shape[0]], histograms.dtype);
82 | }
83 |
84 |
85 | export const computeExtremaAnglesConfig = {//: KernelConfig
86 | kernelName: "ComputeExtremaAngles",
87 | backendName: 'cpu',
88 | kernelFunc: computeExtremaAngles,// as {} as KernelFunc,
89 | };
90 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/computeExtremaFreak.js:
--------------------------------------------------------------------------------
1 | import { FREAKPOINTS } from '../../freak.js';
2 | import * as FakeShader from './fakeShader.js';
3 | const FREAK_EXPANSION_FACTOR = 7.0;
4 |
5 | function GetProgram(prunedExtremasHeight, pyramidImagesLength) {
6 |
7 | const imageVariableNames = [];
8 | for (let i = 1; i < pyramidImagesLength; i++) {
9 | imageVariableNames.push('image' + i);
10 | }
11 |
12 |
13 | const kernel = {
14 | variableNames: [...imageVariableNames, 'extrema', 'angles', 'freakPoints'],
15 | outputShape: [prunedExtremasHeight, FREAKPOINTS.length],
16 | userCode: function () {
17 | const getPixel=(octave, y, x)=> {
18 | const key = 'getImage' + octave;
19 | if (octave < 1 || octave >= pyramidImagesLength)
20 | return 0.0;
21 | return this[key](y, x);
22 | }
23 | const coords = this.getOutputCoords();
24 | const featureIndex = coords[0];
25 | const freakIndex = coords[1];
26 |
27 | //const freakSigma = this.getFreakPoints(freakIndex, 0);
28 | const freakX = this.getFreakPoints(freakIndex, 1);
29 | const freakY = this.getFreakPoints(freakIndex, 2);
30 |
31 | const octave = this.int(this.getExtrema(featureIndex, 1));
32 | const inputY = this.getExtrema(featureIndex, 2);
33 | const inputX = this.getExtrema(featureIndex, 3);
34 | const inputAngle = this.getAngles(featureIndex);
35 | const cos = FREAK_EXPANSION_FACTOR * Math.cos(inputAngle);
36 | const sin = FREAK_EXPANSION_FACTOR * Math.sin(inputAngle);
37 |
38 | const yp = inputY + freakX * sin + freakY * cos;
39 | const xp = inputX + freakX * cos + freakY * -sin;
40 |
41 | const x0 = this.int(Math.floor(xp));
42 | const x1 = x0 + 1;
43 | const y0 = this.int(Math.floor(yp));
44 | const y1 = y0 + 1;
45 |
46 | const f1 = getPixel(octave, y0, x0);
47 | const f2 = getPixel(octave, y0, x1);
48 | const f3 = getPixel(octave, y1, x0);
49 | const f4 = getPixel(octave, y1, x1);
50 |
51 | /* const x1f = float(x1);
52 | const y1f = float(y1);
53 | const x0f = float(x0);
54 | const y0f = float(y0); */
55 |
56 | // ratio for interpolation between four neighbouring points
57 | const value = (x1 - xp) * (y1 - yp) * f1
58 | + (xp - x0) * (y1 - yp) * f2
59 | + (x1 - xp) * (yp - y0) * f3
60 | + (xp - x0) * (yp - y0) * f4;
61 |
62 | this.setOutput(value);
63 | }
64 |
65 | }
66 |
67 | return kernel;
68 |
69 | }
70 |
71 |
72 |
73 | export const computeExtremaFreak = (args) => {
74 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
75 | const { gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT, pyramidImagesLength } = args.inputs;
76 | /** @type {MathBackendCPU} */
77 | const backend = args.backend;
78 | const prog = GetProgram(prunedExtremas.shape[0], pyramidImagesLength);
79 | return FakeShader.runCode(backend, prog, [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT], 'float32');
80 | }
81 |
82 |
83 |
84 | export const computeExtremaFreakConfig = {//: KernelConfig
85 | kernelName: "ComputeExtremaFreak",
86 | backendName: 'cpu',
87 | kernelFunc: computeExtremaFreak,// as {} as KernelFunc,
88 | };
89 |
90 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/computeFreakDescriptors.js:
--------------------------------------------------------------------------------
1 |
2 | import { FREAKPOINTS } from '../../freak.js';
3 |
4 | const FREAK_CONPARISON_COUNT = (FREAKPOINTS.length - 1) * (FREAKPOINTS.length) / 2;
5 | const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8);
6 |
7 |
8 | function computeFreakDescriptorImpl(extremaFreaks,positionT) {
9 | const resultValues = new Float32Array(extremaFreaks.height* descriptorCount);
10 | function getP(y, x) {
11 | return positionT.values[y*positionT.width+x];
12 | }
13 | function getFreak(y, x) {
14 | return extremaFreaks.values[y*extremaFreaks.width+x];
15 | }
16 | function setOutput(y,x,o){
17 | resultValues[y*descriptorCount+x]=o;
18 | }
19 |
20 | for (let featureIndex = 0; featureIndex < extremaFreaks.height; featureIndex++) {
21 | for (let _descIndex = 0; _descIndex < descriptorCount; _descIndex++) {
22 | const descIndex = _descIndex * 8;
23 | let sum = 0;
24 | for (let i = 0; i < 8; i++) {
25 | if (descIndex + i >= FREAK_CONPARISON_COUNT) {
26 | continue;
27 | }
28 |
29 | const p1 = Math.trunc(getP(descIndex + i, 0));
30 | const p2 = Math.trunc(getP(descIndex + i, 1));
31 |
32 | const v1 = getFreak(featureIndex, p1);
33 | const v2 = getFreak(featureIndex, p2);
34 |
35 | if (v1 < v2 + 0.01) {
36 | sum += Math.trunc(Math.pow(2.0, (7.0 - i)));
37 | }
38 | }
39 | setOutput(featureIndex,_descIndex, sum);
40 | }
41 | }
42 |
43 |
44 |
45 | return resultValues;
46 | }
47 |
48 | export const computeFreakDescriptor = (args) => {
49 | const { extremaFreaks, positionT } = args.inputs;
50 | const { backend } = args;
51 |
52 | const freaksData ={values:backend.data.get(extremaFreaks.dataId).values,height:extremaFreaks.shape[0], width:extremaFreaks.shape[1]};
53 | const positionData ={values:backend.data.get(positionT.dataId).values,width:positionT.shape[1]};
54 | //backend.runWebGLProgram(program,[extremaFreaks, positionT],'int32');
55 |
56 | const resultValues=computeFreakDescriptorImpl(freaksData,positionData);
57 | return backend.makeOutput(resultValues, [extremaFreaks.shape[0], descriptorCount], 'int32');
58 | }
59 |
60 | export const computeFreakDescriptorConfig = {//: KernelConfig
61 | kernelName: "ComputeFreakDescriptors",
62 | backendName: 'cpu',
63 | kernelFunc: computeFreakDescriptor,// as {} as KernelFunc,
64 | };
65 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/computeLocalization.js:
--------------------------------------------------------------------------------
1 | import {tensor} from '@tensorflow/tfjs'
2 | import * as FakeShader from './fakeShader.js';
3 |
4 | function GetProgram(numDogPyramidImages, extremasListLength) {
5 |
6 | const dogVariableNames = [];
7 |
8 | for (let i = 1; i < numDogPyramidImages; i++) {
9 | dogVariableNames.push('image' + i);
10 | }
11 |
12 |
13 | const program = {
14 | variableNames: [...dogVariableNames, 'extrema'],
15 | outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema
16 | userCode: function () {
17 | const getPixel = (octave, y, x) => {
18 | const k = 'getImage' + octave
19 | if (!this.hasOwnProperty(k)) {
20 | throw new Error(`ComputeLocalization:: ${k} does not exist`);
21 | }
22 | return this[k](y, x);
23 | }
24 | const coords = this.getOutputCoords();
25 | const featureIndex = coords[0];
26 | const score = this.getExtrema(featureIndex, 0);
27 | if (score == 0.0) {
28 | return;
29 | }
30 |
31 | const dy = coords[1] - 1;
32 | const dx = coords[2] - 1;
33 | const octave = this.int(this.getExtrema(featureIndex, 1));
34 | const y = this.int(this.getExtrema(featureIndex, 2));
35 | const x = this.int(this.getExtrema(featureIndex, 3));
36 | this.setOutput(getPixel(octave, y + dy, x + dx));
37 | }
38 |
39 | };
40 | //}
41 | return program;
42 | }
43 |
44 |
45 |
46 | const int = Math.trunc;
47 | function clamp(n, min, max) {
48 | return Math.min(Math.max(min, n), max - 1);
49 | }
50 |
51 |
52 |
53 |
54 |
55 |
56 | export const computeLocalization = (args) => {
57 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
58 | const { prunedExtremasList, dogPyramidImagesT } = args.inputs;
59 | /** @type {MathBackendCPU} */
60 | const backend = args.backend;
61 |
62 | const program = GetProgram(dogPyramidImagesT.length, prunedExtremasList.length);
63 | const prunedExtremasT = tensor(prunedExtremasList, [prunedExtremasList.length, prunedExtremasList[0].length], 'int32');
64 | return FakeShader.runCode(backend, program, [...dogPyramidImagesT.slice(1), prunedExtremasT], dogPyramidImagesT[0].dtype);
65 | }
66 |
67 | export const computeLocalizationConfig = {//: KernelConfig
68 | kernelName: "ComputeLocalization",
69 | backendName: 'cpu',
70 | kernelFunc: computeLocalization,// as {} as KernelFunc,
71 | };
72 |
73 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/downsampleBilinear.js:
--------------------------------------------------------------------------------
1 | import * as FakeShader from './fakeShader.js';
2 |
3 | export const downsampleBilinear = (args) => {
4 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
5 | const x = args.inputs.image;
6 | /** @type {MathBackendCPU} */
7 | const backend = args.backend;
8 |
9 | const kernel = {
10 | variableNames: ['p'],
11 | outputShape: [Math.floor(x.shape[0] / 2), Math.floor(x.shape[1] / 2)],
12 | userCode:
13 | function () {
14 | const coords = this.getOutputCoords();
15 | const y = coords[0] * 2;
16 | const x = coords[1] * 2;
17 | let sum = new Float32Array(1);
18 | sum[0] = Math.fround(this.getP(y, x) * 0.25);
19 | sum[0] += Math.fround(this.getP(y + 1, x) * 0.25);
20 | sum[0] += Math.fround(this.getP(y, x + 1) * 0.25);
21 | sum[0] += Math.fround(this.getP(y + 1, x + 1) * 0.25);
22 |
23 | this.setOutput(sum[0]);
24 | }
25 | }
26 | return FakeShader.runCode(backend, kernel, [x], x.dtype);
27 | }
28 |
29 | export const downsampleBilinearConfig = {//: KernelConfig
30 | kernelName: "DownsampleBilinear",
31 | backendName: 'cpu',
32 | kernelFunc: downsampleBilinear,// as {} as KernelFunc,
33 | };
34 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/extremaReduction.js:
--------------------------------------------------------------------------------
1 | import * as FakeShader from './fakeShader.js';
2 | function GetProgram(outHeight,outWidth) {
3 | const kernel = {
4 | variableNames: ['extrema'],
5 | outputShape: [outHeight, outWidth],
6 | userCode: function () {
7 | const coords = this.getOutputCoords();
8 | const y = coords[0] * 2;
9 | const x = coords[1] * 2;
10 |
11 | let location = 0.0;
12 | let values = this.getExtrema(y, x);
13 |
14 | if (this.getExtrema(y + 1, x) != 0.0) {
15 | location = 1.0;
16 | values = this.getExtrema(y + 1, x);
17 | }
18 | else if (this.getExtrema(y, x + 1) != 0.0) {
19 | location = 2.0;
20 | values = this.getExtrema(y, x + 1);
21 | }
22 | else if (this.getExtrema(y + 1, x + 1) != 0.0) {
23 | location = 3.0;
24 | values = this.getExtrema(y + 1, x + 1);
25 | }
26 |
27 | if (values < 0.0) {
28 | this.setOutput(location * -1000.0 + values);
29 | } else {
30 | this.setOutput(location * 1000.0 + values);
31 | }
32 | }
33 |
34 | }
35 | return kernel;
36 | }
37 |
38 |
39 | export const extremaReduction = (args) => {
40 | const { extremasResultT } = args.inputs;
41 | /** @type {MathBackendCPU} */
42 | const backend = args.backend;
43 | const extremaHeight = extremasResultT.shape[0];
44 | const extremaWidth = extremasResultT.shape[1];
45 | const outHeight = Math.floor(extremaHeight / 2.0);
46 | const outWidth = Math.floor(extremaWidth / 2.0);
47 | const program=GetProgram(outHeight,outWidth);
48 |
49 | return FakeShader.runCode(backend,program,[extremasResultT],extremasResultT.dtype);
50 | }
51 |
52 | export const extremaReductionConfig = {//: KernelConfig
53 | kernelName: "ExtremaReduction",
54 | backendName: 'cpu',
55 | kernelFunc: extremaReduction,// as {} as KernelFunc,
56 | };
57 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/fakeShader.js:
--------------------------------------------------------------------------------
1 | import {zeros,map, flatten as mathjsflatten} from 'mathjs'
2 | /**
3 | * @typedef {Object} Kernel
4 | * @property {string[]} variableNames
5 | * @property {number[]} outputShape
6 | * @property {Function} userCode
7 | */
8 |
9 | /**
10 | *
11 | * @param {MathBackendCPU} backend
12 | * @param {Kernel} kernel
13 | * @param {Array<.TensorInfo>} inputs
14 | * @param {DataType} dtype
15 | * @returns {Tensor}
16 | */
17 | function runCode(backend, kernel, inputs, dtype) {
18 | const inputData = inputs.map((value) => { return backend.data.get(value.dataId).values; });
19 |
20 | //create getter functions for every variable name, clamping the input.
21 | const tempData = {}
22 | kernel.variableNames.forEach((name, index) => {
23 | const funName=`get${capFirstLetter(name)}`;
24 | //console.log("Making function:",funName,inputs[index].shape);
25 | tempData[funName] = function (...args) {
26 | const inputIndex=index;
27 | for (let i = 0; i < args.length; i++) {
28 | args[i] = clampInt(args[i], 0, inputs[inputIndex].shape[i] );
29 | }
30 | return inputData[index][flatten(args, inputs[inputIndex].shape)];
31 | }
32 | });
33 | tempData.int=Math.trunc;
34 | tempData.atan=Math.atan2;
35 | //create an empty matrix to map the output size, because i'm lazy and want to use Matrix.map(...)
36 | //const temp = new Matrix();
37 | //console.log("Creating output shape:",kernel.outputShape);
38 | const temp=zeros(kernel.outputShape);//reshape([0,0,0],kernel.outputShape);
39 | const output = map(temp,(value, index,matrix) => {
40 |
41 | tempData.getOutputCoords = () => { return index; }
42 | let out;
43 |
44 | tempData.setOutput = (newValue) => { out = Number.isNaN(newValue) ? 0 : Math.fround(newValue); }
45 | //bind the method calls and run the code
46 | kernel.userCode.bind(tempData)();
47 | return out;
48 | })
49 |
50 | //output.flat()
51 | //convert the output from a matrix into a tensor
52 |
53 | return backend.makeOutput(mathjsflatten(output), kernel.outputShape, dtype);
54 | }
55 |
56 | /**
57 | *
58 | * @param {string} word
59 | * @returns
60 | */
61 | function capFirstLetter(word) {
62 | return word[0].toUpperCase() + word.substring(1);
63 | }
64 |
65 | function clampInt(n, min, max) {
66 | return Math.min(Math.max(n, min), max - 1);
67 | }
68 | /**
69 | *
70 | * @param {number[]} input
71 | * @param {number[]} max
72 | * @returns
73 | */
74 | function flatten(input, max) {
75 | return input.reduce((prev, current, index) => {
76 | for (let i = index + 1; i < max.length; i++) {
77 | current *= max[i];
78 | }
79 | return prev + current;
80 | },0);
81 | }
82 |
83 | export { runCode };
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/index.js:
--------------------------------------------------------------------------------
1 | import { registerKernel } from '@tensorflow/tfjs';
2 | import { binomialFilterConfig } from './binomialFilter.js';
3 | import { buildExtremasConfig } from './buildExtremas.js';
4 | import { computeExtremaAnglesConfig } from './computeExtremaAngles.js';
5 | import { computeExtremaFreakConfig } from './computeExtremaFreak.js';
6 | import { computeFreakDescriptorConfig } from './computeFreakDescriptors.js';
7 | import { computeLocalizationConfig } from './computeLocalization.js';
8 | import { computeOrientationHistogramsConfig } from './computeOrientationHistograms.js';
9 | import { downsampleBilinearConfig } from './downsampleBilinear.js';
10 | import { extremaReductionConfig } from './extremaReduction.js';
11 | import { smoothHistogramsConfig } from './smoothHistograms.js';
12 | import { upsampleBilinearConfig } from './upsampleBilinear.js';
13 |
14 | //export function Register(){
15 | registerKernel(binomialFilterConfig);
16 | registerKernel(buildExtremasConfig);
17 | registerKernel(computeExtremaAnglesConfig);
18 | registerKernel(computeExtremaFreakConfig);
19 | registerKernel(computeFreakDescriptorConfig);
20 | registerKernel(computeLocalizationConfig);
21 | registerKernel(computeOrientationHistogramsConfig);
22 | registerKernel(downsampleBilinearConfig);
23 | registerKernel(extremaReductionConfig);
24 | registerKernel(smoothHistogramsConfig);
25 | registerKernel(upsampleBilinearConfig);
26 | //}
27 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/prune.js:
--------------------------------------------------------------------------------
1 | import * as FakeShader from './fakeShader.js';
2 | /*
3 | const kernel = {
4 | variableNames: ['extrema'],
5 | outputShape: [Math.floor(extremaHeight/2), Math.floor(extremaWidth/2)],
6 | userCode: `
7 | void main() {
8 | ivec2 coords = getOutputCoords();
9 | int y = coords[0] * 2;
10 | int x = coords[1] * 2;
11 |
12 | float location = 0.0;
13 | float values = getExtrema(y, x);
14 |
15 | if (getExtrema(y+1, x) != 0.0) {
16 | location = 1.0;
17 | values = getExtrema(y+1, x);
18 | }
19 | else if (getExtrema(y, x+1) != 0.0) {
20 | location = 2.0;
21 | values = getExtrema(y, x+1);
22 | }
23 | else if (getExtrema(y+1, x+1) != 0.0) {
24 | location = 3.0;
25 | values = getExtrema(y+1, x+1);
26 | }
27 |
28 | if (values < 0.0) {
29 | setOutput(location * -1000.0 + values);
30 | } else {
31 | setOutput(location * 1000.0 + values);
32 | }
33 | }
34 | `
35 | }
36 |
37 | */
38 | function clamp(n, min, max) {
39 | return Math.min(Math.max(min, n), max-1);
40 | }
41 |
42 | const pruneImpl=(vals,width,height)=>{
43 | const w=Math.floor(width/2);
44 | const h=Math.floor(height/2);
45 | const resultValues = new Float32Array(w*h);
46 | function getExtrema(x,y){
47 | x=clamp(x,0,width);
48 | y=clamp(y,0,height);
49 | return vals[y*width+x];
50 | }
51 | function setOutput(x,y,o){
52 | resultValues[y*width+x]=o;
53 | }
54 |
55 | for(let x=0;x{
89 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
90 | const x = args.inputs.x;
91 | /** @type {MathBackendCPU} */
92 | const cpuBackend = args.backend;
93 | const imageHeight = x.shape[0];
94 | const imageWidth = x.shape[1];
95 | /** @type {TypedArray} */
96 | const values = cpuBackend.data.get(x.dataId).values;
97 |
98 | const resultValues = pruneImpl(values,imageWidth,imageHeight);
99 |
100 | return cpuBackend.makeOutput(resultValues, [Math.floor(imageHeight/2), Math.floor(imageWidth/2)], 'float32');
101 | }
102 |
103 | const pruneConfig = {//: KernelConfig
104 | kernelName: "Prune",
105 | backendName: 'cpu',
106 | kernelFunc: prune,// as {} as KernelFunc,
107 | };
108 |
109 | module.exports={
110 | pruneConfig,
111 | prune,
112 | pruneImpl
113 | }
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/smoothHistograms.js:
--------------------------------------------------------------------------------
1 |
2 | const ORIENTATION_NUM_BINS = 36;
3 | const ORIENTATION_SMOOTHING_ITERATIONS = 5;
4 |
5 |
6 | function smoothHistogramsImpl(histograms) {
7 | const resultValues = new Float32Array(histograms.height * ORIENTATION_NUM_BINS);
8 | function getHistogram(y, x) {
9 | return histograms.values[y * histograms.width + x];
10 | }
11 | function setOutput(y, x, o) {
12 | resultValues[y * ORIENTATION_NUM_BINS + x] = o;
13 | }
14 | function imod(x,y){
15 | return Math.trunc(x - y * Math.floor(x/y));
16 | }
17 | for (let featureIndex = 0; featureIndex < histograms.height; featureIndex++) {
18 | for (let binIndex = 0; binIndex < ORIENTATION_NUM_BINS; binIndex++) {
19 | const prevBin = imod(binIndex - 1 + ORIENTATION_NUM_BINS, ORIENTATION_NUM_BINS);
20 | const nextBin = imod(binIndex + 1, ORIENTATION_NUM_BINS);
21 | const result = 0.274068619061197 * getHistogram(featureIndex, prevBin) + 0.451862761877606 * getHistogram(featureIndex, binIndex) + 0.274068619061197 * getHistogram(featureIndex, nextBin);
22 |
23 | setOutput(featureIndex, binIndex, result);
24 | }
25 | }
26 |
27 | return resultValues;
28 | }
29 |
30 | export const smoothHistograms = (args) => {
31 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
32 | const { histograms } = args.inputs;
33 | /** @type {MathBackendCPU} */
34 | const backend = args.backend;
35 | const histogramsData = { values: backend.data.get(histograms.dataId).values, height: histograms.shape[0], width: histograms.shape[1] };
36 | //const program = GetProgram(histograms);
37 | for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) {
38 | histogramsData.values = smoothHistogramsImpl(histogramsData);//backend.runWebGLProgram(program, [histograms], histograms.dtype);//this._compileAndRun(program, [histograms]);
39 | }
40 | return backend.makeOutput(histogramsData.values,[histograms.shape[0],ORIENTATION_NUM_BINS],histograms.dtype);
41 |
42 | }
43 |
44 |
45 |
46 | export const smoothHistogramsConfig = {//: KernelConfig
47 | kernelName: "SmoothHistograms",
48 | backendName: 'cpu',
49 | kernelFunc: smoothHistograms,// as {} as KernelFunc,
50 | };
51 |
52 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/cpu/upsampleBilinear.js:
--------------------------------------------------------------------------------
1 | import * as FakeShader from './fakeShader.js';
2 | function getProgram(targetImage) {
3 | const kernel = {
4 | variableNames: ['p'],
5 | outputShape: [targetImage.shape[0], targetImage.shape[1]],
6 | userCode: function () {
7 | const coords = this.getOutputCoords();
8 | const j = coords[0];
9 | const i = coords[1];
10 |
11 | const sj = Math.fround(0.5 * j) - 0.25;
12 | const si = Math.fround(0.5 * i) - 0.25;
13 |
14 | const sj0 = Math.floor(sj);
15 | const sj1 = Math.ceil(sj);
16 | const si0 = Math.floor(si);
17 | const si1 = Math.ceil(si);
18 |
19 | const sj0I = this.int(sj0);
20 | const sj1I = this.int(sj1);
21 | const si0I = this.int(si0);
22 | const si1I = this.int(si1);
23 |
24 | let sum = 0.0;
25 | sum += this.getP(sj0I, si0I) * Math.fround((si1 - si) * (sj1 - sj));
26 | sum += this.getP(sj1I, si0I) * Math.fround((si1 - si) * (sj - sj0));
27 | sum += this.getP(sj0I, si1I) * Math.fround((si - si0) * (sj1 - sj));
28 | sum += this.getP(sj1I, si1I) * Math.fround((si - si0) * (sj - sj0));
29 | this.setOutput(sum);
30 | }
31 |
32 | };
33 | return kernel;
34 | }
35 |
36 | export const upsampleBilinear = (args) => {
37 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
38 | const { image, targetImage } = args.inputs;
39 |
40 | /** @type {MathBackendCPU} */
41 | const cpuBackend = args.backend;
42 |
43 | const program = getProgram(targetImage);
44 | return FakeShader.runCode(cpuBackend,program,[image],image.dtype);
45 |
46 | }
47 |
48 | export const upsampleBilinearConfig = {//: KernelConfig
49 | kernelName: "UpsampleBilinear",
50 | backendName: 'cpu',
51 | kernelFunc: upsampleBilinear,// as {} as KernelFunc,
52 | };
53 |
54 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/index.js:
--------------------------------------------------------------------------------
1 | import './webgl/index.js';
2 | import './cpu/index.js';
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/binomialFilter.js:
--------------------------------------------------------------------------------
1 | const cache={};
2 |
3 | /**
4 | *
5 | * @param {string} key
6 | * @param {TensorInfo} image
7 | * @returns {[GPGPUProgram,GPGPUProgram]}
8 | */
9 | function GetKernels(image){
10 | const imageWidth = image.shape[1];
11 | const imageHeight = image.shape[0];
12 | const key = 'w' + imageWidth + "h" + imageHeight;
13 | if(!cache.hasOwnProperty(key)){
14 | const kernel1 = {
15 | variableNames: ['p'],
16 | outputShape: [imageHeight, imageWidth],
17 | userCode: `
18 | void main() {
19 | ivec2 coords = getOutputCoords();
20 |
21 | float sum = getP(coords[0], coords[1]-2);
22 | sum += getP(coords[0], coords[1]-1) * 4.;
23 | sum += getP(coords[0], coords[1]) * 6.;
24 | sum += getP(coords[0], coords[1]+1) * 4.;
25 | sum += getP(coords[0], coords[1]+2);
26 | setOutput(sum);
27 | }
28 | `
29 | };
30 | const kernel2 = {
31 | variableNames: ['p'],
32 | outputShape: [imageHeight, imageWidth],
33 | userCode: `
34 | void main() {
35 | ivec2 coords = getOutputCoords();
36 |
37 | float sum = getP(coords[0]-2, coords[1]);
38 | sum += getP(coords[0]-1, coords[1]) * 4.;
39 | sum += getP(coords[0], coords[1]) * 6.;
40 | sum += getP(coords[0]+1, coords[1]) * 4.;
41 | sum += getP(coords[0]+2, coords[1]);
42 | sum /= 256.;
43 | setOutput(sum);
44 | }
45 | `
46 | };
47 | cache[key]=[kernel1,kernel2];
48 | }
49 | return cache[key];
50 | }
51 |
52 | export const binomialFilter = (args) =>{//{inputs: UnaryInputs, backend: MathBackendCPU}
53 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
54 | const image = args.inputs.image;
55 | /** @type {MathBackendWebGL} */
56 | const backend = args.backend;
57 |
58 | const[kernel1,kernel2]=GetKernels(image);
59 |
60 | const result1=backend.runWebGLProgram(kernel1,[image],image.dtype);
61 | const result2=backend.runWebGLProgram(kernel2,[result1],image.dtype);
62 | backend.disposeIntermediateTensorInfo(result1);
63 | return result2;
64 | }
65 |
66 |
67 |
68 |
69 | export const binomialFilterConfig = {//: KernelConfig
70 | kernelName: "BinomialFilter",
71 | backendName: 'webgl',
72 | kernelFunc: binomialFilter,// as {} as KernelFunc,
73 | };
74 |
75 |
76 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/buildExtremas.js:
--------------------------------------------------------------------------------
1 | import {engine} from '@tensorflow/tfjs';
2 |
3 | const FREAK_EXPANSION_FACTOR = 7.0;
4 |
5 | const LAPLACIAN_THRESHOLD = 3.0;
6 | const LAPLACIAN_SQR_THRESHOLD = LAPLACIAN_THRESHOLD * LAPLACIAN_THRESHOLD;
7 |
8 | const EDGE_THRESHOLD = 4.0;
9 | const EDGE_HESSIAN_THRESHOLD = ((EDGE_THRESHOLD+1) * (EDGE_THRESHOLD+1) / EDGE_THRESHOLD);
10 |
11 |
12 | const cache={};
13 | function GetProgram(image){
14 | const imageWidth = image.shape[1];
15 | const imageHeight = image.shape[0];
16 | const kernelKey = 'w' + imageWidth + "h" + imageHeight;
17 | if(!cache.hasOwnProperty(kernelKey)){
18 | const kernel = {
19 | variableNames: ['image0', 'image1', 'image2'],
20 | outputShape: [imageHeight, imageWidth],
21 | userCode: `
22 | void main() {
23 | ivec2 coords = getOutputCoords();
24 |
25 | int y = coords[0];
26 | int x = coords[1];
27 |
28 | float value = getImage1(y, x);
29 |
30 | // Step 1: find local maxima/minima
31 | if (value * value < ${LAPLACIAN_SQR_THRESHOLD}.) {
32 | setOutput(0.);
33 | return;
34 | }
35 | if (y < ${FREAK_EXPANSION_FACTOR} || y > ${imageHeight - 1 - FREAK_EXPANSION_FACTOR}) {
36 | setOutput(0.);
37 | return;
38 | }
39 | if (x < ${FREAK_EXPANSION_FACTOR} || x > ${imageWidth - 1 - FREAK_EXPANSION_FACTOR}) {
40 | setOutput(0.);
41 | return;
42 | }
43 |
44 | bool isMax = true;
45 | bool isMin = true;
46 | for (int dy = -1; dy <= 1; dy++) {
47 | for (int dx = -1; dx <= 1; dx++) {
48 | float value0 = getImage0(y+dy, x+dx);
49 | float value1 = getImage1(y+dy, x+dx);
50 | float value2 = getImage2(y+dy, x+dx);
51 |
52 | if (value < value0 || value < value1 || value < value2) {
53 | isMax = false;
54 | }
55 | if (value > value0 || value > value1 || value > value2) {
56 | isMin = false;
57 | }
58 | }
59 | }
60 |
61 | if (!isMax && !isMin) {
62 | setOutput(0.);
63 | return;
64 | }
65 |
66 | // compute edge score and reject based on threshold
67 | float dxx = getImage1(y, x+1) + getImage1(y, x-1) - 2. * getImage1(y, x);
68 | float dyy = getImage1(y+1, x) + getImage1(y-1, x) - 2. * getImage1(y, x);
69 | float dxy = 0.25 * (getImage1(y-1,x-1) + getImage1(y+1,x+1) - getImage1(y-1,x+1) - getImage1(y+1,x-1));
70 |
71 | float det = (dxx * dyy) - (dxy * dxy);
72 |
73 | if (abs(det) < 0.0001) { // determinant undefined. no solution
74 | setOutput(0.);
75 | return;
76 | }
77 |
78 | float edgeScore = (dxx + dyy) * (dxx + dyy) / det;
79 |
80 | if (abs(edgeScore) >= ${EDGE_HESSIAN_THRESHOLD} ) {
81 | setOutput(0.);
82 | return;
83 | }
84 | setOutput(getImage1(y,x));
85 | }
86 | `
87 | };
88 | cache[kernelKey]=kernel;
89 | }
90 | return cache[kernelKey];
91 | }
92 |
93 | export const buildExtremas=(args)=>{
94 | let {image0,image1,image2}=args.inputs;
95 | /** @type {MathBackendWebGL} */
96 | const backend = args.backend;
97 |
98 | const program=GetProgram(image1);
99 |
100 | image0=engine().runKernel('DownsampleBilinear',{image:image0});
101 | image2=engine().runKernel('UpsampleBilinear',{image:image2,targetImage:image1});
102 | return backend.runWebGLProgram(program,[image0,image1,image2],image1.dtype);
103 | }
104 |
105 | export const buildExtremasConfig = {//: KernelConfig
106 | kernelName: "BuildExtremas",
107 | backendName: 'webgl',
108 | kernelFunc: buildExtremas,// as {} as KernelFunc,
109 | };
110 |
111 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/computeExtremaAngles.js:
--------------------------------------------------------------------------------
1 |
2 | const ORIENTATION_NUM_BINS = 36;
3 |
4 | const cache={};
5 | function GetProgram(histograms){
6 | const key=histograms.shape[0];
7 | if(!cache.hasOwnProperty(key)){
8 | const kernel = {
9 | variableNames: ['histogram'],
10 | outputShape: [histograms.shape[0]],
11 | userCode: `
12 | void main() {
13 | int featureIndex = getOutputCoords();
14 |
15 | int maxIndex = 0;
16 | for (int i = 1; i < ${ORIENTATION_NUM_BINS}; i++) {
17 | if (getHistogram(featureIndex, i) > getHistogram(featureIndex, maxIndex)) {
18 | maxIndex = i;
19 | }
20 | }
21 |
22 | int prev = imod(maxIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
23 | int next = imod(maxIndex + 1, ${ORIENTATION_NUM_BINS});
24 |
25 | /**
26 | * Fit a quatratic to 3 points. The system of equations is:
27 | *
28 | * y0 = A*x0^2 + B*x0 + C
29 | * y1 = A*x1^2 + B*x1 + C
30 | * y2 = A*x2^2 + B*x2 + C
31 | *
32 | * This system of equations is solved for A,B,C.
33 | */
34 | float p10 = float(maxIndex - 1);
35 | float p11 = getHistogram(featureIndex, prev);
36 | float p20 = float(maxIndex);
37 | float p21 = getHistogram(featureIndex, maxIndex);
38 | float p30 = float(maxIndex + 1);
39 | float p31 = getHistogram(featureIndex, next);
40 |
41 | float d1 = (p30-p20)*(p30-p10);
42 | float d2 = (p10-p20)*(p30-p10);
43 | float d3 = p10-p20;
44 |
45 | // If any of the denominators are zero then, just use maxIndex.
46 | float fbin = float(maxIndex);
47 | if ( abs(d1) > 0.00001 && abs(d2) > 0.00001 && abs(d3) > 0.00001) {
48 | float a = p10*p10;
49 | float b = p20*p20;
50 |
51 | // Solve for the coefficients A,B,C
52 | float A = ((p31-p21)/d1)-((p11-p21)/d2);
53 | float B = ((p11-p21)+(A*(b-a)))/d3;
54 | float C = p11-(A*a)-(B*p10);
55 | fbin = -B / (2. * A);
56 | }
57 |
58 | float an = 2.0 *${Math.PI} * (fbin + 0.5) / ${ORIENTATION_NUM_BINS}. - ${Math.PI};
59 | setOutput(an);
60 | }
61 | `
62 | }
63 | cache[key]=kernel;
64 | }
65 | return cache[key];
66 | }
67 |
68 | export const computeExtremaAngles=(args)=>{
69 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
70 | const {histograms} = args.inputs;
71 | /** @type {MathBackendWebGL} */
72 | const backend = args.backend;
73 |
74 | const program = GetProgram(histograms);
75 | return backend.runWebGLProgram(program,[histograms],histograms.dtype);
76 | }
77 |
78 |
79 | export const computeExtremaAnglesConfig = {//: KernelConfig
80 | kernelName: "ComputeExtremaAngles",
81 | backendName: 'webgl',
82 | kernelFunc: computeExtremaAngles,// as {} as KernelFunc,
83 | };
84 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/computeExtremaFreak.js:
--------------------------------------------------------------------------------
1 | import { FREAKPOINTS } from '../../freak.js'
2 |
3 | const FREAK_EXPANSION_FACTOR = 7.0;
4 | const cache = {};
5 | function GetProgram(imageCount, prunedExtremas) {
6 | const key = `${imageCount}|${prunedExtremas.shape[0]}`;
7 | if (!cache.hasOwnProperty(key)) {
8 | const imageVariableNames = [];
9 | for (let i = 1; i < imageCount; i++) {
10 | imageVariableNames.push('image' + i);
11 | }
12 |
13 | let pixelsSubCodes = `float getPixel(int octave, int y, int x) {`;
14 | for (let i = 1; i < imageCount; i++) {
15 | pixelsSubCodes += `
16 | if (octave == ${i}) {
17 | return getImage${i}(y, x);
18 | }
19 | `
20 | }
21 | pixelsSubCodes += `}`;
22 |
23 | const kernel = {
24 | variableNames: [...imageVariableNames, 'extrema', 'angles', 'freakPoints'],
25 | outputShape: [prunedExtremas.shape[0], FREAKPOINTS.length],
26 | userCode: `
27 | ${pixelsSubCodes}
28 | void main() {
29 | ivec2 coords = getOutputCoords();
30 | int featureIndex = coords[0];
31 | int freakIndex = coords[1];
32 |
33 | float freakSigma = getFreakPoints(freakIndex, 0);
34 | float freakX = getFreakPoints(freakIndex, 1);
35 | float freakY = getFreakPoints(freakIndex, 2);
36 |
37 | int octave = int(getExtrema(featureIndex, 1));
38 | float inputY = getExtrema(featureIndex, 2);
39 | float inputX = getExtrema(featureIndex, 3);
40 | float inputAngle = getAngles(featureIndex);
41 | float cos = ${FREAK_EXPANSION_FACTOR}. * cos(inputAngle);
42 | float sin = ${FREAK_EXPANSION_FACTOR}. * sin(inputAngle);
43 |
44 | float yp = inputY + freakX * sin + freakY * cos;
45 | float xp = inputX + freakX * cos + freakY * -sin;
46 |
47 | int x0 = int(floor(xp));
48 | int x1 = x0 + 1;
49 | int y0 = int(floor(yp));
50 | int y1 = y0 + 1;
51 |
52 | float f1 = getPixel(octave, y0, x0);
53 | float f2 = getPixel(octave, y0, x1);
54 | float f3 = getPixel(octave, y1, x0);
55 | float f4 = getPixel(octave, y1, x1);
56 |
57 | float x1f = float(x1);
58 | float y1f = float(y1);
59 | float x0f = float(x0);
60 | float y0f = float(y0);
61 |
62 | // ratio for interpolation between four neighbouring points
63 | float value = (x1f - xp) * (y1f - yp) * f1
64 | + (xp - x0f) * (y1f - yp) * f2
65 | + (x1f - xp) * (yp - y0f) * f3
66 | + (xp - x0f) * (yp - y0f) * f4;
67 |
68 | setOutput(value);
69 | }
70 | `
71 | }
72 |
73 | cache[key] = kernel;
74 | }
75 |
76 | return cache[key];
77 | }
78 |
79 | export const computeExtremaFreak = (args) => {
80 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
81 | const { gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT,pyramidImagesLength } = args.inputs;
82 | /** @type {MathBackendWebGL} */
83 | const backend = args.backend;
84 |
85 | const program = GetProgram(pyramidImagesLength, prunedExtremas);
86 |
87 | return backend.runWebGLProgram(program, [...gaussianImagesT, prunedExtremas, prunedExtremasAngles, freakPointsT], 'float32');
88 | }
89 |
90 | export const computeExtremaFreakConfig = {//: KernelConfig
91 | kernelName: "ComputeExtremaFreak",
92 | backendName: 'webgl',
93 | kernelFunc: computeExtremaFreak,// as {} as KernelFunc,
94 | };
95 |
96 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/computeFreakDescriptors.js:
--------------------------------------------------------------------------------
1 |
2 | import { FREAKPOINTS } from '../../freak.js';
3 |
4 | const FREAK_CONPARISON_COUNT = (FREAKPOINTS.length - 1) * (FREAKPOINTS.length) / 2;
5 | const descriptorCount = Math.ceil(FREAK_CONPARISON_COUNT / 8);
6 |
7 | const cache={};
8 | function GetProgram(extremaFreaks){
9 | const key=`${extremaFreaks.shape[0]}`;
10 | if (!cache.hasOwnProperty(key)) {
11 | const kernel = {
12 | variableNames: ['freak', 'p'],
13 | outputShape: [extremaFreaks.shape[0], descriptorCount],
14 | userCode: `
15 | void main() {
16 | ivec2 coords = getOutputCoords();
17 | int featureIndex = coords[0];
18 | int descIndex = coords[1] * 8;
19 |
20 | int sum = 0;
21 | for (int i = 0; i < 8; i++) {
22 | if (descIndex + i >= ${FREAK_CONPARISON_COUNT}) {
23 | continue;
24 | }
25 |
26 | int p1 = int(getP(descIndex + i, 0));
27 | int p2 = int(getP(descIndex + i, 1));
28 |
29 | float v1 = getFreak(featureIndex, p1);
30 | float v2 = getFreak(featureIndex, p2);
31 |
32 | if (v1 < v2 + 0.01) {
33 | sum += int(pow(2.0, float(7 - i)));
34 | }
35 | }
36 | setOutput(float(sum));
37 | }
38 | `
39 | }
40 | cache[key]=kernel;
41 | }
42 | return cache[key];
43 | }
44 |
45 | export const computeFreakDescriptor=(args)=>{
46 | const {extremaFreaks, positionT} =args.inputs;
47 | const {backend} = args;
48 | const program = GetProgram(extremaFreaks);
49 | return backend.runWebGLProgram(program,[extremaFreaks, positionT],'int32');
50 | }
51 |
52 | export const computeFreakDescriptorConfig = {//: KernelConfig
53 | kernelName: "ComputeFreakDescriptors",
54 | backendName: 'webgl',
55 | kernelFunc: computeFreakDescriptor,// as {} as KernelFunc,
56 | };
57 |
58 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/computeLocalization.js:
--------------------------------------------------------------------------------
1 | import {tensor} from '@tensorflow/tfjs'
2 |
3 | const cache={};
4 | function GetProgram(numDogPyramidImages,extremasListLength){
5 | const kernelKey=`${numDogPyramidImages}|${extremasListLength}`;
6 | if(!cache.hasOwnProperty(kernelKey)){
7 | const dogVariableNames = [];
8 | let dogSubCodes = `float getPixel(int octave, int y, int x) {`;
9 | for (let i = 1; i < numDogPyramidImages; i++) { // extrema starts from second octave
10 | dogVariableNames.push('image' + i);
11 | dogSubCodes += `
12 | if (octave == ${i}) {
13 | return getImage${i}(y, x);
14 | }
15 | `;
16 | }
17 | dogSubCodes += `}`;
18 |
19 | cache[kernelKey] = {
20 | variableNames: [...dogVariableNames, 'extrema'],
21 | outputShape: [extremasListLength, 3, 3], // 3x3 pixels around the extrema
22 | userCode: `
23 | ${dogSubCodes}
24 |
25 | void main() {
26 | ivec3 coords = getOutputCoords();
27 | int featureIndex = coords[0];
28 | float score = getExtrema(featureIndex, 0);
29 | if (score == 0.0) {
30 | return;
31 | }
32 |
33 | int dy = coords[1]-1;
34 | int dx = coords[2]-1;
35 | int octave = int(getExtrema(featureIndex, 1));
36 | int y = int(getExtrema(featureIndex, 2));
37 | int x = int(getExtrema(featureIndex, 3));
38 | setOutput(getPixel(octave, y+dy, x+dx));
39 | }
40 | `
41 | };
42 | }
43 | return cache[kernelKey];
44 | }
45 |
46 | export const computeLocalization=(args)=>{
47 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
48 | const {prunedExtremasList, dogPyramidImagesT} = args.inputs;
49 | /** @type {MathBackendWebGL} */
50 | const backend = args.backend;
51 | const program = GetProgram(dogPyramidImagesT.length,prunedExtremasList.length);
52 | const prunedExtremasT = tensor(prunedExtremasList, [prunedExtremasList.length, prunedExtremasList[0].length], 'int32');
53 | return backend.runWebGLProgram(program, [...dogPyramidImagesT.slice(1), prunedExtremasT],dogPyramidImagesT[0].dtype);
54 |
55 | }
56 |
57 | export const computeLocalizationConfig = {//: KernelConfig
58 | kernelName: "ComputeLocalization",
59 | backendName: 'webgl',
60 | kernelFunc: computeLocalization,// as {} as KernelFunc,
61 | };
62 |
63 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/downsampleBilinear.js:
--------------------------------------------------------------------------------
1 | const cache={};
2 | /**
3 | *
4 | * @param {TensorInfo} image
5 | * @returns {GPGPUProgram}
6 | */
7 | function GetProgram(image){
8 | const imageWidth = image.shape[1];
9 | const imageHeight = image.shape[0];
10 | const kernelKey = 'w' + imageWidth + "h" + imageHeight;
11 | if(!cache.hasOwnProperty(kernelKey)){
12 | const kernel = {
13 | variableNames: ['p'],
14 | outputShape: [Math.floor(imageHeight/2), Math.floor(imageWidth/2)],
15 | userCode: `
16 | void main() {
17 | ivec2 coords = getOutputCoords();
18 | int y = coords[0] * 2;
19 | int x = coords[1] * 2;
20 |
21 | float sum = getP(y, x) * 0.25;
22 | sum += getP(y+1,x) * 0.25;
23 | sum += getP(y, x+1) * 0.25;
24 | sum += getP(y+1,x+1) * 0.25;
25 | setOutput(sum);
26 | }
27 | `
28 | };
29 | cache[kernelKey]=kernel;
30 | }
31 | return cache[kernelKey];
32 | }
33 |
34 |
35 | export const downsampleBilinear =(args)=>{
36 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
37 | const image = args.inputs.image;
38 | /** @type {MathBackendWebGL} */
39 | const backend = args.backend;
40 |
41 | const program=GetProgram(image);
42 |
43 | return backend.runWebGLProgram(program,[image],image.dtype);
44 | }
45 |
46 | export const downsampleBilinearConfig = {//: KernelConfig
47 | kernelName: "DownsampleBilinear",
48 | backendName: 'webgl',
49 | kernelFunc: downsampleBilinear,// as {} as KernelFunc,
50 | };
51 |
52 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/extremaReduction.js:
--------------------------------------------------------------------------------
1 | export const extremaReduction=(args)=>{
2 | /** @type {import('@tensorflow/tfjs').TensorInfo[]} */
3 | const {extremasResultT} = args.inputs;
4 | /** @type {MathBackendWebGL} */
5 | const backend = args.backend;
6 | const extremaHeight = extremasResultT.shape[0];
7 | const extremaWidth = extremasResultT.shape[1];
8 | const kernel = {
9 | variableNames: ['extrema'],
10 | outputShape: [Math.floor(extremaHeight/2), Math.floor(extremaWidth/2)],
11 | userCode: `
12 | void main() {
13 | ivec2 coords = getOutputCoords();
14 | int y = coords[0] * 2;
15 | int x = coords[1] * 2;
16 |
17 | float location = 0.0;
18 | float values = getExtrema(y, x);
19 |
20 | if (getExtrema(y+1, x) != 0.0) {
21 | location = 1.0;
22 | values = getExtrema(y+1, x);
23 | }
24 | else if (getExtrema(y, x+1) != 0.0) {
25 | location = 2.0;
26 | values = getExtrema(y, x+1);
27 | }
28 | else if (getExtrema(y+1, x+1) != 0.0) {
29 | location = 3.0;
30 | values = getExtrema(y+1, x+1);
31 | }
32 |
33 | if (values < 0.0) {
34 | setOutput(location * -1000.0 + values);
35 | } else {
36 | setOutput(location * 1000.0 + values);
37 | }
38 | }
39 | `
40 | }
41 |
42 | return backend.runWebGLProgram(kernel,[extremasResultT],extremasResultT.dtype);
43 | }
44 |
45 | export const extremaReductionConfig = {//: KernelConfig
46 | kernelName: "ExtremaReduction",
47 | backendName: 'webgl',
48 | kernelFunc: extremaReduction,// as {} as KernelFunc,
49 | };
50 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/index.js:
--------------------------------------------------------------------------------
1 | import { registerKernel } from '@tensorflow/tfjs';
2 | import { binomialFilterConfig } from './binomialFilter.js';
3 | import { buildExtremasConfig } from './buildExtremas.js';
4 | import { computeExtremaAnglesConfig } from './computeExtremaAngles.js';
5 | import { computeExtremaFreakConfig } from './computeExtremaFreak.js'
6 | import { computeFreakDescriptorConfig } from './computeFreakDescriptors.js'
7 | import { computeLocalizationConfig } from './computeLocalization.js'
8 | import { computeOrientationHistogramsConfig } from './computeOrientationHistograms.js'
9 | import { downsampleBilinearConfig } from './downsampleBilinear.js';
10 | import { extremaReductionConfig } from './extremaReduction.js';
11 | import { smoothHistogramsConfig } from './smoothHistograms.js';
12 | import { upsampleBilinearConfig } from './upsampleBilinear.js';
13 |
14 | //export function Register(){
15 | registerKernel(binomialFilterConfig);
16 | registerKernel(buildExtremasConfig);
17 | registerKernel(computeExtremaAnglesConfig);
18 | registerKernel(computeExtremaFreakConfig);
19 | registerKernel(computeFreakDescriptorConfig);
20 | registerKernel(computeLocalizationConfig);
21 | registerKernel(computeOrientationHistogramsConfig);
22 | registerKernel(downsampleBilinearConfig);
23 | registerKernel(extremaReductionConfig);
24 | registerKernel(smoothHistogramsConfig);
25 | registerKernel(upsampleBilinearConfig);
26 | //}
27 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/smoothHistograms.js:
--------------------------------------------------------------------------------
1 |
2 | const ORIENTATION_NUM_BINS = 36;
3 | const ORIENTATION_SMOOTHING_ITERATIONS = 5;
4 |
5 | const cache={};
6 | function GetProgram(histograms){
7 | const kernelKey=`h${histograms.shape[0]}`;
8 | if(!cache.hasOwnProperty(kernelKey)){
9 | const kernel = {
10 | variableNames: ['histogram'],
11 | outputShape: [histograms.shape[0], ORIENTATION_NUM_BINS],
12 | userCode: `
13 | void main() {
14 | ivec2 coords = getOutputCoords();
15 |
16 | int featureIndex = coords[0];
17 | int binIndex = coords[1];
18 |
19 | int prevBin = imod(binIndex - 1 + ${ORIENTATION_NUM_BINS}, ${ORIENTATION_NUM_BINS});
20 | int nextBin = imod(binIndex + 1, ${ORIENTATION_NUM_BINS});
21 | float result = 0.274068619061197 * getHistogram(featureIndex, prevBin) + 0.451862761877606 * getHistogram(featureIndex, binIndex) + 0.274068619061197 * getHistogram(featureIndex, nextBin);
22 |
23 | setOutput(result);
24 | }
25 | `
26 | };
27 | cache[kernelKey]=kernel;
28 | }
29 | return cache[kernelKey];
30 | }
31 |
32 | export const smoothHistograms=(args)=>{
33 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
34 | let {histograms} = args.inputs;
35 | /** @type {MathBackendWebGL} */
36 | const backend = args.backend;
37 |
38 | const program = GetProgram(histograms);
39 | for (let i = 0; i < ORIENTATION_SMOOTHING_ITERATIONS; i++) {
40 | const _histograms = histograms;
41 | histograms = backend.runWebGLProgram(program,[histograms],histograms.dtype);//this._compileAndRun(program, [histograms]);
42 | if (i > 0) {
43 | backend.disposeIntermediateTensorInfo(_histograms);
44 | }
45 | }
46 | return histograms;
47 |
48 | }
49 |
50 |
51 |
52 | export const smoothHistogramsConfig = {//: KernelConfig
53 | kernelName: "SmoothHistograms",
54 | backendName: 'webgl',
55 | kernelFunc: smoothHistograms,// as {} as KernelFunc,
56 | };
57 |
58 |
--------------------------------------------------------------------------------
/src/image-target/detector/kernels/webgl/upsampleBilinear.js:
--------------------------------------------------------------------------------
1 | import {MathBackendWebGL} from '@tensorflow/tfjs-backend-webgl';
2 |
3 | const cache={};
4 | function GetProgram(image,targetImage){
5 | const targetImageWidth = targetImage.shape[1];
6 | const targetImageHeight = targetImage.shape[0];
7 | const kernelKey = 'w' + targetImageWidth + "h" + targetImageHeight;
8 | if(!cache.hasOwnProperty(kernelKey)){
9 | const kernel = {
10 | variableNames: ['p'],
11 | outputShape: [targetImageHeight, targetImageWidth],
12 | userCode: `
13 | void main() {
14 | ivec2 coords = getOutputCoords();
15 | int j = coords[0];
16 | int i = coords[1];
17 |
18 | float sj = 0.5 * float(j) - 0.25;
19 | float si = 0.5 * float(i) - 0.25;
20 |
21 | float sj0 = floor(sj);
22 | float sj1 = ceil(sj);
23 | float si0 = floor(si);
24 | float si1 = ceil(si);
25 |
26 | int sj0I = int(sj0);
27 | int sj1I = int(sj1);
28 | int si0I = int(si0);
29 | int si1I = int(si1);
30 |
31 | float sum = 0.0;
32 | sum += getP(sj0I, si0I) * (si1 - si) * (sj1 - sj);
33 | sum += getP(sj1I, si0I) * (si1 - si) * (sj - sj0);
34 | sum += getP(sj0I, si1I) * (si - si0) * (sj1 - sj);
35 | sum += getP(sj1I, si1I) * (si - si0) * (sj - sj0);
36 | setOutput(sum);
37 | }
38 | `
39 | };
40 | cache[kernelKey]=kernel;
41 | }
42 |
43 | return cache[kernelKey];
44 | }
45 |
46 | export const upsampleBilinear =(args)=>{
47 | /** @type {import('@tensorflow/tfjs').TensorInfo} */
48 | const {image,targetImage} = args.inputs;
49 |
50 |
51 | /** @type {MathBackendWebGL} */
52 | const backend = args.backend;
53 |
54 | const program = GetProgram(image,targetImage);
55 | return backend.runWebGLProgram(program,[image],image.dtype);
56 |
57 |
58 | }
59 |
60 | export const upsampleBilinearConfig = {//: KernelConfig
61 | kernelName: "UpsampleBilinear",
62 | backendName: 'webgl',
63 | kernelFunc: upsampleBilinear,// as {} as KernelFunc,
64 | };
65 |
--------------------------------------------------------------------------------
/src/image-target/estimation/estimate.js:
--------------------------------------------------------------------------------
1 | import {Matrix, inverse} from 'ml-matrix';
2 | import {solveHomography} from '../utils/homography.js';
3 |
4 | // build world matrix with list of matching worldCoords|screenCoords
5 | //
6 | // Step 1. estimate homography with list of pairs
7 | // Ref: https://www.uio.no/studier/emner/matnat/its/TEK5030/v19/lect/lecture_4_3-estimating-homographies-from-feature-correspondences.pdf (Basic homography estimation from points)
8 | //
9 | // Step 2. decompose homography into rotation and translation matrixes (i.e. world matrix)
10 | // Ref: can anyone provide reference?
11 | const estimate = ({screenCoords, worldCoords, projectionTransform}) => {
12 | const Harray = solveHomography(worldCoords.map((p) => [p.x, p.y]), screenCoords.map((p) => [p.x, p.y]));
13 | const H = new Matrix([
14 | [Harray[0], Harray[1], Harray[2]],
15 | [Harray[3], Harray[4], Harray[5]],
16 | [Harray[6], Harray[7], Harray[8]],
17 | ]);
18 |
19 | const K = new Matrix(projectionTransform);
20 | const KInv = inverse(K);
21 |
22 | const _KInvH = KInv.mmul(H);
23 | const KInvH = _KInvH.to1DArray();
24 |
25 | const norm1 = Math.sqrt( KInvH[0] * KInvH[0] + KInvH[3] * KInvH[3] + KInvH[6] * KInvH[6]);
26 | const norm2 = Math.sqrt( KInvH[1] * KInvH[1] + KInvH[4] * KInvH[4] + KInvH[7] * KInvH[7]);
27 | const tnorm = (norm1 + norm2) / 2;
28 |
29 | const rotate = [];
30 | rotate[0] = KInvH[0] / norm1;
31 | rotate[3] = KInvH[3] / norm1;
32 | rotate[6] = KInvH[6] / norm1;
33 |
34 | rotate[1] = KInvH[1] / norm2;
35 | rotate[4] = KInvH[4] / norm2;
36 | rotate[7] = KInvH[7] / norm2;
37 |
38 | rotate[2] = rotate[3] * rotate[7] - rotate[6] * rotate[4];
39 | rotate[5] = rotate[6] * rotate[1] - rotate[0] * rotate[7];
40 | rotate[8] = rotate[0] * rotate[4] - rotate[1] * rotate[3];
41 |
42 | const norm3 = Math.sqrt(rotate[2] * rotate[2] + rotate[5] * rotate[5] + rotate[8] * rotate[8]);
43 | rotate[2] /= norm3;
44 | rotate[5] /= norm3;
45 | rotate[8] /= norm3;
46 |
47 | // TODO: artoolkit has check_rotation() that somehow switch the rotate vector. not sure what that does. Can anyone advice?
48 | // https://github.com/artoolkitx/artoolkit5/blob/5bf0b671ff16ead527b9b892e6aeb1a2771f97be/lib/SRC/ARICP/icpUtil.c#L215
49 |
50 | const tran = []
51 | tran[0] = KInvH[2] / tnorm;
52 | tran[1] = KInvH[5] / tnorm;
53 | tran[2] = KInvH[8] / tnorm;
54 |
55 | let initialModelViewTransform = [
56 | [rotate[0], rotate[1], rotate[2], tran[0]],
57 | [rotate[3], rotate[4], rotate[5], tran[1]],
58 | [rotate[6], rotate[7], rotate[8], tran[2]]
59 | ];
60 |
61 | return initialModelViewTransform;
62 | };
63 |
64 | export {
65 | estimate
66 | }
67 |
--------------------------------------------------------------------------------
/src/image-target/estimation/estimator.js:
--------------------------------------------------------------------------------
1 | import {estimate} from './estimate.js';
2 | import {refineEstimate} from './refine-estimate.js';
3 |
4 | class Estimator {
5 | constructor(projectionTransform) {
6 | this.projectionTransform = projectionTransform;
7 | }
8 |
9 | // Solve homography between screen points and world points using Direct Linear Transformation
10 | // then decompose homography into rotation and translation matrix (i.e. modelViewTransform)
11 | estimate({screenCoords, worldCoords}) {
12 | const modelViewTransform = estimate({screenCoords, worldCoords, projectionTransform: this.projectionTransform});
13 | return modelViewTransform;
14 | }
15 |
16 | // Given an initial guess of the modelViewTransform and new pairs of screen-world coordinates,
17 | // use Iterative Closest Point to refine the transformation
18 | //refineEstimate({initialModelViewTransform, screenCoords, worldCoords}) {
19 | refineEstimate({initialModelViewTransform, worldCoords, screenCoords}) {
20 | const updatedModelViewTransform = refineEstimate({initialModelViewTransform, worldCoords, screenCoords, projectionTransform: this.projectionTransform});
21 | return updatedModelViewTransform;
22 | }
23 | }
24 |
25 | export {
26 | Estimator,
27 | }
28 |
--------------------------------------------------------------------------------
/src/image-target/estimation/utils.js:
--------------------------------------------------------------------------------
1 | const buildModelViewProjectionTransform = (projectionTransform, modelViewTransform) => {
2 | // assume the projectTransform has the following format:
3 | // [[fx, 0, cx],
4 | // [0, fy, cy]
5 | // [0, 0, 1]]
6 | const modelViewProjectionTransform = [
7 | [
8 | projectionTransform[0][0] * modelViewTransform[0][0] + projectionTransform[0][2] * modelViewTransform[2][0],
9 | projectionTransform[0][0] * modelViewTransform[0][1] + projectionTransform[0][2] * modelViewTransform[2][1],
10 | projectionTransform[0][0] * modelViewTransform[0][2] + projectionTransform[0][2] * modelViewTransform[2][2],
11 | projectionTransform[0][0] * modelViewTransform[0][3] + projectionTransform[0][2] * modelViewTransform[2][3],
12 | ],
13 | [
14 | projectionTransform[1][1] * modelViewTransform[1][0] + projectionTransform[1][2] * modelViewTransform[2][0],
15 | projectionTransform[1][1] * modelViewTransform[1][1] + projectionTransform[1][2] * modelViewTransform[2][1],
16 | projectionTransform[1][1] * modelViewTransform[1][2] + projectionTransform[1][2] * modelViewTransform[2][2],
17 | projectionTransform[1][1] * modelViewTransform[1][3] + projectionTransform[1][2] * modelViewTransform[2][3],
18 | ],
19 | [
20 | modelViewTransform[2][0],
21 | modelViewTransform[2][1],
22 | modelViewTransform[2][2],
23 | modelViewTransform[2][3],
24 | ]
25 | ];
26 | return modelViewProjectionTransform;
27 |
28 | /*
29 | // this is the full computation if the projectTransform does not look like the expected format, but more computations
30 | //
31 | const modelViewProjectionTransform = [[],[],[]];
32 | for (let j = 0; j < 3; j++ ) {
33 | for (let i = 0; i < 4; i++) {
34 | modelViewProjectionTransform[j][i] = projectionTransform[j][0] * modelViewTransform[0][i]
35 | + projectionTransform[j][1] * modelViewTransform[1][i]
36 | + projectionTransform[j][2] * modelViewTransform[2][i];
37 | }
38 | }
39 | return modelViewProjectionTransform;
40 | */
41 | }
42 |
43 | const applyModelViewProjectionTransform = (modelViewProjectionTransform, x, y, z) => {
44 | // assume z is zero
45 | const ux = modelViewProjectionTransform[0][0] * x + modelViewProjectionTransform[0][1] * y + modelViewProjectionTransform[0][3];
46 | const uy = modelViewProjectionTransform[1][0] * x + modelViewProjectionTransform[1][1] * y + modelViewProjectionTransform[1][3];
47 | const uz = modelViewProjectionTransform[2][0] * x + modelViewProjectionTransform[2][1] * y + modelViewProjectionTransform[2][3];
48 | return {x: ux, y: uy, z: uz};
49 | }
50 |
51 | const computeScreenCoordiate = (modelViewProjectionTransform, x, y, z) => {
52 | const {x: ux, y: uy, z: uz} = applyModelViewProjectionTransform(modelViewProjectionTransform, x, y, z);
53 | //if( Math.abs(uz) < 0.000001 ) return null;
54 | return {x: ux/uz, y: uy/uz};
55 | }
56 |
57 | const screenToMarkerCoordinate = (modelViewProjectionTransform, sx, sy) => {
58 | const c11 = modelViewProjectionTransform[2][0] * sx - modelViewProjectionTransform[0][0];
59 | const c12 = modelViewProjectionTransform[2][1] * sx - modelViewProjectionTransform[0][1];
60 | const c21 = modelViewProjectionTransform[2][0] * sy - modelViewProjectionTransform[1][0];
61 | const c22 = modelViewProjectionTransform[2][1] * sy - modelViewProjectionTransform[1][1];
62 | const b1 = modelViewProjectionTransform[0][3] - modelViewProjectionTransform[2][3] * sx;
63 | const b2 = modelViewProjectionTransform[1][3] - modelViewProjectionTransform[2][3] * sy;
64 |
65 | const m = c11 * c22 - c12 * c21;
66 | return {
67 | x: (c22 * b1 - c12 * b2) / m,
68 | y: (c11 * b2 - c21 * b1) / m
69 | }
70 | }
71 |
72 | export {
73 | buildModelViewProjectionTransform,
74 | applyModelViewProjectionTransform,
75 | computeScreenCoordiate,
76 | }
77 |
--------------------------------------------------------------------------------
/src/image-target/image-list.js:
--------------------------------------------------------------------------------
1 | import {resize} from "./utils/images.js";
2 |
3 | const MIN_IMAGE_PIXEL_SIZE = 100;
4 |
5 | // Build a list of image {data, width, height, scale} with different scales
6 | const buildImageList = (inputImage) => {
7 | const minScale = MIN_IMAGE_PIXEL_SIZE / Math.min(inputImage.width, inputImage.height);
8 |
9 | const scaleList = [];
10 | let c = minScale;
11 | while (true) {
12 | scaleList.push(c);
13 | c *= Math.pow(2.0, 1.0/3.0);
14 | if (c >= 0.95) {
15 | c = 1;
16 | break;
17 | }
18 | }
19 | scaleList.push(c);
20 | scaleList.reverse();
21 |
22 | const imageList = [];
23 | for (let i = 0; i < scaleList.length; i++) {
24 | const w = inputImage.width * scaleList[i];
25 | const h = inputImage.height * scaleList[i];
26 | imageList.push(Object.assign(resize({image: inputImage, ratio: scaleList[i]}), {scale: scaleList[i]}));
27 | }
28 | return imageList;
29 | }
30 |
31 | const buildTrackingImageList = (inputImage) => {
32 | const minDimension = Math.min(inputImage.width, inputImage.height);
33 | const scaleList = [];
34 | const imageList = [];
35 | scaleList.push( 256.0 / minDimension);
36 | scaleList.push( 128.0 / minDimension);
37 | for (let i = 0; i < scaleList.length; i++) {
38 | imageList.push(Object.assign(resize({image: inputImage, ratio: scaleList[i]}), {scale: scaleList[i]}));
39 | }
40 | return imageList;
41 | }
42 |
43 | export {
44 | buildImageList,
45 | buildTrackingImageList
46 | }
47 |
--------------------------------------------------------------------------------
/src/image-target/index.js:
--------------------------------------------------------------------------------
1 | import {Controller} from './controller.js';
2 | import {Compiler} from './compiler.js';
3 | import {UI} from '../ui/ui.js';
4 |
5 | export {
6 | Controller,
7 | Compiler,
8 | UI
9 | }
10 |
11 | if (!window.MINDAR) {
12 | window.MINDAR = {};
13 | }
14 |
15 | window.MINDAR.IMAGE = {
16 | Controller,
17 | Compiler,
18 | UI
19 | };
20 |
--------------------------------------------------------------------------------
/src/image-target/input-loader.js:
--------------------------------------------------------------------------------
1 | import * as tf from '@tensorflow/tfjs';
2 |
3 | // More efficient implementation for tf.browser.fromPixels
4 | // original implementation: /node_modules/@tensorflow/tfjs-backend-webgl/src/kernels/FromPixels.ts
5 | //
6 | // This implementation return grey scale instead of RGBA in the orignal implementation
7 |
8 | class InputLoader {
9 | constructor(width, height) {
10 | this.width = width;
11 | this.height = height;
12 | this.texShape = [height, width];
13 |
14 | const context = document.createElement('canvas').getContext('2d');
15 | context.canvas.width = width;
16 | context.canvas.height = height;
17 | this.context = context;
18 |
19 | this.program = this.buildProgram(width, height);
20 |
21 | const backend = tf.backend();
22 | //this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'int32');
23 | this.tempPixelHandle = backend.makeTensorInfo(this.texShape, 'float32');
24 | // warning!!!
25 | // usage type should be TextureUsage.PIXELS, but tfjs didn't export this enum type, so we hard-coded 2 here
26 | // i.e. backend.texData.get(tempPixelHandle.dataId).usage = TextureUsage.PIXELS;
27 | backend.texData.get(this.tempPixelHandle.dataId).usage = 2;
28 | }
29 |
30 | // old method
31 | _loadInput(input) {
32 | return tf.tidy(() => {
33 | let inputImage = tf.browser.fromPixels(input);
34 | inputImage = inputImage.mean(2);
35 | return inputImage;
36 | });
37 | }
38 |
39 | // input is instance of HTMLVideoElement or HTMLImageElement
40 | loadInput(input) {
41 | const context = this.context;
42 | context.clearRect(0, 0, this.context.canvas.width, this.context.canvas.height);
43 |
44 | const isInputRotated = input.width === this.height && input.height === this.width;
45 | if (isInputRotated) { // rotate 90 degree and draw
46 | let x = this.context.canvas.width / 2;
47 | let y = this.context.canvas.height / 2;
48 | let angleInDegrees = 90;
49 |
50 | context.save(); // save the current context state
51 | context.translate(x, y); // move the context origin to the center of the image
52 | context.rotate(angleInDegrees * Math.PI / 180); // rotate the context
53 |
54 | // draw the image with its center at the origin
55 | context.drawImage(input, -input.width / 2, -input.height / 2);
56 | context.restore(); // restore the context to its original state
57 | } else {
58 | this.context.drawImage(input, 0, 0, input.width, input.height);
59 | }
60 |
61 | const backend = tf.backend();
62 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(this.tempPixelHandle.dataId), this.context.canvas);
63 |
64 | //const res = backend.compileAndRun(this.program, [this.tempPixelHandle]);
65 | const res = this._compileAndRun(this.program, [this.tempPixelHandle]);
66 | //const res = this._runWebGLProgram(this.program, [this.tempPixelHandle], 'float32');
67 | //backend.disposeData(tempPixelHandle.dataId);
68 | return res;
69 | }
70 |
71 | buildProgram(width, height) {
72 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D';
73 |
74 | const program = {
75 | variableNames: ['A'],
76 | outputShape: this.texShape,
77 | userCode:`
78 | void main() {
79 | ivec2 coords = getOutputCoords();
80 | int texR = coords[0];
81 | int texC = coords[1];
82 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0);
83 |
84 | vec4 values = ${textureMethod}(A, uv);
85 | setOutput((0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0);
86 | }
87 | `
88 | }
89 | return program;
90 | }
91 |
92 | _compileAndRun(program, inputs) {
93 | const outInfo = tf.backend().compileAndRun(program, inputs);
94 | return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
95 | }
96 |
97 | _runWebGLProgram(program, inputs, outputType) {
98 | const outInfo = tf.backend().runWebGLProgram(program, inputs, outputType);
99 | return tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
100 | }
101 | }
102 |
103 | export {
104 | InputLoader
105 | };
106 |
--------------------------------------------------------------------------------
/src/image-target/matching/hamming-distance.js:
--------------------------------------------------------------------------------
1 | // Fast computation on number of bit sets
2 | // Ref: https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
3 | const compute = (options) => {
4 | const {v1, v2} = options;
5 | let d = 0;
6 |
7 | for (let i = 0; i < v1.length; i++) {
8 | let x = (v1[i] ^ v2[i]) >>> 0;
9 | d += bitCount(x);
10 | }
11 | return d;
12 | }
13 |
14 | const bitCount = (v) => {
15 | var c = v - ((v >> 1) & 0x55555555);
16 | c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
17 | c = ((c >> 4) + c) & 0x0F0F0F0F;
18 | c = ((c >> 8) + c) & 0x00FF00FF;
19 | c = ((c >> 16) + c) & 0x0000FFFF;
20 | return c;
21 | }
22 |
23 | export {
24 | compute
25 | };
26 |
--------------------------------------------------------------------------------
/src/image-target/matching/hierarchical-clustering.js:
--------------------------------------------------------------------------------
1 | import {compute as hammingCompute} from './hamming-distance.js';
2 | import {createRandomizer} from '../utils/randomizer.js';
3 |
4 | const MIN_FEATURE_PER_NODE = 16;
5 | const NUM_ASSIGNMENT_HYPOTHESES = 128;
6 | const NUM_CENTERS = 8;
7 |
8 |
9 | const _computeKMedoids = (options) => {
10 | const {points, pointIndexes, randomizer} = options;
11 |
12 | const randomPointIndexes = [];
13 | for (let i = 0; i < pointIndexes.length; i++) {
14 | randomPointIndexes.push(i);
15 | }
16 |
17 | let bestSumD = Number.MAX_SAFE_INTEGER;
18 | let bestAssignmentIndex = -1;
19 |
20 | const assignments = [];
21 | for (let i = 0; i < NUM_ASSIGNMENT_HYPOTHESES; i++) {
22 | randomizer.arrayShuffle({arr: randomPointIndexes, sampleSize: NUM_CENTERS});
23 |
24 | let sumD = 0;
25 | const assignment = [];
26 | for (let j = 0; j < pointIndexes.length; j++) {
27 | let bestD = Number.MAX_SAFE_INTEGER;
28 | for (let k = 0; k < NUM_CENTERS; k++) {
29 | const centerIndex = pointIndexes[randomPointIndexes[k]];
30 | const d = hammingCompute({v1: points[pointIndexes[j]].descriptors, v2: points[centerIndex].descriptors});
31 | if (d < bestD) {
32 | assignment[j] = randomPointIndexes[k];
33 | bestD = d;
34 | }
35 | }
36 | sumD += bestD;
37 | }
38 | assignments.push(assignment);
39 |
40 | if (sumD < bestSumD) {
41 | bestSumD = sumD;
42 | bestAssignmentIndex = i;
43 | }
44 | }
45 | return assignments[bestAssignmentIndex];
46 | }
47 |
48 | // kmedoids clustering of points, with hamming distance of FREAK descriptor
49 | //
50 | // node = {
51 | // isLeaf: bool,
52 | // children: [], list of children node
53 | // pointIndexes: [], list of int, point indexes
54 | // centerPointIndex: int
55 | // }
56 | const build = ({points}) => {
57 | const pointIndexes = [];
58 | for (let i = 0; i < points.length; i++) {
59 | pointIndexes.push(i);
60 | }
61 |
62 | const randomizer = createRandomizer();
63 |
64 | const rootNode = _build({points: points, pointIndexes: pointIndexes, centerPointIndex: null, randomizer});
65 | return {rootNode};
66 | }
67 |
68 | // recursive build hierarchy clusters
69 | const _build = (options) => {
70 | const {points, pointIndexes, centerPointIndex, randomizer} = options;
71 |
72 | let isLeaf = false;
73 |
74 | if (pointIndexes.length <= NUM_CENTERS || pointIndexes.length <= MIN_FEATURE_PER_NODE) {
75 | isLeaf = true;
76 | }
77 |
78 | const clusters = {};
79 | if (!isLeaf) {
80 | // compute clusters
81 | const assignment = _computeKMedoids({points, pointIndexes, randomizer});
82 |
83 | for (let i = 0; i < assignment.length; i++) {
84 | if (clusters[pointIndexes[assignment[i]]] === undefined) {
85 | clusters[pointIndexes[assignment[i]]] = [];
86 | }
87 | clusters[pointIndexes[assignment[i]]].push(pointIndexes[i]);
88 | }
89 | }
90 | if (Object.keys(clusters).length === 1) {
91 | isLeaf = true;
92 | }
93 |
94 | const node = {
95 | centerPointIndex: centerPointIndex
96 | }
97 |
98 | if (isLeaf) {
99 | node.leaf = true;
100 | node.pointIndexes = [];
101 | for (let i = 0; i < pointIndexes.length; i++) {
102 | node.pointIndexes.push(pointIndexes[i]);
103 | }
104 | return node;
105 | }
106 |
107 | // recursive build children
108 | node.leaf = false;
109 | node.children = [];
110 |
111 | Object.keys(clusters).forEach((centerIndex) => {
112 | node.children.push(_build({points: points, pointIndexes: clusters[centerIndex], centerPointIndex: centerIndex, randomizer}));
113 | });
114 | return node;
115 | }
116 |
117 | export {
118 | build,
119 | };
120 |
121 |
--------------------------------------------------------------------------------
/src/image-target/matching/matcher.js:
--------------------------------------------------------------------------------
1 | import {match} from './matching.js';
2 |
3 | class Matcher {
4 | constructor(queryWidth, queryHeight, debugMode = false) {
5 | this.queryWidth = queryWidth;
6 | this.queryHeight = queryHeight;
7 | this.debugMode = debugMode;
8 | }
9 |
10 | matchDetection(keyframes, featurePoints) {
11 | let debugExtra = {frames: []};
12 |
13 | let bestResult = null;
14 | for (let i = 0; i < keyframes.length; i++) {
15 | const {H, matches, debugExtra: frameDebugExtra} = match({keyframe: keyframes[i], querypoints: featurePoints, querywidth: this.queryWidth, queryheight: this.queryHeight, debugMode: this.debugMode});
16 | debugExtra.frames.push(frameDebugExtra);
17 |
18 | if (H) {
19 | if (bestResult === null || bestResult.matches.length < matches.length) {
20 | bestResult = {keyframeIndex: i, H, matches};
21 | }
22 | }
23 | }
24 |
25 | if (bestResult === null) {
26 | return {keyframeIndex: -1, debugExtra};
27 | }
28 |
29 | const screenCoords = [];
30 | const worldCoords = [];
31 | const keyframe = keyframes[bestResult.keyframeIndex];
32 | for (let i = 0; i < bestResult.matches.length; i++) {
33 | const querypoint = bestResult.matches[i].querypoint;
34 | const keypoint = bestResult.matches[i].keypoint;
35 | screenCoords.push({
36 | x: querypoint.x,
37 | y: querypoint.y,
38 | })
39 | worldCoords.push({
40 | x: (keypoint.x + 0.5) / keyframe.scale,
41 | y: (keypoint.y + 0.5) / keyframe.scale,
42 | z: 0,
43 | })
44 | }
45 | return {screenCoords, worldCoords, keyframeIndex: bestResult.keyframeIndex, debugExtra};
46 | }
47 | }
48 |
49 | export {
50 | Matcher
51 | }
52 |
--------------------------------------------------------------------------------
/src/image-target/offline-compiler.js:
--------------------------------------------------------------------------------
1 | import {CompilerBase} from './compiler-base.js'
2 | import { buildTrackingImageList } from './image-list.js';
3 | import { extractTrackingFeatures } from './tracker/extract-utils.js';
4 | import { createCanvas } from 'canvas'
5 | import './detector/kernels/cpu/index.js'
6 |
7 | export class OfflineCompiler extends CompilerBase {
8 | createProcessCanvas(img) {
9 | const processCanvas = createCanvas(img.width, img.height);
10 | return processCanvas;
11 | }
12 |
13 | compileTrack({progressCallback, targetImages, basePercent}) {
14 | return new Promise((resolve, reject) => {
15 | const percentPerImage = (100-basePercent) / targetImages.length;
16 | let percent = 0;
17 | const list = [];
18 | for (let i = 0; i < targetImages.length; i++) {
19 | const targetImage = targetImages[i];
20 | const imageList = buildTrackingImageList(targetImage);
21 | const percentPerAction = percentPerImage / imageList.length;
22 |
23 | //console.log("compiling tracking...", i);
24 | const trackingData = extractTrackingFeatures(imageList, (index) => {
25 | //console.log("done tracking", i, index);
26 | percent += percentPerAction;
27 | progressCallback(basePercent+percent);
28 | });
29 | list.push(trackingData);
30 | }
31 | resolve(list);
32 | });
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/image-target/tracker/extract-utils.js:
--------------------------------------------------------------------------------
1 | import { extract } from './extract.js';
2 |
3 | export const extractTrackingFeatures = (imageList, doneCallback) => {
4 | const featureSets = [];
5 | for (let i = 0; i < imageList.length; i++) {
6 | const image = imageList[i];
7 | const points = extract(image);
8 |
9 | const featureSet = {
10 | data: image.data,
11 | scale: image.scale,
12 | width: image.width,
13 | height: image.height,
14 | points,
15 | };
16 | featureSets.push(featureSet);
17 |
18 | doneCallback(i);
19 | }
20 | return featureSets;
21 | }
22 |
--------------------------------------------------------------------------------
/src/image-target/utils/cumsum.js:
--------------------------------------------------------------------------------
1 | // fast 2D submatrix sum using cumulative sum algorithm
2 | class Cumsum {
3 | constructor(data, width, height) {
4 | this.cumsum = [];
5 | for (let j = 0; j < height; j++) {
6 | this.cumsum.push([]);
7 | for (let i = 0; i < width; i++) {
8 | this.cumsum[j].push(0);
9 | }
10 | }
11 |
12 | this.cumsum[0][0] = data[0];
13 | for (let i = 1; i < width; i++) {
14 | this.cumsum[0][i] = this.cumsum[0][i-1] + data[i];
15 | }
16 | for (let j = 1; j < height; j++) {
17 | this.cumsum[j][0] = this.cumsum[j-1][0] + data[j*width];
18 | }
19 |
20 | for (let j = 1; j < height; j++) {
21 | for (let i = 1; i < width; i++) {
22 | this.cumsum[j][i] = data[j*width+i]
23 | + this.cumsum[j-1][i]
24 | + this.cumsum[j][i-1]
25 | - this.cumsum[j-1][i-1];
26 | }
27 | }
28 | }
29 |
30 | query(x1, y1, x2, y2) {
31 | let ret = this.cumsum[y2][x2];
32 | if (y1 > 0) ret -= this.cumsum[y1-1][x2];
33 | if (x1 > 0) ret -= this.cumsum[y2][x1-1];
34 | if (x1 > 0 && y1 > 0) ret += this.cumsum[y1-1][x1-1];
35 | return ret;
36 | }
37 | }
38 |
39 | export {
40 | Cumsum
41 | }
42 |
--------------------------------------------------------------------------------
/src/image-target/utils/geometry.js:
--------------------------------------------------------------------------------
1 | // check which side point C on the line from A to B
2 | const linePointSide = (A, B, C) => {
3 | return ((B[0]-A[0])*(C[1]-A[1])-(B[1]-A[1])*(C[0]-A[0]));
4 | }
5 |
6 | // srcPoints, dstPoints: array of four elements [x, y]
7 | const checkFourPointsConsistent = (x1, x2, x3, x4, x1p, x2p, x3p, x4p) => {
8 | if ((linePointSide(x1, x2, x3) > 0) !== (linePointSide(x1p, x2p, x3p) > 0)) return false;
9 | if ((linePointSide(x2, x3, x4) > 0) !== (linePointSide(x2p, x3p, x4p) > 0)) return false;
10 | if ((linePointSide(x3, x4, x1) > 0) !== (linePointSide(x3p, x4p, x1p) > 0)) return false;
11 | if ((linePointSide(x4, x1, x2) > 0) !== (linePointSide(x4p, x1p, x2p) > 0)) return false;
12 | return true;
13 | }
14 |
15 | const checkThreePointsConsistent = (x1, x2, x3, x1p, x2p, x3p) => {
16 | if ((linePointSide(x1, x2, x3) > 0) !== (linePointSide(x1p, x2p, x3p) > 0)) return false;
17 | return true;
18 | }
19 |
20 | const determinant = (A) => {
21 | const C1 = A[4] * A[8] - A[5] * A[7];
22 | const C2 = A[3] * A[8] - A[5] * A[6];
23 | const C3 = A[3] * A[7] - A[4] * A[6];
24 | return A[0] * C1 - A[1] * C2 + A[2] * C3;
25 | }
26 |
27 | const matrixInverse33 = (A, threshold) => {
28 | const det = determinant(A);
29 | if (Math.abs(det) <= threshold) return null;
30 | const oneOver = 1.0 / det;
31 |
32 | const B = [
33 | (A[4] * A[8] - A[5] * A[7]) * oneOver,
34 | (A[2] * A[7] - A[1] * A[8]) * oneOver,
35 | (A[1] * A[5] - A[2] * A[4]) * oneOver,
36 | (A[5] * A[6] - A[3] * A[8]) * oneOver,
37 | (A[0] * A[8] - A[2] * A[6]) * oneOver,
38 | (A[2] * A[3] - A[0] * A[5]) * oneOver,
39 | (A[3] * A[7] - A[4] * A[6]) * oneOver,
40 | (A[1] * A[6] - A[0] * A[7]) * oneOver,
41 | (A[0] * A[4] - A[1] * A[3]) * oneOver,
42 | ];
43 | return B;
44 | }
45 |
46 | const matrixMul33 = (A, B) => {
47 | const C = [];
48 | C[0] = A[0]*B[0] + A[1]*B[3] + A[2]*B[6];
49 | C[1] = A[0]*B[1] + A[1]*B[4] + A[2]*B[7];
50 | C[2] = A[0]*B[2] + A[1]*B[5] + A[2]*B[8];
51 | C[3] = A[3]*B[0] + A[4]*B[3] + A[5]*B[6];
52 | C[4] = A[3]*B[1] + A[4]*B[4] + A[5]*B[7];
53 | C[5] = A[3]*B[2] + A[4]*B[5] + A[5]*B[8];
54 | C[6] = A[6]*B[0] + A[7]*B[3] + A[8]*B[6];
55 | C[7] = A[6]*B[1] + A[7]*B[4] + A[8]*B[7];
56 | C[8] = A[6]*B[2] + A[7]*B[5] + A[8]*B[8];
57 | return C;
58 | }
59 |
60 | const multiplyPointHomographyInhomogenous = (x, H) => {
61 | const w = H[6]*x[0] + H[7]*x[1] + H[8];
62 | const xp = [];
63 | xp[0] = (H[0]*x[0] + H[1]*x[1] + H[2])/w;
64 | xp[1] = (H[3]*x[0] + H[4]*x[1] + H[5])/w;
65 | return xp;
66 | }
67 |
68 | const smallestTriangleArea = (x1, x2, x3, x4) => {
69 | const v12 = _vector(x2, x1);
70 | const v13 = _vector(x3, x1);
71 | const v14 = _vector(x4, x1);
72 | const v32 = _vector(x2, x3);
73 | const v34 = _vector(x4, x3);
74 | const a1 = _areaOfTriangle(v12, v13);
75 | const a2 = _areaOfTriangle(v13, v14);
76 | const a3 = _areaOfTriangle(v12, v14);
77 | const a4 = _areaOfTriangle(v32, v34);
78 | return Math.min(Math.min(Math.min(a1, a2), a3), a4);
79 | }
80 |
81 | // check if four points form a convex quadrilaternal.
82 | // all four combinations should have same sign
83 | const quadrilateralConvex = (x1, x2, x3, x4) => {
84 | const first = linePointSide(x1, x2, x3) <= 0;
85 | if ( (linePointSide(x2, x3, x4) <= 0) !== first) return false;
86 | if ( (linePointSide(x3, x4, x1) <= 0) !== first) return false;
87 | if ( (linePointSide(x4, x1, x2) <= 0) !== first) return false;
88 |
89 | //if (linePointSide(x1, x2, x3) <= 0) return false;
90 | //if (linePointSide(x2, x3, x4) <= 0) return false;
91 | //if (linePointSide(x3, x4, x1) <= 0) return false;
92 | //if (linePointSide(x4, x1, x2) <= 0) return false;
93 | return true;
94 | }
95 |
96 | const _vector = (a, b) => {
97 | return [
98 | a[0] - b[0],
99 | a[1] - b[1]
100 | ]
101 | }
102 |
103 | const _areaOfTriangle = (u, v) => {
104 | const a = u[0]*v[1] - u[1]*v[0];
105 | return Math.abs(a) * 0.5;
106 | }
107 |
108 | export {
109 | matrixInverse33,
110 | matrixMul33,
111 | quadrilateralConvex,
112 | smallestTriangleArea,
113 | multiplyPointHomographyInhomogenous,
114 | checkThreePointsConsistent,
115 | checkFourPointsConsistent,
116 | determinant
117 | }
118 |
119 |
--------------------------------------------------------------------------------
/src/image-target/utils/homography.js:
--------------------------------------------------------------------------------
1 | import {Matrix, inverse} from 'ml-matrix';
2 |
3 | const solveHomography = (srcPoints, dstPoints) => {
4 | const {normPoints: normSrcPoints, param: srcParam} = _normalizePoints(srcPoints);
5 | const {normPoints: normDstPoints, param: dstParam} = _normalizePoints(dstPoints);
6 |
7 | const num = normDstPoints.length;
8 | const AData = [];
9 | const BData = [];
10 | for (let j = 0; j < num; j++) {
11 | const row1 = [
12 | normSrcPoints[j][0],
13 | normSrcPoints[j][1],
14 | 1,
15 | 0,
16 | 0,
17 | 0,
18 | -(normSrcPoints[j][0] * normDstPoints[j][0]),
19 | -(normSrcPoints[j][1] * normDstPoints[j][0]),
20 | ];
21 | const row2 = [
22 | 0,
23 | 0,
24 | 0,
25 | normSrcPoints[j][0],
26 | normSrcPoints[j][1],
27 | 1,
28 | -(normSrcPoints[j][0] * normDstPoints[j][1]),
29 | -(normSrcPoints[j][1] * normDstPoints[j][1]),
30 | ];
31 | AData.push(row1);
32 | AData.push(row2);
33 |
34 | BData.push([normDstPoints[j][0]]);
35 | BData.push([normDstPoints[j][1]]);
36 | }
37 |
38 | try {
39 | const A = new Matrix(AData);
40 | const B = new Matrix(BData);
41 | const AT = A.transpose();
42 | const ATA = AT.mmul(A);
43 | const ATB = AT.mmul(B);
44 | const ATAInv = inverse(ATA);
45 | const C = ATAInv.mmul(ATB).to1DArray();
46 | const H = _denormalizeHomography(C, srcParam, dstParam);
47 | return H;
48 | } catch (e) {
49 | return null;
50 | }
51 | }
52 |
53 | // centroid at origin and avg distance from origin is sqrt(2)
54 | const _normalizePoints = (coords) => {
55 | //return {normalizedCoords: coords, param: {meanX: 0, meanY: 0, s: 1}}; // skip normalization
56 |
57 | let sumX = 0;
58 | let sumY = 0;
59 | for (let i = 0; i < coords.length; i++) {
60 | sumX += coords[i][0];
61 | sumY += coords[i][1];
62 | }
63 | let meanX = sumX / coords.length;
64 | let meanY = sumY / coords.length;
65 |
66 | let sumDiff = 0;
67 | for (let i = 0; i < coords.length; i++) {
68 | const diffX = coords[i][0] - meanX;
69 | const diffY = coords[i][1] - meanY;
70 | sumDiff += Math.sqrt(diffX * diffX + diffY * diffY);
71 | }
72 | let s = Math.sqrt(2) * coords.length / sumDiff;
73 |
74 | const normPoints = [];
75 | for (let i = 0; i < coords.length; i++) {
76 | normPoints.push([
77 | (coords[i][0] - meanX) * s,
78 | (coords[i][1] - meanY) * s,
79 | ]);
80 | }
81 | return {normPoints, param: {meanX, meanY, s}};
82 | }
83 |
84 | // Denormalize homography
85 | // where T is the normalization matrix, i.e.
86 | //
87 | // [1 0 -meanX]
88 | // T = [0 1 -meanY]
89 | // [0 0 1/s]
90 | //
91 | // [1 0 s*meanX]
92 | // inv(T) = [0 1 s*meanY]
93 | // [0 0 s]
94 | //
95 | // H = inv(Tdst) * Hn * Tsrc
96 | //
97 | // @param {
98 | // nH: normH,
99 | // srcParam: param of src transform,
100 | // dstParam: param of dst transform
101 | // }
102 | const _denormalizeHomography = (nH, srcParam, dstParam) => {
103 | /*
104 | Matrix version
105 | const normH = new Matrix([
106 | [nH[0], nH[1], nH[2]],
107 | [nH[3], nH[4], nH[5]],
108 | [nH[6], nH[7], 1],
109 | ]);
110 | const Tsrc = new Matrix([
111 | [1, 0, -srcParam.meanX],
112 | [0, 1, -srcParam.meanY],
113 | [0, 0, 1/srcParam.s],
114 | ]);
115 |
116 | const invTdst = new Matrix([
117 | [1, 0, dstParam.s * dstParam.meanX],
118 | [0, 1, dstParam.s * dstParam.meanY],
119 | [0, 0, dstParam.s],
120 | ]);
121 | const H = invTdst.mmul(normH).mmul(Tsrc);
122 | */
123 |
124 | // plain implementation of the above using Matrix
125 | const sMeanX = dstParam.s * dstParam.meanX;
126 | const sMeanY = dstParam.s * dstParam.meanY;
127 |
128 | const H = [
129 | nH[0] + sMeanX * nH[6],
130 | nH[1] + sMeanX * nH[7],
131 | (nH[0] + sMeanX * nH[6]) * -srcParam.meanX + (nH[1] + sMeanX * nH[7]) * -srcParam.meanY + (nH[2] + sMeanX) / srcParam.s,
132 | nH[3] + sMeanY * nH[6],
133 | nH[4] + sMeanY * nH[7],
134 | (nH[3] + sMeanY * nH[6]) * -srcParam.meanX + (nH[4] + sMeanY * nH[7]) * -srcParam.meanY + (nH[5] + sMeanY) / srcParam.s,
135 | dstParam.s * nH[6],
136 | dstParam.s * nH[7],
137 | dstParam.s * nH[6] * -srcParam.meanX + dstParam.s * nH[7] * -srcParam.meanY + dstParam.s / srcParam.s,
138 | ];
139 |
140 | // make H[8] === 1;
141 | for (let i = 0; i < 9; i++) {
142 | H[i] = H[i] / H[8];
143 | }
144 | return H;
145 | }
146 |
147 | export {
148 | solveHomography
149 | }
150 |
--------------------------------------------------------------------------------
/src/image-target/utils/images.js:
--------------------------------------------------------------------------------
1 | // simpler version of upsampling. better performance
2 | const _upsampleBilinear = ({image, padOneWidth, padOneHeight}) => {
3 | const {width, height, data} = image;
4 | const dstWidth = image.width * 2 + (padOneWidth?1:0);
5 | const dstHeight = image.height * 2 + (padOneHeight?1:0);
6 | const temp = new Float32Array(dstWidth * dstHeight);
7 |
8 | for (let i = 0; i < width; i++) {
9 | for (let j = 0; j < height; j++) {
10 | const v = 0.25 * data[j * width + i];
11 | const ii = Math.floor(i/2);
12 | const jj = Math.floor(j/2);
13 | const pos = Math.floor(j/2) * dstWidth + Math.floor(i/2);
14 | temp[pos] += v;
15 | temp[pos+1] += v;
16 | temp[pos+dstWidth] += v;
17 | temp[pos+dstWidth+1] += v;
18 | }
19 | }
20 | return {data: temp, width: dstWidth, height: dstHeight};
21 | }
22 |
23 | // artoolkit version. slower. is it necessary?
24 | const upsampleBilinear = ({image, padOneWidth, padOneHeight}) => {
25 | const {width, height, data} = image;
26 |
27 | const dstWidth = image.width * 2 + (padOneWidth?1:0);
28 | const dstHeight = image.height * 2 + (padOneHeight?1:0);
29 |
30 | const temp = new Float32Array(dstWidth * dstHeight);
31 | for (let i = 0; i < dstWidth; i++) {
32 | const si = 0.5 * i - 0.25;
33 | let si0 = Math.floor(si);
34 | let si1 = Math.ceil(si);
35 | if (si0 < 0) si0 = 0; // border
36 | if (si1 >= width) si1 = width - 1; // border
37 |
38 | for (let j = 0; j < dstHeight; j++) {
39 | const sj = 0.5 * j - 0.25;
40 | let sj0 = Math.floor(sj);
41 | let sj1 = Math.ceil(sj);
42 | if (sj0 < 0) sj0 = 0; // border
43 | if (sj1 >= height) sj1 = height - 1; //border
44 |
45 | const value = (si1 - si) * (sj1 - sj) * data[ sj0 * width + si0 ] +
46 | (si1 - si) * (sj - sj0) * data[ sj1 * width + si0 ] +
47 | (si - si0) * (sj1 - sj) * data[ sj0 * width + si1 ] +
48 | (si - si0) * (sj - sj0) * data[ sj1 * width + si1 ];
49 |
50 | temp[j * dstWidth + i] = value;
51 | }
52 | }
53 |
54 | return {data: temp, width: dstWidth, height: dstHeight};
55 | }
56 |
57 | const downsampleBilinear = ({image}) => {
58 | const {data, width, height} = image;
59 |
60 | const dstWidth = Math.floor(width / 2);
61 | const dstHeight = Math.floor(height / 2);
62 |
63 | const temp = new Float32Array(dstWidth * dstHeight);
64 | const offsets = [0, 1, width, width+1];
65 |
66 | for (let j = 0; j < dstHeight; j++) {
67 | for (let i = 0; i < dstWidth; i++) {
68 | let srcPos = j*2 * width + i*2;
69 | let value = 0.0;
70 | for (let d = 0; d < offsets.length; d++) {
71 | value += data[srcPos+ offsets[d]];
72 | }
73 | value *= 0.25;
74 | temp[j*dstWidth+i] = value;
75 | }
76 | }
77 | return {data: temp, width: dstWidth, height: dstHeight};
78 | }
79 |
80 | const resize = ({image, ratio}) => {
81 | const width = Math.round(image.width * ratio);
82 | const height = Math.round(image.height * ratio);
83 |
84 | //const imageData = new Float32Array(width * height);
85 | const imageData = new Uint8Array(width * height);
86 | for (let i = 0; i < width; i++) {
87 | let si1 = Math.round(1.0 * i / ratio);
88 | let si2 = Math.round(1.0 * (i+1) / ratio) - 1;
89 | if (si2 >= image.width) si2 = image.width - 1;
90 |
91 | for (let j = 0; j < height; j++) {
92 | let sj1 = Math.round(1.0 * j / ratio);
93 | let sj2 = Math.round(1.0 * (j+1) / ratio) - 1;
94 | if (sj2 >= image.height) sj2 = image.height - 1;
95 |
96 | let sum = 0;
97 | let count = 0;
98 | for (let ii = si1; ii <= si2; ii++) {
99 | for (let jj = sj1; jj <= sj2; jj++) {
100 | sum += (1.0 * image.data[jj * image.width + ii]);
101 | count += 1;
102 | }
103 | }
104 | imageData[j * width + i] = Math.floor(sum / count);
105 | }
106 | }
107 | return {data: imageData, width: width, height: height};
108 | }
109 |
110 | export {
111 | downsampleBilinear,
112 | upsampleBilinear,
113 | resize,
114 | }
115 |
116 |
--------------------------------------------------------------------------------
/src/image-target/utils/randomizer.js:
--------------------------------------------------------------------------------
1 | const mRandSeed = 1234;
2 |
3 | const createRandomizer = () => {
4 | const randomizer = {
5 | seed: mRandSeed,
6 |
7 | arrayShuffle(options) {
8 | const {arr, sampleSize} = options;
9 | for (let i = 0; i < sampleSize; i++) {
10 |
11 | this.seed = (214013 * this.seed + 2531011) % (1 << 31);
12 | let k = (this.seed >> 16) & 0x7fff;
13 | k = k % arr.length;
14 |
15 | let tmp = arr[i];
16 | arr[i] = arr[k];
17 | arr[k] = tmp;
18 | }
19 | },
20 |
21 | nextInt(maxValue) {
22 | this.seed = (214013 * this.seed + 2531011) % (1 << 31);
23 | let k = (this.seed >> 16) & 0x7fff;
24 | k = k % maxValue;
25 | return k;
26 | }
27 | }
28 | return randomizer;
29 | }
30 | export {
31 | createRandomizer
32 | }
33 |
--------------------------------------------------------------------------------
/src/libs/one-euro-filter.js:
--------------------------------------------------------------------------------
1 | // Ref: https://jaantollander.com/post/noise-filtering-using-one-euro-filter/#mjx-eqn%3A1
2 |
3 | const smoothingFactor = (te, cutoff) => {
4 | const r = 2 * Math.PI * cutoff * te;
5 | return r / (r+1);
6 | }
7 |
8 | const exponentialSmoothing = (a, x, xPrev) => {
9 | return a * x + (1 - a) * xPrev;
10 | }
11 |
12 | class OneEuroFilter {
13 | constructor({minCutOff, beta}) {
14 | this.minCutOff = minCutOff;
15 | this.beta = beta;
16 | this.dCutOff = 0.001; // period in milliseconds, so default to 0.001 = 1Hz
17 |
18 | this.xPrev = null;
19 | this.dxPrev = null;
20 | this.tPrev = null;
21 | this.initialized = false;
22 | }
23 |
24 | reset() {
25 | this.initialized = false;
26 | }
27 |
28 | filter(t, x) {
29 | if (!this.initialized) {
30 | this.initialized = true;
31 | this.xPrev = x;
32 | this.dxPrev = x.map(() => 0);
33 | this.tPrev = t;
34 | return x;
35 | }
36 |
37 | const {xPrev, tPrev, dxPrev} = this;
38 |
39 | //console.log("filter", x, xPrev, x.map((xx, i) => x[i] - xPrev[i]));
40 |
41 | const te = t - tPrev;
42 |
43 | const ad = smoothingFactor(te, this.dCutOff);
44 |
45 | const dx = [];
46 | const dxHat = [];
47 | const xHat = [];
48 | for (let i = 0; i < x.length; i++) {
49 | // The filtered derivative of the signal.
50 | dx[i] = (x[i] - xPrev[i]) / te;
51 | dxHat[i] = exponentialSmoothing(ad, dx[i], dxPrev[i]);
52 |
53 | // The filtered signal
54 | const cutOff = this.minCutOff + this.beta * Math.abs(dxHat[i]);
55 | const a = smoothingFactor(te, cutOff);
56 | xHat[i] = exponentialSmoothing(a, x[i], xPrev[i]);
57 | }
58 |
59 | // update prev
60 | this.xPrev = xHat;
61 | this.dxPrev = dxHat;
62 | this.tPrev = t;
63 |
64 | return xHat;
65 | }
66 | }
67 |
68 | export {
69 | OneEuroFilter
70 | }
71 |
--------------------------------------------------------------------------------
/src/libs/opencv-helper.js:
--------------------------------------------------------------------------------
1 | import {cv} from './opencv.js'
2 | let initialized = false;
3 |
4 | const _cv = {};
5 |
6 | const waitResolves = [];
7 |
8 | export const waitCV = async() => {
9 | if (initialized) return true;
10 | return new Promise((resolve, reject) => {
11 | waitResolves.push(resolve);
12 | });
13 | }
14 |
15 | cv().then((target) => {
16 | initialized = true;
17 | Object.assign(_cv, target);
18 | waitResolves.forEach((resolve) => {
19 | resolve();
20 | });
21 | });
22 |
23 | export const opencv=_cv;
24 | //module.exports={waitCV,opencv}
25 |
--------------------------------------------------------------------------------
/src/ui/compatibility.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
Failed to launch :(
4 |
5 | Looks like your device/browser is not compatible.
6 |
7 |
8 |
9 |
10 |
11 | Please try the following recommended browsers:
12 |
13 |
14 | For Android device - Chrome
15 |
16 |
17 | For iOS device - Safari
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/src/ui/loading.html:
--------------------------------------------------------------------------------
1 |
4 |
--------------------------------------------------------------------------------
/src/ui/scanning.html:
--------------------------------------------------------------------------------
1 |
8 |
--------------------------------------------------------------------------------
/src/ui/ui.js:
--------------------------------------------------------------------------------
1 | //import "./ui.scss";
2 | import loadingHTML from './loading.html?raw';
3 | import compatibilityHTML from './compatibility.html?raw';
4 | import scanningHTML from './scanning.html?raw';
5 |
6 | const css=`.mindar-ui-overlay{display:flex;align-items:center;justify-content:center;position:absolute;left:0;right:0;top:0;bottom:0;background:transparent;z-index:2}.mindar-ui-overlay.hidden{display:none}.mindar-ui-loading .loader{border:16px solid #222;border-top:16px solid white;opacity:.8;border-radius:50%;width:120px;height:120px;animation:spin 2s linear infinite}@keyframes spin{0%{transform:rotate(0)}to{transform:rotate(360deg)}}.mindar-ui-compatibility .content{background:black;color:#fff;opacity:.8;text-align:center;margin:20px;padding:20px;min-height:50vh}@media (min-aspect-ratio: 1/1){.mindar-ui-scanning .scanning{width:50vh;height:50vh}}@media (max-aspect-ratio: 1/1){.mindar-ui-scanning .scanning{width:80vw;height:80vw}}.mindar-ui-scanning .scanning .inner{position:relative;width:100%;height:100%;opacity:.8;background:linear-gradient(to right,white 10px,transparent 10px) 0 0,linear-gradient(to right,white 10px,transparent 10px) 0 100%,linear-gradient(to left,white 10px,transparent 10px) 100% 0,linear-gradient(to left,white 10px,transparent 10px) 100% 100%,linear-gradient(to bottom,white 10px,transparent 10px) 0 0,linear-gradient(to bottom,white 10px,transparent 10px) 100% 0,linear-gradient(to top,white 10px,transparent 10px) 0 100%,linear-gradient(to top,white 10px,transparent 10px) 100% 100%;background-repeat:no-repeat;background-size:40px 40px}.mindar-ui-scanning .scanning .inner .scanline{position:absolute;width:100%;height:10px;background:white;animation:move 2s linear infinite}@keyframes move{0%,to{top:0%}50%{top:calc(100% - 10px)}}`;
7 |
8 | export class UI {
9 | constructor({uiLoading, uiScanning, uiError}) {
10 | const cssBlock=document.createElement('style');
11 | cssBlock.innerText=css;
12 | document.head.appendChild(cssBlock);
13 | if (uiLoading === 'yes') {
14 | this.loadingModal = this._loadHTML(loadingHTML);
15 | } else if (uiLoading !== 'no') {
16 | this.loadingModal = document.querySelector(uiLoading);
17 | }
18 |
19 | if (uiError === 'yes') {
20 | this.compatibilityModal = this._loadHTML(compatibilityHTML);
21 | } else if (uiError !== 'no') {
22 | this.compatibilityModal = document.querySelector(uiError);
23 | }
24 |
25 | if (uiScanning === 'yes') {
26 | this.scanningMask = this._loadHTML(scanningHTML);
27 | } else if (uiScanning !== 'no') {
28 | this.scanningMask = document.querySelector(uiScanning);
29 | }
30 |
31 | this.hideLoading();
32 | this.hideCompatibility();
33 | this.hideScanning();
34 | }
35 |
36 | showLoading() {
37 | if (!this.loadingModal) return;
38 | this.loadingModal.classList.remove("hidden");
39 | }
40 | hideLoading() {
41 | if (!this.loadingModal) return;
42 | this.loadingModal.classList.add("hidden");
43 | }
44 | showCompatibility() {
45 | if (!this.compatibilityModal) return;
46 | this.compatibilityModal.classList.remove("hidden");
47 | }
48 | hideCompatibility() {
49 | if (!this.compatibilityModal) return;
50 | this.compatibilityModal.classList.add("hidden");
51 | }
52 | showScanning() {
53 | if (!this.scanningMask) return;
54 | this.scanningMask.classList.remove("hidden");
55 | }
56 | hideScanning() {
57 | if (!this.scanningMask) return;
58 | this.scanningMask.classList.add("hidden");
59 | }
60 |
61 | _loadHTML(html) {
62 | const e = document.createElement('template');
63 | e.innerHTML = html.trim();
64 | const rootNode = e.content.firstChild;
65 | document.getElementsByTagName('body')[0].appendChild(rootNode);
66 | return rootNode;
67 | }
68 | }
69 |
70 |
71 |
72 |
--------------------------------------------------------------------------------
/src/ui/ui.scss:
--------------------------------------------------------------------------------
1 | $scan-mask-border: 10px;
2 | $primary-color: white;
3 |
4 | .mindar-ui-overlay {
5 | display: flex;
6 | align-items: center;
7 | justify-content: center;
8 | position: absolute;
9 | left: 0;
10 | right: 0;
11 | top: 0;
12 | bottom: 0;
13 | background: transparent;
14 | z-index: 2;
15 |
16 | &.hidden {
17 | display: none;
18 | }
19 | }
20 |
21 | .mindar-ui-loading {
22 | .loader {
23 | border: 16px solid #222; // background color
24 | border-top: 16px solid $primary-color; // foreground color
25 | opacity: 0.8;
26 | border-radius: 50%;
27 | width: 120px;
28 | height: 120px;
29 | animation: spin 2s linear infinite;
30 | }
31 |
32 | @keyframes spin {
33 | 0% { transform: rotate(0deg); }
34 | 100% { transform: rotate(360deg); }
35 | }
36 | }
37 |
38 | .mindar-ui-compatibility {
39 | .content {
40 | background: black;
41 | color: white;
42 | opacity: 0.8;
43 | text-align: center;
44 |
45 | margin: 20px;
46 | padding: 20px;
47 | min-height: 50vh;
48 | }
49 | }
50 |
51 | .mindar-ui-scanning {
52 | .scanning {
53 | @media (min-aspect-ratio: "1/1") {
54 | width: 50vh;
55 | height: 50vh;
56 | }
57 | @media (max-aspect-ratio: "1/1") {
58 | width: 80vw;
59 | height: 80vw;
60 | }
61 | .inner {
62 | position: relative;
63 | width: 100%;
64 | height: 100%;
65 | opacity: 0.8;
66 |
67 | background:
68 | linear-gradient(to right, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 0,
69 | linear-gradient(to right, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 100%,
70 | linear-gradient(to left, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 0,
71 | linear-gradient(to left, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 100%,
72 | linear-gradient(to bottom, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 0,
73 | linear-gradient(to bottom, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 0,
74 | linear-gradient(to top, $primary-color $scan-mask-border, transparent $scan-mask-border) 0 100%,
75 | linear-gradient(to top, $primary-color $scan-mask-border, transparent $scan-mask-border) 100% 100%;
76 |
77 | background-repeat: no-repeat;
78 | background-size: 40px 40px;
79 |
80 | .scanline {
81 | position: absolute;
82 | width: 100%;
83 | height: 10px;
84 | background: $primary-color;
85 | animation: move 2s linear infinite;
86 |
87 | @keyframes move {
88 | 0%, 100% { top: 0% }
89 | 50% { top: calc(100% - 10px) }
90 | }
91 | }
92 | }
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/testing/common.js:
--------------------------------------------------------------------------------
1 | const utils = {
2 | loadImage: async (src) => {
3 | const img = new Image();
4 | return new Promise((resolve, reject) => {
5 | let img = new Image()
6 | img.onload = () => resolve(img);
7 | img.onerror = reject;
8 | img.src = src;
9 | })
10 | },
11 |
12 | drawPoint: (ctx, color, centerX, centerY, radius=1) => {
13 | ctx.beginPath();
14 | ctx.arc(centerX, centerY, radius, 0, 2 * Math.PI, false);
15 | ctx.fillStyle = color;
16 | //ctx.lineWidth = 1;
17 | ctx.strokeStyle = color;
18 | ctx.stroke();
19 | },
20 |
21 | drawRect: (ctx, color, centerX, centerY, width) => {
22 | ctx.beginPath();
23 | ctx.strokeStyle = color;
24 | ctx.rect(centerX - width/2, centerY - width /2 , width, width);
25 | ctx.stroke();
26 | },
27 |
28 | drawLine: (ctx, color, fromX, fromY, toX, toY) => {
29 | ctx.beginPath();
30 | ctx.lineWidth = 2;
31 | ctx.fillStyle = color;
32 | ctx.strokeStyle = color;
33 | ctx.moveTo(fromX, fromY);
34 | ctx.lineTo(toX, toY);
35 | ctx.stroke();
36 | },
37 |
38 | pixel2DToImageData: (pixels) => {
39 | const height = pixels.length;
40 | const width = pixels[0].length;
41 | const data = new Uint8ClampedArray(height * width * 4);
42 | for (let j = 0; j < height; j++) {
43 | for (let i = 0; i < width; i++) {
44 | const pos = j * width + i;
45 | data[pos*4 + 0] = pixels[j][i];
46 | data[pos*4 + 1] = pixels[j][i];
47 | data[pos*4 + 2] = pixels[j][i];
48 | data[pos*4 + 3] = 255;
49 | }
50 | }
51 | const imageData = new ImageData(data, width, height);
52 | return imageData;
53 | },
54 |
55 | pixel3DToImageData: (pixels) => {
56 | const height = pixels.length;
57 | const width = pixels[0].length;
58 | const data = new Uint8ClampedArray(height * width * 4);
59 | for (let j = 0; j < height; j++) {
60 | for (let i = 0; i < width; i++) {
61 | const pos = j * width + i;
62 | data[pos*4 + 0] = pixels[j][i][0];
63 | data[pos*4 + 1] = pixels[j][i][1];
64 | data[pos*4 + 2] = pixels[j][i][2];
65 | data[pos*4 + 3] = 255;
66 | }
67 | }
68 | const imageData = new ImageData(data, width, height);
69 | return imageData;
70 | },
71 |
72 | pixel1DToImageData: (pixels, width, height) => {
73 | const data = new Uint8ClampedArray(pixels.length * 4);
74 | for (let j = 0; j < pixels.length; j++) {
75 | data[j*4 + 0] = pixels[j];
76 | data[j*4 + 1] = pixels[j];
77 | data[j*4 + 2] = pixels[j];
78 | data[j*4 + 3] = 255;
79 | }
80 | const imageData = new ImageData(data, width, height);
81 | return imageData;
82 | },
83 |
84 | matrixInverse33: (A, threshold) => {
85 | const det = utils.determinant(A);
86 | if (Math.abs(det) <= threshold) return null;
87 | const oneOver = 1.0 / det;
88 |
89 | const B = [
90 | (A[4] * A[8] - A[5] * A[7]) * oneOver,
91 | (A[2] * A[7] - A[1] * A[8]) * oneOver,
92 | (A[1] * A[5] - A[2] * A[4]) * oneOver,
93 | (A[5] * A[6] - A[3] * A[8]) * oneOver,
94 | (A[0] * A[8] - A[2] * A[6]) * oneOver,
95 | (A[2] * A[3] - A[0] * A[5]) * oneOver,
96 | (A[3] * A[7] - A[4] * A[6]) * oneOver,
97 | (A[1] * A[6] - A[0] * A[7]) * oneOver,
98 | (A[0] * A[4] - A[1] * A[3]) * oneOver,
99 | ];
100 | return B;
101 | },
102 | determinant: (A) => {
103 | const C1 = A[4] * A[8] - A[5] * A[7];
104 | const C2 = A[3] * A[8] - A[5] * A[6];
105 | const C3 = A[3] * A[7] - A[4] * A[6];
106 | return A[0] * C1 - A[1] * C2 + A[2] * C3;
107 | }
108 | }
109 |
--------------------------------------------------------------------------------
/testing/detection.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
19 |
20 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
--------------------------------------------------------------------------------
/testing/glsl/applyFilter.js:
--------------------------------------------------------------------------------
1 | const applyFilter = (image) => {
2 | const imageHeight = image.shape[0];
3 | const imageWidth = image.shape[1];
4 |
5 | const imaxmin = `
6 | int imax(int a, int b) {
7 | if (a > b) return a;
8 | return b;
9 | }
10 | int imin(int a, int b) {
11 | if (a < b) return a;
12 | return b;
13 | }
14 |
15 | `
16 | const kernel1 = {
17 | variableNames: ['p'],
18 | outputShape: [imageHeight, imageWidth],
19 | userCode: `
20 | ${imaxmin}
21 | void main() {
22 | ivec2 coords = getOutputCoords();
23 |
24 | float sum = getP(coords[0], imax(0, coords[1]-2));
25 | sum += getP(coords[0], imax(0, coords[1]-1)) * 4.;
26 | sum += getP(coords[0], coords[1]) * 6.;
27 | sum += getP(coords[0], imin(${imageWidth}-1, coords[1]+1)) * 4.;
28 | sum += getP(coords[0], imin(${imageWidth}-1, coords[1]+2));
29 | setOutput(sum);
30 |
31 | }
32 | `
33 | };
34 |
35 | const kernel2 = {
36 | variableNames: ['p'],
37 | outputShape: [imageHeight, imageWidth],
38 | userCode: `
39 | ${imaxmin}
40 | void main() {
41 | ivec2 coords = getOutputCoords();
42 |
43 | float sum = getP(imax(coords[0]-2, 0), coords[1]);
44 | sum += getP(imax(coords[0]-1, 0), coords[1]) * 4.;
45 | sum += getP(coords[0], coords[1]) * 6.;
46 | sum += getP(imin(coords[0]+1, ${imageHeight}-1), coords[1]) * 4.;
47 | sum += getP(imin(coords[0]+2, ${imageHeight}-1), coords[1]);
48 | sum /= 256.;
49 | setOutput(sum);
50 | }
51 | `
52 | };
53 |
54 | return tf.tidy(() => {
55 | const result1 = runWebGLProgram(kernel1, [image], 'float32');
56 | const result2 = runWebGLProgram(kernel2, [result1], 'float32');
57 | return result2;
58 | });
59 | }
60 |
--------------------------------------------------------------------------------
/testing/glsl/applyFilter2.js:
--------------------------------------------------------------------------------
1 | const applyFilter2 = (image) => {
2 | const imageHeight = image.shape[0];
3 | const imageWidth = image.shape[1];
4 |
5 | const imaxmin = `
6 | int imax(int a, int b) {
7 | if (a > b) return a;
8 | return b;
9 | }
10 | int imin(int a, int b) {
11 | if (a < b) return a;
12 | return b;
13 | }
14 |
15 | `
16 | const kernel1 = {
17 | variableNames: ['p'],
18 | outputShape: [imageHeight, imageWidth],
19 | userCode: `
20 | ${imaxmin}
21 | void main() {
22 | ivec2 coords = getOutputCoords();
23 | float sum = getP(coords[0], coords[1]-1) * 0.25;
24 | sum += getP(coords[0], coords[1]) * 0.5;
25 | sum += getP(coords[0], coords[1]+1) * 0.25;
26 | setOutput(sum);
27 | }
28 | `
29 | };
30 | const kernel2 = {
31 | variableNames: ['p'],
32 | outputShape: [imageHeight, imageWidth],
33 | userCode: `
34 | ${imaxmin}
35 | void main() {
36 | ivec2 coords = getOutputCoords();
37 | float sum = getP(coords[0]-1, coords[1]) * 0.25;
38 | sum += getP(coords[0], coords[1]) * 0.5;
39 | sum += getP(coords[0]+1, coords[1]) * 0.25;
40 | setOutput(sum);
41 | }
42 | `
43 | };
44 |
45 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D';
46 |
47 | const dummyKernel = {
48 | variableNames: ['p'],
49 | outputShape: [imageHeight, imageWidth],
50 | packedOutput: true,
51 | userCode: `
52 | void main() {
53 | ivec2 coords = getOutputCoords();
54 |
55 | vec4 values = ${textureMethod}(p, resultUV);
56 | vec4 v = vec4(getP(coords[0], coords[1]), 0, 0, 0);
57 | setOutput(v);
58 | }
59 | `
60 | }
61 |
62 | return tf.tidy(() => {
63 | //const resultDummy = runWebGLProgram(dummyKernel, [image], 'int32');
64 | //const result = compileAndRun(kernel1, [resultDummy]);
65 | //return result;
66 |
67 | const out1 = tf.backend().runWebGLProgram(kernel1, [image], 'float32');
68 | const out2 = tf.backend().runWebGLProgram(kernel2, [out1], 'float32');
69 | tf.backend().disposeIntermediateTensorInfo(out1);
70 | return tf.engine().makeTensorFromDataId(out2.dataId, out2.shape, out2.dtype);
71 |
72 | const result1 = compileAndRun(kernel1, [image]);
73 | return result1;
74 | const result2 = compileAndRun(kernel2, [result1]);
75 | return result2;
76 | });
77 | }
78 |
--------------------------------------------------------------------------------
/testing/glsl/loadImageToTensor.js:
--------------------------------------------------------------------------------
1 | const loadImageToTensor = (canvas) => {
2 | const width = canvas.width;
3 | const height = canvas.height;
4 | const backend = tf.backend();
5 |
6 | const texShape = [height, width];
7 |
8 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D';
9 | const program = {
10 | variableNames: ['A'],
11 | outputShape: texShape,
12 | userCode:`
13 | void main() {
14 | ivec2 coords = getOutputCoords();
15 | int texR = coords[0];
16 | int texC = coords[1];
17 |
18 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(${width}.0, ${height}.0);
19 | vec4 values = ${textureMethod}(A, uv);
20 | float res = (values.r + values.g + values.b) * 255.0 / 3.0;
21 |
22 | setOutput(res);
23 | }
24 | `
25 | }
26 |
27 | if (!caches.loadImageToTensor) {
28 | const tempPixelHandle = backend.makeTensorInfo(texShape, 'int32');
29 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(tempPixelHandle.dataId), canvas);
30 | backend.texData.get(tempPixelHandle.dataId).usage = 2;
31 |
32 | caches.loadImageToTensor = {
33 | tempPixelHandle
34 | }
35 | }
36 |
37 | const {tempPixelHandle} = caches.loadImageToTensor;
38 |
39 | const inputs = [tempPixelHandle];
40 |
41 | const result = runWebGLProgram(program, inputs, 'int32');
42 | return result;
43 |
44 | //const outInfo = tf.backend().compileAndRun(program, inputs);
45 | //const res = tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
46 | //return res;
47 | }
48 |
--------------------------------------------------------------------------------
/testing/glsl/loadImageToTensor2.js:
--------------------------------------------------------------------------------
1 | const loadImageToTensor2 = (canvas) => {
2 | const width = canvas.width;
3 | const height = canvas.height;
4 | const backend = tf.backend();
5 |
6 | const texShape = [height, width];
7 |
8 | const textureMethod = tf.env().getNumber('WEBGL_VERSION') === 2? 'texture': 'texture2D';
9 | const program = {
10 | variableNames: ['A'],
11 | outputShape: texShape,
12 | userCode:`
13 | void main() {
14 | vec4 values = ${textureMethod}(A, resultUV);
15 | float res = (0.299 * values.r + 0.587 * values.g + 0.114 * values.b) * 255.0;
16 | setOutput(res);
17 | }
18 | `
19 | }
20 |
21 | if (!caches.loadImageToTensor2) {
22 | const tempPixelHandle = backend.makeTensorInfo(texShape, 'int32');
23 | backend.gpgpu.uploadPixelDataToTexture(backend.getTexture(tempPixelHandle.dataId), canvas);
24 | backend.texData.get(tempPixelHandle.dataId).usage = 2;
25 |
26 | caches.loadImageToTensor2 = {
27 | tempPixelHandle
28 | }
29 | }
30 |
31 | const {tempPixelHandle} = caches.loadImageToTensor2;
32 |
33 | const inputs = [tempPixelHandle];
34 |
35 | const outInfo = tf.backend().compileAndRun(program, inputs);
36 | const res = tf.engine().makeTensorFromDataId(outInfo.dataId, outInfo.shape, outInfo.dtype);
37 |
38 | return res;
39 | }
40 |
41 |
--------------------------------------------------------------------------------
/testing/glsl/packKernel.js:
--------------------------------------------------------------------------------
1 | const packKernel = (image) => {
2 | //const imageHeight = image.shape[0];
3 | const imageHeight = 10;
4 | const imageWidth = image.shape[1];
5 |
6 | const kernel = {
7 | variableNames: ['p'],
8 | outputShape: [imageHeight],
9 | packedOutput: true,
10 | userCode: `
11 | void main() {
12 | int coords = getOutputCoords();
13 | //setOutput(float(coords));
14 | setOutput(vec4(1,0,0,0));
15 | }
16 | `
17 | };
18 |
19 | const dummyKernel = {
20 | variableNames: ['a'],
21 | outputShape: [imageHeight],
22 | packedInputs: true,
23 | userCode: `
24 | void main() {
25 | int coords = getOutputCoords();
26 |
27 | int texNumC = 10;
28 | int texNumR = 1;
29 | int texelIndex = coords;
30 | int texR = texelIndex / texNumC;
31 | int texC = texelIndex - texR * texNumC;
32 | vec2 uv = (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
33 |
34 | vec4 values = texture(a, vec2(0.3, 0.1));
35 | //vec4 values = texture(a, uv);
36 | //vec4 values = texture(a, resultUV);
37 | setOutput(values.r);
38 |
39 | //setOutput(getA(coords).r);
40 | //setOutput(getAAtOuttCoords().r);
41 | }
42 |
43 | `
44 | }
45 |
46 | return tf.tidy(() => {
47 | //const resultDummy = runWebGLProgram(dummyKernel, [image], 'int32');
48 | //const result = compileAndRun(kernel1, [resultDummy]);
49 | //return result;
50 |
51 | const out1 = tf.backend().runWebGLProgram(kernel, [image], 'int32');
52 | const out2 = tf.backend().runWebGLProgram(dummyKernel, [out1], 'int32');
53 | const result = tf.engine().makeTensorFromDataId(out1.dataId, out1.shape, out1.dtype);
54 | const result2 = tf.engine().makeTensorFromDataId(out2.dataId, out2.shape, out2.dtype);
55 |
56 | console.log("test pack", result, result.arraySync());
57 | console.log("test pack", result2, result2.arraySync());
58 |
59 | return result;
60 | });
61 | }
62 |
--------------------------------------------------------------------------------
/testing/glsl/randomImage.js:
--------------------------------------------------------------------------------
1 | const randomImage = (height, width) => {
2 | const mRandSeed = 1234;
3 |
4 | const createRandomizer = () => {
5 | const randomizer = {
6 | seed: mRandSeed,
7 |
8 | arrayShuffle(options) {
9 | const {arr, sampleSize} = options;
10 | for (let i = 0; i < sampleSize; i++) {
11 |
12 | this.seed = (214013 * this.seed + 2531011) % (1 << 31);
13 | let k = (this.seed >> 16) & 0x7fff;
14 | k = k % arr.length;
15 |
16 | let tmp = arr[i];
17 | arr[i] = arr[k];
18 | arr[k] = tmp;
19 | }
20 | },
21 |
22 | nextInt(maxValue) {
23 | this.seed = (214013 * this.seed + 2531011) % (1 << 31);
24 | let k = (this.seed >> 16) & 0x7fff;
25 | k = k % maxValue;
26 | return k;
27 | }
28 | }
29 | return randomizer;
30 | }
31 |
32 | const randomizer = createRandomizer();
33 |
34 | const tData = new Uint8ClampedArray(height * width * 4);
35 | for (let j = 0; j < height; j++) {
36 | for (let i = 0; i < width; i++) {
37 | const index = j * width + i;
38 | tData[index*4] = randomizer.nextInt(255);
39 | tData[index*4+1] = randomizer.nextInt(255);
40 | tData[index*4+2] = randomizer.nextInt(255);
41 | tData[index*4+3] = 255;
42 | }
43 | }
44 | const imageData = new ImageData(tData, width, height);
45 | const canvas = document.createElement('canvas');
46 | canvas.width = width;
47 | canvas.height = height;
48 | const ctx = canvas.getContext('2d');
49 | ctx.putImageData(imageData, 0, 0);
50 |
51 | console.log("tData", tData);
52 |
53 | return new Promise((resolve, reject) => {
54 | const img = document.createElement("img");
55 | img.onload = () => {
56 | resolve(img);
57 | }
58 | img.src = canvas.toDataURL("image/png");
59 | });
60 | }
61 |
--------------------------------------------------------------------------------
/testing/homography.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
21 |
22 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/testing/matching-all.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
44 |
45 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/testing/matching.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
29 |
30 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/testing/mobile-console.css:
--------------------------------------------------------------------------------
1 | #js-mobile-console{
2 | width: 100%;
3 | position: fixed;
4 | bottom: 10px;
5 | z-index: 10000;
6 | }
7 |
8 | #jsmc-log{
9 | border-top: 1px solid #808080;
10 | max-height: 500px;
11 | overflow-y: auto;
12 | margin-right: 10px;
13 | font-size: 1em;
14 | }
15 |
16 | #jsmc-input-container{
17 | position: relative;
18 | }
19 |
20 | #jsmc-input{
21 | width: 100%;
22 | }
23 |
24 | #jsmc-button{
25 | position: fixed;
26 | bottom: 31px;
27 | right: 48px;
28 | }
29 |
30 | .jsmc-log-el{
31 | color: red;
32 | }
33 |
34 | .jsmc-log-error .jsmc-log-text{
35 | color: red;
36 | }
37 |
38 | .jsmc-log-command .jsmc-log-text{
39 | color: #009EE4;
40 | }
41 |
42 | .jsmc-log-text{
43 | color: black;
44 | }
45 |
46 | .jsmc-log-target{
47 | color: #7979EF;
48 | }
49 |
50 | .jsmc-log-target{
51 | clear: both;
52 | }
53 |
54 | #jsmc-content{
55 | background: white;
56 | clear: both;
57 | }
58 |
59 | #jsmc-collapse,
60 | #jsmc-commands,
61 | #jsmc-clear{
62 | cursor: pointer;
63 | width: 25px;
64 | background: white;
65 | text-align: center;
66 | color: gray;
67 | border: 1px solid #808080;
68 | display: inline-block;
69 | }
70 |
71 | #jsmc-clear{
72 | font-weight: bolder;
73 | height: 19px;
74 | }
75 |
76 | #jsmc-commands{
77 | display: none;
78 | }
79 |
80 | #jsmc-commands-container{
81 | background: #FFF;
82 | display: none;
83 | }
84 |
85 | .jsmc-command-wrapper{
86 | border-bottom: 1px solid #808080;
87 | padding: 0px 14px;
88 | cursor: pointer;
89 | }
90 |
91 | .jsmc-command:last-child{
92 | border-bottom: none;
93 | }
94 |
--------------------------------------------------------------------------------
/testing/speed.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
21 |
22 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/testing/speed.js:
--------------------------------------------------------------------------------
1 | const { useEffect, useMemo, useRef, useState, useCallback } = React;
2 |
3 | let queryImages = [];
4 | let controller;
5 |
6 | const Main = () => {
7 | const [numRun, setNumRun] = useState(10);
8 | const [ready, setReady] = useState(false);
9 | const [result, setResult] = useState(null);
10 |
11 | useEffect(() => {
12 | const init = async () => {
13 | //for (let i = 11; i <= 15; i+=1) {
14 | // queryImages.push(await utils.loadImage('../tests/video2/out' + i + '.png'));
15 | //}
16 | //queryImages.push(await utils.loadImage('../tests/video2/out01_540.png'));
17 | //queryImages.push(await utils.loadImage('../tests/video2/out01_540.png'));
18 | queryImages.push(await utils.loadImage('../tests/video2/out01.png'));
19 | queryImages.push(await utils.loadImage('../tests/video2/out02.png'));
20 | const queryImage0 = queryImages[0];
21 |
22 | const inputWidth = queryImage0.width;
23 | const inputHeight = queryImage0.height;
24 | controller = new MINDAR.Controller({inputWidth, inputHeight, debugMode: false});
25 | const {dimensions, matchingDataList, trackingDataList, imageListList} = await controller.addImageTargets('../examples/assets/card-example/card.mind');
26 | controller.dummyRun(queryImage0);
27 | setReady(true);
28 | }
29 | init();
30 | }, []);
31 |
32 | const start = useCallback(async () => {
33 | const targetIndex = 0;
34 | console.log("start", numRun);
35 |
36 | let timeSpent = {
37 | detect: [],
38 | match: [],
39 | track: [],
40 | trackUpdate: [],
41 | };
42 | let _start = null;
43 |
44 | for (let i = 0; i < numRun; i++) {
45 | _start = new Date();
46 | const {featurePoints} = await controller.detect(queryImages[0]);
47 | timeSpent['detect'].push(new Date() - _start);
48 |
49 | _start = new Date();
50 | const {modelViewTransform: firstModelViewTransform, allMatchResults, debugExtra} = await controller.match(featurePoints, targetIndex);
51 |
52 | /*
53 | const sumTime = {};
54 | for (let j = 0; j < debugExtras[0].length; j++) {
55 | Object.keys(debugExtras[0][j].time).forEach((k) => {
56 | if (!sumTime[k]) sumTime[k] = 0;
57 | sumTime[k] += debugExtras[0][j].time[k];
58 | });
59 | }
60 | const timeDiv = document.createElement("div");
61 | timeDiv.innerHTML = JSON.stringify(sumTime);
62 | document.body.appendChild(timeDiv);
63 | */
64 |
65 | timeSpent['match'].push(new Date() - _start);
66 |
67 | if (!firstModelViewTransform) {
68 | timeSpent['track'].push(0);
69 | timeSpent['trackUpdate'].push(0);
70 | continue;
71 | }
72 |
73 | _start = new Date();
74 | const nKeyframes = 1;
75 | const trackResult = await controller.track(queryImages[1], firstModelViewTransform, targetIndex);
76 | timeSpent['track'].push(new Date() - _start);
77 | //console.log("trackResultsf", trackResult);
78 |
79 | _start = new Date();
80 | const bestSelectedFeatures = trackResult;
81 | const newModelViewTransform = await controller.trackUpdate(firstModelViewTransform, bestSelectedFeatures);
82 | timeSpent['trackUpdate'].push(new Date() - _start);
83 |
84 | controller.showTFStats();
85 | }
86 |
87 | const avg = {};
88 | Object.keys(timeSpent).forEach((key) => {
89 | const sum = timeSpent[key].reduce((acc, t) => acc + t, 0.0);
90 | avg[key] = sum / timeSpent[key].length;
91 | });
92 |
93 | setResult({timeSpent, avg});
94 | });
95 |
96 | console.log("result", result);
97 |
98 | return (
99 |
100 |
Speed Test
101 | {!ready &&
Initializing...}
102 | {ready && (
103 |
104 | setNumRun(e.target.value)}/>
105 |
106 |
107 | )}
108 |
109 | {result && (
110 |
111 |
112 |
113 |
114 | |
115 | {result.timeSpent['detect'].map((spent, index) => (
116 | {index+1} |
117 | ))}
118 | avg |
119 |
120 |
121 |
122 | {Object.keys(result.timeSpent).map((key, keyIndex) => (
123 |
124 | {key} |
125 | {result.timeSpent[key].map((t, index) => (
126 | {t} |
127 | ))}
128 | {result.avg[key]} |
129 |
130 | ))}
131 |
132 |
133 |
134 | )}
135 |
136 | )
137 | }
138 |
139 | ReactDOM.render(
140 | ,
141 | document.getElementById('root')
142 | );
143 |
--------------------------------------------------------------------------------
/testing/tracking.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/vite.config.dev.js:
--------------------------------------------------------------------------------
1 | import {defineConfig,build} from 'vite'
2 | import * as fs from 'fs/promises';
3 | import * as path from 'path';
4 | import basicSsl from '@vitejs/plugin-basic-ssl'
5 |
6 | const outDir = 'dist-dev'
7 |
8 | const moduleConfig={
9 | mode: 'development',
10 | assetsInclude:'**/*.html',
11 | base:'./',
12 | plugins:[
13 | basicSsl()
14 | ],
15 | build: {
16 | outDir: outDir,
17 | emptyOutDir:false,
18 | sourcemap:'inline' ,
19 | lib: {
20 | fileName:"[name]",
21 | entry:'index.js',
22 | formats:['es']
23 | },
24 | rollupOptions:{
25 | external:(id)=>(id==='three'||id.includes('three/examples/jsm/')||id.includes('three/addons/')),
26 | input:{
27 | 'mindar-image': './src/image-target/index.js',
28 | 'mindar-image-three': './src/image-target/three.js',
29 | 'mindar-face': './src/face-target/index.js',
30 | 'mindar-face-three': './src/face-target/three.js',
31 | }
32 | },
33 | },
34 | resolve:{
35 | alias:{
36 | 'three/addons/':'three/examples/jsm/'
37 | }
38 | }
39 | };
40 | const faceAframeConfig=defineConfig({
41 | mode: 'development',
42 | build: {
43 | outDir: outDir,
44 | emptyOutDir:false,
45 | sourcemap:'inline' ,
46 | minify: false,
47 | lib: {
48 | name:"MINDAR",
49 | fileName:"[name]",
50 | entry:'index.js',
51 | formats:['iife']
52 | },
53 | rollupOptions:{
54 | input:{
55 | 'mindar-face-aframe': './src/face-target/aframe.js',
56 | },
57 |
58 | }
59 | }
60 | })
61 | /** @type {import('vite').UserConfig} */
62 | const imageAframeConfig=defineConfig({
63 | mode: 'development',
64 | build: {
65 | outDir: outDir,
66 | emptyOutDir:false,
67 | sourcemap:'inline' ,
68 | minify: false,
69 | lib: {
70 | name:"MINDAR",
71 | fileName:"[name]",
72 | entry:'index.js',
73 | formats:['iife'],
74 |
75 | },
76 | rollupOptions:{
77 | input:{
78 | 'mindar-image-aframe': './src/image-target/aframe.js'
79 | }
80 | }
81 | }
82 | })
83 |
84 | export default defineConfig(async ({ command, mode }) => {
85 | await fs.rm(outDir,{recursive:true,force:true});
86 | if (command === 'build') {
87 | await build(imageAframeConfig);
88 | await build(faceAframeConfig);
89 | const files=await fs.readdir(outDir);
90 | //rename the aframe builds
91 | await Promise.all(files.map(async (filename)=>{
92 | if(filename.includes(".iife.js")){
93 | const newName=filename.replace(".iife.js",".js");
94 | console.log(filename,"->",newName)
95 | await fs.rename(path.join(outDir,filename),path.join(outDir,newName));
96 | }
97 | }));
98 | }
99 | return moduleConfig
100 | })
101 |
--------------------------------------------------------------------------------
/vite.config.prod.js:
--------------------------------------------------------------------------------
1 | import {defineConfig,build} from 'vite'
2 | import * as fs from 'fs/promises';
3 | import * as path from 'path';
4 |
5 | const outDir = 'dist'
6 |
7 | /** @type {import('vite').UserConfig} */
8 | const moduleConfig= defineConfig({
9 | mode: 'production',
10 | publicDir:false,
11 | base:'./',
12 | build: {
13 | outDir:outDir,
14 | emptyOutDir:false,
15 | copyPublicDir:false,
16 | lib: {
17 | fileName:"[name].prod",
18 | entry:'index.js',
19 | formats:["es"],
20 | },
21 | rollupOptions:{
22 | external:(id)=>(id==='three'||id.includes('three/examples/jsm/')||id.includes('three/addons/')),
23 | input:{
24 | 'mindar-image': './src/image-target/index.js',
25 | 'mindar-image-three': './src/image-target/three.js',
26 | 'mindar-face': './src/face-target/index.js',
27 | 'mindar-face-three': './src/face-target/three.js'
28 | }
29 | }
30 | },
31 | resolve:{
32 | alias:{
33 | 'three/addons/':'three/examples/jsm/'
34 | }
35 | }
36 | });
37 | const faceAframeConfig=defineConfig({
38 | mode: 'production',
39 | build: {
40 | outDir: outDir,
41 | emptyOutDir:false,
42 | lib: {
43 | name:"MINDAR",
44 | fileName:"[name].prod",
45 | entry:'index.js',
46 | formats:['iife']
47 | },
48 | rollupOptions:{
49 | input:{
50 | 'mindar-face-aframe': './src/face-target/aframe.js',
51 | },
52 |
53 | }
54 | }
55 | })
56 | /** @type {import('vite').UserConfig} */
57 | const imageAframeConfig=defineConfig({
58 | mode: 'production',
59 | build: {
60 | outDir: outDir,
61 | emptyOutDir:false,
62 | lib: {
63 | name:"MINDAR",
64 | fileName:"[name].prod",
65 | entry:'index.js',
66 | formats:['iife'],
67 |
68 | },
69 | rollupOptions:{
70 | input:{
71 | 'mindar-image-aframe': './src/image-target/aframe.js'
72 | }
73 | }
74 | }
75 | })
76 |
77 | export default defineConfig(async ({ command, mode }) => {
78 | await fs.rm(outDir,{recursive:true,force:true});
79 | if (command === 'build') {
80 | await build(imageAframeConfig);
81 | await build(faceAframeConfig);
82 | const files=await fs.readdir(outDir);
83 | //rename the aframe builds
84 | await Promise.all(files.map(async (filename)=>{
85 | if(filename.includes(".iife.js")){
86 | const newName=filename.replace(".iife.js",".js");
87 | console.log(filename,"->",newName)
88 | await fs.rename(path.join(outDir,filename),path.join(outDir,newName));
89 | }
90 | }));
91 | }
92 | return moduleConfig
93 | })
94 |
--------------------------------------------------------------------------------