├── .babelrc ├── .editorconfig ├── .gitattributes ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── package-lock.json ├── package.json ├── src ├── core.ts ├── helper │ ├── basis.ts │ ├── canvas.ts │ ├── clock.ts │ ├── debug.ts │ ├── filter.ts │ ├── landmark.ts │ ├── quaternion.ts │ ├── test.ts │ └── utils.ts ├── index.ts ├── mediapipe.ts ├── types.d.ts ├── v3d-web.ts └── worker │ └── pose-processing.ts ├── test ├── css │ ├── control_utils.css │ ├── main.css │ └── normalize.css ├── img │ ├── .gitignore │ └── icon.png └── index.html ├── tracking.md ├── tsconfig.json ├── webpack.config.js └── webpack.test.config.js /.babelrc: -------------------------------------------------------------------------------- 1 | {"presets": ["@babel/preset-env"]} 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_size = 4 8 | indent_style = space 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ## GITATTRIBUTES FOR WEB PROJECTS 2 | # 3 | # These settings are for any web project. 4 | # 5 | # Details per file setting: 6 | # text These files should be normalized (i.e. convert CRLF to LF). 7 | # binary These files are binary and should be left untouched. 8 | # 9 | # Note that binary is a macro for -text -diff. 10 | ###################################################################### 11 | 12 | ## AUTO-DETECT 13 | ## Handle line endings automatically for files detected as 14 | ## text and leave all files detected as binary untouched. 15 | ## This will handle all files NOT defined below. 16 | * text=auto 17 | 18 | ## SOURCE CODE 19 | *.bat text eol=crlf 20 | *.coffee text 21 | *.css text 22 | *.htm text 23 | *.html text 24 | *.inc text 25 | *.ini text 26 | *.js text 27 | *.json text 28 | *.jsx text 29 | *.less text 30 | *.od text 31 | *.onlydata text 32 | *.php text 33 | *.pl text 34 | *.py text 35 | *.rb text 36 | *.sass text 37 | *.scm text 38 | *.scss text 39 | *.sh text eol=lf 40 | *.sql text 41 | *.styl text 42 | *.tag text 43 | *.ts text 44 | *.tsx text 45 | *.xml text 46 | *.xhtml text 47 | 48 | ## DOCKER 49 | *.dockerignore text 50 | Dockerfile text 51 | 52 | ## DOCUMENTATION 53 | *.markdown text 54 | *.md text 55 | *.mdwn text 56 | *.mdown text 57 | *.mkd text 58 | *.mkdn text 59 | *.mdtxt text 60 | *.mdtext text 61 | *.txt text 62 | AUTHORS text 63 | CHANGELOG text 64 | CHANGES text 65 | CONTRIBUTING text 66 | COPYING text 67 | copyright text 68 | *COPYRIGHT* text 69 | INSTALL text 70 | license text 71 | LICENSE text 72 | NEWS text 73 | readme text 74 | *README* text 75 | TODO text 76 | 77 | ## TEMPLATES 78 | *.dot text 79 | *.ejs text 80 | *.haml text 81 | *.handlebars text 82 | *.hbs text 83 | *.hbt text 84 | *.jade text 85 | *.latte text 86 | *.mustache text 87 | *.njk text 88 | *.phtml text 89 | *.tmpl text 90 | *.tpl text 91 | *.twig text 92 | 93 | ## LINTERS 94 | .babelrc text 95 | .csslintrc text 96 | .eslintrc text 97 | .htmlhintrc text 98 | .jscsrc text 99 | .jshintrc text 100 | .jshintignore text 101 | .prettierrc text 102 | .stylelintrc text 103 | 104 | ## CONFIGS 105 | *.bowerrc text 106 | *.cnf text 107 | *.conf text 108 | *.config text 109 | .browserslistrc text 110 | .editorconfig text 111 | .gitattributes text 112 | .gitconfig text 113 | .gitignore text 114 | .htaccess text 115 | *.npmignore text 116 | *.yaml text 117 | *.yml text 118 | browserslist text 119 | Makefile text 120 | makefile text 121 | 122 | ## HEROKU 123 | Procfile text 124 | .slugignore text 125 | 126 | ## GRAPHICS 127 | *.ai binary 128 | *.bmp binary 129 | *.eps binary 130 | *.gif binary 131 | *.ico binary 132 | *.jng binary 133 | *.jp2 binary 134 | *.jpg binary 135 | *.jpeg binary 136 | *.jpx binary 137 | *.jxr binary 138 | *.pdf binary 139 | *.png binary 140 | *.psb binary 141 | *.psd binary 142 | *.svg text 143 | *.svgz binary 144 | *.tif binary 145 | *.tiff binary 146 | *.wbmp binary 147 | *.webp binary 148 | 149 | ## AUDIO 150 | *.kar binary 151 | *.m4a binary 152 | *.mid binary 153 | *.midi binary 154 | *.mp3 binary 155 | *.ogg binary 156 | *.ra binary 157 | 158 | ## VIDEO 159 | *.3gpp binary 160 | *.3gp binary 161 | *.as binary 162 | *.asf binary 163 | *.asx binary 164 | *.fla binary 165 | *.flv binary 166 | *.m4v binary 167 | *.mng binary 168 | *.mov binary 169 | *.mp4 binary 170 | *.mpeg binary 171 | *.mpg binary 172 | *.ogv binary 173 | *.swc binary 174 | *.swf binary 175 | *.webm binary 176 | 177 | ## ARCHIVES 178 | *.7z binary 179 | *.gz binary 180 | *.jar binary 181 | *.rar binary 182 | *.tar binary 183 | *.zip binary 184 | 185 | ## FONTS 186 | *.ttf binary 187 | *.eot binary 188 | *.otf binary 189 | *.woff binary 190 | *.woff2 binary 191 | 192 | ## EXECUTABLES 193 | *.exe binary 194 | *.pyc binary 195 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Include your project-specific ignores in this file 2 | # Read about how to use .gitignore: https://help.github.com/articles/ignoring-files 3 | # Useful .gitignore templates: https://github.com/github/gitignore 4 | /.idea/ 5 | 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | lerna-debug.log* 13 | 14 | # Diagnostic reports (https://nodejs.org/api/report.html) 15 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 16 | 17 | # Runtime data 18 | pids 19 | *.pid 20 | *.seed 21 | *.pid.lock 22 | 23 | # Directory for instrumented libs generated by jscoverage/JSCover 24 | lib-cov 25 | 26 | # Coverage directory used by tools like istanbul 27 | coverage 28 | *.lcov 29 | 30 | # nyc test coverage 31 | .nyc_output 32 | 33 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 34 | .grunt 35 | 36 | # Bower dependency directory (https://bower.io/) 37 | bower_components 38 | 39 | # node-waf configuration 40 | .lock-wscript 41 | 42 | # Compiled binary addons (https://nodejs.org/api/addons.html) 43 | build/ 44 | build/Release 45 | 46 | # Dependency directories 47 | node_modules/ 48 | jspm_packages/ 49 | 50 | # TypeScript v1 declaration files 51 | typings/ 52 | 53 | # TypeScript cache 54 | *.tsbuildinfo 55 | 56 | # Optional npm cache directory 57 | .npm 58 | 59 | # Optional eslint cache 60 | .eslintcache 61 | 62 | # Microbundle cache 63 | .rpt2_cache/ 64 | .rts2_cache_cjs/ 65 | .rts2_cache_es/ 66 | .rts2_cache_umd/ 67 | 68 | # Optional REPL history 69 | .node_repl_history 70 | 71 | # Output of 'npm pack' 72 | *.tgz 73 | 74 | # Yarn Integrity file 75 | .yarn-integrity 76 | 77 | # dotenv environment variables file 78 | .env 79 | .env.test 80 | 81 | # parcel-bundler cache (https://parceljs.org/) 82 | .cache 83 | 84 | # Next.js build output 85 | .next 86 | 87 | # Nuxt.js build / generate output 88 | .nuxt 89 | 90 | # Gatsby files 91 | .cache/ 92 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 93 | # https://nextjs.org/blog/next-9-1#public-directory-support 94 | # public 95 | 96 | # vuepress build output 97 | .vuepress/dist 98 | 99 | # Serverless directories 100 | .serverless/ 101 | 102 | # FuseBox cache 103 | .fusebox/ 104 | 105 | # DynamoDB Local files 106 | .dynamodb/ 107 | 108 | # TernJS port file 109 | .tern-port 110 | 111 | # exclude working directories 112 | dist/ 113 | test/testfiles/ 114 | /src/index-*.ts 115 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## What type of pull request do we accept? 4 | 5 | * Bug fixes 6 | * Documentation fixes 7 | 8 | For new feature additions, please open an issue first to discuss the feature. 9 | 10 | ## Submit your own code 11 | 12 | We'd love to accept your patches! However, 13 | 14 | ***Please note that*** only original source code from you and other people can be accepted into this repository. 15 | 16 | ## Contributing code 17 | 18 | If you have bug fixes and documentation fixes to v3d-web, send us your pull requests! For those just getting started, GitHub has a [howto](https://help.github.com/articles/using-pull-requests/). 19 | 20 | Once you create your pull request, we will review the bug/documentation fixes as soon as possible. Once verified, we will acknowledge your contribution in the pull request comments, manually merge the fixes into our codebase, and close the pull request. 21 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published by 637 | the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # v3d-web 2 | 3 | Single camera motion capture/humanoid rigging in browser. 4 | 5 | ## Objectives 6 | 7 | v3d-web is a project aiming to bridge the gap between our reality and the virtual world. With the power of latest XR and web technologies, you can render and rig your favorite 3D humanoid avatars right inside your browser. Try it out now! 8 | 9 | ## Features 10 | 11 | - Contains a complete solution from image capturing to model rendering, all inside browsers. 12 | - Thanks to the latest machine learning technologies, we can achieve a fairly accurate facial and pose estimation from a single camera video stream. 13 | - It also comes with a complete WebGL rendering engine, in which VRM models can be present with highly complicated backgrounds. VRM model rendering is done with [v3d-core](https://github.com/phantom-software-AZ/v3d-core). 14 | - There is a [demo site](https://www.phantom-dev.com) showing how this project can be seamlessly embedded into a modern UI framework like React.js. 15 | 16 | ## Usage 17 | 18 | ### Install from NPM 19 | 20 | ```s 21 | npm install v3d-web 22 | ``` 23 | 24 | ### In browser 25 | 26 | *Only latest FireFox, Chrome/Chromium, Edge and Safari are supported. WebGL and WebAssembly are necessary for this project to work correctly.* 27 | 28 | In the simplest case, all you need is: 29 | ```s 30 | const vrmFile = 'testfile.vrm'; 31 | try { 32 | this.v3DWeb = new V3DWeb(vrmFile); 33 | } catch (e: any) { 34 | console.error(e); 35 | } 36 | ``` 37 | 38 | You will need HTML elements with certain `id`s. See [index.html](test/index.html). 39 | 40 | A more complicated example can be found at the [repo for our demo site](https://github.com/phantom-software-AZ/v3d-web-demo). 41 | 42 | ## Contributing 43 | 44 | See [CONTRIBUTING.md](./CONTRIBUTING.md). 45 | 46 | ## Build 47 | 48 | 1. Clone this repo and submodules: 49 | 50 | ```s 51 | git clone https://github.com/phantom-software-AZ/v3d-web.git 52 | ``` 53 | 54 | 2. Build v3d-web 55 | 56 | ```s 57 | npm install 58 | npm run build && tsc 59 | ``` 60 | 61 | ## Debugging 62 | 63 | Go to root folder of this repository and run: 64 | 65 | ```s 66 | $ npm run debug 67 | ``` 68 | 69 | The debug page can be opened in any browser with `http://localhost:8080/`. 70 | 71 | ## Demo Site 72 | 73 | See [this demo site](https://www.phantom-dev.com/demo) for a live example. 74 | 75 | ## Credits 76 | 77 | - [MediaPipe](https://mediapipe.dev/) 78 | - [babylon.js](https://github.com/BabylonJS/Babylon.js) 79 | - [babylon-vrm-loader](https://github.com/virtual-cast/babylon-vrm-loader) 80 | - [babylon-mtoon-material](https://github.com/virtual-cast/babylon-mtoon-material) 81 | - [VRM Consortium](https://vrm.dev/en/) 82 | - Demo model used: `Kaori/ Kaori` by `ClaValLuis` From VRoid Hub. Used according to [VRM PUBLIC LICENSE 1.0](https://vrm.dev/licenses/1.0/en/). 83 | 84 | ## Acknowledgement 85 | 86 | An adorable and perseverant individual who keeps on pursuing dreams. 87 | 88 | ## Licenses 89 | 90 | see [LICENSE](./LICENSE). 91 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "v3d-web", 3 | "version": "0.1.0", 4 | "description": "Single camera motion-tracking in browser", 5 | "keywords": [ 6 | "babylon.js", 7 | "VRM", 8 | "MediaPipe", 9 | "WebGL" 10 | ], 11 | "files": [ 12 | "dist", 13 | "dist/src/index.d.ts", 14 | "dist/src/index.js", 15 | "dist/src/index.js.map", 16 | "dist/src/core.d.ts", 17 | "dist/src/core.js", 18 | "dist/src/core.js.map", 19 | "dist/src/mediapipe.d.ts", 20 | "dist/src/mediapipe.js", 21 | "dist/src/mediapipe.js.map", 22 | "dist/src/v3d-web.d.ts", 23 | "dist/src/v3d-web.js", 24 | "dist/src/v3d-web.js.map", 25 | "dist/src/helper", 26 | "dist/src/worker" 27 | ], 28 | "license": "AGPL-3.0-only", 29 | "author": "Phantom Development", 30 | "scripts": { 31 | "build": "webpack --config webpack.config.js", 32 | "debug": "webpack-dev-server --config webpack.test.config.js", 33 | "start": "npm run build && npm run dev", 34 | "test": "echo \"Error: no test specified\" && exit 1" 35 | }, 36 | "devDependencies": { 37 | "@types/chroma-js": "^2.1.3", 38 | "@types/node": "^16.11.9", 39 | "@types/webpack": "^5.28.0", 40 | "chroma-js": "^2.1.2", 41 | "copy-webpack-plugin": "^10.0.0", 42 | "kalmanjs": "^1.1.0", 43 | "ts-loader": "^9.2.6", 44 | "typescript": "^4.5.2", 45 | "webpack": "^5.64.2", 46 | "webpack-cli": "^4.9.1", 47 | "webpack-dev-server": "^4.5.0" 48 | }, 49 | "dependencies": { 50 | "@mediapipe/camera_utils": "^0.3.1632432234", 51 | "@mediapipe/control_utils": "^0.6.1629159505", 52 | "@mediapipe/drawing_utils": "^0.3.1620248257", 53 | "@mediapipe/holistic": "^0.5.1635989137", 54 | "comlink": "^4.3.1", 55 | "v3d-core": "^1.0.15" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/core.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {V3DCore} from "v3d-core/dist/src"; 18 | import {ArcRotateCamera, Nullable, Quaternion, Scene} from "@babylonjs/core"; 19 | import {Color3, Vector3} from "@babylonjs/core/Maths"; 20 | import {Engine} from "@babylonjs/core/Engines"; 21 | import {Camera} from "@babylonjs/core"; 22 | import {DebugInfo} from "./helper/debug"; 23 | 24 | // Debug 25 | import "@babylonjs/core/Debug"; 26 | import "@babylonjs/gui"; 27 | import "@babylonjs/inspector"; 28 | 29 | import * as Comlink from "comlink"; 30 | import {Poses} from "./worker/pose-processing"; 31 | import {Clock} from "./helper/clock"; 32 | import {VRMManager} from "v3d-core/dist/src/importer/babylon-vrm-loader/src"; 33 | import {HAND_LANDMARKS_BONE_MAPPING} from "./helper/landmark"; 34 | import {HumanoidBone} from "v3d-core/dist/src/importer/babylon-vrm-loader/src/humanoid-bone"; 35 | import {KeysMatching, LR} from "./helper/utils"; 36 | import { 37 | CloneableQuaternionMap, 38 | cloneableQuaternionToQuaternion, 39 | } from "./helper/quaternion"; 40 | import {Holistic} from "@mediapipe/holistic"; 41 | import {BoneOptions, BoneState, HolisticState} from "./v3d-web"; 42 | 43 | const IS_DEBUG = false; 44 | const clock = new Clock(), textDecode = new TextDecoder(); 45 | export let debugInfo: Nullable; 46 | 47 | // Can only have one VRM model at this time 48 | export async function createScene( 49 | engine: Engine, 50 | workerPose: Nullable>, 51 | boneState: BoneState, 52 | boneOptions: BoneOptions, 53 | holistic: Holistic, 54 | holisticState: HolisticState, 55 | vrmFile: File | string, 56 | videoElement: HTMLVideoElement): Promise> { 57 | 58 | if (!workerPose) return null; 59 | 60 | // Create v3d core 61 | const v3DCore = new V3DCore(engine, new Scene(engine)); 62 | await v3DCore.AppendAsync('', vrmFile); 63 | 64 | // Get managers 65 | const vrmManager = v3DCore.getVRMManagerByURI((vrmFile as File).name ? (vrmFile as File).name : (vrmFile as string)); 66 | 67 | // Camera 68 | // v3DCore.attachCameraTo(vrmManager); 69 | const mainCamera = (v3DCore.mainCamera as ArcRotateCamera); 70 | mainCamera.setPosition(new Vector3(0, 1.05, 4.5)); 71 | mainCamera.setTarget( 72 | vrmManager.rootMesh.getWorldMatrix().getTranslation().subtractFromFloats(0, -1.25, 0)); 73 | mainCamera.fovMode = Camera.FOVMODE_HORIZONTAL_FIXED; 74 | 75 | // Lights and Skybox 76 | v3DCore.addAmbientLight(new Color3(1, 1, 1)); 77 | v3DCore.setBackgroundColor(Color3.FromHexString('#e7a2ff')); 78 | 79 | v3DCore.renderingPipeline.depthOfFieldEnabled = false; 80 | 81 | // Pose web worker 82 | await workerPose.setBonesHierarchyTree(vrmManager.transformNodeTree); 83 | 84 | // Disable auto animation 85 | v3DCore.springBonesAutoUpdate = false; 86 | 87 | // Update functions 88 | v3DCore.updateBeforeRenderFunction( 89 | () => { 90 | // Half input fps. This version of Holistic is heavy on CPU time. 91 | // Wait until they fix web worker (https://github.com/google/mediapipe/issues/2506). 92 | if (holisticState.holisticUpdate && holisticState.ready && !videoElement.paused && videoElement.readyState > 2) { 93 | holistic.send({image: videoElement}) 94 | } 95 | holisticState.holisticUpdate = !holisticState.holisticUpdate; 96 | } 97 | ); 98 | v3DCore.updateAfterRenderFunction( 99 | () => { 100 | if (boneState.bonesNeedUpdate) { 101 | updatePose(vrmManager, boneState, boneOptions); 102 | updateSpringBones(vrmManager); 103 | boneState.bonesNeedUpdate = false; 104 | } 105 | } 106 | ); 107 | 108 | // Render loop 109 | engine.runRenderLoop(() => { 110 | v3DCore.scene?.render(); 111 | }); 112 | 113 | // Model Transformation 114 | vrmManager.rootMesh.rotationQuaternion = Quaternion.RotationYawPitchRoll(0, 0, 0); 115 | 116 | // Debug 117 | if (IS_DEBUG && v3DCore.scene) { 118 | debugInfo = new DebugInfo(v3DCore.scene); 119 | } 120 | 121 | engine.hideLoadingUI(); 122 | 123 | return [v3DCore, vrmManager]; 124 | } 125 | 126 | export function updateSpringBones(vrmManager: VRMManager) { 127 | const deltaTime = clock.getDelta() * 1000; 128 | vrmManager.update(deltaTime); 129 | } 130 | 131 | export function updateBuffer(data: Uint8Array, boneState: BoneState) { 132 | let jsonStr = textDecode.decode(data); 133 | let boneRotationsData: CloneableQuaternionMap = JSON.parse(jsonStr); 134 | boneState.boneRotations = boneRotationsData; 135 | boneState.bonesNeedUpdate = true; 136 | (data as any) = null; 137 | (jsonStr as any) = null; 138 | } 139 | 140 | export function updatePose( 141 | vrmManager: VRMManager, 142 | boneState: BoneState, 143 | boneOptions: BoneOptions 144 | ) { 145 | // Wait for buffer to fill 146 | if (!boneState.boneRotations) return; 147 | 148 | const resultBoneRotations = boneState.boneRotations; 149 | 150 | vrmManager.morphing('A', resultBoneRotations['mouth'].x); 151 | 152 | const resetExpressions = () => { 153 | vrmManager.morphing('Neutral', 0); 154 | vrmManager.morphing('Happy', 0); 155 | vrmManager.morphing('Joy', 0); 156 | vrmManager.morphing('Angry', 0); 157 | vrmManager.morphing('Sad', 0); 158 | vrmManager.morphing('Sorrow', 0); 159 | vrmManager.morphing('Relaxed', 0); 160 | vrmManager.morphing('Fun', 0); 161 | vrmManager.morphing('Surprised', 0); 162 | } 163 | 164 | // Update expression 165 | resetExpressions(); 166 | switch (boneOptions.expression) { 167 | case "Angry": 168 | vrmManager.morphing('Angry', 1); 169 | break; 170 | case "Happy": 171 | vrmManager.morphing('Happy', 1); 172 | vrmManager.morphing('Joy', 1); 173 | break; 174 | case "Relaxed": 175 | vrmManager.morphing('Relaxed', 1); 176 | vrmManager.morphing('Fun', 1); 177 | break; 178 | case "Sad": 179 | vrmManager.morphing('Sad', 1); 180 | vrmManager.morphing('Sorrow', 1); 181 | break; 182 | case "Surprised": 183 | vrmManager.morphing('Surprised', 1); 184 | break; 185 | case "Neutral": // fall through 186 | default: 187 | vrmManager.morphing('Neutral', 1); 188 | break; 189 | } 190 | 191 | if (boneOptions.blinkLinkLR) { 192 | vrmManager.morphing('Blink', resultBoneRotations['blink'].z) 193 | } else { 194 | vrmManager.morphing('Blink_L', resultBoneRotations['blink'].x) 195 | vrmManager.morphing('Blink_R', resultBoneRotations['blink'].y) 196 | } 197 | 198 | if (vrmManager.humanoidBone.leftEye) 199 | vrmManager.humanoidBone.leftEye.rotationQuaternion = cloneableQuaternionToQuaternion( 200 | resultBoneRotations['iris']); 201 | if (vrmManager.humanoidBone.rightEye) 202 | vrmManager.humanoidBone.rightEye.rotationQuaternion = cloneableQuaternionToQuaternion( 203 | resultBoneRotations['iris']); 204 | 205 | for (const d of LR) { 206 | for (const k of Object.keys(HAND_LANDMARKS_BONE_MAPPING)) { 207 | const key = d + k as keyof Omit>; 208 | if (vrmManager.humanoidBone[key]) { 209 | vrmManager.humanoidBone[key]!.rotationQuaternion = cloneableQuaternionToQuaternion( 210 | resultBoneRotations[key]); 211 | } 212 | } 213 | } 214 | 215 | vrmManager.humanoidBone.hips.rotationQuaternion = cloneableQuaternionToQuaternion( 216 | resultBoneRotations['hips']); 217 | vrmManager.humanoidBone.spine.rotationQuaternion = cloneableQuaternionToQuaternion( 218 | resultBoneRotations['spine']); 219 | vrmManager.humanoidBone.neck.rotationQuaternion = cloneableQuaternionToQuaternion( 220 | resultBoneRotations['neck']); 221 | vrmManager.humanoidBone.head.rotationQuaternion = cloneableQuaternionToQuaternion( 222 | resultBoneRotations['head']); 223 | 224 | // Arms 225 | for (const k of LR) { 226 | const upperArmKey = `${k}UpperArm` as unknown as keyof Omit>; 227 | const lowerArmKey = `${k}LowerArm` as unknown as keyof Omit>; 228 | vrmManager.humanoidBone[upperArmKey]!.rotationQuaternion = cloneableQuaternionToQuaternion( 229 | resultBoneRotations[upperArmKey]); 230 | vrmManager.humanoidBone[lowerArmKey]!.rotationQuaternion = cloneableQuaternionToQuaternion( 231 | resultBoneRotations[lowerArmKey]); 232 | // Legs 233 | const upperLegKey = `${k}UpperLeg` as unknown as keyof Omit>; 234 | const lowerLegKey = `${k}LowerLeg` as unknown as keyof Omit>; 235 | vrmManager.humanoidBone[upperLegKey]!.rotationQuaternion = cloneableQuaternionToQuaternion( 236 | resultBoneRotations[upperLegKey]); 237 | vrmManager.humanoidBone[lowerLegKey]!.rotationQuaternion = cloneableQuaternionToQuaternion( 238 | resultBoneRotations[lowerLegKey]); 239 | // Feet 240 | const footKey = `${k}Foot` as unknown as keyof Omit>; 241 | vrmManager.humanoidBone[footKey]!.rotationQuaternion = cloneableQuaternionToQuaternion( 242 | resultBoneRotations[footKey]); 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/helper/basis.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2022 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | // Calculate 3D rotations 18 | import {Nullable, Plane, Quaternion, Vector3} from "@babylonjs/core"; 19 | import {AXIS, vectorsSameDirWithinEps} from "./quaternion"; 20 | import {setEqual, validVector3} from "./utils"; 21 | 22 | export type Vector33 = [Vector3, Vector3, Vector3]; 23 | 24 | export class Basis { 25 | private static readonly ORIGINAL_CARTESIAN_BASIS_VECTORS: Vector33 = [ 26 | new Vector3(1, 0, 0), 27 | new Vector3(0, 1, 0), 28 | new Vector3(0, 0, 1), 29 | ]; 30 | 31 | private readonly _data: Vector33 = Basis.getOriginalCoordVectors(); 32 | 33 | get x(): Vector3 { 34 | return this._data[0]; 35 | } 36 | 37 | get y(): Vector3 { 38 | return this._data[1]; 39 | } 40 | 41 | get z(): Vector3 { 42 | return this._data[2]; 43 | } 44 | 45 | constructor( 46 | v33: Nullable, 47 | private readonly leftHanded = true, 48 | private eps = 1e-6 49 | ) { 50 | if (v33 && v33.every((v) => validVector3(v))) 51 | this.set(v33); 52 | this._data.forEach((v) => { 53 | Object.freeze(v); 54 | }) 55 | } 56 | 57 | public get() { 58 | return this._data; 59 | } 60 | 61 | private set(v33: Vector33) { 62 | this.x.copyFrom(v33[0]); 63 | this.y.copyFrom(v33[1]); 64 | this.z.copyFrom(v33[2]); 65 | 66 | this.verifyBasis(); 67 | } 68 | 69 | public verifyBasis() { 70 | const z = this.leftHanded ? this.z : this.z.negate(); 71 | if (!vectorsSameDirWithinEps(this.x.cross(this.y), z, this.eps)) 72 | throw Error("Basis is not correct!"); 73 | } 74 | 75 | public rotateByQuaternion(q: Quaternion): Basis { 76 | const newBasisVectors: Vector33 = [Vector3.Zero(), Vector3.Zero(), Vector3.Zero()]; 77 | this._data.map((v, i) => { 78 | v.rotateByQuaternionToRef(q, newBasisVectors[i]); 79 | }); 80 | return new Basis(newBasisVectors); 81 | } 82 | 83 | // Basis validity is not checked! 84 | public negateAxes(axis: AXIS) { 85 | const x = this.x.clone(); 86 | const y = this.y.clone(); 87 | const z = this.z.clone(); 88 | switch (axis) { 89 | case AXIS.x: 90 | x.negateInPlace(); 91 | break; 92 | case AXIS.y: 93 | y.negateInPlace(); 94 | break; 95 | case AXIS.z: 96 | z.negateInPlace(); 97 | break; 98 | case AXIS.xy: 99 | x.negateInPlace(); 100 | y.negateInPlace(); 101 | break; 102 | case AXIS.yz: 103 | y.negateInPlace(); 104 | z.negateInPlace(); 105 | break; 106 | case AXIS.xz: 107 | x.negateInPlace(); 108 | z.negateInPlace(); 109 | break; 110 | case AXIS.xyz: 111 | x.negateInPlace(); 112 | y.negateInPlace(); 113 | z.negateInPlace(); 114 | break; 115 | default: 116 | throw Error("Unknown axis!"); 117 | } 118 | 119 | return new Basis([x, y, z]); 120 | } 121 | 122 | public transpose(order: [number, number, number]) { 123 | // Sanity check 124 | if (!setEqual(new Set(order), new Set([0, 1, 2]))) { 125 | console.error("Basis transpose failed: wrong input."); 126 | return this; 127 | } 128 | 129 | const data = [this.x.clone(), this.y.clone(), this.z.clone()]; 130 | const newData = order.map(i => data[i]) as Vector33; 131 | 132 | return new Basis(newData); 133 | } 134 | 135 | private static getOriginalCoordVectors(): Vector33 { 136 | return Basis.ORIGINAL_CARTESIAN_BASIS_VECTORS.map(v => v.clone()) as Vector33; 137 | } 138 | } 139 | 140 | export function quaternionBetweenBases( 141 | basis1: Basis, 142 | basis2: Basis, 143 | prevQuaternion?: Quaternion 144 | ) { 145 | let thisBasis1 = basis1, thisBasis2 = basis2; 146 | if (prevQuaternion !== undefined) { 147 | const extraQuaternionR = Quaternion.Inverse(prevQuaternion); 148 | thisBasis1 = basis1.rotateByQuaternion(extraQuaternionR); 149 | thisBasis2 = basis2.rotateByQuaternion(extraQuaternionR); 150 | } 151 | const rotationBasis1 = Quaternion.RotationQuaternionFromAxis( 152 | thisBasis1.x.clone(), 153 | thisBasis1.y.clone(), 154 | thisBasis1.z.clone()); 155 | const rotationBasis2 = Quaternion.RotationQuaternionFromAxis( 156 | thisBasis2.x.clone(), 157 | thisBasis2.y.clone(), 158 | thisBasis2.z.clone()); 159 | 160 | const quaternion31 = rotationBasis1.clone().normalize(); 161 | const quaternion31R = Quaternion.Inverse(quaternion31); 162 | const quaternion32 = rotationBasis2.clone().normalize(); 163 | return quaternion32.multiply(quaternion31R); 164 | } 165 | 166 | /* 167 | * Left handed for BJS. 168 | * Each object is defined by 3 points. 169 | * Assume a is origin, b points to +x, abc forms XY plane. 170 | */ 171 | export function getBasis(obj: Vector33): Basis { 172 | const [a, b, c] = obj; 173 | const planeXY = Plane.FromPoints(a, b, c).normalize(); 174 | const axisX = b.subtract(a).normalize(); 175 | const axisZ = planeXY.normal; 176 | // Project c onto ab 177 | const cp = a.add( 178 | axisX.scale(Vector3.Dot(c.subtract(a), axisX) / Vector3.Dot(axisX, axisX)) 179 | ); 180 | const axisY = c.subtract(cp).normalize(); 181 | return new Basis([axisX, axisY, axisZ]); 182 | } 183 | 184 | // Project points to an average plane 185 | export function calcAvgPlane(pts: Vector3[], normal: Vector3): Vector3[] { 186 | if (pts.length === 0) return [Vector3.Zero()]; 187 | const avgPt = pts.reduce((prev, curr) => { 188 | return prev.add(curr); 189 | }).scale(1 / pts.length); 190 | 191 | const ret = pts.map((v) => { 192 | return v.subtract(normal.scale(Vector3.Dot(normal, v.subtract(avgPt)))) 193 | }); 194 | 195 | return ret; 196 | } 197 | -------------------------------------------------------------------------------- /src/helper/canvas.ts: -------------------------------------------------------------------------------- 1 | function fit(contains: boolean) { 2 | return (parentWidth: number, parentHeight: number, childWidth: number, childHeight: number, 3 | scale: number = 1, offsetX: number = 0.5, offsetY: number = 0.5) => { 4 | const childRatio = childWidth / childHeight 5 | const parentRatio = parentWidth / parentHeight 6 | let width = parentWidth * scale 7 | let height = parentHeight * scale 8 | 9 | if (contains ? (childRatio > parentRatio) : (childRatio < parentRatio)) { 10 | height = width / childRatio 11 | } else { 12 | width = height * childRatio 13 | } 14 | 15 | return { 16 | width, 17 | height, 18 | offsetX: (parentWidth - width) * offsetX, 19 | offsetY: (parentHeight - height) * offsetY 20 | } 21 | } 22 | } 23 | 24 | export const contain = fit(true) 25 | export const cover = fit(false) 26 | -------------------------------------------------------------------------------- /src/helper/clock.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Adapted from three.js Clock class 3 | */ 4 | export class Clock { 5 | private _autoStart: boolean; 6 | get autoStart(): boolean { 7 | return this._autoStart; 8 | } 9 | 10 | private _startTime: number; 11 | get startTime(): number { 12 | return this._startTime; 13 | } 14 | 15 | private _oldTime: number; 16 | get oldTime(): number { 17 | return this._oldTime; 18 | } 19 | 20 | private _elapsedTime: number; 21 | get elapsedTime(): number { 22 | return this._elapsedTime; 23 | } 24 | 25 | private _running: boolean; 26 | get running(): boolean { 27 | return this._running; 28 | } 29 | 30 | constructor(autoStart = true) { 31 | this._autoStart = autoStart; 32 | 33 | this._startTime = 0; 34 | this._oldTime = 0; 35 | this._elapsedTime = 0; 36 | 37 | this._running = false; 38 | } 39 | 40 | public start() { 41 | this._startTime = now(); 42 | 43 | this._oldTime = this._startTime; 44 | this._elapsedTime = 0; 45 | this._running = true; 46 | } 47 | 48 | public stop() { 49 | this.getElapsedTime(); 50 | this._running = false; 51 | this._autoStart = false; 52 | } 53 | 54 | /** 55 | * Calculate how much time has passed from Clock start to input timestamp. 56 | * Unlike getElapsedTime, this method does not tick oldTime. 57 | * @param time input time 58 | */ 59 | public getElapsedTimeAny(time: number) { 60 | return (time - this._startTime) / 1000; 61 | } 62 | 63 | public getElapsedTime() { 64 | this.getDelta(); 65 | return this._elapsedTime; 66 | } 67 | 68 | public getDelta() { 69 | let diff = 0; 70 | 71 | if (this._autoStart && !this._running) { 72 | this.start(); 73 | return 0; 74 | } 75 | 76 | if (this._running) { 77 | const newTime = now(); 78 | diff = (newTime - this._oldTime) / 1000; 79 | this._oldTime = newTime; 80 | this._elapsedTime += diff; 81 | } 82 | 83 | return diff; 84 | } 85 | } 86 | 87 | function now() { 88 | return (typeof performance === 'undefined' ? Date : performance).now(); // see #10732 89 | } 90 | -------------------------------------------------------------------------------- /src/helper/debug.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {Mesh, MeshBuilder, Nullable, Scene, StandardMaterial, Vector3} from "@babylonjs/core"; 18 | import {Color3, Vector4} from "@babylonjs/core/Maths"; 19 | import { 20 | initArray, 21 | } from "./utils"; 22 | import chroma from "chroma-js"; 23 | import {NormalizedLandmark, NormalizedLandmarkList, POSE_LANDMARKS} from "@mediapipe/holistic"; 24 | import {Poses} from "../worker/pose-processing"; 25 | import {CloneableQuaternion, CloneableQuaternionList, cloneableQuaternionToQuaternion} from "./quaternion"; 26 | import {HAND_LANDMARK_LENGTH, HAND_LANDMARKS, normalizedLandmarkToVector, POSE_LANDMARK_LENGTH} from "./landmark"; 27 | 28 | type createSphereOptions = { 29 | segments?: number; 30 | diameter?: number; 31 | diameterX?: number; 32 | diameterY?: number; 33 | diameterZ?: number; 34 | arc?: number; 35 | slice?: number; 36 | sideOrientation?: number; 37 | frontUVs?: Vector4; 38 | backUVs?: Vector4; 39 | updatable?: boolean; 40 | }; 41 | 42 | export class Arrow3D { 43 | //Shape profile in XY plane 44 | private readonly myShape: Vector3[] = []; 45 | private readonly myPath: Vector3[] = []; 46 | private arrowInstance: Nullable = null; 47 | 48 | private _arrowRadius: number; 49 | get arrowRadius(): number { 50 | return this._arrowRadius; 51 | } 52 | 53 | set arrowRadius(value: number) { 54 | this._arrowRadius = value; 55 | this.updateTopShape(); 56 | } 57 | 58 | private _n: number; 59 | get n(): number { 60 | return this._n; 61 | } 62 | 63 | set n(value: number) { 64 | this._n = value; 65 | this.updateTopShape(); 66 | } 67 | 68 | private _arrowHeadLength: number; 69 | get arrowHeadLength(): number { 70 | return this._arrowHeadLength; 71 | } 72 | 73 | set arrowHeadLength(value: number) { 74 | this._arrowHeadLength = value; 75 | this.updatePath(); 76 | } 77 | 78 | private _arrowHeadMaxSize: number; 79 | 80 | get arrowStart(): Vector3 { 81 | return this._arrowStart; 82 | } 83 | 84 | set arrowStart(value: Vector3) { 85 | this._arrowStart = value; 86 | this.updatePath(); 87 | } 88 | 89 | private _arrowLength: number; 90 | get arrowLength(): number { 91 | return this._arrowLength; 92 | } 93 | 94 | set arrowLength(value: number) { 95 | this._arrowLength = value; 96 | this.updatePath(); 97 | } 98 | 99 | private _arrowStart: Vector3; 100 | 101 | get arrowHeadMaxSize(): number { 102 | return this._arrowHeadMaxSize; 103 | } 104 | 105 | set arrowHeadMaxSize(value: number) { 106 | this._arrowHeadMaxSize = value; 107 | this.updatePath(); 108 | } 109 | 110 | private _arrowDirection: Vector3; 111 | get arrowDirection(): Vector3 { 112 | return this._arrowDirection; 113 | } 114 | 115 | set arrowDirection(value: Vector3) { 116 | this._arrowDirection = value; 117 | this.updatePath(); 118 | } 119 | 120 | private _material: StandardMaterial; 121 | get material(): StandardMaterial { 122 | return this._material; 123 | } 124 | 125 | constructor( 126 | private scene: Scene, 127 | arrowRadius = 0.5, 128 | n = 30, 129 | arrowHeadLength = 1.5, 130 | arrowHeadMaxSize = 1.5, 131 | arrowLength = 10, 132 | arrowStart: Vector3, 133 | arrowDirection: Vector3, 134 | color?: number | string, 135 | ) { 136 | this._arrowRadius = arrowRadius; 137 | this._n = n; 138 | this._arrowHeadLength = arrowHeadLength; 139 | this._arrowHeadMaxSize = arrowHeadMaxSize; 140 | this._arrowLength = arrowLength; 141 | this._arrowStart = arrowStart; 142 | this._arrowDirection = arrowDirection; 143 | this.updateTopShape(); 144 | this.updatePath(); 145 | this._material = new StandardMaterial("sphereMaterial", scene); 146 | if (color) { 147 | if (typeof color === 'number') color = `#${color.toString(16)}`; 148 | this._material.diffuseColor = Color3.FromHexString(color); 149 | } 150 | } 151 | 152 | private updateTopShape() { 153 | const deltaAngle = 2 * Math.PI / this.n; 154 | this.myShape.length = 0; 155 | for (let i = 0; i <= this.n; i++) { 156 | this.myShape.push(new Vector3( 157 | this.arrowRadius * Math.cos(i * deltaAngle), 158 | this.arrowRadius * Math.sin(i * deltaAngle), 159 | 0)) 160 | } 161 | this.myShape.push(this.myShape[0]); //close profile 162 | } 163 | 164 | private updatePath() { 165 | const arrowBodyLength = this.arrowLength - this.arrowHeadLength; 166 | this.arrowDirection.normalize(); 167 | const arrowBodyEnd = this.arrowStart.add(this.arrowDirection.scale(arrowBodyLength)); 168 | const arrowHeadEnd = arrowBodyEnd.add(this.arrowDirection.scale(this.arrowHeadLength)); 169 | 170 | this.myPath.length = 0; 171 | this.myPath.push(this.arrowStart); 172 | this.myPath.push(arrowBodyEnd); 173 | this.myPath.push(arrowBodyEnd); 174 | this.myPath.push(arrowHeadEnd); 175 | 176 | if (!this.arrowInstance) 177 | this.arrowInstance = MeshBuilder.ExtrudeShapeCustom( 178 | "arrow", 179 | { 180 | shape: this.myShape, 181 | path: this.myPath, 182 | updatable: true, 183 | scaleFunction: this.scaling.bind(this), 184 | sideOrientation: Mesh.DOUBLESIDE 185 | }, this.scene); 186 | else 187 | this.arrowInstance = MeshBuilder.ExtrudeShapeCustom( 188 | "arrow", 189 | { 190 | shape: this.myShape, 191 | path: this.myPath, 192 | scaleFunction: this.scaling.bind(this), 193 | instance: this.arrowInstance 194 | }, this.scene); 195 | this.arrowInstance.material = this.material; 196 | } 197 | 198 | private scaling(index: number, distance: number): number { 199 | switch (index) { 200 | case 0: 201 | case 1: 202 | return 1; 203 | case 2: 204 | return this.arrowHeadMaxSize / this.arrowRadius; 205 | case 3: 206 | return 0; 207 | default: 208 | return 1; 209 | } 210 | } 211 | 212 | public updateStartAndDirection(arrowStart: Vector3, arrowDirection: Vector3) { 213 | this._arrowStart = arrowStart; 214 | this._arrowDirection = arrowDirection.length() === 0 ? 215 | Vector3.One() : arrowDirection; 216 | this.updatePath(); 217 | } 218 | } 219 | 220 | export function makeSphere( 221 | scene: Scene, 222 | pos?: Vector3, 223 | color?: number | string, 224 | options?: createSphereOptions): Mesh { 225 | const sphere = MeshBuilder.CreateSphere("sphere", 226 | options || { 227 | diameterX: 1, diameterY: 0.5, diameterZ: 0.5 228 | }, scene); 229 | const material = new StandardMaterial("sphereMaterial", scene); 230 | if (color) { 231 | if (typeof color === 'number') color = `#${color.toString(16)}`; 232 | material.diffuseColor = Color3.FromHexString(color); 233 | } 234 | sphere.material = material; 235 | 236 | if (pos) 237 | sphere.position = pos; 238 | 239 | return sphere; 240 | } 241 | 242 | export function quaternionToDirectionVector( 243 | base: Vector3, 244 | resultQuaternion: CloneableQuaternion 245 | ): Vector3 { 246 | const quaternion = cloneableQuaternionToQuaternion(resultQuaternion); 247 | const result = Vector3.Zero(); 248 | base.rotateByQuaternionToRef(quaternion, result); 249 | return result.normalize(); 250 | } 251 | 252 | export class DebugInfo { 253 | private readonly poseLandmarkSpheres: Mesh[]; 254 | private readonly faceNormalArrows: Arrow3D[]; 255 | private faceMeshLandmarkSpheres: Nullable = null; 256 | private readonly leftHandLandmarkSpheres: Mesh[]; 257 | private readonly rightHandLandmarkSpheres: Mesh[]; 258 | private readonly irisNormalArrows: Arrow3D[]; 259 | private readonly leftHandNormalArrow: Arrow3D; 260 | private readonly rightHandNormalArrow: Arrow3D; 261 | private readonly leftHandNormalArrows: Arrow3D[]; 262 | private readonly rightHandNormalArrows: Arrow3D[]; 263 | private readonly poseNormalArrows: Arrow3D[]; 264 | 265 | constructor( 266 | private readonly scene: Scene 267 | ) { 268 | this.poseLandmarkSpheres = this.initLandmarks(POSE_LANDMARK_LENGTH); 269 | this.faceNormalArrows = this.initNormalArrows(1); 270 | this.leftHandLandmarkSpheres = this.initLandmarks(HAND_LANDMARK_LENGTH, '#ff0000'); 271 | this.rightHandLandmarkSpheres = this.initLandmarks(HAND_LANDMARK_LENGTH, '#0022ff'); 272 | this.irisNormalArrows = this.initIrisNormalArrows(); 273 | this.poseNormalArrows = this.initNormalArrows(3); 274 | 275 | this.leftHandNormalArrow = new Arrow3D(this.scene, 276 | 0.02, 32, 0.06, 0.06, 277 | 0.5, Vector3.Zero(), Vector3.One()); 278 | this.rightHandNormalArrow = new Arrow3D(this.scene, 279 | 0.02, 32, 0.06, 0.06, 280 | 0.5, Vector3.Zero(), Vector3.One()); 281 | 282 | this.leftHandNormalArrows = this.initNormalArrows(6); 283 | this.rightHandNormalArrows = this.initNormalArrows(6); 284 | 285 | if (!scene.debugLayer.isVisible()) { 286 | scene.debugLayer.show({ 287 | globalRoot: document.getElementById('wrapper') as HTMLElement, 288 | handleResize: true, 289 | }); 290 | } 291 | } 292 | 293 | private initLandmarks( 294 | length: number, 295 | color?: number | string 296 | ) { 297 | const colors = chroma.scale('Spectral') 298 | .colors(length, 'hex'); 299 | return initArray( 300 | length, 301 | (i) => makeSphere( 302 | this.scene, Vector3.One(), colors[i], {diameter: 0.03})); 303 | } 304 | 305 | private initNormalArrows(length: number) { 306 | const colors = chroma.scale('Spectral') 307 | .colors(length / 2, 'hex'); 308 | return initArray( 309 | length, // Temp magical number 310 | (i) => new Arrow3D(this.scene, 311 | 0.01, 32, 0.02, 0.02, 312 | 0.2, Vector3.Zero(), Vector3.One(), colors[i % (length / 2)])); 313 | } 314 | 315 | private initIrisNormalArrows() { 316 | return initArray( 317 | 2, 318 | () => new Arrow3D(this.scene, 319 | 0.01, 32, 0.04, 0.04, 320 | 0.25, Vector3.Zero(), Vector3.One())); 321 | } 322 | 323 | private initFaceMeshLandmarks(indexList: number[][]) { 324 | return initArray( 325 | indexList.length, 326 | (i) => { 327 | return initArray( 328 | indexList[i].length, 329 | ((() => { 330 | const colors = chroma.scale('Spectral') 331 | .colors(indexList[i].length, 'hex'); 332 | return (i) => { 333 | return makeSphere( 334 | this.scene, Vector3.One(), colors[i], {diameter: 0.01}) 335 | }; 336 | })())); 337 | }); 338 | } 339 | 340 | public updatePoseLandmarkSpheres(resultPoseLandmarks: NormalizedLandmarkList) { 341 | if (resultPoseLandmarks.length !== POSE_LANDMARK_LENGTH) return; 342 | for (let i = 0; i < POSE_LANDMARK_LENGTH; ++i) { 343 | if ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 18, 19, 20, 21, 22].includes(i)) continue; 344 | this.poseLandmarkSpheres[i].position.set( 345 | resultPoseLandmarks[i].x, 346 | resultPoseLandmarks[i].y, 347 | resultPoseLandmarks[i].z 348 | ); 349 | } 350 | } 351 | 352 | public updateHandLandmarkSpheres( 353 | resultHandLandmarks: NormalizedLandmarkList, 354 | left: boolean 355 | ) { 356 | const landmarkSpheres = left ? 357 | this.leftHandLandmarkSpheres : 358 | this.rightHandLandmarkSpheres; 359 | if (resultHandLandmarks.length !== HAND_LANDMARK_LENGTH) return; 360 | for (let i = 0; i < HAND_LANDMARK_LENGTH; ++i) { 361 | landmarkSpheres[i].position.set( 362 | resultHandLandmarks[i].x, 363 | resultHandLandmarks[i].y, 364 | resultHandLandmarks[i].z 365 | ); 366 | } 367 | } 368 | 369 | public updateFaceNormalArrows( 370 | resultFaceNormals: NormalizedLandmarkList, 371 | resultPoseLandmarks: NormalizedLandmarkList 372 | ) { 373 | if (resultFaceNormals.length !== this.faceNormalArrows.length) return; 374 | for (let i = 0; i < this.faceNormalArrows.length; ++i) { 375 | this.faceNormalArrows[i].updateStartAndDirection( 376 | normalizedLandmarkToVector( 377 | resultPoseLandmarks[POSE_LANDMARKS.NOSE]), 378 | normalizedLandmarkToVector(resultFaceNormals[i]), 379 | ); 380 | } 381 | } 382 | 383 | public updateIrisQuaternionArrows( 384 | resultIrisQuaternions: CloneableQuaternionList, 385 | resultPoseLandmarks: NormalizedLandmarkList, 386 | resultFaceNormal: NormalizedLandmark 387 | ) { 388 | if (resultIrisQuaternions.length !== 2 && resultIrisQuaternions.length !== 3) return; 389 | this.irisNormalArrows[0].updateStartAndDirection( 390 | normalizedLandmarkToVector( 391 | resultPoseLandmarks[POSE_LANDMARKS.LEFT_EYE]), 392 | quaternionToDirectionVector( 393 | normalizedLandmarkToVector(resultFaceNormal), 394 | resultIrisQuaternions[0]), 395 | ); 396 | this.irisNormalArrows[1].updateStartAndDirection( 397 | normalizedLandmarkToVector( 398 | resultPoseLandmarks[POSE_LANDMARKS.RIGHT_EYE]), 399 | quaternionToDirectionVector( 400 | normalizedLandmarkToVector(resultFaceNormal), 401 | resultIrisQuaternions[1]), 402 | ); 403 | } 404 | 405 | public updateFaceMeshLandmarkSpheres( 406 | resultFaceMeshIndexLandmarks: number[][], 407 | resultFaceMeshLandmarks: NormalizedLandmarkList[]) { 408 | const toShow = [ 409 | [0], [0], 410 | [9, 4, 15, 12], [0, 4, 8, 12], 411 | [0], [0], 412 | [24, 25, 26, 34, 35, 36], 413 | [0, 6, 18, 30] 414 | ]; 415 | if (resultFaceMeshIndexLandmarks.length !== 0 && !this.faceMeshLandmarkSpheres) 416 | this.faceMeshLandmarkSpheres = this.initFaceMeshLandmarks(resultFaceMeshIndexLandmarks); 417 | if (!this.faceMeshLandmarkSpheres || 418 | resultFaceMeshLandmarks.length !== this.faceMeshLandmarkSpheres.length) return; 419 | for (let i = 0; i < this.faceMeshLandmarkSpheres.length; ++i) { 420 | for (let j = 0; j < this.faceMeshLandmarkSpheres[i].length; ++j) { 421 | if (toShow[i].length > 0 && !toShow[i].includes(j)) continue; 422 | this.faceMeshLandmarkSpheres[i][j].position.set( 423 | resultFaceMeshLandmarks[i][j].x, 424 | resultFaceMeshLandmarks[i][j].y, 425 | resultFaceMeshLandmarks[i][j].z 426 | ); 427 | } 428 | } 429 | } 430 | 431 | public updateHandWristNormalArrows( 432 | resultLeftHandBoneRotations: CloneableQuaternionList, 433 | resultRightHandBoneRotations: CloneableQuaternionList, 434 | resultPoseLandmarks: NormalizedLandmarkList 435 | ) { 436 | this.leftHandNormalArrow.updateStartAndDirection( 437 | normalizedLandmarkToVector( 438 | resultPoseLandmarks[POSE_LANDMARKS.LEFT_WRIST]), 439 | quaternionToDirectionVector( 440 | Poses.HAND_BASE_ROOT_NORMAL, resultLeftHandBoneRotations[HAND_LANDMARKS.WRIST]), 441 | ); 442 | this.rightHandNormalArrow.updateStartAndDirection( 443 | normalizedLandmarkToVector( 444 | resultPoseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]), 445 | quaternionToDirectionVector( 446 | Poses.HAND_BASE_ROOT_NORMAL, resultRightHandBoneRotations[HAND_LANDMARKS.WRIST]), 447 | ); 448 | } 449 | 450 | public updateHandNormalArrows( 451 | resultLeftHandNormals: Nullable, 452 | resultRightHandNormals: Nullable, 453 | resultPoseLandmarks: NormalizedLandmarkList 454 | ) { 455 | if (resultLeftHandNormals) { 456 | for (let i = 0; i < Math.min(this.leftHandNormalArrows.length, 457 | resultLeftHandNormals.length); ++i) { 458 | this.leftHandNormalArrows[i].updateStartAndDirection( 459 | // normalizedLandmarkToVector( 460 | // resultPoseLandmarks[POSE_LANDMARKS.LEFT_WRIST]), 461 | i < 3 ? Vector3.Zero() : new Vector3(0, 1, 0), 462 | normalizedLandmarkToVector(resultLeftHandNormals[i]), 463 | ); 464 | } 465 | } 466 | if (resultRightHandNormals) { 467 | for (let i = 0; i < Math.min(this.rightHandNormalArrows.length, 468 | resultRightHandNormals.length); ++i) { 469 | this.rightHandNormalArrows[i].updateStartAndDirection( 470 | // normalizedLandmarkToVector( 471 | // resultPoseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]), 472 | i < 3 ? Vector3.One() : new Vector3(0, 1, 0), 473 | normalizedLandmarkToVector(resultRightHandNormals[i]), 474 | ); 475 | } 476 | } 477 | } 478 | 479 | public updatePoseNormalArrows( 480 | resultPoseNormals: NormalizedLandmarkList, 481 | resultPoseLandmarks: NormalizedLandmarkList 482 | ) { 483 | if (resultPoseNormals.length !== this.poseNormalArrows.length) return; 484 | for (let i = 0; i < this.poseNormalArrows.length; ++i) { 485 | this.poseNormalArrows[i].updateStartAndDirection( 486 | // normalizedLandmarkToVector( 487 | // resultPoseLandmarks[POSE_LANDMARKS.NOSE]), 488 | Vector3.Zero(), 489 | normalizedLandmarkToVector(resultPoseNormals[i]), 490 | ); 491 | } 492 | } 493 | } 494 | -------------------------------------------------------------------------------- /src/helper/filter.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2022 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import { Vector3} from "@babylonjs/core"; 18 | import KalmanFilter from "kalmanjs"; 19 | 20 | export const VISIBILITY_THRESHOLD: number = 0.6; 21 | 22 | export interface FilterParams { 23 | R?: number, 24 | Q?: number, 25 | oneEuroCutoff?: number, 26 | oneEuroBeta?: number, 27 | type: string, 28 | gaussianSigma?: number, 29 | } 30 | 31 | // 1D Gaussian Kernel 32 | export const gaussianKernel1d = (function () { 33 | let sqr2pi = Math.sqrt(2 * Math.PI); 34 | 35 | return function gaussianKernel1d (size: number, sigma: number) { 36 | // ensure size is even and prepare variables 37 | let width = (size / 2) | 0, 38 | kernel = new Array(width * 2 + 1), 39 | norm = 1.0 / (sqr2pi * sigma), 40 | coefficient = 2 * sigma * sigma, 41 | total = 0, 42 | x; 43 | 44 | // set values and increment total 45 | for (x = -width; x <= width; x++) { 46 | total += kernel[width + x] = norm * Math.exp(-x * x / coefficient); 47 | } 48 | 49 | // divide by total to make sure the sum of all the values is equal to 1 50 | for (x = 0; x < kernel.length; x++) { 51 | kernel[x] /= total; 52 | } 53 | 54 | return kernel; 55 | }; 56 | }()); 57 | 58 | /* 59 | * Converted from https://github.com/jaantollander/OneEuroFilter. 60 | */ 61 | export class OneEuroVectorFilter { 62 | constructor( 63 | public t_prev: number, 64 | public x_prev: Vector3, 65 | private dx_prev = Vector3.Zero(), 66 | public min_cutoff = 1.0, 67 | public beta = 0.0, 68 | public d_cutoff = 1.0 69 | ) { 70 | } 71 | 72 | private static smoothing_factor(t_e: number, cutoff: number) { 73 | const r = 2 * Math.PI * cutoff * t_e; 74 | return r / (r + 1); 75 | } 76 | 77 | private static exponential_smoothing(a: number, x: Vector3, x_prev: Vector3) { 78 | return x.scale(a).addInPlace(x_prev.scale((1 - a))); 79 | } 80 | 81 | public next(t: number, x: Vector3) { 82 | const t_e = t - this.t_prev; 83 | 84 | // The filtered derivative of the signal. 85 | const a_d = OneEuroVectorFilter.smoothing_factor(t_e, this.d_cutoff); 86 | const dx = x.subtract(this.x_prev).scaleInPlace(1 / t_e); 87 | const dx_hat = OneEuroVectorFilter.exponential_smoothing(a_d, dx, this.dx_prev); 88 | 89 | // The filtered signal. 90 | const cutoff = this.min_cutoff + this.beta * dx_hat.length(); 91 | const a = OneEuroVectorFilter.smoothing_factor(t_e, cutoff); 92 | const x_hat = OneEuroVectorFilter.exponential_smoothing(a, x, this.x_prev); 93 | 94 | // Memorize the previous values. 95 | this.x_prev = x_hat; 96 | this.dx_prev = dx_hat; 97 | this.t_prev = t; 98 | 99 | return x_hat; 100 | } 101 | } 102 | export class KalmanVectorFilter { 103 | private readonly kalmanFilterX; 104 | private readonly kalmanFilterY; 105 | private readonly kalmanFilterZ; 106 | constructor( 107 | public R = 0.1, 108 | public Q = 3, 109 | ) { 110 | this.kalmanFilterX = new KalmanFilter({Q: Q, R: R}); 111 | this.kalmanFilterY = new KalmanFilter({Q: Q, R: R}); 112 | this.kalmanFilterZ = new KalmanFilter({Q: Q, R: R}); 113 | } 114 | 115 | public next(t: number, vec: Vector3) { 116 | const newValues = [ 117 | this.kalmanFilterX.filter(vec.x), 118 | this.kalmanFilterY.filter(vec.y), 119 | this.kalmanFilterZ.filter(vec.z), 120 | ] 121 | 122 | return Vector3.FromArray(newValues); 123 | } 124 | } 125 | 126 | export class GaussianVectorFilter { 127 | private _values: Vector3[] = []; 128 | get values(): Vector3[] { 129 | return this._values; 130 | } 131 | private readonly kernel: number[]; 132 | 133 | constructor( 134 | public readonly size: number, 135 | private readonly sigma: number 136 | ) { 137 | if (size < 2) throw RangeError("Filter size too short"); 138 | this.size = Math.floor(size); 139 | this.kernel = gaussianKernel1d(size, sigma); 140 | } 141 | 142 | public push(v: Vector3) { 143 | this.values.push(v); 144 | 145 | if (this.values.length === this.size + 1) { 146 | this.values.shift(); 147 | } else if (this.values.length > this.size + 1) { 148 | console.warn(`Internal queue has length longer than size: ${this.size}`); 149 | this.values.slice(-this.size); 150 | } 151 | } 152 | 153 | public reset() { 154 | this.values.length = 0; 155 | } 156 | 157 | public apply() { 158 | if (this.values.length !== this.size) return Vector3.Zero(); 159 | const ret = Vector3.Zero(); 160 | const len0 = ret.length(); 161 | for (let i = 0; i < this.size; ++i) { 162 | ret.addInPlace(this.values[i].scale(this.kernel[i])); 163 | } 164 | const len1 = ret.length(); 165 | // Normalize to original length 166 | ret.scaleInPlace(len0 / len1); 167 | 168 | return ret; 169 | } 170 | } 171 | 172 | export class EuclideanHighPassFilter { 173 | private _value: Vector3 = Vector3.Zero(); 174 | get value(): Vector3 { 175 | return this._value; 176 | } 177 | 178 | constructor( 179 | private readonly threshold: number 180 | ) {} 181 | 182 | public update(v: Vector3) { 183 | if (this.value.subtract(v).length() > this.threshold) { 184 | this._value = v; 185 | } 186 | } 187 | 188 | public reset() { 189 | this._value = Vector3.Zero(); 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/helper/landmark.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2022 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {NormalizedLandmark, Results} from "@mediapipe/holistic"; 18 | import {Nullable, Vector3} from "@babylonjs/core"; 19 | import { 20 | FilterParams, 21 | GaussianVectorFilter, 22 | KalmanVectorFilter, 23 | OneEuroVectorFilter, 24 | VISIBILITY_THRESHOLD 25 | } from "./filter"; 26 | import {objectFlip} from "./utils"; 27 | 28 | export class FilteredLandmarkVector { 29 | private mainFilter: OneEuroVectorFilter | KalmanVectorFilter; 30 | private readonly gaussianVectorFilter: Nullable = null; 31 | 32 | private _t = 0; 33 | get t(): number { 34 | return this._t; 35 | } 36 | 37 | set t(value: number) { 38 | this._t = value; 39 | } 40 | 41 | private _pos = Vector3.Zero(); 42 | get pos(): Vector3 { 43 | return this._pos; 44 | } 45 | 46 | public visibility: number | undefined = 0; 47 | 48 | constructor( 49 | params: FilterParams = { 50 | oneEuroCutoff: 0.01, 51 | oneEuroBeta: 0, 52 | type: 'OneEuro' 53 | } 54 | ) { 55 | if (params.type === "Kalman") 56 | this.mainFilter = new KalmanVectorFilter(params.R, params.Q); 57 | else if (params.type === "OneEuro") 58 | this.mainFilter = new OneEuroVectorFilter( 59 | this.t, 60 | this.pos, 61 | Vector3.Zero(), 62 | params.oneEuroCutoff, 63 | params.oneEuroBeta); 64 | else 65 | throw Error("Wrong filter type!"); 66 | if (params.gaussianSigma) 67 | this.gaussianVectorFilter = new GaussianVectorFilter(5, params.gaussianSigma); 68 | } 69 | 70 | public updatePosition(pos: Vector3, visibility?: number) { 71 | this.t += 1; 72 | 73 | // Face Mesh has no visibility 74 | if (visibility === undefined || visibility > VISIBILITY_THRESHOLD) { 75 | pos = this.mainFilter.next(this.t, pos); 76 | 77 | if (this.gaussianVectorFilter) { 78 | this.gaussianVectorFilter.push(pos); 79 | pos = this.gaussianVectorFilter.apply(); 80 | } 81 | 82 | this._pos = pos; 83 | 84 | this.visibility = visibility; 85 | } 86 | } 87 | } 88 | 89 | export type FilteredLandmarkVectorList = FilteredLandmarkVector[]; 90 | 91 | export type FilteredLandmarkVector3 = [ 92 | FilteredLandmarkVector, 93 | FilteredLandmarkVector, 94 | FilteredLandmarkVector, 95 | ]; 96 | 97 | export interface CloneableResults extends Omit { 98 | } 99 | 100 | export const POSE_LANDMARK_LENGTH = 33; 101 | export const FACE_LANDMARK_LENGTH = 478; 102 | export const HAND_LANDMARK_LENGTH = 21; 103 | 104 | export const normalizedLandmarkToVector = ( 105 | l: NormalizedLandmark, 106 | scaling = 1., 107 | reverseY = false) => { 108 | return new Vector3( 109 | l.x * scaling, 110 | reverseY ? -l.y * scaling : l.y * scaling, 111 | l.z * scaling); 112 | } 113 | export const vectorToNormalizedLandmark = (l: Vector3): NormalizedLandmark => { 114 | return {x: l.x, y: l.y, z: l.z}; 115 | }; 116 | 117 | export const HAND_LANDMARKS = { 118 | WRIST: 0, 119 | THUMB_CMC: 1, 120 | THUMB_MCP: 2, 121 | THUMB_IP: 3, 122 | THUMB_TIP: 4, 123 | INDEX_FINGER_MCP: 5, 124 | INDEX_FINGER_PIP: 6, 125 | INDEX_FINGER_DIP: 7, 126 | INDEX_FINGER_TIP: 8, 127 | MIDDLE_FINGER_MCP: 9, 128 | MIDDLE_FINGER_PIP: 10, 129 | MIDDLE_FINGER_DIP: 11, 130 | MIDDLE_FINGER_TIP: 12, 131 | RING_FINGER_MCP: 13, 132 | RING_FINGER_PIP: 14, 133 | RING_FINGER_DIP: 15, 134 | RING_FINGER_TIP: 16, 135 | PINKY_MCP: 17, 136 | PINKY_PIP: 18, 137 | PINKY_DIP: 19, 138 | PINKY_TIP: 20, 139 | }; 140 | 141 | export const HAND_LANDMARKS_BONE_MAPPING = { 142 | Hand: HAND_LANDMARKS.WRIST, 143 | ThumbProximal: HAND_LANDMARKS.THUMB_CMC, 144 | ThumbIntermediate: HAND_LANDMARKS.THUMB_MCP, 145 | ThumbDistal: HAND_LANDMARKS.THUMB_IP, 146 | IndexProximal: HAND_LANDMARKS.INDEX_FINGER_MCP, 147 | IndexIntermediate: HAND_LANDMARKS.INDEX_FINGER_PIP, 148 | IndexDistal: HAND_LANDMARKS.INDEX_FINGER_DIP, 149 | MiddleProximal: HAND_LANDMARKS.MIDDLE_FINGER_MCP, 150 | MiddleIntermediate: HAND_LANDMARKS.MIDDLE_FINGER_PIP, 151 | MiddleDistal: HAND_LANDMARKS.MIDDLE_FINGER_DIP, 152 | RingProximal: HAND_LANDMARKS.RING_FINGER_MCP, 153 | RingIntermediate: HAND_LANDMARKS.RING_FINGER_PIP, 154 | RingDistal: HAND_LANDMARKS.RING_FINGER_DIP, 155 | LittleProximal: HAND_LANDMARKS.PINKY_MCP, 156 | LittleIntermediate: HAND_LANDMARKS.PINKY_PIP, 157 | LittleDistal: HAND_LANDMARKS.PINKY_DIP, 158 | }; 159 | export const HAND_LANDMARKS_BONE_REVERSE_MAPPING: { [key: number]: string } = objectFlip(HAND_LANDMARKS_BONE_MAPPING); 160 | export type HandBoneMappingKey = keyof typeof HAND_LANDMARKS_BONE_MAPPING; 161 | 162 | export function handLandMarkToBoneName(landmark: number, isLeft: boolean) { 163 | if (!(landmark in HAND_LANDMARKS_BONE_REVERSE_MAPPING)) throw Error("Wrong landmark given!"); 164 | return (isLeft ? 'left' : 'right') + HAND_LANDMARKS_BONE_REVERSE_MAPPING[landmark]; 165 | } 166 | 167 | /* 168 | * Depth-first search/walk of a generic tree. 169 | * Also returns a map for backtracking from leaf. 170 | */ 171 | export function depthFirstSearch( 172 | rootNode: any, 173 | f: (n: any) => boolean 174 | ): [any, any] { 175 | const stack = []; 176 | const parentMap: Map = new Map(); 177 | stack.push(rootNode); 178 | 179 | while (stack.length !== 0) { 180 | // remove the first child in the stack 181 | const currentNode: any = stack.splice(-1, 1)[0]; 182 | const retVal = f(currentNode); 183 | if (retVal) return [currentNode, parentMap]; 184 | 185 | const currentChildren = currentNode.children; 186 | // add any children in the node at the top of the stack 187 | if (currentChildren !== null) { 188 | for (let index = 0; index < currentChildren.length; index++) { 189 | const child = currentChildren[index]; 190 | stack.push(child); 191 | if (!(parentMap.has(child))) { 192 | parentMap.set(child, currentNode); 193 | } 194 | } 195 | } 196 | } 197 | return [null, null]; 198 | } 199 | -------------------------------------------------------------------------------- /src/helper/quaternion.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2022 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {Nullable, Quaternion, Angle, Vector3, Plane} from "@babylonjs/core"; 18 | import {rangeCap} from "./utils"; 19 | import {Basis, quaternionBetweenBases} from "./basis"; 20 | import {vectorToNormalizedLandmark} from "./landmark"; 21 | import { 22 | FilterParams, 23 | GaussianVectorFilter, 24 | KalmanVectorFilter, 25 | } from "./filter"; 26 | 27 | export class CloneableQuaternionLite { 28 | public x: number = 0; 29 | public y: number = 0; 30 | public z: number = 0; 31 | public w: number = 1; 32 | 33 | constructor( 34 | q: Nullable, 35 | ) { 36 | if (q) { 37 | this.x = q.x; 38 | this.y = q.y; 39 | this.z = q.z; 40 | this.w = q.w; 41 | } 42 | } 43 | } 44 | 45 | export class CloneableQuaternion extends CloneableQuaternionLite { 46 | private readonly _baseBasis: Basis; 47 | get baseBasis(): Basis { 48 | return this._baseBasis; 49 | } 50 | 51 | constructor( 52 | q: Nullable, 53 | basis?: Basis 54 | ) { 55 | super(q); 56 | this._baseBasis = basis ? basis : new Basis(null); 57 | } 58 | 59 | public set(q: Quaternion) { 60 | this.x = q.x; 61 | this.y = q.y; 62 | this.z = q.z; 63 | this.w = q.w; 64 | } 65 | 66 | public rotateBasis(q: Quaternion): Basis { 67 | return this._baseBasis.rotateByQuaternion(q); 68 | } 69 | } 70 | 71 | export interface CloneableQuaternionMap { 72 | [key: string]: CloneableQuaternion 73 | } 74 | 75 | export type CloneableQuaternionList = CloneableQuaternion[]; 76 | export const cloneableQuaternionToQuaternion = (q: CloneableQuaternionLite): Quaternion => { 77 | const ret = new Quaternion(q.x, q.y, q.z, q.w); 78 | return ret; 79 | }; 80 | 81 | export class FilteredQuaternion { 82 | private mainFilter: KalmanVectorFilter; 83 | private readonly gaussianVectorFilter: Nullable = null; 84 | 85 | private _t = 0; 86 | get t(): number { 87 | return this._t; 88 | } 89 | set t(value: number) { 90 | this._t = value; 91 | } 92 | 93 | private _rot = Quaternion.Identity(); 94 | get rot(): Quaternion { 95 | return this._rot; 96 | } 97 | 98 | constructor( 99 | params: FilterParams = { 100 | R: 1, 101 | Q: 1, 102 | type: 'Kalman' 103 | } 104 | ) { 105 | if (params.type === "Kalman") 106 | this.mainFilter = new KalmanVectorFilter(params.R, params.Q); 107 | else 108 | throw Error("Wrong filter type!"); 109 | if (params.gaussianSigma) 110 | this.gaussianVectorFilter = new GaussianVectorFilter(5, params.gaussianSigma); 111 | } 112 | 113 | public updateRotation(rot: Quaternion) { 114 | this.t += 1; 115 | let angles = rot.toEulerAngles(); 116 | angles = this.mainFilter.next(this.t, angles); 117 | 118 | if (this.gaussianVectorFilter) { 119 | this.gaussianVectorFilter.push(angles); 120 | angles = this.gaussianVectorFilter.apply(); 121 | } 122 | 123 | this._rot = Quaternion.FromEulerVector(angles); 124 | } 125 | } 126 | 127 | export type FilteredQuaternionList = FilteredQuaternion[]; 128 | 129 | 130 | export enum AXIS { 131 | x, 132 | y, 133 | z, 134 | xy, 135 | yz, 136 | xz, 137 | xyz, 138 | none = 10 139 | } 140 | 141 | // Convenience functions 142 | export const RadToDeg = (r: number) => { 143 | return Angle.FromRadians(r).degrees(); 144 | } 145 | export const DegToRad = (d: number) => { 146 | return Angle.FromDegrees(d).radians(); 147 | } 148 | 149 | /** 150 | * Check a quaternion is valid 151 | * @param q Input quaternion 152 | */ 153 | export function checkQuaternion(q: Quaternion) { 154 | return Number.isFinite(q.x) && Number.isFinite(q.y) && Number.isFinite(q.z) && Number.isFinite(q.w); 155 | } 156 | 157 | // Similar to three.js Quaternion.setFromUnitVectors 158 | export const quaternionBetweenVectors = ( 159 | v1: Vector3, v2: Vector3, 160 | ): Quaternion => { 161 | const angle = Vector3.GetAngleBetweenVectors(v1, v2, Vector3.Cross(v1, v2)) 162 | const axis = Vector3.Cross(v1, v2); 163 | axis.normalize(); 164 | return Quaternion.RotationAxis(axis, angle); 165 | }; 166 | /** 167 | * Same as above, Euler angle version 168 | * @param v1 Input rotation in degrees 1 169 | * @param v2 Input rotation in degrees 2 170 | * @param remapDegree Whether re-map degrees 171 | */ 172 | export const degreeBetweenVectors = ( 173 | v1: Vector3, v2: Vector3, remapDegree = false 174 | ) => { 175 | return quaternionToDegrees(quaternionBetweenVectors(v1, v2), remapDegree); 176 | }; 177 | /** 178 | * Re-map degrees to -180 to 180 179 | * @param deg Input angle in Degrees 180 | */ 181 | export const remapDegreeWithCap = (deg: number) => { 182 | deg = rangeCap(deg, 0, 360); 183 | return deg < 180 ? deg : deg - 360; 184 | } 185 | /** 186 | * Convert quaternions to degrees 187 | * @param q Input quaternion 188 | * @param remapDegree whether re-map degrees 189 | */ 190 | export const quaternionToDegrees = ( 191 | q: Quaternion, 192 | remapDegree = false, 193 | ) => { 194 | const angles = q.toEulerAngles(); 195 | const remapFn = remapDegree ? remapDegreeWithCap : (x: number) => x; 196 | return new Vector3( 197 | remapFn(RadToDeg(angles.x)), 198 | remapFn(RadToDeg(angles.y)), 199 | remapFn(RadToDeg(angles.z)), 200 | ); 201 | }; 202 | 203 | /** 204 | * Check whether two directions are close enough within a small values 205 | * @param v1 Input direction 1 206 | * @param v2 Input direction 2 207 | * @param eps Error threshold 208 | */ 209 | export function vectorsSameDirWithinEps(v1: Vector3, v2: Vector3, eps = 1e-6) { 210 | return v1.cross(v2).length() < eps && Vector3.Dot(v1, v2) > 0; 211 | } 212 | 213 | /** 214 | * Test whether two quaternions have equal effects 215 | * @param q1 Input quaternion 1 216 | * @param q2 Input quaternion 2 217 | */ 218 | export function testQuaternionEqualsByVector(q1: Quaternion, q2: Quaternion) { 219 | const testVec = Vector3.One(); 220 | const testVec1 = Vector3.Zero(); 221 | const testVec2 = Vector3.One(); 222 | testVec.rotateByQuaternionToRef(q1, testVec1); 223 | testVec.rotateByQuaternionToRef(q2, testVec2); 224 | return vectorsSameDirWithinEps(testVec1, testVec2); 225 | } 226 | 227 | /** 228 | * Same as above, Euler angle version 229 | * @param d1 Input degrees 1 230 | * @param d2 Input degrees 2 231 | */ 232 | export function degreesEqualInQuaternion( 233 | d1: Vector3, d2: Vector3 234 | ) { 235 | const q1 = Quaternion.FromEulerAngles(DegToRad(d1.x), DegToRad(d1.y), DegToRad(d1.z)); 236 | const q2 = Quaternion.FromEulerAngles(DegToRad(d2.x), DegToRad(d2.y), DegToRad(d2.z)); 237 | return testQuaternionEqualsByVector(q1, q2); 238 | } 239 | 240 | /** 241 | * Reverse rotation Euler angles on given axes 242 | * @param q Input quaternion 243 | * @param axis Axes to reverse 244 | */ 245 | export const reverseRotation = (q: Quaternion, axis: AXIS) => { 246 | if (axis === AXIS.none) return q; 247 | const angles = q.toEulerAngles(); 248 | switch (axis) { 249 | case AXIS.x: 250 | angles.x = -angles.x; 251 | break; 252 | case AXIS.y: 253 | angles.y = -angles.y; 254 | break; 255 | case AXIS.z: 256 | angles.z = -angles.z; 257 | break; 258 | case AXIS.xy: 259 | angles.x = -angles.x; 260 | angles.y = -angles.y; 261 | break; 262 | case AXIS.yz: 263 | angles.y = -angles.y; 264 | angles.z = -angles.z; 265 | break; 266 | case AXIS.xz: 267 | angles.x = -angles.x; 268 | angles.z = -angles.z; 269 | break; 270 | case AXIS.xyz: 271 | angles.x = -angles.x; 272 | angles.y = -angles.y; 273 | angles.z = -angles.z; 274 | break; 275 | default: 276 | throw Error("Unknown axis!"); 277 | } 278 | return Quaternion.RotationYawPitchRoll(angles.y, angles.x, angles.z); 279 | } 280 | /** 281 | * Remove rotation on given axes. 282 | * Optionally capping rotation (in Euler angles) on two axes. 283 | * This operation assumes re-mapped degrees. 284 | * @param q Input quaternion 285 | * @param axis Axes to remove 286 | * @param capAxis1 Capping axis 1 287 | * @param capLow1 Axis 1 lower range 288 | * @param capHigh1 Axis 1 higher range 289 | * @param capAxis2 Capping axis 2 290 | * @param capLow2 Axis 2 lower range 291 | * @param capHigh2 Axis 2 higher range 292 | */ 293 | export const removeRotationAxisWithCap = ( 294 | q: Quaternion, 295 | axis: AXIS, 296 | capAxis1?: AXIS, 297 | capLow1?: number, 298 | capHigh1?: number, 299 | capAxis2?: AXIS, 300 | capLow2?: number, 301 | capHigh2?: number, 302 | ) => { 303 | const angles = quaternionToDegrees(q, true); 304 | switch (axis) { 305 | case AXIS.none: 306 | break; 307 | case AXIS.x: 308 | angles.x = 0; 309 | break; 310 | case AXIS.y: 311 | angles.y = 0; 312 | break; 313 | case AXIS.z: 314 | angles.z = 0; 315 | break; 316 | case AXIS.xy: 317 | angles.x = 0; 318 | angles.y = 0; 319 | break; 320 | case AXIS.yz: 321 | angles.y = 0; 322 | angles.z = 0; 323 | break; 324 | case AXIS.xz: 325 | angles.x = 0; 326 | angles.z = 0; 327 | break; 328 | case AXIS.xyz: 329 | angles.x = 0; 330 | angles.y = 0; 331 | angles.z = 0; 332 | break; 333 | default: 334 | throw Error("Unknown axis!"); 335 | } 336 | if (capAxis1 !== undefined && capLow1 !== undefined && capHigh1 !== undefined) { 337 | switch (capAxis1 as AXIS) { 338 | case AXIS.x: 339 | angles.x = rangeCap(angles.x, capLow1, capHigh1); 340 | break; 341 | case AXIS.y: 342 | angles.y = rangeCap(angles.y, capLow1, capHigh1); 343 | break; 344 | case AXIS.z: 345 | angles.z = rangeCap(angles.z, capLow1, capHigh1); 346 | break; 347 | default: 348 | throw Error("Unknown cap axis!"); 349 | } 350 | } 351 | if (capAxis2 !== undefined && capLow2 !== undefined && capHigh2 !== undefined) { 352 | switch (capAxis2 as AXIS) { 353 | case AXIS.x: 354 | angles.x = rangeCap(angles.x, capLow2, capHigh2); 355 | break; 356 | case AXIS.y: 357 | angles.y = rangeCap(angles.y, capLow2, capHigh2); 358 | break; 359 | case AXIS.z: 360 | angles.z = rangeCap(angles.z, capLow2, capHigh2); 361 | break; 362 | default: 363 | throw Error("Unknown cap axis!"); 364 | } 365 | } 366 | return Quaternion.RotationYawPitchRoll( 367 | DegToRad(angles.y), 368 | DegToRad(angles.x), 369 | DegToRad(angles.z)); 370 | } 371 | /** 372 | * Switch rotation axes. 373 | * @param q Input quaternion 374 | * @param axis1 Axis 1 to switch 375 | * @param axis2 Axis 2 to switch 376 | */ 377 | export const exchangeRotationAxis = ( 378 | q: Quaternion, 379 | axis1: AXIS, 380 | axis2: AXIS, 381 | ) => { 382 | const angles: number[] = []; 383 | q.toEulerAngles().toArray(angles); 384 | const angle1 = angles[axis1]; 385 | const angle2 = angles[axis2]; 386 | const temp = angle1; 387 | angles[axis1] = angle2; 388 | angles[axis2] = temp; 389 | return Quaternion.FromEulerAngles( 390 | angles[0], 391 | angles[1], 392 | angles[2]); 393 | } 394 | 395 | export function printQuaternion(q: Quaternion, s?: string) { 396 | console.log(s, vectorToNormalizedLandmark(quaternionToDegrees(q, true))); 397 | } 398 | 399 | 400 | /** 401 | * Result is in Radian on unit sphere (r = 1). 402 | * Canonical ISO 80000-2:2019 convention. 403 | * @param pos Euclidean local position 404 | * @param basis Local coordinate system basis 405 | */ 406 | export function calcSphericalCoord( 407 | pos: Vector3, basis: Basis, 408 | ) { 409 | const qToOriginal = Quaternion.Inverse(Quaternion.RotationQuaternionFromAxis( 410 | basis.x.clone(), basis.y.clone(), basis.z.clone())).normalize(); 411 | const posInOriginal = Vector3.Zero(); 412 | pos.rotateByQuaternionToRef(qToOriginal, posInOriginal); 413 | posInOriginal.normalize(); 414 | 415 | // Calculate theta and phi 416 | const x = posInOriginal.x; 417 | const y = posInOriginal.y; 418 | const z = posInOriginal.z; 419 | 420 | const theta = Math.acos(z); 421 | const phi = Math.atan2(y, x); 422 | 423 | return [theta, phi]; 424 | } 425 | 426 | /** 427 | * Assuming rotation starts from (1, 0, 0) in local coordinate system. 428 | * @param basis Local coordinate system basis 429 | * @param theta Polar angle 430 | * @param phi Azimuthal angle 431 | * @param prevQuaternion Parent quaternion to the local system 432 | */ 433 | export function sphericalToQuaternion( 434 | basis: Basis, theta: number, phi: number, 435 | prevQuaternion: Quaternion) { 436 | const xTz = Quaternion.RotationAxis(basis.y.clone(), -Math.PI / 2); 437 | const xTzBasis = basis.rotateByQuaternion(xTz); 438 | const q1 = Quaternion.RotationAxis(xTzBasis.x.clone(), phi); 439 | const q1Basis = xTzBasis.rotateByQuaternion(q1); 440 | const q2 = Quaternion.RotationAxis(q1Basis.y.clone(), theta); 441 | const q2Basis = q1Basis.rotateByQuaternion(q2); 442 | 443 | // Force result to face front 444 | const planeXZ = Plane.FromPositionAndNormal(Vector3.Zero(), basis.y.clone()); 445 | // const intermBasis = basis.rotateByQuaternion(xTz.multiply(q1).multiplyInPlace(q2)); 446 | const intermBasis = q2Basis; 447 | const newBasisZ = Vector3.Cross(intermBasis.x.clone(), planeXZ.normal); 448 | const newBasisY = Vector3.Cross(newBasisZ, intermBasis.x.clone()); 449 | const newBasis = new Basis([intermBasis.x, newBasisY, newBasisZ]); 450 | 451 | return quaternionBetweenBases(basis, newBasis, prevQuaternion); 452 | } 453 | 454 | // Scale rotation angles in place 455 | export function scaleRotation(quaternion: Quaternion, scale: number) { 456 | const angles = quaternion.toEulerAngles(); 457 | angles.scaleInPlace(scale); 458 | return Quaternion.FromEulerVector(angles); 459 | } 460 | -------------------------------------------------------------------------------- /src/helper/test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2022 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {Basis, getBasis, quaternionBetweenBases, Vector33} from "./basis"; 18 | import {Quaternion, Vector3} from "@babylonjs/core"; 19 | import { 20 | calcSphericalCoord, 21 | degreesEqualInQuaternion, 22 | DegToRad, 23 | quaternionBetweenVectors, 24 | quaternionToDegrees, RadToDeg, sphericalToQuaternion, vectorsSameDirWithinEps 25 | } from "./quaternion"; 26 | import {vectorToNormalizedLandmark} from "./landmark"; 27 | 28 | export function test_quaternionBetweenBases3() { 29 | console.log("Testing quaternionBetweenBases3"); 30 | 31 | const remap = false; 32 | const basis0: Basis = new Basis(null); 33 | 34 | // X 90 35 | const deg10 = new Vector3(90, 0, 0); 36 | const basis1 = basis0.rotateByQuaternion(Quaternion.FromEulerAngles(DegToRad(deg10.x), DegToRad(deg10.y), DegToRad(deg10.z))); 37 | basis1.verifyBasis(); 38 | const degrees1 = quaternionToDegrees(quaternionBetweenBases(basis0, basis1), remap); 39 | console.log(vectorToNormalizedLandmark(degrees1), degreesEqualInQuaternion(deg10, degrees1)); 40 | 41 | // Y 225 42 | const deg20 = new Vector3(0, 225, 0); 43 | const basis2 = basis0.rotateByQuaternion( 44 | Quaternion.FromEulerAngles(DegToRad(deg20.x), DegToRad(deg20.y), DegToRad(deg20.z))); 45 | basis2.verifyBasis(); 46 | const degrees2 = quaternionToDegrees(quaternionBetweenBases(basis0, basis2), remap); 47 | console.log(vectorToNormalizedLandmark(degrees2), degreesEqualInQuaternion(deg20, degrees2)); 48 | 49 | // Z 135 50 | const deg30 = new Vector3(0, 0, 135); 51 | const basis3 = basis0.rotateByQuaternion( 52 | Quaternion.FromEulerAngles(DegToRad(deg30.x), DegToRad(deg30.y), DegToRad(deg30.z))); 53 | basis3.verifyBasis(); 54 | const degrees3 = quaternionToDegrees(quaternionBetweenBases(basis0, basis3), remap); 55 | console.log(vectorToNormalizedLandmark(degrees3), degreesEqualInQuaternion(deg30, degrees3)); 56 | 57 | // X Y 135 58 | const deg40 = new Vector3(135, 135, 0); 59 | const basis4 = basis0.rotateByQuaternion( 60 | Quaternion.FromEulerAngles(DegToRad(deg40.x), DegToRad(deg40.y), DegToRad(deg40.z))); 61 | basis4.verifyBasis(); 62 | const degrees4 = quaternionToDegrees(quaternionBetweenBases(basis0, basis4), remap); 63 | console.log(vectorToNormalizedLandmark(degrees4), degreesEqualInQuaternion(deg40, degrees4)); 64 | 65 | // X Z 90 66 | const deg50 = new Vector3(90, 0, 90); 67 | const basis5 = basis0.rotateByQuaternion( 68 | Quaternion.FromEulerAngles(DegToRad(deg50.x), DegToRad(deg50.y), DegToRad(deg50.z))); 69 | basis5.verifyBasis(); 70 | const degrees5 = quaternionToDegrees(quaternionBetweenBases(basis0, basis5), remap); 71 | console.log(vectorToNormalizedLandmark(degrees5), degreesEqualInQuaternion(deg50, degrees5)); 72 | 73 | // Y Z 225 74 | const deg60 = new Vector3(0, 225, 225); 75 | const basis6 = basis0.rotateByQuaternion( 76 | Quaternion.FromEulerAngles(DegToRad(deg60.x), DegToRad(deg60.y), DegToRad(deg60.z))); 77 | basis6.verifyBasis(); 78 | const degrees6 = quaternionToDegrees(quaternionBetweenBases(basis0, basis6), remap); 79 | console.log(vectorToNormalizedLandmark(degrees6), degreesEqualInQuaternion(deg60, degrees6)); 80 | 81 | // X Y Z 135 82 | const deg70 = new Vector3(135, 135, 135); 83 | const basis7 = basis0.rotateByQuaternion( 84 | Quaternion.FromEulerAngles(DegToRad(deg70.x), DegToRad(deg70.y), DegToRad(deg70.z))); 85 | basis7.verifyBasis(); 86 | const degrees7 = quaternionToDegrees(quaternionBetweenBases(basis0, basis7), remap); 87 | console.log(vectorToNormalizedLandmark(degrees7), degreesEqualInQuaternion(deg70, degrees7)); 88 | } 89 | 90 | export function test_getBasis() { 91 | console.log("Testing getBasis"); 92 | 93 | const axes0: Vector33 = [ 94 | new Vector3(0, 0, 0), 95 | new Vector3(2, 0, 0), 96 | new Vector3(1, 1, 0) 97 | ]; 98 | const axes = getBasis(axes0); 99 | console.log(vectorToNormalizedLandmark(axes.x)); 100 | console.log(vectorToNormalizedLandmark(axes.y)); 101 | console.log(vectorToNormalizedLandmark(axes.z)); 102 | } 103 | 104 | export function test_quaternionBetweenVectors() { 105 | console.log("Testing quaternionBetweenVectors"); 106 | 107 | const remap = true; 108 | const vec0 = Vector3.One(); 109 | 110 | // X 90 111 | const vec1 = Vector3.Zero(); 112 | const deg10 = new Vector3(90, 0, 0); 113 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 114 | DegToRad(deg10.x), DegToRad(deg10.y), DegToRad(deg10.z), 115 | ), vec1); 116 | const deg11 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec1), remap); 117 | console.log(vectorToNormalizedLandmark(deg11), degreesEqualInQuaternion(deg10, deg11)); 118 | 119 | // Y 225 120 | const vec2 = Vector3.Zero(); 121 | const deg20 = new Vector3(0, 225, 0); 122 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 123 | DegToRad(deg20.x), DegToRad(deg20.y), DegToRad(deg20.z), 124 | ), vec2); 125 | const deg21 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec2), remap); 126 | console.log(vectorToNormalizedLandmark(deg21), degreesEqualInQuaternion(deg20, deg21)); 127 | 128 | // Z 135 129 | const vec3 = Vector3.Zero(); 130 | const deg30 = new Vector3(0, 0, 135); 131 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 132 | DegToRad(deg30.x), DegToRad(deg30.y), DegToRad(deg30.z), 133 | ), vec3); 134 | const deg31 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec3), remap); 135 | console.log(vectorToNormalizedLandmark(deg31), degreesEqualInQuaternion(deg30, deg31)); 136 | 137 | // X Y 90 138 | const vec4 = Vector3.Zero(); 139 | const deg40 = new Vector3(90, 90, 0); 140 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 141 | DegToRad(deg40.x), DegToRad(deg40.y), DegToRad(deg40.z), 142 | ), vec4); 143 | const deg41 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec4), remap); 144 | console.log(vectorToNormalizedLandmark(deg41), degreesEqualInQuaternion(deg40, deg41)); 145 | 146 | // X Z 135 147 | const vec5 = Vector3.Zero(); 148 | const deg50 = new Vector3(135, 0, 135); 149 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 150 | DegToRad(deg50.x), DegToRad(deg50.y), DegToRad(deg50.z), 151 | ), vec5); 152 | const deg51 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec5), remap); 153 | console.log(vectorToNormalizedLandmark(deg51), degreesEqualInQuaternion(deg50, deg51)); 154 | 155 | // Y Z 45 156 | const vec6 = Vector3.Zero(); 157 | const deg60 = new Vector3(0, 45, 45); 158 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 159 | DegToRad(deg60.x), DegToRad(deg60.y), DegToRad(deg60.z), 160 | ), vec6); 161 | const deg61 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec6), remap); 162 | console.log(vectorToNormalizedLandmark(deg61), degreesEqualInQuaternion(deg60, deg61)); 163 | 164 | // X Y Z 135 165 | const vec7 = Vector3.Zero(); 166 | const deg70 = new Vector3(135, 135, 135); 167 | vec0.rotateByQuaternionToRef(Quaternion.FromEulerAngles( 168 | DegToRad(deg70.x), DegToRad(deg70.y), DegToRad(deg70.z), 169 | ), vec7); 170 | const deg71 = quaternionToDegrees(quaternionBetweenVectors(vec0, vec7), remap); 171 | console.log(vectorToNormalizedLandmark(deg71), degreesEqualInQuaternion(deg70, deg71)); 172 | } 173 | 174 | export function test_calcSphericalCoord(rotationVector = Vector3.Zero()) { 175 | // TODO sphericalToQuaternion is using wrong basis 176 | console.log("Testing calcSphericalCoord"); 177 | 178 | const vec0 = new Vector3(1, 1, 1); 179 | const basisOriginal = new Basis(null); 180 | const parentQuaternion = Quaternion.FromEulerVector(rotationVector); 181 | const basis0 = basisOriginal.rotateByQuaternion(parentQuaternion); 182 | 183 | // X 90 184 | const deg10 = new Vector3(90, 0, 0); 185 | const q11 = Quaternion.FromEulerAngles( 186 | DegToRad(deg10.x), DegToRad(deg10.y), DegToRad(deg10.z), 187 | ); 188 | const vec10 = Vector3.Zero(); 189 | vec0.rotateByQuaternionToRef(q11, vec10); 190 | const [rady1, radz1] = calcSphericalCoord(vec10, basis0); 191 | const vec11 = Vector3.Zero(); 192 | const q12 = sphericalToQuaternion(basis0, rady1, radz1, parentQuaternion); 193 | basis0.x.rotateByQuaternionToRef(q12, vec11); 194 | console.log(RadToDeg(rady1), RadToDeg(radz1), vectorsSameDirWithinEps(vec10, vec11)); 195 | 196 | // Y 225 197 | const deg20 = new Vector3(0, 225, 0); 198 | const q21 = Quaternion.FromEulerAngles( 199 | DegToRad(deg20.x), DegToRad(deg20.y), DegToRad(deg20.z), 200 | ); 201 | const vec20 = Vector3.Zero(); 202 | vec0.rotateByQuaternionToRef(q21, vec20); 203 | const [rady2, radz2] = calcSphericalCoord(vec20, basis0); 204 | const vec21 = Vector3.Zero(); 205 | const q22 = sphericalToQuaternion(basis0, rady2, radz2, parentQuaternion); 206 | basis0.x.rotateByQuaternionToRef(q22, vec21); 207 | console.log(RadToDeg(rady2), RadToDeg(radz2), vectorsSameDirWithinEps(vec20, vec21)); 208 | 209 | // Z 135 210 | const deg30 = new Vector3(0, 0, 135); 211 | const q31 = Quaternion.FromEulerAngles( 212 | DegToRad(deg30.x), DegToRad(deg30.y), DegToRad(deg30.z), 213 | ); 214 | const vec30 = Vector3.Zero(); 215 | vec0.rotateByQuaternionToRef(q31, vec30); 216 | const [rady3, radz3] = calcSphericalCoord(vec30, basis0); 217 | const vec31 = Vector3.Zero(); 218 | const q32 = sphericalToQuaternion(basis0, rady3, radz3, parentQuaternion); 219 | basis0.x.rotateByQuaternionToRef(q32, vec31); 220 | console.log(RadToDeg(rady3), RadToDeg(radz3), vectorsSameDirWithinEps(vec30, vec31)); 221 | 222 | // X Y 90 223 | const deg40 = new Vector3(90, 90, 0); 224 | const q41 = Quaternion.FromEulerAngles( 225 | DegToRad(deg40.x), DegToRad(deg40.y), DegToRad(deg40.z), 226 | ); 227 | const vec40 = Vector3.Zero(); 228 | vec0.rotateByQuaternionToRef(q41, vec40); 229 | const [rady4, radz4] = calcSphericalCoord(vec40, basis0); 230 | const vec41 = Vector3.Zero(); 231 | const q42 = sphericalToQuaternion(basis0, rady4, radz4, parentQuaternion); 232 | basis0.x.rotateByQuaternionToRef(q42, vec41); 233 | console.log(RadToDeg(rady4), RadToDeg(radz4), vectorsSameDirWithinEps(vec40, vec41)); 234 | 235 | // X Z 135 236 | const deg50 = new Vector3(135, 0, 135); 237 | const q51 = Quaternion.FromEulerAngles( 238 | DegToRad(deg50.x), DegToRad(deg50.y), DegToRad(deg50.z), 239 | ); 240 | const vec50 = Vector3.Zero(); 241 | vec0.rotateByQuaternionToRef(q51, vec50); 242 | const [rady5, radz5] = calcSphericalCoord(vec50, basis0); 243 | const vec51 = Vector3.Zero(); 244 | const q52 = sphericalToQuaternion(basis0, rady5, radz5, parentQuaternion); 245 | basis0.x.rotateByQuaternionToRef(q52, vec51); 246 | console.log(RadToDeg(rady5), RadToDeg(radz5), vectorsSameDirWithinEps(vec50, vec51)); 247 | 248 | // Y Z 45 249 | const deg60 = new Vector3(0, 45, 45); 250 | const q61 = Quaternion.FromEulerAngles( 251 | DegToRad(deg60.x), DegToRad(deg60.y), DegToRad(deg60.z), 252 | ); 253 | const vec60 = Vector3.Zero(); 254 | vec0.rotateByQuaternionToRef(q61, vec60); 255 | const [rady6, radz6] = calcSphericalCoord(vec60, basis0); 256 | const vec61 = Vector3.Zero(); 257 | const q62 = sphericalToQuaternion(basis0, rady6, radz6, parentQuaternion); 258 | basis0.x.rotateByQuaternionToRef(q62, vec61); 259 | console.log(RadToDeg(rady6), RadToDeg(radz6), vectorsSameDirWithinEps(vec60, vec61)); 260 | 261 | // X Y Z 135 262 | const deg70 = new Vector3(135, 135, 135); 263 | const q71 = Quaternion.FromEulerAngles( 264 | DegToRad(deg70.x), DegToRad(deg70.y), DegToRad(deg70.z), 265 | ); 266 | const vec70 = Vector3.Zero(); 267 | vec0.rotateByQuaternionToRef(q71, vec70); 268 | const [rady7, radz7] = calcSphericalCoord(vec70, basis0); 269 | const vec71 = Vector3.Zero(); 270 | const q72 = sphericalToQuaternion(basis0, rady7, radz7, parentQuaternion); 271 | basis0.x.rotateByQuaternionToRef(q72, vec71); 272 | console.log(RadToDeg(rady7), RadToDeg(radz7), vectorsSameDirWithinEps(vec70, vec71)); 273 | } 274 | -------------------------------------------------------------------------------- /src/helper/utils.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import {Plane, Vector3, Curve3, ILoadingScreen} from "@babylonjs/core"; 18 | 19 | export function initArray(length: number, initializer: (i: number) => T) { 20 | let arr = new Array(length); 21 | for (let i = 0; i < length; i++) 22 | arr[i] = initializer(i); 23 | return arr; 24 | } 25 | 26 | export function range(start: number, end: number, step: number) { 27 | return Array.from( 28 | {length: Math.ceil((end - start) / step)}, 29 | (_, i) => start + i * step 30 | ); 31 | } 32 | 33 | export function linspace(start: number, end: number, div: number) { 34 | const step = (end - start) / div; 35 | return Array.from( 36 | {length: div}, 37 | (_, i) => start + i * step 38 | ); 39 | } 40 | 41 | export function objectFlip(obj: any) { 42 | const ret: any = {}; 43 | Object.keys(obj).forEach((key: any) => { 44 | ret[obj[key]] = key; 45 | }); 46 | return ret; 47 | } 48 | 49 | export const rangeCap = ( 50 | v: number, 51 | min: number, 52 | max: number 53 | ) => { 54 | if (min > max) { 55 | const tmp = max; 56 | max = min; 57 | min = tmp; 58 | } 59 | return Math.max(Math.min(v, max), min); 60 | } 61 | export const remapRange = ( 62 | v: number, 63 | src_low: number, 64 | src_high: number, 65 | dst_low: number, 66 | dst_high: number 67 | ) => { 68 | return dst_low + (v - src_low) * (dst_high - dst_low) / (src_high - src_low); 69 | }; 70 | export const remapRangeWithCap = ( 71 | v: number, 72 | src_low: number, 73 | src_high: number, 74 | dst_low: number, 75 | dst_high: number 76 | ) => { 77 | const v1 = rangeCap(v, src_low, src_high); 78 | return dst_low + (v1 - src_low) * (dst_high - dst_low) / (src_high - src_low); 79 | }; 80 | export const remapRangeNoCap = ( 81 | v: number, 82 | src_low: number, 83 | src_high: number, 84 | dst_low: number, 85 | dst_high: number 86 | ) => { 87 | return dst_low + (v - src_low) * (dst_high - dst_low) / (src_high - src_low); 88 | }; 89 | export function validVector3(v: Vector3) { 90 | return Number.isFinite(v.x) && Number.isFinite(v.y) && Number.isFinite(v.z); 91 | } 92 | 93 | export type KeysMatching = { [K in keyof T]-?: T[K] extends V ? K : never }[keyof T]; 94 | 95 | // type MethodKeysOfA = KeysMatching; 96 | 97 | export type IfEquals = 98 | (() => T extends X ? 1 : 2) extends (() => T extends Y ? 1 : 2) ? A : B; 99 | export type ReadonlyKeys = { 100 | [P in keyof T]-?: IfEquals<{ [Q in P]: T[P] }, { -readonly [Q in P]: T[P] }, never, P> 101 | }[keyof T]; 102 | 103 | // type ReadonlyKeysOfA = ReadonlyKeys; 104 | 105 | export function setEqual(as: Set, bs: Set) { 106 | if (as.size !== bs.size) return false; 107 | for (const a of as) if (!bs.has(a)) return false; 108 | return true; 109 | } 110 | 111 | export function projectVectorOnPlane(projPlane: Plane, vec: Vector3) { 112 | return vec.subtract(projPlane.normal.scale(Vector3.Dot(vec, projPlane.normal))); 113 | } 114 | 115 | export function round(value: number, precision: number) { 116 | const multiplier = Math.pow(10, precision || 0); 117 | return Math.round(value * multiplier) / multiplier; 118 | } 119 | 120 | /** 121 | * Simple fixed length FIFO queue. 122 | */ 123 | export class fixedLengthQueue { 124 | private _values: T[] = []; 125 | get values(): T[] { 126 | return this._values; 127 | } 128 | 129 | constructor(public readonly size: number) { 130 | } 131 | 132 | public push(v: T) { 133 | this.values.push(v); 134 | 135 | if (this.values.length === this.size + 1) { 136 | this.values.shift(); 137 | } else if (this.values.length > this.size + 1) { 138 | console.warn(`Internal queue has length longer than size ${this.size}: Got length ${this.values.length}`); 139 | this._values = this.values.slice(-this.size); 140 | } 141 | } 142 | 143 | public concat(arr: T[]) { 144 | this._values = this.values.concat(arr); 145 | 146 | if (this.values.length > this.size) { 147 | this._values = this.values.slice(-this.size); 148 | } 149 | } 150 | 151 | public pop() { 152 | return this.values.shift(); 153 | } 154 | 155 | public first() { 156 | if (this._values.length > 0) 157 | return this.values[0]; 158 | else 159 | return null; 160 | } 161 | 162 | public last() { 163 | if (this._values.length > 0) 164 | return this._values[this._values.length - 1]; 165 | else 166 | return null; 167 | } 168 | 169 | public reset() { 170 | this.values.length = 0; 171 | } 172 | 173 | public length() { 174 | return this.values.length; 175 | } 176 | } 177 | 178 | export function findPoint(curve: Curve3, x: number, eps = 0.001) { 179 | const pts = curve.getPoints(); 180 | if (x > pts[0].x) return pts[0].y; 181 | else if (x < pts[pts.length - 1].x) return pts[pts.length - 1].y; 182 | for (let i = 0; i < pts.length; ++i) { 183 | if (Math.abs(x - pts[i].x) < eps) return pts[i].y; 184 | } 185 | return 0; 186 | } 187 | 188 | export const LR = ["left", "right"]; 189 | 190 | export class CustomLoadingScreen implements ILoadingScreen { 191 | //optional, but needed due to interface definitions 192 | public loadingUIBackgroundColor: string = ''; 193 | public loadingUIText: string = ''; 194 | 195 | constructor( 196 | private readonly renderingCanvas: HTMLCanvasElement, 197 | private loadingDiv: HTMLDivElement 198 | ) {} 199 | 200 | public displayLoadingUI() { 201 | if (!this.loadingDiv) return; 202 | if (this.loadingDiv.style.display === 'none') { 203 | // Do not add a loading screen if there is already one 204 | this.loadingDiv.style.display = "initial"; 205 | } 206 | 207 | // this._resizeLoadingUI(); 208 | // window.addEventListener("resize", this._resizeLoadingUI); 209 | } 210 | 211 | public hideLoadingUI() { 212 | if (this.loadingDiv) 213 | this.loadingDiv.style.display = "none"; 214 | } 215 | 216 | // private _resizeLoadingUI = () => { 217 | // const canvasRect = this.renderingCanvas.getBoundingClientRect(); 218 | // const canvasPositioning = window.getComputedStyle(this.renderingCanvas).position; 219 | // 220 | // if (!this._loadingDiv) { 221 | // return; 222 | // } 223 | // 224 | // this._loadingDiv.style.position = (canvasPositioning === "fixed") ? "fixed" : "absolute"; 225 | // this._loadingDiv.style.left = canvasRect.left + "px"; 226 | // this._loadingDiv.style.top = canvasRect.top + "px"; 227 | // this._loadingDiv.style.width = canvasRect.width + "px"; 228 | // this._loadingDiv.style.height = canvasRect.height + "px"; 229 | // } 230 | } 231 | 232 | export function pointLineDistance( 233 | point: Vector3, 234 | lineEndA: Vector3, lineEndB: Vector3 235 | ) { 236 | const lineDir = lineEndB.subtract(lineEndA).normalize(); 237 | const pProj = lineEndA.add( 238 | lineDir.scale( 239 | Vector3.Dot(point.subtract(lineEndA), lineDir) 240 | / Vector3.Dot(lineDir, lineDir))); 241 | return point.subtract(pProj).length(); 242 | } 243 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | export * from "./v3d-web"; 18 | export type {CloneableQuaternionMap} from "./helper/quaternion"; 19 | export type {HolisticOptions} from "./mediapipe"; 20 | -------------------------------------------------------------------------------- /src/mediapipe.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import * as Comlink from "comlink"; 18 | import { 19 | ControlPanel, 20 | FPS, 21 | OptionMap, 22 | Slider, 23 | StaticText, 24 | Toggle 25 | } from "@mediapipe/control_utils"; 26 | import { 27 | FACEMESH_FACE_OVAL, 28 | FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, 29 | FACEMESH_LEFT_IRIS, FACEMESH_LIPS, 30 | FACEMESH_RIGHT_EYE, FACEMESH_RIGHT_EYEBROW, 31 | FACEMESH_RIGHT_IRIS, 32 | HAND_CONNECTIONS, 33 | Holistic, 34 | NormalizedLandmark, 35 | NormalizedLandmarkList, 36 | Options, POSE_CONNECTIONS, 37 | POSE_LANDMARKS, POSE_LANDMARKS_LEFT, POSE_LANDMARKS_RIGHT, 38 | Results 39 | } from "@mediapipe/holistic"; 40 | import {Data, drawConnectors, drawLandmarks, lerp} from "@mediapipe/drawing_utils"; 41 | import {Poses} from "./worker/pose-processing"; 42 | import {debugInfo, updateBuffer} from "./core"; 43 | import {Vector3, Nullable} from "@babylonjs/core"; 44 | import {VRMManager} from "v3d-core/dist/src/importer/babylon-vrm-loader/src"; 45 | 46 | function removeElements( 47 | landmarks: NormalizedLandmarkList, elements: number[]) { 48 | for (const element of elements) { 49 | delete landmarks[element]; 50 | } 51 | } 52 | 53 | function removeLandmarks(results: Results) { 54 | if (results.poseLandmarks) { 55 | removeElements( 56 | results.poseLandmarks, 57 | [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22]); 58 | } 59 | } 60 | 61 | function connect( 62 | ctx: CanvasRenderingContext2D, 63 | connectors: 64 | Array<[NormalizedLandmark, NormalizedLandmark]>): 65 | void { 66 | const canvas = ctx.canvas; 67 | for (const connector of connectors) { 68 | const from = connector[0]; 69 | const to = connector[1]; 70 | if (from && to) { 71 | if (from.visibility && to.visibility && 72 | (from.visibility < 0.1 || to.visibility < 0.1)) { 73 | continue; 74 | } 75 | ctx.beginPath(); 76 | ctx.moveTo(from.x * canvas.width, from.y * canvas.height); 77 | ctx.lineTo(to.x * canvas.width, to.y * canvas.height); 78 | ctx.stroke(); 79 | } 80 | } 81 | } 82 | 83 | export interface HolisticOptions extends Options, OptionMap { 84 | cameraOn: boolean; 85 | useCpuInference: boolean; // This is actually official 86 | effect: string; 87 | } 88 | 89 | export const InitHolisticOptions: HolisticOptions = Object.freeze({ 90 | selfieMode: true, 91 | modelComplexity: 1, 92 | useCpuInference: false, 93 | cameraOn: true, 94 | smoothLandmarks: true, 95 | enableSegmentation: false, 96 | smoothSegmentation: false, 97 | refineFaceLandmarks: true, 98 | minDetectionConfidence: 0.5, 99 | minTrackingConfidence: 0.55, 100 | effect: 'background', 101 | }); 102 | 103 | export function onResults( 104 | results: Results, 105 | vrmManager: VRMManager, 106 | videoCanvasElement: Nullable | undefined, 107 | workerPose: Comlink.Remote, 108 | activeEffect: string, 109 | proxiedCallback: ((data: Uint8Array) => void) & Comlink.ProxyMarked, 110 | fpsControl: Nullable 111 | ): void { 112 | 113 | // notify loaded. 114 | document.body.classList.add('loaded'); 115 | 116 | // @ts-ignore: delete camera input to prevent accidental paint 117 | delete results.image; 118 | 119 | // Worker process 120 | workerPose.process( 121 | (({segmentationMask, image, ...o}) => o)(results), // Remove canvas properties 122 | ) 123 | .then(async (r) => { 124 | if (debugInfo) { 125 | const resultPoseLandmarks = await workerPose.cloneablePoseLandmarks; 126 | const resultFaceNormal = await workerPose.faceNormal; 127 | const resultFaceMeshIndexLandmarks = await workerPose.faceMeshLandmarkIndexList; 128 | const resultFaceMeshLandmarks = await workerPose.faceMeshLandmarkList; 129 | const resultLeftHandNormals = await workerPose.leftHandNormals; 130 | const resultRightHandNormals = await workerPose.rightHandNormals; 131 | const resultPoseNormals = await workerPose.poseNormals; 132 | debugInfo.updatePoseLandmarkSpheres(resultPoseLandmarks); 133 | debugInfo.updateFaceNormalArrows( 134 | [resultFaceNormal], resultPoseLandmarks); 135 | debugInfo.updateFaceMeshLandmarkSpheres( 136 | resultFaceMeshIndexLandmarks, resultFaceMeshLandmarks); 137 | debugInfo.updateHandNormalArrows( 138 | resultLeftHandNormals, resultRightHandNormals, resultPoseLandmarks); 139 | debugInfo.updatePoseNormalArrows(resultPoseNormals, resultPoseLandmarks); 140 | } 141 | 142 | workerPose.midHipPos.then((v) => { 143 | if (v) 144 | vrmManager.rootMesh.position = new Vector3(-v.x, v.y, -v.z); 145 | }); 146 | }); 147 | 148 | // Remove landmarks we don't want to draw. 149 | removeLandmarks(results); 150 | 151 | // Update the frame rate. 152 | if (fpsControl) fpsControl.tick(); 153 | 154 | // Get canvas context 155 | if (!videoCanvasElement) return; 156 | const videoCanvasCtx = videoCanvasElement.getContext('2d'); 157 | if (!videoCanvasCtx) return; 158 | 159 | // Draw the overlays. 160 | videoCanvasCtx.save(); 161 | videoCanvasCtx.clearRect(0, 0, videoCanvasElement.width, videoCanvasElement.height); 162 | 163 | // Draw safe area 164 | videoCanvasCtx.fillStyle = 'rgba(34,34,34,0.5)'; 165 | videoCanvasCtx.strokeStyle = 'red'; 166 | videoCanvasCtx.lineWidth = 3; 167 | const paddingX = 0.04, paddingY = 0.08; 168 | videoCanvasCtx.fillRect(videoCanvasElement.width * paddingX, 169 | videoCanvasElement.height * paddingY, 170 | videoCanvasElement.width * (1 - 2 * paddingX), 171 | videoCanvasElement.height * (1 - 2 * paddingY)); 172 | videoCanvasCtx.strokeRect(videoCanvasElement.width * paddingX, 173 | videoCanvasElement.height * paddingY, 174 | videoCanvasElement.width * (1 - 2 * paddingX), 175 | videoCanvasElement.height * (1 - 2 * paddingY)); 176 | 177 | if (results.segmentationMask) { 178 | videoCanvasCtx.drawImage( 179 | results.segmentationMask, 0, 0, videoCanvasElement.width, 180 | videoCanvasElement.height); 181 | 182 | // Only overwrite existing pixels. 183 | if (activeEffect === 'mask' || activeEffect === 'both') { 184 | videoCanvasCtx.globalCompositeOperation = 'source-in'; 185 | // This can be a color or a texture or whatever... 186 | videoCanvasCtx.fillStyle = '#00FF007F'; 187 | videoCanvasCtx.fillRect(0, 0, videoCanvasElement.width, videoCanvasElement.height); 188 | } else { 189 | videoCanvasCtx.globalCompositeOperation = 'source-out'; 190 | videoCanvasCtx.fillStyle = '#0000FF7F'; 191 | videoCanvasCtx.fillRect(0, 0, videoCanvasElement.width, videoCanvasElement.height); 192 | } 193 | 194 | // Only overwrite missing pixels. 195 | videoCanvasCtx.globalCompositeOperation = 'destination-atop'; 196 | if (results.image) 197 | videoCanvasCtx.drawImage( 198 | results.image, 0, 0, videoCanvasElement.width, videoCanvasElement.height); 199 | 200 | videoCanvasCtx.globalCompositeOperation = 'source-over'; 201 | } else { 202 | if (results.image) 203 | videoCanvasCtx.drawImage( 204 | results.image, 0, 0, videoCanvasElement.width, videoCanvasElement.height); 205 | } 206 | 207 | // Connect elbows to hands. Do this first so that the other graphics will draw 208 | // on top of these marks. 209 | videoCanvasCtx.lineWidth = 5; 210 | if (!!results.poseLandmarks) { 211 | if (results.rightHandLandmarks) { 212 | videoCanvasCtx.strokeStyle = 'white'; 213 | connect(videoCanvasCtx, [[ 214 | results.poseLandmarks[POSE_LANDMARKS.RIGHT_ELBOW], 215 | results.rightHandLandmarks[0] 216 | ]]); 217 | } 218 | if (results.leftHandLandmarks) { 219 | videoCanvasCtx.strokeStyle = 'white'; 220 | connect(videoCanvasCtx, [[ 221 | results.poseLandmarks[POSE_LANDMARKS.LEFT_ELBOW], 222 | results.leftHandLandmarks[0] 223 | ]]); 224 | } 225 | 226 | // Pose... 227 | drawConnectors( 228 | videoCanvasCtx, results.poseLandmarks, POSE_CONNECTIONS, 229 | {color: 'white'}); 230 | drawLandmarks( 231 | videoCanvasCtx, 232 | Object.values(POSE_LANDMARKS_LEFT) 233 | .map(index => results.poseLandmarks[index]), 234 | {visibilityMin: 0.55, color: 'white', fillColor: 'rgb(255,138,0)'}); 235 | drawLandmarks( 236 | videoCanvasCtx, 237 | Object.values(POSE_LANDMARKS_RIGHT) 238 | .map(index => results.poseLandmarks[index]), 239 | {visibilityMin: 0.55, color: 'white', fillColor: 'rgb(0,217,231)'}); 240 | 241 | // Hands... 242 | drawConnectors( 243 | videoCanvasCtx, results.rightHandLandmarks, HAND_CONNECTIONS, 244 | {color: 'white'}); 245 | drawLandmarks(videoCanvasCtx, results.rightHandLandmarks, { 246 | color: 'white', 247 | fillColor: 'rgb(0,217,231)', 248 | lineWidth: 2, 249 | radius: (data: Data) => { 250 | return lerp(data.from!.z!, -0.15, .1, 10, 1); 251 | } 252 | }); 253 | drawConnectors( 254 | videoCanvasCtx, results.leftHandLandmarks, HAND_CONNECTIONS, 255 | {color: 'white'}); 256 | drawLandmarks(videoCanvasCtx, results.leftHandLandmarks, { 257 | color: 'white', 258 | fillColor: 'rgb(255,138,0)', 259 | lineWidth: 2, 260 | radius: (data: Data) => { 261 | return lerp(data.from!.z!, -0.15, .1, 10, 1); 262 | } 263 | }); 264 | 265 | // Face... 266 | // drawConnectors( 267 | // videoCanvasCtx, results.faceLandmarks, FACEMESH_TESSELATION, 268 | // {color: '#C0C0C070', lineWidth: 1}); 269 | drawConnectors( 270 | videoCanvasCtx, results.faceLandmarks, FACEMESH_RIGHT_IRIS, 271 | {color: 'rgb(0,217,231)'}); 272 | drawConnectors( 273 | videoCanvasCtx, results.faceLandmarks, FACEMESH_RIGHT_EYE, 274 | {color: 'rgb(0,217,231)'}); 275 | drawConnectors( 276 | videoCanvasCtx, results.faceLandmarks, FACEMESH_RIGHT_EYEBROW, 277 | {color: 'rgb(0,217,231)'}); 278 | drawConnectors( 279 | videoCanvasCtx, results.faceLandmarks, FACEMESH_LEFT_IRIS, 280 | {color: 'rgb(255,138,0)'}); 281 | drawConnectors( 282 | videoCanvasCtx, results.faceLandmarks, FACEMESH_LEFT_EYE, 283 | {color: 'rgb(255,138,0)'}); 284 | drawConnectors( 285 | videoCanvasCtx, results.faceLandmarks, FACEMESH_LEFT_EYEBROW, 286 | {color: 'rgb(255,138,0)'}); 287 | drawConnectors( 288 | videoCanvasCtx, results.faceLandmarks, FACEMESH_FACE_OVAL, 289 | {color: '#E0E0E0', lineWidth: 5}); 290 | drawConnectors( 291 | videoCanvasCtx, results.faceLandmarks, FACEMESH_LIPS, 292 | {color: '#E0E0E0', lineWidth: 5}); 293 | } 294 | 295 | videoCanvasCtx.restore(); 296 | } 297 | 298 | export function setHolisticOptions(x: OptionMap, videoElement: HTMLVideoElement, activeEffect: string, holistic: Holistic) { 299 | const options = x as HolisticOptions; 300 | videoElement.classList.toggle('selfie', options.selfieMode); 301 | if (options.cameraOn) { 302 | videoElement.play(); 303 | } else { 304 | videoElement.pause(); 305 | } 306 | activeEffect = options.effect; 307 | holistic.setOptions(options); 308 | } 309 | 310 | export function createControlPanel( 311 | holistic: Holistic, 312 | videoElement: HTMLVideoElement, 313 | controlsElement: HTMLDivElement, 314 | activeEffect: string, 315 | fpsControl: Nullable) { 316 | if (!controlsElement || !fpsControl) return; 317 | 318 | const holisticOptions = Object.assign({}, InitHolisticOptions); 319 | new ControlPanel(controlsElement, holisticOptions) 320 | .add([ 321 | new StaticText({title: 'Options'}), 322 | fpsControl, 323 | new Toggle({title: 'Selfie Mode', field: 'selfieMode'}), 324 | new Slider({ 325 | title: 'Model Complexity', 326 | field: 'modelComplexity', 327 | discrete: ['Lite', 'Full', 'Heavy'], 328 | }), 329 | new Toggle( 330 | {title: 'Use CPU Only', field: 'useCpuInference'}), 331 | new Toggle( 332 | {title: 'Camera on', field: 'cameraOn'}), 333 | new Toggle( 334 | {title: 'Smooth Landmarks', field: 'smoothLandmarks'}), 335 | new Toggle( 336 | {title: 'Enable Segmentation', field: 'enableSegmentation'}), 337 | new Toggle( 338 | {title: 'Smooth Segmentation', field: 'smoothSegmentation'}), 339 | new Toggle( 340 | {title: 'Refine Face Landmarks', field: 'refineFaceLandmarks'}), 341 | new Slider({ 342 | title: 'Min Detection Confidence', 343 | field: 'minDetectionConfidence', 344 | range: [0, 1], 345 | step: 0.01 346 | }), 347 | new Slider({ 348 | title: 'Min Tracking Confidence', 349 | field: 'minTrackingConfidence', 350 | range: [0, 1], 351 | step: 0.01 352 | }), 353 | new Slider({ 354 | title: 'Effect', 355 | field: 'effect', 356 | discrete: {'background': 'Background', 'mask': 'Foreground'}, 357 | }), 358 | ]) 359 | .on((x) => { 360 | setHolisticOptions(x, videoElement, activeEffect, holistic); 361 | }); 362 | } 363 | -------------------------------------------------------------------------------- /src/types.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'kalmanjs'; 2 | -------------------------------------------------------------------------------- /src/v3d-web.ts: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (C) 2021 The v3d Authors. 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License as published by 6 | the Free Software Foundation, version 3. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | import * as Comlink from "comlink"; 18 | 19 | import "@babylonjs/core/Loading/loadingScreen"; 20 | 21 | // Register plugins (side effect) 22 | import "@babylonjs/core/Loading/Plugins/babylonFileLoader"; 23 | import "@babylonjs/core/Materials"; 24 | import "@babylonjs/loaders/glTF/glTFFileLoader"; 25 | 26 | import {Engine, Nullable, ArcRotateCamera, Vector3, Quaternion} from "@babylonjs/core"; 27 | 28 | import {FPS} from "@mediapipe/control_utils"; 29 | import {Holistic, HolisticConfig, Results} from "@mediapipe/holistic"; 30 | 31 | import {Poses, poseWrapper} from "./worker/pose-processing"; 32 | import {createScene, updateBuffer, updatePose, updateSpringBones} from "./core"; 33 | import {createControlPanel, HolisticOptions, InitHolisticOptions, onResults, setHolisticOptions} from "./mediapipe"; 34 | import {VRMManager} from "v3d-core/dist/src/importer/babylon-vrm-loader/src"; 35 | import {V3DCore} from "v3d-core/dist/src"; 36 | import {CloneableQuaternionMap} from "./helper/quaternion"; 37 | import {CustomLoadingScreen} from "./helper/utils"; 38 | 39 | 40 | export interface HolisticState { 41 | ready: boolean; 42 | activeEffect: string; 43 | holisticUpdate: boolean; 44 | } 45 | export interface BoneState { 46 | boneRotations: Nullable; 47 | bonesNeedUpdate: boolean; 48 | } 49 | export interface BoneOptions { 50 | blinkLinkLR: boolean; 51 | expression: "Neutral" | "Happy" | "Angry" | "Sad" | "Relaxed" | "Surprised"; 52 | irisLockX: boolean; 53 | lockFinger: boolean; 54 | lockArm: boolean; 55 | lockLeg: boolean; 56 | resetInvisible: boolean; 57 | } 58 | 59 | export class V3DWeb { 60 | private readonly worker: Worker; 61 | private workerPose: Nullable> = null; 62 | private boneState: BoneState = { 63 | boneRotations: null, 64 | bonesNeedUpdate: false, 65 | } 66 | private _boneOptions: BoneOptions = { 67 | blinkLinkLR: true, 68 | expression: "Neutral", 69 | irisLockX: true, 70 | lockFinger: false, 71 | lockArm: false, 72 | lockLeg: false, 73 | resetInvisible: false, 74 | } 75 | get boneOptions(): BoneOptions { 76 | return this._boneOptions; 77 | } 78 | set boneOptions(value: BoneOptions) { 79 | this._boneOptions = value; 80 | this.workerPose?.updateBoneOptions(this._boneOptions); 81 | } 82 | private readonly _updateBufferCallback = Comlink.proxy((data: Uint8Array) => { 83 | updateBuffer(data, this.boneState) 84 | }); 85 | 86 | private customLoadingScreen: Nullable = null; 87 | 88 | private _v3DCore: Nullable = null; 89 | get v3DCore(): Nullable{ 90 | return this._v3DCore; 91 | } 92 | private _vrmManager: Nullable = null; 93 | get vrmManager(): Nullable{ 94 | return this._vrmManager; 95 | } 96 | private readonly engine: Engine; 97 | private _vrmFile: File | string; 98 | set vrmFile(value: File | string) { 99 | this._vrmFile = value; 100 | this.switchModel(); 101 | } 102 | 103 | private readonly fpsControl = this.controlsElement ? new FPS() : null; 104 | private readonly holistic = new Holistic(this.holisticConfig); 105 | private holisticState: HolisticState = { 106 | ready: false, 107 | activeEffect: 'mask', 108 | holisticUpdate: false, 109 | }; 110 | private _holisticOptions = Object.assign({}, InitHolisticOptions); 111 | get holisticOptions(): HolisticOptions { 112 | return this._holisticOptions; 113 | } 114 | set holisticOptions(value: HolisticOptions) { 115 | this._holisticOptions = value; 116 | setHolisticOptions(value, this.videoElement!, this.holisticState.activeEffect, this.holistic); 117 | } 118 | 119 | private _cameraList: MediaDeviceInfo[] = []; 120 | get cameraList(): MediaDeviceInfo[] { 121 | return this._cameraList; 122 | } 123 | 124 | constructor( 125 | vrmFilePath:string, 126 | public readonly videoElement?: Nullable, 127 | public readonly webglCanvasElement?: Nullable, 128 | public readonly videoCanvasElement?: Nullable, 129 | public readonly controlsElement?: Nullable, 130 | private readonly holisticConfig?: HolisticConfig, 131 | private readonly loadingDiv?: Nullable, 132 | afterInitCallback?: (...args : any[]) => any, 133 | ) { 134 | let globalInit = false; 135 | if (!this.videoElement || !this.webglCanvasElement) { 136 | globalInit = true; 137 | this.videoElement = 138 | document.getElementsByClassName('input_video')[0] as HTMLVideoElement; 139 | this.videoCanvasElement = 140 | document.getElementById('video-canvas') as HTMLCanvasElement; 141 | this.webglCanvasElement = 142 | document.getElementById('webgl-canvas') as HTMLCanvasElement; 143 | this.controlsElement = 144 | document.getElementsByClassName('control-panel')[0] as HTMLDivElement; 145 | } 146 | if (!this.videoElement || !this.webglCanvasElement) throw Error("Canvas or Video elements not found!"); 147 | 148 | this._vrmFile = vrmFilePath; 149 | 150 | /** 151 | * Babylonjs 152 | */ 153 | if (Engine.isSupported()) { 154 | this.engine = new Engine(this.webglCanvasElement, true); 155 | // engine.loadingScreen = new CustomLoadingScreen(webglCanvasElement); 156 | } else { 157 | throw Error("WebGL is not supported in this browser!"); 158 | } 159 | // Loading screen 160 | if (this.loadingDiv) { 161 | this.customLoadingScreen = 162 | new CustomLoadingScreen(this.webglCanvasElement, this.loadingDiv); 163 | } 164 | this.customLoadingScreen?.displayLoadingUI(); 165 | 166 | /** 167 | * Comlink/workers 168 | */ 169 | this.worker = new Worker( 170 | new URL("./worker/pose-processing", import.meta.url), 171 | {type: 'module'}); 172 | const posesRemote = Comlink.wrap(this.worker); 173 | const Poses = new posesRemote.poses( 174 | this.boneOptions, this._updateBufferCallback); 175 | Poses.then((v) => { 176 | if (!v) throw Error('Worker start failed!'); 177 | this.workerPose = v; 178 | 179 | createScene( 180 | this.engine, this.workerPose, 181 | this.boneState, this.boneOptions, 182 | this.holistic, this.holisticState, 183 | this._vrmFile, this.videoElement! 184 | ).then((value) => { 185 | if (!value) throw Error("VRM Manager initialization failed!"); 186 | 187 | const [v3DCore, vrmManager] = value; 188 | this._v3DCore = v3DCore; 189 | this._vrmManager = vrmManager; 190 | 191 | // Camera 192 | this.getVideoDevices().then((devices) => { 193 | if (devices.length < 1) { 194 | throw Error("No camera found!"); 195 | } else { 196 | this._cameraList = devices; 197 | this.getCamera(0).then(() => { 198 | /** 199 | * MediaPipe 200 | */ 201 | const mainOnResults = (results: Results) => onResults( 202 | results, 203 | vrmManager, 204 | this.videoCanvasElement, 205 | this.workerPose!, 206 | this.holisticState.activeEffect, 207 | this._updateBufferCallback, 208 | this.fpsControl 209 | ); 210 | this.holistic.initialize().then(() => { 211 | // Set initial options 212 | setHolisticOptions(this.holisticOptions, this.videoElement!, 213 | this.holisticState.activeEffect, this.holistic); 214 | 215 | this.holistic.onResults(mainOnResults); 216 | this.holisticState.ready = true; 217 | 218 | this.customLoadingScreen?.hideLoadingUI(); 219 | 220 | if (afterInitCallback) afterInitCallback(); 221 | }); 222 | }); 223 | } 224 | }); 225 | }); 226 | }); 227 | 228 | // Present a control panel through which the user can manipulate the solution 229 | // options. 230 | if (this.controlsElement) { 231 | createControlPanel(this.holistic, this.videoElement, this.controlsElement, 232 | this.holisticState.activeEffect, this.fpsControl); 233 | } 234 | } 235 | 236 | public async getVideoDevices() { 237 | // Ask permission 238 | await navigator.mediaDevices.getUserMedia({video: true}); 239 | 240 | const devices = await navigator.mediaDevices.enumerateDevices(); 241 | return devices.filter(device => device.kind === 'videoinput'); 242 | } 243 | 244 | public async getCamera(idx: number) { 245 | await navigator.mediaDevices.getUserMedia({ 246 | video: { 247 | width: 640, 248 | height: 480, 249 | deviceId: { 250 | exact: this.cameraList[idx].deviceId 251 | } 252 | } 253 | }) 254 | .then(stream => { 255 | if (!this.videoElement) throw Error("Video Element not found!"); 256 | this.videoElement.srcObject = stream; 257 | // let source = document.createElement('source'); 258 | // 259 | // source.setAttribute('src', 'testfiles/dance5.webm'); 260 | // source.setAttribute('type', 'video/webm'); 261 | // 262 | // this.videoElement.appendChild(source); 263 | this.videoElement.play(); 264 | }); 265 | // .catch(e => console.error(e)); 266 | } 267 | 268 | /** 269 | * Close and dispose the application (BabylonJS and MediaPipe) 270 | */ 271 | public close() { 272 | this.holisticState.ready = false; 273 | this.holistic.close().then(() => { 274 | this.worker.terminate(); 275 | (this._updateBufferCallback as any) = null; 276 | this.videoElement!.srcObject = null; 277 | }); 278 | this._vrmManager?.dispose(); 279 | this.engine.dispose(); 280 | } 281 | 282 | /** 283 | * Reset poses and holistic 284 | */ 285 | public reset() { 286 | this.workerPose?.resetBoneRotations(true); 287 | this.holistic.reset(); 288 | } 289 | 290 | public switchSource(idx: number) { 291 | if (idx >= this.cameraList.length) return; 292 | 293 | this.holisticState.ready = false; 294 | this.getCamera(idx).then(() => { 295 | this.reset(); 296 | this.holisticState.ready = true; 297 | }); 298 | } 299 | 300 | private async switchModel() { 301 | if (!this.v3DCore || !this.workerPose) return; 302 | this.v3DCore.updateAfterRenderFunction(() => {}); 303 | this.vrmManager?.dispose(); 304 | this._vrmManager = null; 305 | 306 | await this.v3DCore.AppendAsync('', this._vrmFile); 307 | this._vrmManager = this.v3DCore.getVRMManagerByURI((this._vrmFile as File).name ? 308 | (this._vrmFile as File).name : (this._vrmFile as string)); 309 | 310 | if (!this._vrmManager) throw Error("VRM model loading failed!"); 311 | 312 | // Reset camera 313 | const mainCamera = (this.v3DCore.mainCamera as ArcRotateCamera); 314 | mainCamera.setPosition(new Vector3(0, 1.05, 4.5)); 315 | mainCamera.setTarget( 316 | this._vrmManager.rootMesh.getWorldMatrix().getTranslation().subtractFromFloats(0, -1.25, 0)); 317 | await this.workerPose.setBonesHierarchyTree(this._vrmManager.transformNodeTree, true); 318 | this.workerPose.resetBoneRotations(); 319 | this.v3DCore.updateAfterRenderFunction( 320 | () => { 321 | if (this.boneState.bonesNeedUpdate) { 322 | updatePose(this._vrmManager!, this.boneState, this.boneOptions); 323 | updateSpringBones(this._vrmManager!); 324 | this.boneState.bonesNeedUpdate = false; 325 | } 326 | } 327 | ); 328 | this._vrmManager.rootMesh.rotationQuaternion = Quaternion.RotationYawPitchRoll(0, 0, 0); 329 | } 330 | } 331 | 332 | export default V3DWeb; 333 | -------------------------------------------------------------------------------- /test/css/control_utils.css: -------------------------------------------------------------------------------- 1 | .control-panel-entry{width:250px;background-color:#8d8d8d;border:2px solid #fff;border-radius:15px;box-shadow:1px 1px 10px #000000a0;cursor:pointer;font-family:"Titillium Web",sans-serif;font-size:15px;line-height:25px;opacity:.8;padding:5px 25px;position:relative;z-index:100}.control-panel-text{cursor:default;font-size:20px;font-weight:600;text-align:center}.control-panel-toggle .label{position:relative}.control-panel-toggle .value{position:absolute;right:25px}.control-panel-toggle.yes{background-color:#398c39}.control-panel-toggle.no{background-color:#f92c2c}.control-panel-toggle.yes .value{color:#a3faa3}.control-panel-toggle.no .value{color:#f19e9e}.control-panel-slider .value{position:relative;width:100%}.control-panel-slider .callout{position:absolute;right:25px}.control-panel-fps{display:flex;justify-content:center;align-items:center}.control-panel-fps canvas{height:50px;width:100%;background-color:#222;border-radius:10px;margin-top:5px}.control-panel-fps .fps-text{position:absolute;font-size:20px;font-weight:600;text-align:center;backface-visibility:hidden}.control-panel-source-picker{display:flex;flex-direction:column;align-items:center}.control-panel-source-picker .inputs{display:none}.control-panel-source-picker .file-selection div{display:flex;align-items:center;justify-items:center}.source-selection{display:flex;flex-direction:row;align-items:center;width:100%}.video-controls{width:100%;display:none;flex-direction:row;align-items:center;justify-content:center;position:relative}.video-controls *{display:inline-block}.pause-button{height:100%}.video-track{height:5px;width:100%;background-color:#fff;flex-grow:1;margin-right:10px}.video-slider-ball{height:10px;width:10px;background-color:#fff;box-shadow:0 0 2px 2px #888;position:absolute;display:none;border-radius:50%}.video-time{padding-left:.1rem}.dropdown-wrapper{position:relative;user-select:none;width:100%}.dropdown{position:relative;display:flex;flex-direction:column;border-width:0 2px;border-style:solid;border-color:#394a6d}.dropdown-trigger{position:relative;display:flex;align-items:center;white-space:nowrap;text-overflow:ellipsis;justify-content:space-between;padding:0 10px;font-size:16px;font-weight:300;color:#3b3b3b;height:30px;line-height:30px;background:#fff;cursor:pointer;border-width:2px 0;border-style:solid;border-color:#394a6d}.dropdown-trigger span{max-width:calc(100% - 30px);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.dropdown-options{position:absolute;display:block;top:100%;left:0;right:0;border:2px solid #394a6d;border-top:0;background:#fff;transition:all .5s;opacity:0;visibility:hidden;pointer-events:none;z-index:2}.dropdown.open .dropdown-options{opacity:1;visibility:visible;pointer-events:all}.dropdown-option{position:relative;display:flex;flex-direction:row;align-items:center;justify-content:left;padding:0 5px;font-size:16px;font-weight:300;color:#3b3b3b;line-height:30px;cursor:pointer;transition:all .5s}.dropdown-option *{padding-right:.5rem}.dropdown-option:hover{cursor:pointer;background-color:#b2b2b2}.dropdown-option.selected{color:#fff;background-color:#305c91}.arrow{position:relative;height:15px;width:10.6066017178px;margin-left:10.6066017178px}.arrow::before,.arrow::after{content:"";position:absolute;bottom:0px;width:.15rem;height:100%;transition:all .5s;backface-visibility:hidden}.arrow::before{left:-5px;transform:rotate(135deg);background-color:#394a6d}.arrow::after{left:5px;transform:rotate(-135deg);background-color:#394a6d}.open .arrow::before{left:-5px;transform:rotate(45deg)}.open .arrow::after{left:5px;transform:rotate(-45deg);backface-visibility:hidden}.fps-30{position:absolute;font-size:8px;top:45%;left:10px}.fps-60{position:absolute;font-size:8px;top:15%;left:10px} 2 | -------------------------------------------------------------------------------- /test/css/main.css: -------------------------------------------------------------------------------- 1 | /*! HTML5 Boilerplate v8.0.0 | MIT License | https://html5boilerplate.com/ */ 2 | 3 | /* main.css 2.1.0 | MIT License | https://github.com/h5bp/main.css#readme */ 4 | /* 5 | * What follows is the result of much research on cross-browser styling. 6 | * Credit left inline and big thanks to Nicolas Gallagher, Jonathan Neal, 7 | * Kroc Camen, and the H5BP dev community and team. 8 | */ 9 | 10 | /* ========================================================================== 11 | Base styles: opinionated defaults 12 | ========================================================================== */ 13 | 14 | .abs { 15 | position: absolute; 16 | } 17 | 18 | a { 19 | color: white; 20 | text-decoration: none; 21 | } 22 | a:hover { 23 | color: lightblue; 24 | } 25 | 26 | body { 27 | bottom: 0; 28 | font-family: 'Titillium Web', sans-serif; 29 | color: white; 30 | left: 0; 31 | margin: 0; 32 | position: absolute; 33 | right: 0; 34 | top: 0; 35 | transform-origin: 0px 0px; 36 | overflow: hidden; 37 | } 38 | 39 | .container { 40 | position: absolute; 41 | width: 100%; 42 | max-height: 100%; 43 | height: 100vh; 44 | } 45 | 46 | .input_video { 47 | display: block; 48 | position: absolute; 49 | top: 0; 50 | left: 0; 51 | right: 0; 52 | bottom: 0; 53 | } 54 | .input_video.selfie { 55 | transform: scale(-1, 1); 56 | } 57 | 58 | .input_image { 59 | position: absolute; 60 | } 61 | 62 | .canvas-container { 63 | display:flex; 64 | height: 100vh; 65 | width: 100vw; 66 | justify-content: center; 67 | align-items:center; 68 | } 69 | 70 | .output_canvas { 71 | width: 100vw; 72 | display: block; 73 | position: relative; 74 | height: 100vh; 75 | z-index: 0; 76 | } 77 | 78 | .video_output_canvas { 79 | width: 20vw; 80 | max-height: 30vh; 81 | display: block; 82 | position: absolute; 83 | top: 2rem; 84 | right: 3rem; 85 | background: #3d3d3d; 86 | opacity: 0.8; 87 | z-index: 1; 88 | } 89 | 90 | .control-panel { 91 | position: absolute; 92 | right: 150px; 93 | top: 10px; 94 | } 95 | 96 | .loading { 97 | display: flex; 98 | position: absolute; 99 | top: 0; 100 | right: 0; 101 | bottom: 0; 102 | left: 0; 103 | align-items: center; 104 | backface-visibility: hidden; 105 | justify-content: center; 106 | opacity: 1; 107 | transition: opacity 1s; 108 | } 109 | 110 | .message { 111 | font-size: x-large; 112 | } 113 | 114 | .lds-ring { 115 | display: inline-block; 116 | position: relative; 117 | width: 80px; 118 | height: 80px; 119 | z-index: 10; 120 | } 121 | .lds-ring div { 122 | box-sizing: border-box; 123 | display: block; 124 | position: absolute; 125 | width: 64px; 126 | height: 64px; 127 | margin: 8px; 128 | border: 8px solid #fff; 129 | border-radius: 50%; 130 | animation: lds-ring 1.2s cubic-bezier(0.5, 0, 0.5, 1) infinite; 131 | border-color: #fff transparent transparent transparent; 132 | } 133 | .lds-ring div:nth-child(1) { 134 | animation-delay: -0.45s; 135 | } 136 | .lds-ring div:nth-child(2) { 137 | animation-delay: -0.3s; 138 | } 139 | .lds-ring div:nth-child(3) { 140 | animation-delay: -0.15s; 141 | } 142 | @keyframes lds-ring { 143 | 0% { 144 | transform: rotate(0deg); 145 | } 146 | 100% { 147 | transform: rotate(360deg); 148 | } 149 | } 150 | 151 | .dark-primary-color { background: #512DA8; } 152 | .default-primary-color { background: #673AB7; } 153 | .light-primary-color { background: #D1C4E9; } 154 | .text-primary-color { color: #FFFFFF; } 155 | .accent-color { background: #607D8B; } 156 | .primary-text-color { color: #212121; } 157 | .secondary-text-color { color: #757575; } 158 | .divider-color { border-color: #BDBDBD; } 159 | -------------------------------------------------------------------------------- /test/css/normalize.css: -------------------------------------------------------------------------------- 1 | /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */ 2 | 3 | /* Document 4 | ========================================================================== */ 5 | 6 | /** 7 | * 1. Correct the line height in all browsers. 8 | * 2. Prevent adjustments of font size after orientation changes in iOS. 9 | */ 10 | 11 | html { 12 | line-height: 1.15; /* 1 */ 13 | -webkit-text-size-adjust: 100%; /* 2 */ 14 | } 15 | 16 | /* Sections 17 | ========================================================================== */ 18 | 19 | /** 20 | * Remove the margin in all browsers. 21 | */ 22 | 23 | body { 24 | margin: 0; 25 | } 26 | 27 | /** 28 | * Render the `main` element consistently in IE. 29 | */ 30 | 31 | main { 32 | display: block; 33 | } 34 | 35 | /** 36 | * Correct the font size and margin on `h1` elements within `section` and 37 | * `article` contexts in Chrome, Firefox, and Safari. 38 | */ 39 | 40 | h1 { 41 | font-size: 2em; 42 | margin: 0.67em 0; 43 | } 44 | 45 | /* Grouping content 46 | ========================================================================== */ 47 | 48 | /** 49 | * 1. Add the correct box sizing in Firefox. 50 | * 2. Show the overflow in Edge and IE. 51 | */ 52 | 53 | hr { 54 | box-sizing: content-box; /* 1 */ 55 | height: 0; /* 1 */ 56 | overflow: visible; /* 2 */ 57 | } 58 | 59 | /** 60 | * 1. Correct the inheritance and scaling of font size in all browsers. 61 | * 2. Correct the odd `em` font sizing in all browsers. 62 | */ 63 | 64 | pre { 65 | font-family: monospace, monospace; /* 1 */ 66 | font-size: 1em; /* 2 */ 67 | } 68 | 69 | /* Text-level semantics 70 | ========================================================================== */ 71 | 72 | /** 73 | * Remove the gray background on active links in IE 10. 74 | */ 75 | 76 | a { 77 | background-color: transparent; 78 | } 79 | 80 | /** 81 | * 1. Remove the bottom border in Chrome 57- 82 | * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. 83 | */ 84 | 85 | abbr[title] { 86 | border-bottom: none; /* 1 */ 87 | text-decoration: underline; /* 2 */ 88 | text-decoration: underline dotted; /* 2 */ 89 | } 90 | 91 | /** 92 | * Add the correct font weight in Chrome, Edge, and Safari. 93 | */ 94 | 95 | b, 96 | strong { 97 | font-weight: bolder; 98 | } 99 | 100 | /** 101 | * 1. Correct the inheritance and scaling of font size in all browsers. 102 | * 2. Correct the odd `em` font sizing in all browsers. 103 | */ 104 | 105 | code, 106 | kbd, 107 | samp { 108 | font-family: monospace, monospace; /* 1 */ 109 | font-size: 1em; /* 2 */ 110 | } 111 | 112 | /** 113 | * Add the correct font size in all browsers. 114 | */ 115 | 116 | small { 117 | font-size: 80%; 118 | } 119 | 120 | /** 121 | * Prevent `sub` and `sup` elements from affecting the line height in 122 | * all browsers. 123 | */ 124 | 125 | sub, 126 | sup { 127 | font-size: 75%; 128 | line-height: 0; 129 | position: relative; 130 | vertical-align: baseline; 131 | } 132 | 133 | sub { 134 | bottom: -0.25em; 135 | } 136 | 137 | sup { 138 | top: -0.5em; 139 | } 140 | 141 | /* Embedded content 142 | ========================================================================== */ 143 | 144 | /** 145 | * Remove the border on images inside links in IE 10. 146 | */ 147 | 148 | img { 149 | border-style: none; 150 | } 151 | 152 | /* Forms 153 | ========================================================================== */ 154 | 155 | /** 156 | * 1. Change the font styles in all browsers. 157 | * 2. Remove the margin in Firefox and Safari. 158 | */ 159 | 160 | button, 161 | input, 162 | optgroup, 163 | select, 164 | textarea { 165 | font-family: inherit; /* 1 */ 166 | font-size: 100%; /* 1 */ 167 | line-height: 1.15; /* 1 */ 168 | margin: 0; /* 2 */ 169 | } 170 | 171 | /** 172 | * Show the overflow in IE. 173 | * 1. Show the overflow in Edge. 174 | */ 175 | 176 | button, 177 | input { /* 1 */ 178 | overflow: visible; 179 | } 180 | 181 | /** 182 | * Remove the inheritance of text transform in Edge, Firefox, and IE. 183 | * 1. Remove the inheritance of text transform in Firefox. 184 | */ 185 | 186 | button, 187 | select { /* 1 */ 188 | text-transform: none; 189 | } 190 | 191 | /** 192 | * Correct the inability to style clickable types in iOS and Safari. 193 | */ 194 | 195 | button, 196 | [type="button"], 197 | [type="reset"], 198 | [type="submit"] { 199 | -webkit-appearance: button; 200 | } 201 | 202 | /** 203 | * Remove the inner border and padding in Firefox. 204 | */ 205 | 206 | button::-moz-focus-inner, 207 | [type="button"]::-moz-focus-inner, 208 | [type="reset"]::-moz-focus-inner, 209 | [type="submit"]::-moz-focus-inner { 210 | border-style: none; 211 | padding: 0; 212 | } 213 | 214 | /** 215 | * Restore the focus styles unset by the previous rule. 216 | */ 217 | 218 | button:-moz-focusring, 219 | [type="button"]:-moz-focusring, 220 | [type="reset"]:-moz-focusring, 221 | [type="submit"]:-moz-focusring { 222 | outline: 1px dotted ButtonText; 223 | } 224 | 225 | /** 226 | * Correct the padding in Firefox. 227 | */ 228 | 229 | fieldset { 230 | padding: 0.35em 0.75em 0.625em; 231 | } 232 | 233 | /** 234 | * 1. Correct the text wrapping in Edge and IE. 235 | * 2. Correct the color inheritance from `fieldset` elements in IE. 236 | * 3. Remove the padding so developers are not caught out when they zero out 237 | * `fieldset` elements in all browsers. 238 | */ 239 | 240 | legend { 241 | box-sizing: border-box; /* 1 */ 242 | color: inherit; /* 2 */ 243 | display: table; /* 1 */ 244 | max-width: 100%; /* 1 */ 245 | padding: 0; /* 3 */ 246 | white-space: normal; /* 1 */ 247 | } 248 | 249 | /** 250 | * Add the correct vertical alignment in Chrome, Firefox, and Opera. 251 | */ 252 | 253 | progress { 254 | vertical-align: baseline; 255 | } 256 | 257 | /** 258 | * Remove the default vertical scrollbar in IE 10+. 259 | */ 260 | 261 | textarea { 262 | overflow: auto; 263 | } 264 | 265 | /** 266 | * 1. Add the correct box sizing in IE 10. 267 | * 2. Remove the padding in IE 10. 268 | */ 269 | 270 | [type="checkbox"], 271 | [type="radio"] { 272 | box-sizing: border-box; /* 1 */ 273 | padding: 0; /* 2 */ 274 | } 275 | 276 | /** 277 | * Correct the cursor style of increment and decrement buttons in Chrome. 278 | */ 279 | 280 | [type="number"]::-webkit-inner-spin-button, 281 | [type="number"]::-webkit-outer-spin-button { 282 | height: auto; 283 | } 284 | 285 | /** 286 | * 1. Correct the odd appearance in Chrome and Safari. 287 | * 2. Correct the outline style in Safari. 288 | */ 289 | 290 | [type="search"] { 291 | -webkit-appearance: textfield; /* 1 */ 292 | outline-offset: -2px; /* 2 */ 293 | } 294 | 295 | /** 296 | * Remove the inner padding in Chrome and Safari on macOS. 297 | */ 298 | 299 | [type="search"]::-webkit-search-decoration { 300 | -webkit-appearance: none; 301 | } 302 | 303 | /** 304 | * 1. Correct the inability to style clickable types in iOS and Safari. 305 | * 2. Change font properties to `inherit` in Safari. 306 | */ 307 | 308 | ::-webkit-file-upload-button { 309 | -webkit-appearance: button; /* 1 */ 310 | font: inherit; /* 2 */ 311 | } 312 | 313 | /* Interactive 314 | ========================================================================== */ 315 | 316 | /* 317 | * Add the correct display in Edge, IE 10+, and Firefox. 318 | */ 319 | 320 | details { 321 | display: block; 322 | } 323 | 324 | /* 325 | * Add the correct display in all browsers. 326 | */ 327 | 328 | summary { 329 | display: list-item; 330 | } 331 | 332 | /* Misc 333 | ========================================================================== */ 334 | 335 | /** 336 | * Add the correct display in IE 10+. 337 | */ 338 | 339 | template { 340 | display: none; 341 | } 342 | 343 | /** 344 | * Add the correct display in IE 10. 345 | */ 346 | 347 | [hidden] { 348 | display: none; 349 | } 350 | -------------------------------------------------------------------------------- /test/img/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phantom-software-AZ/v3d-web/3e37a66ffdcbf03a18fb977dff53a7afe4a3f930/test/img/.gitignore -------------------------------------------------------------------------------- /test/img/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phantom-software-AZ/v3d-web/3e37a66ffdcbf03a18fb977dff53a7afe4a3f930/test/img/icon.png -------------------------------------------------------------------------------- /test/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 |
35 | 36 |
37 | 38 | 39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 |
50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /tracking.md: -------------------------------------------------------------------------------- 1 | 1. Keep your face visible to the camera. 2 | 2. It will be better if your webcam has a refresh rate of at least 30FPS. Also make sure your environment is bright enough. 3 | 3. Avoid overly complicated backgrounds. 4 | 4. Try to stay inside the safe area. 5 | 5. Try not do poses pointing directly to camera (because camera cannot see behind things). 6 | 6. When tracking arms, make sure your elbows are visible. 7 | 7. Leg and foot tracking won't start until full leg and foot can be seen. This is to prevent strange moves. 8 | 8. If eye tracking doesn't work, try removing your glasses and make sure nothing is covering your eyes. 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Basic Options */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | "target": "es2021", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', 'ES2021', or 'ESNEXT'. */ 8 | "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ 9 | "lib": [ 10 | "es6", 11 | "es7", 12 | "dom", 13 | "webworker", 14 | "scripthost" 15 | ], 16 | /* Specify library files to be included in the compilation. */ 17 | "allowJs": true, /* Allow javascript files to be compiled. */ 18 | // "checkJs": true, /* Report errors in .js files. */ 19 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', 'react', 'react-jsx' or 'react-jsxdev'. */ 20 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 21 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 22 | "sourceMap": true, /* Generates corresponding '.map' file. */ 23 | // "outFile": "./", /* Concatenate and emit output to single file. */ 24 | "outDir": "dist", /* Redirect output structure to the directory. */ 25 | "rootDir": ".", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 26 | // "composite": true, /* Enable project compilation */ 27 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 28 | // "removeComments": true, /* Do not emit comments to output. */ 29 | "noEmit": false, /* Do not emit outputs. */ 30 | "importHelpers": true, /* Import emit helpers from 'tslib'. */ 31 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 32 | "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 33 | 34 | /* Strict Type-Checking Options */ 35 | "strict": true, /* Enable all strict type-checking options. */ 36 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 37 | "strictNullChecks": true, /* Enable strict null checks. */ 38 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 39 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 40 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 41 | "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 42 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 43 | 44 | /* Additional Checks */ 45 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 46 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 47 | "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 48 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 49 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 50 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an 'override' modifier. */ 51 | // "noPropertyAccessFromIndexSignature": true, /* Require undeclared properties from index signatures to use element accesses. */ 52 | 53 | /* Module Resolution Options */ 54 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 55 | "baseUrl": ".", /* Base directory to resolve non-absolute module names. */ 56 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 57 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 58 | // "typeRoots": [], /* List of folders to include type definitions from. */ 59 | // "types": [], /* Type declaration files to be included in compilation. */ 60 | "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 61 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 62 | "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 63 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 64 | 65 | /* Source Map Options */ 66 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 67 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 68 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 69 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 70 | 71 | /* Experimental Options */ 72 | "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 73 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 74 | 75 | /* Advanced Options */ 76 | "resolveJsonModule": true, /* Include modules imported with '.json' extension */ 77 | "skipLibCheck": true, /* Skip type checking of declaration files. */ 78 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 79 | }, 80 | "exclude": [ 81 | // exclude output dir 82 | "dist", 83 | "examples", 84 | "lib", 85 | "node_modules", 86 | "test", 87 | "babel.config.js", 88 | "jest.config.js", 89 | ".prettierrc.js" 90 | ] 91 | } 92 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require( 'path' ); 2 | const Merge = require('webpack-merge'); 3 | const terser = require('terser-webpack-plugin'); 4 | 5 | const baseConfig = { 6 | mode: 'production', 7 | entry: { 8 | v3dweb: path.resolve(__dirname, 'src', 'index'), 9 | }, 10 | module: { 11 | rules: [ 12 | { 13 | test: /\.ts$/, 14 | use: 'ts-loader', 15 | }, 16 | { 17 | test: /\.m?js$/, 18 | resolve: { 19 | fullySpecified: false, 20 | }, 21 | }, 22 | ], 23 | }, 24 | resolve: { 25 | modules: [path.resolve(__dirname, 'node_modules')], 26 | extensions: ['.js', '.ts'], 27 | symlinks: false, 28 | }, 29 | experiments: { 30 | topLevelAwait: true, 31 | }, 32 | optimization: { 33 | minimize: true, 34 | minimizer: [new terser({ 35 | extractComments: false, 36 | })], 37 | concatenateModules: true, 38 | }, 39 | target: ['web'], 40 | }; 41 | 42 | const config = [ 43 | // UMD 44 | Merge.merge(baseConfig, { 45 | output: { 46 | library: { 47 | name: 'v3d-web', 48 | type: 'umd', 49 | }, 50 | filename: '[name].module.js', 51 | path: path.resolve(__dirname, 'dist'), 52 | }, 53 | }), 54 | // ES6 55 | Merge.merge(baseConfig, { 56 | output: { 57 | library: { 58 | type: 'module', 59 | }, 60 | filename: '[name].es6.js', 61 | path: path.resolve(__dirname, 'dist'), 62 | environment: { module: true }, 63 | }, 64 | experiments: { 65 | outputModule: true, 66 | }, 67 | externalsType: 'module', 68 | }), 69 | // browser global 70 | Merge.merge(baseConfig, { 71 | output: { 72 | library: { 73 | name: 'v3d-web', 74 | type: 'window', 75 | }, 76 | filename: '[name].js', 77 | path: path.resolve(__dirname, 'dist'), 78 | }, 79 | }), 80 | ]; 81 | 82 | module.exports = config; 83 | -------------------------------------------------------------------------------- /webpack.test.config.js: -------------------------------------------------------------------------------- 1 | const path = require( 'path' ); 2 | const CopyPlugin = require( 'copy-webpack-plugin' ); 3 | 4 | const test_folder = 'test' 5 | 6 | const config = { 7 | mode: 'development', 8 | devtool: 'inline-source-map', 9 | entry: path.resolve(__dirname, 'src', 'index-test'), 10 | output: { 11 | library: { 12 | name: 'v3d-web', 13 | type: 'umd', 14 | }, 15 | filename: '[name].test.js', 16 | path: path.resolve(__dirname, test_folder), 17 | }, 18 | module: { 19 | rules: [ 20 | { 21 | test: /\.ts$/, 22 | use: 'ts-loader', 23 | }, 24 | { 25 | test: /\.m?js$/, 26 | resolve: { 27 | fullySpecified: false, 28 | }, 29 | }, 30 | ], 31 | }, 32 | resolve: { 33 | modules: [path.resolve(__dirname, 'node_modules')], 34 | extensions: ['.js', '.ts'], 35 | symlinks: false, 36 | }, 37 | experiments: { 38 | topLevelAwait: true, 39 | }, 40 | target: ['web'], 41 | devServer: { 42 | allowedHosts: 'localhost', 43 | static: { 44 | directory: path.resolve(__dirname, test_folder), 45 | }, 46 | compress: true, 47 | port: 8080, 48 | }, 49 | plugins: [ 50 | new CopyPlugin({ 51 | patterns: [ 52 | {from: "node_modules/@mediapipe/holistic", to: "."}, 53 | ], 54 | }), 55 | ], 56 | }; 57 | 58 | module.exports = config; 59 | --------------------------------------------------------------------------------