├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .npmignore ├── .vscode ├── settings.json └── tasks.json ├── README.md ├── cloudbuild.yml ├── demo ├── mobilenet │ ├── README.md │ ├── app.js │ ├── app.json │ ├── model │ │ ├── classifier.ts │ │ └── mobilenet.ts │ ├── package.json │ ├── pages │ │ └── index │ │ │ ├── index.json │ │ │ ├── index.ts │ │ │ ├── index.wxml │ │ │ └── index.wxss │ ├── project.config.json │ ├── sitemap.json │ ├── tsconfig.json │ └── yarn.lock └── posenet │ ├── app.js │ ├── app.json │ ├── package.json │ ├── pages │ └── index │ │ ├── index.ts │ │ ├── index.wxml │ │ └── index.wxss │ ├── posenet │ ├── posenet.ts │ └── util.ts │ ├── sitemap.json │ ├── tsconfig.json │ └── yarn.lock ├── doc ├── README.md └── setting.png ├── karma.conf.js ├── package.json ├── project.config.json ├── src └── plugin │ ├── .gitignore │ ├── api │ └── config.ts │ ├── index.ts │ ├── package.json │ ├── plugin.json │ ├── test │ ├── file_storage_test.ts │ ├── local_storage_test.ts │ └── wechat_platform_test.ts │ ├── typings │ └── abab.d.ts │ ├── utils │ ├── file_storage.ts │ ├── local_storage.ts │ ├── model_artifacts.ts │ └── wechat_platform.ts │ └── yarn.lock ├── tsconfig.json ├── tslint.json └── yarn.lock /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | To get help from the community, we encourage using Stack Overflow and the tensorflow.js tag. 11 | 12 | TensorFlow.js version 13 | tfjs-wechat plugin version 14 | WeChat version 15 | WeChat base API version 16 | WeChat IDE version 17 | Describe the problem or feature request 18 | Code to reproduce the bug / link to feature request 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **node_modules/ 2 | coverage/ 3 | npm-debug.log 4 | yarn-error.log 5 | .DS_Store 6 | dist/ 7 | demo/dist 8 | demo/mobilenet/**/*.d.ts 9 | demo/mobilenet/**/*.js.map 10 | demo/mobilenet/**/*.js 11 | .idea/ 12 | *.tgz 13 | .cache 14 | package/ 15 | 16 | bazel-* 17 | 18 | **.yalc 19 | **yalc.lock 20 | 21 | src/*/miniprogram_npm/ 22 | demo/*/miniprogram_npm/ 23 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .rpt2_cache 2 | .vscode 3 | .DS_Store 4 | .babelrc 5 | .idea/ 6 | demo/ 7 | docs/ 8 | scripts/ 9 | /src/ 10 | node_modules/ 11 | coverage/ 12 | package-lock.json 13 | npm-debug.log 14 | *.tgz 15 | *.txt 16 | karma.conf.js 17 | dist/**/*_test.js 18 | dist/**/*_test.d.ts 19 | tslint.json 20 | tsconfig.json 21 | yarn.lock 22 | yarn-error.log 23 | rollup.config.js 24 | test_results 25 | rollup.config.google.js 26 | **.yalc 27 | **yalc.lock 28 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "search.exclude": { 4 | "**/node_modules": true, 5 | "**/bower_components": true, 6 | "coverage/": true, 7 | "dist/": true, 8 | "**/bundle.js": true, 9 | "**/yarn.lock": true 10 | }, 11 | "tslint.enable": true, 12 | "tslint.run": "onType", 13 | "tslint.configFile": "tslint.json", 14 | "files.trimTrailingWhitespace": true, 15 | "editor.tabSize": 2, 16 | "editor.insertSpaces": true, 17 | "[typescript]": { 18 | "editor.formatOnSave": true 19 | }, 20 | "clang-format.style": "Google", 21 | "files.insertFinalNewline": true, 22 | "editor.detectIndentation": false, 23 | "editor.rulers": [80], 24 | "editor.wrappingIndent": "none", 25 | "typescript.tsdk": "node_modules/typescript/lib", 26 | "clang-format.executable": "${workspaceRoot}/node_modules/.bin/clang-format", 27 | "python.formatting.provider": "autopep8", 28 | "python.pythonPath": "python3", 29 | "python.formatting.autopep8Args": ["--indent-size=2"] 30 | } 31 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "command": "yarn", 6 | "label": "lint", 7 | "type": "shell", 8 | "args": ["lint"], 9 | "problemMatcher": { 10 | "base": "$tslint5", 11 | "owner": "tslint-type-checked", 12 | "fileLocation": "absolute" 13 | } 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow.js 微信小程序插件 2 | [TensorFlow.js](https://github.com/tensorflow/tfjs)是谷歌开发的机器学习开源项目，致力于为javascript提供具有硬件加速的机器学习模型训练和部署。 3 | TensorFlow.js 微信小程序插件封装了TensorFlow.js库，用于提供给第三方小程序调用。 4 | 例子可以看TFJS Mobilenet [物体识别小程序](https://github.com/tensorflow/tfjs-wechat/tree/master/demo/mobilenet) 5 | ## 添加插件 6 | 在使用插件前，首先要在小程序管理后台的“设置-第三方服务-插件管理”中添加插件。开发者可登录小程序管理后台，通过 appid [wx6afed118d9e81df9] 查找插件并添加。本插件无需申请，添加后可直接使用。 7 | 8 | ### 引入插件代码包 9 | 使用插件前，使用者要在 app.json 中声明需要使用的插件，例如： 10 | 11 | 代码示例： 12 | ``` 13 | { 14 | ... 15 | "plugins": { 16 | "tfjsPlugin": { 17 | "version": "0.0.6", 18 | "provider": "wx6afed118d9e81df9" 19 | } 20 | } 21 | ... 22 | } 23 | ``` 24 | ### 引入TensorFlow.js npm 25 | TensorFlow.js 最新版本是以npm包的形式发布，小程序需要使用npm或者yarn来载入TensorFlow.js npm包。也可以手动修改 package.json 文件来加入。 26 | 27 | TensorFlow.js v2.0 有一个联合包 - @tensorflow/tfjs，包含了六个分npm包： 28 | - tfjs-core: 基础包 29 | - tfjs-converter: GraphModel 导入和执行包 30 | - tfjs-layers: LayersModel 创建，导入和执行包 31 | - tfjs-backend-webgl: webgl 后端 32 | - tfjs-backend-cpu: cpu 后端 33 | - tfjs-data：数据流 34 | 35 | 36 | 对于小程序而言，由于有2M的app大小限制，不建议直接使用联合包，而是按照需求加载分包。 37 | - 如果小程序只需要导入和运行GraphModel模型的的话，建议至少加入tfjs-core， tfjs-converter， tfjs-backend-webgl 和tfjs-backend-cpu包。这样可以尽量减少导入包的大小。 38 | - 如果需要创建,导入或训练LayersModel模型，需要再加入 tfjs-layers包。 39 | 40 | 下面的例子是只用到tfjs-core， tfjs-converter，tfjs-backend-webgl 和tfjs-backend-cpu包。代码示例： 41 | ``` 42 | { 43 | "name": "yourProject", 44 | "version": "0.0.1", 45 | "main": "dist/index.js", 46 | "license": "Apache-2.0", 47 | "dependencies": { 48 | "@tensorflow/tfjs-core": "3.5.0"， 49 | "@tensorflow/tfjs-converter": "3.5.0"， 50 | "@tensorflow/tfjs-backend-webgl": "3.5.0" 51 | } 52 | } 53 | ``` 54 | 55 | 参考小程序npm工具[文档](https://developers.weixin.qq.com/miniprogram/dev/devtools/npm.html)如何编译npm包到小程序中。 56 | 57 | __注意__ 58 | 请从微信小程序[开发版Nightly Build更新日志](https://developers.weixin.qq.com/miniprogram/dev/devtools/nightly.html)下载最新的微信开发者工具，保证版本号>=v1.02.1907022. 59 | 60 | ### Polyfill fetch 函数 61 | 如果需要使用tf.loadGraphModel或tf.loadLayersModel API来载入模型，小程序需要按以下流程填充fetch函数： 62 | 63 | 1. 如果你使用npm, 你可以载入fetch-wechat npm 包 64 | 65 | ``` 66 | { 67 | "name": "yourProject", 68 | "version": "0.0.1", 69 | "main": "dist/index.js", 70 | "license": "Apache-2.0", 71 | "dependencies": { 72 | "@tensorflow/tfjs-core": "3.5.0"， 73 | "@tensorflow/tfjs-converter": "3.5.0"， 74 | "@tensorflow/tfjs-backend-webgl": "3.5.0" 75 | "fetch-wechat": "0.0.3" 76 | } 77 | } 78 | ``` 79 | 80 | 2. 也可以直接拷贝以下文件到你的javascript源目录： 81 | https://cdn.jsdelivr.net/npm/fetch-wechat@0.0.3/dist/fetch_wechat.min.js 82 | 83 | 84 | ### 在app.js的onLaunch里调用插件configPlugin函数 85 | 86 | ``` 87 | var fetchWechat = require('fetch-wechat'); 88 | var tf = require('@tensorflow/tfjs-core'); 89 | var webgl = require('@tensorflow/tfjs-backend-webgl'); 90 | var plugin = requirePlugin('tfjsPlugin'); 91 | //app.js 92 | App({ 93 | onLaunch: function () { 94 | plugin.configPlugin({ 95 | // polyfill fetch function 96 | fetchFunc: fetchWechat.fetchFunc(), 97 | // inject tfjs runtime 98 | tf, 99 | // inject webgl backend 100 | webgl, 101 | // provide webgl canvas 102 | canvas: wx.createOffscreenCanvas() 103 | }); 104 | } 105 | }); 106 | ``` 107 | 108 | ### 支持模型localStorage缓存 109 | 采用localStorage缓存可以减少模型下载耗费带宽和时间。由于微信小程序对于localStorage有10MB的限制，这个方法适用于小于10MB的模型。 110 | 步骤如下： 111 | 1. 在app.js中提供localStorageHandler函数. 112 | 113 | ``` 114 | var fetchWechat = require('fetch-wechat'); 115 | var tf = require('@tensorflow/tfjs-core'); 116 | var plugin = requirePlugin('tfjsPlugin'); 117 | //app.js 118 | App({ 119 | // expose localStorage handler 120 | globalData: {localStorageIO: plugin.localStorageIO}, 121 | ... 122 | }); 123 | ``` 124 | 125 | 2. 在模型加载时加入localStorageHandler逻辑。 126 | 127 | ``` 128 | const LOCAL_STORAGE_KEY = 'mobilenet_model'; 129 | export class MobileNet { 130 | private model: tfc.GraphModel; 131 | constructor() { } 132 | 133 | async load() { 134 | 135 | const localStorageHandler = getApp().globalData.localStorageIO(LOCAL_STORAGE_KEY); 136 | try { 137 | this.model = await tfc.loadGraphModel(localStorageHandler); 138 | } catch (e) { 139 | this.model = 140 | await tfc.loadGraphModel(MODEL_URL); 141 | this.model.save(localStorageHandler); 142 | } 143 | } 144 | 145 | ``` 146 | 147 | ### 支持模型保存为用户文件 148 | 微信也支持保存模型为文件。同localStorage, 微信小程序对于本地文件也有10MB的限制，这个方法适用于小于10MB的模型。由于最终模型是按 binary 保存，较 localstorage 保存为 base64 string 更为节省空间。 149 | 150 | 步骤如下： 151 | 1. 在app.js中提供 fileStorageHandler 函数. 152 | 153 | ```js 154 | var fetchWechat = require('fetch-wechat'); 155 | var tf = require('@tensorflow/tfjs-core'); 156 | var plugin = requirePlugin('tfjsPlugin'); 157 | //app.js 158 | App({ 159 | // expose fileStorage handler 160 | globalData: {fileStorageIO: plugin.fileStorageIO}, 161 | ... 162 | }); 163 | ``` 164 | 165 | 2. 在模型加载时加入 fileStorageHandler 逻辑。 166 | 167 | ```js 168 | const FILE_STORAGE_PATH = 'mobilenet_model'; 169 | export class MobileNet { 170 | private model: tfc.GraphModel; 171 | constructor() { } 172 | 173 | async load() { 174 | 175 | const fileStorageHandler = getApp().globalData.fileStorageIO( 176 | FILE_STORAGE_PATH, wx.getFileSystemManager()); 177 | try { 178 | this.model = await tfc.loadGraphModel(fileStorageHandler); 179 | } catch (e) { 180 | this.model = 181 | await tfc.loadGraphModel(MODEL_URL); 182 | this.model.save(fileStorageHandler); 183 | } 184 | } 185 | } 186 | ``` 187 | 188 | ### 使用 WebAssembly backend 189 | 微信小程序在 Android 手机上提供 WebAssembly的支持。TensorFlow.js的WASM backend非常适合在中低端Android手机上使用。 190 | 中低端手机的GPU往往相对CPU要弱一些，而WASM backend是跑在CPU上的，这就为中低端手机提供了另一个加速平台。而且WASM的能耗一般会更低。 191 | 使用WASM backend需要修改package.json文件： 192 | 193 | ``` 194 | { 195 | "name": "yourProject", 196 | "version": "0.0.1", 197 | "main": "dist/index.js", 198 | "license": "Apache-2.0", 199 | "dependencies": { 200 | "@tensorflow/tfjs-core": "2.0.0"， 201 | "@tensorflow/tfjs-converter": "2.0.0"， 202 | "@tensorflow/tfjs-backend-wasm": "2.0.0", 203 | ... 204 | } 205 | } 206 | ``` 207 | 208 | 然后在app.js中设置 wasm backend, 你可以自行host wasm file以提高下载速度, 下面例子中的 `wasmUrl`可以替代成你host的URL。 209 | ``` 210 | const info = wx.getSystemInfoSync(); 211 | const wasmUrl = 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@2.0.0/wasm-out/tfjs-backend-wasm.wasm'; 212 | const usePlatformFetch = true; 213 | console.log(info.platform); 214 | if (info.platform == 'android') { 215 | setWasmPath(wasmUrl, usePlatformFetch); 216 | tf.setBackend('wasm').then(() => console.log('set wasm backend')); 217 | } 218 | ``` 219 | 220 | __注意__ 221 | WASM backend is broken due to bundle imcompatible with WeChat npm loader, will update here when it is fixed. 222 | 223 | 224 | __注意__ 225 | 由于最新版本的WeChat的OffscreenCanvas会随页面跳转而失效，在app.js的 onLaunch 函数中设置 tfjs 会导致小程序退出或页面跳转之后操作出错。建议在使用tfjs的page的onLoad中调用 configPlugin 函数。 226 | WeChat的12月版本会修复这个问题。 227 | 228 | ``` 229 | var fetchWechat = require('fetch-wechat'); 230 | var tf = require('@tensorflow/tfjs-core'); 231 | var plugin = requirePlugin('tfjsPlugin'); 232 | //index.js 233 | Page({ 234 | onLoad: function () { 235 | plugin.configPlugin({ 236 | // polyfill fetch function 237 | fetchFunc: fetchWechat.fetchFunc(), 238 | // inject tfjs runtime 239 | tf, 240 | // provide webgl canvas 241 | canvas: wx.createOffscreenCanvas(), 242 | backendName: 'wechat-webgl-' + Date.now() 243 | }); 244 | ... 245 | } 246 | }); 247 | ``` 248 | 249 | 组件设置完毕就可以开始使用 TensorFlow.js库的[API](https://js.tensorflow.org/api/latest/)了。 250 | 251 | ### 使用 [tfjs-models](https://github.com/tensorflow/tfjs-models) 模型库注意事项 252 | 模型库提供了一系列训练好的模型，方便大家快速的给小程序注入ML功能。模型分类包括 253 | - 图像识别 254 | - 语音识别 255 | - 人体姿态识别 256 | - 物体识别 257 | - 文字分类 258 | 259 | 由于这些API默认模型文件都存储在谷歌云上，直接使用会导致中国用户无法直接读取。在小程序内使用模型API时要提供 modelUrl 的参数，可以指向我们在谷歌中国的镜像服务器。 260 | 谷歌云的base url是 https://storage.googleapis.com， 中国镜像的base url是https://www.gstaticcnapps.cn 261 | 模型的url path是一致的，比如 262 | - posenet模型的谷歌云地址是： 263 | https://storage.googleapis.com/tfjs-models/savedmodel/posenet/mobilenet/float/050/model-stride16.json 264 | - 中国镜像的地址为 https://www.gstaticcnapps.cn/tfjs-models/savedmodel/posenet/mobilenet/float/050/model-stride16.json 265 | 266 | 他们的 URL Path 都是 /tfjs-models/savedmodel/posenet/mobilenet/float/050/model-stride16.json 267 | 268 | 下面是加载posenet模型的例子： 269 | 270 | ``` 271 | import * as posenet from '@tensorflow-models/posenet'; 272 | 273 | const POSENET_URL = 274 | 'https://www.gstaticcnapps.cn/tfjs-models/savedmodel/posenet/mobilenet/float/050/model-stride16.json'; 275 | 276 | const model = await posenet.load({ 277 | architecture: 'MobileNetV1', 278 | outputStride: 16, 279 | inputResolution: 193, 280 | multiplier: 0.5, 281 | modelUrl: POSENET_URL 282 | }); 283 | ``` 284 | 285 | ## [tfjs-examples](https://github.com/tensorflow/tfjs-examples) tfjs例子库 286 | tfjs API 使用实例。 287 | 288 | ## 版本需求 289 | - 微信基础库版本 >= 2.7.3 290 | - 微信开发者工具 >= v1.02.1907022 291 | - tfjs-core >= 1.5.2 292 | - tfjs-converter >= 1.5.2 如果使用localStorage模型缓存 293 | 294 | __注意__ 295 | 在微信开发者工具 v1.02.19070300 中，你需要在通用设置中打开硬件加速，从而在TensorFlow.js中启用WebGL加速。 296 | ![setting](https://raw.githubusercontent.com/tensorflow/tfjs-wechat/master/doc/setting.png) 297 | ## 更新说明 298 | - 0.0.2 plugin不再映射TensorFlow.js API库，由小程序端提供。 299 | - 0.0.3 使用offscreen canvas，小程序无需加入plugin component。 300 | - 0.0.5 修改例子程序使用tfjs分包来降低小程序大小。 301 | - 0.0.6 支持 tfjs-core版本1.2.7。 302 | - 0.0.7 允许用户设置webgl backend name, 这可以解决小程序offscreen canvas会失效的问题。 303 | - 0.0.8 加入localStorage支持，允许小于10M模型在localStorage内缓存。 304 | - 0.0.9 加入fileSystem支持，允许小于10M模型在local file system内缓存。fixed missing kernel bug. 305 | - 0.1.0 支持 tfjs版本2.0.x。 306 | - 0.2.0 支持 tfjs版本3.x。 307 | -------------------------------------------------------------------------------- /cloudbuild.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'node:10' 3 | entrypoint: 'yarn' 4 | id: 'yarn' 5 | args: ['install'] 6 | - name: 'node:10' 7 | entrypoint: 'yarn' 8 | id: 'build' 9 | args: ['build'] 10 | waitFor: ['yarn'] 11 | - name: 'node:10' 12 | entrypoint: 'yarn' 13 | id: 'test' 14 | args: ['test-ci'] 15 | waitFor: ['build'] 16 | env: ['BROWSERSTACK_USERNAME=deeplearnjs1'] 17 | secretEnv: ['BROWSERSTACK_KEY'] 18 | secrets: 19 | - kmsKeyName: projects/learnjs-174218/locations/global/keyRings/tfjs/cryptoKeys/enc 20 | secretEnv: 21 | BROWSERSTACK_KEY: CiQAkwyoIW0LcnxymzotLwaH4udVTQFBEN4AEA5CA+a3+yflL2ASPQAD8BdZnGARf78MhH5T9rQqyz9HNODwVjVIj64CTkFlUCGrP1B2HX9LXHWHLmtKutEGTeFFX9XhuBzNExA= 22 | timeout: 1800s 23 | logsBucket: 'gs://tfjs-build-logs' 24 | substitutions: 25 | _NIGHTLY: '' 26 | options: 27 | logStreamingOption: 'STREAM_ON' 28 | machineType: 'N1_HIGHCPU_8' 29 | substitution_option: 'ALLOW_LOOSE' 30 | -------------------------------------------------------------------------------- /demo/mobilenet/README.md: -------------------------------------------------------------------------------- 1 | 1. 安装 npm 2 | 2. 执行以下命令 3 | ``` 4 | $ npm install 5 | ``` 6 | 3. 点击微信开发工具中的‘npm构建’菜单。 7 | 4. 执行以下命令 8 | ``` 9 | $ npm run build 10 | ``` 11 | 5. 用你的小程序 appId 替代 project.config.json 中的 appid. 12 | 6. 在你的小程序管理员界面里加入 tfjs-wechat plugin, 你可以搜索 tensorflow.js 或者 wx6afed118d9e81df9 13 | 7. 将 https://www.gstaticcnapps.cn 加入到你的小程序request合法域名列表中。 14 | -------------------------------------------------------------------------------- /demo/mobilenet/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | const fetchWechat = require('fetch-wechat'); 19 | import * as tf from '@tensorflow/tfjs-core'; 20 | import '@tensorflow/tfjs-backend-wasm'; 21 | import {setWasmPath} from '@tensorflow/tfjs-backend-wasm'; 22 | 23 | const plugin = requirePlugin('tfjsPlugin'); 24 | const ENABLE_DEBUG = true; 25 | // app.js 26 | App({ 27 | globalData: { 28 | localStorageIO: plugin.localStorageIO, 29 | fileStorageIO: plugin.fileStorageIO, 30 | }, 31 | onLaunch: async function() { 32 | plugin.configPlugin( 33 | { 34 | fetchFunc: fetchWechat.fetchFunc(), 35 | tf, 36 | canvas: wx.createOffscreenCanvas() 37 | }, 38 | ENABLE_DEBUG); 39 | const info = wx.getSystemInfoSync(); 40 | console.log(info.platform); 41 | if (info.platform == 'android') { 42 | setWasmPath( 43 | 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-wasm@2.0.0/wasm-out/tfjs-backend-wasm.wasm', 44 | true); 45 | await tf.setBackend('wasm'); 46 | console.log('set wasm as backend'); 47 | } 48 | } 49 | }) 50 | -------------------------------------------------------------------------------- /demo/mobilenet/app.json: -------------------------------------------------------------------------------- 1 | { 2 | "pages": [ 3 | "pages/index/index" 4 | ], 5 | "plugins": { 6 | "tfjsPlugin": { 7 | "version": "dev", 8 | "provider": "wx6afed118d9e81df9" 9 | } 10 | }, 11 | "sitemapLocation": "sitemap.json" 12 | } 13 | -------------------------------------------------------------------------------- /demo/mobilenet/model/classifier.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import * as tf from '@tensorflow/tfjs-core'; 19 | 20 | import {MobileNet} from './mobilenet'; 21 | 22 | export interface CameraSize { 23 | width: number; 24 | height: number; 25 | } 26 | 27 | export class Classifier { 28 | private mobileNet: MobileNet; 29 | constructor(private page: WechatMiniprogram.Page.Instance) {} 30 | async load() { 31 | this.mobileNet = new MobileNet(); 32 | this.page.setData({result: 'loading model...'}); 33 | const start = Date.now(); 34 | await this.mobileNet.load(); 35 | const result = `model loaded: ${Date.now() - start}ms\n`; 36 | this.page.setData({result}); 37 | } 38 | 39 | classify(ab: ArrayBuffer, size: CameraSize) { 40 | const data = new Uint8Array(ab); 41 | let result = ''; 42 | const start = Date.now(); 43 | tf.tidy(() => { 44 | const temp = tf.browser.fromPixels({data, ...size}, 4); 45 | const pixels = 46 | temp.slice([0, 0, 0], [-1, -1, 3]).resizeBilinear([224, 224]); 47 | const tensor = this.mobileNet.predict(pixels) as tf.Tensor; 48 | 49 | const topIndex = tensor.argMax().dataSync()[0]; 50 | result += `prediction: ${Date.now() - start}ms\n`; 51 | result += `${tensor.dataSync()[topIndex]}`; 52 | return result; 53 | }); 54 | this.page.setData({result}); 55 | } 56 | dispose() { 57 | this.mobileNet.dispose(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /demo/mobilenet/model/mobilenet.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | import * as tfc from '@tensorflow/tfjs-converter'; 18 | import * as tf from '@tensorflow/tfjs-core'; 19 | 20 | const GOOGLE_CLOUD_STORAGE_DIR = 21 | 'https://tfhub.dev/google/tfjs-model/imagenet/'; 22 | const MODEL_FILE_URL = 'mobilenet_v2_050_224/feature_vector/3/default/1'; 23 | const PREPROCESS_DIVISOR = 255 / 2; 24 | const STORAGE_KEY = 'mobilenet_model'; 25 | 26 | export interface TopKValue { 27 | label: string; 28 | value: number; 29 | } 30 | export class MobileNet { 31 | private model: tfc.GraphModel; 32 | constructor() {} 33 | 34 | async load() { 35 | // save model into local storage as base64 string 36 | // const storageHandler = getApp().globalData.localStorageIO(STORAGE_KEY); 37 | // save model into files (weight binary) 38 | const storageHandler = getApp().globalData.fileStorageIO( 39 | STORAGE_KEY, wx.getFileSystemManager()); 40 | 41 | try { 42 | this.model = await tfc.loadGraphModel(storageHandler); 43 | } catch (e) { 44 | this.model = await tfc.loadGraphModel( 45 | GOOGLE_CLOUD_STORAGE_DIR + MODEL_FILE_URL, {fromTFHub: true}); 46 | this.model.save(storageHandler); 47 | } 48 | } 49 | 50 | dispose() { 51 | if (this.model) { 52 | this.model.dispose(); 53 | } 54 | } 55 | /** 56 | * Infer through MobileNet. This does standard ImageNet pre-processing before 57 | * inferring through the model. This method returns named activations as well 58 | * as softmax logits. 59 | * 60 | * @param input un-preprocessed input Array. 61 | * @return The softmax logits. 62 | */ 63 | predict(input: tf.Tensor) { 64 | const preprocessedInput = tf.div( 65 | tf.sub(input.asType('float32'), PREPROCESS_DIVISOR), 66 | PREPROCESS_DIVISOR); 67 | const reshapedInput = 68 | preprocessedInput.reshape([1, ...preprocessedInput.shape]); 69 | return this.model.predict(reshapedInput); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /demo/mobilenet/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tfjs-wechat-plugin-mobile-example", 3 | "version": "0.0.1", 4 | "scripts": { 5 | "build": "tsc" 6 | }, 7 | "main": "dist/index.js", 8 | "license": "Apache-2.0", 9 | "devDependencies": { 10 | "typescript": "^3.3.3333", 11 | "miniprogram-api-typings": "^2.8.1" 12 | }, 13 | "dependencies": { 14 | "@tensorflow/tfjs-core": "2.8.6", 15 | "@tensorflow/tfjs-converter": "2.8.6", 16 | "@tensorflow/tfjs-backend-wasm": "2.8.6", 17 | "fetch-wechat": "0.0.3" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /demo/mobilenet/pages/index/index.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /demo/mobilenet/pages/index/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import { Classifier } from '../../model/classifier'; 19 | Page({ 20 | data: { result: '' }, 21 | mobilenetModel: undefined, 22 | mobilenet() { 23 | if (this.mobilenetModel == null) { 24 | const model = new Classifier(this); 25 | model.load().then(() => { 26 | this.setData({ result: 'loading mobilenet model...' }); 27 | this.mobilenetModel = model; 28 | this.setData({ result: 'model loaded.' }); 29 | }); 30 | } 31 | }, 32 | executeMobilenet(frame) { 33 | if (this.mobilenetModel) { 34 | this.mobilenetModel.classify( 35 | frame.data, { width: frame.width, height: frame.height }); 36 | } 37 | }, 38 | async onReady() { 39 | this.mobilenet(); 40 | 41 | // Start the camera API to feed the captured images to the models. 42 | // @ts-ignore the ts definition for this method is worng. 43 | const context = wx.createCameraContext(); 44 | let count = 0; 45 | const listener = (context as any).onCameraFrame((frame) => { 46 | count++; 47 | if (count === 3) { 48 | this.executeMobilenet(frame); 49 | count = 0; 50 | } 51 | }); 52 | listener.start(); 53 | }, 54 | onUnload() { 55 | if (this.mobilenetModel) { 56 | this.mobilenetModel.dispose(); 57 | } 58 | } 59 | }); 60 | -------------------------------------------------------------------------------- /demo/mobilenet/pages/index/index.wxml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | {{result}} 11 | 12 | -------------------------------------------------------------------------------- /demo/mobilenet/pages/index/index.wxss: -------------------------------------------------------------------------------- 1 | /* pages/index/index.wxss */ 2 | .buttons { 3 | display: flex; 4 | flex-direction: row; 5 | align-items: stretch; 6 | } 7 | 8 | .buttons .button { 9 | width: 100px; 10 | height: 36px; 11 | font-size: 15px; 12 | } 13 | 14 | .image-canvas { 15 | width: 224px; 16 | height: 224px; 17 | position: absolute; 18 | top: -300px; 19 | } 20 | .container { 21 | height: 100%; 22 | display: flex; 23 | flex-direction: column; 24 | align-items: center; 25 | justify-content: space-between; 26 | padding: 20rpx 0; 27 | box-sizing: border-box; 28 | position: relative; 29 | } -------------------------------------------------------------------------------- /demo/mobilenet/project.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packOptions": { 3 | "ignore": [ 4 | { 5 | "type": "folder", 6 | "value": ".vscode" 7 | }, 8 | { 9 | "type": "folder", 10 | "value": "node_modules" 11 | }, 12 | { 13 | "type": "folder", 14 | "value": "src" 15 | }, 16 | { 17 | "type": "folder", 18 | "value": "dist/plugin/test" 19 | } 20 | ] 21 | }, 22 | "compileType": "miniprogram", 23 | "setting": { 24 | "urlCheck": false, 25 | "es6": true, 26 | "postcss": true, 27 | "minified": true, 28 | "newFeature": true, 29 | "coverView": true, 30 | "nodeModules": true, 31 | "autoAudits": false, 32 | "checkInvalidKey": true, 33 | "checkSiteMap": true, 34 | "uploadWithSourceMap": true, 35 | "babelSetting": { 36 | "ignore": [], 37 | "disablePlugins": [], 38 | "outputPath": "" 39 | } 40 | }, 41 | "appid": "wx5809925511b85ad7", 42 | "projectname": "TensorFlowJS", 43 | "scripts": { 44 | "beforeCompile": "", 45 | "beforePreview": "", 46 | "beforeUpload": "" 47 | }, 48 | "libVersion": "2.7.5", 49 | "simulatorType": "wechat", 50 | "simulatorPluginLibVersion": {}, 51 | "condition": {} 52 | } 53 | -------------------------------------------------------------------------------- /demo/mobilenet/sitemap.json: -------------------------------------------------------------------------------- 1 | { 2 | "desc": "关于本文件的更多信息，请参考文档 https://developers.weixin.qq.com/miniprogram/dev/framework/sitemap.html", 3 | "rules": [{ 4 | "action": "allow", 5 | "page": "*" 6 | }] 7 | } -------------------------------------------------------------------------------- /demo/mobilenet/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "moduleResolution": "node", 5 | "noImplicitAny": false, 6 | "sourceMap": true, 7 | "removeComments": false, 8 | "preserveConstEnums": true, 9 | "declaration": true, 10 | "target": "es5", 11 | "skipLibCheck": true, 12 | "lib": [ 13 | "es2015", 14 | "dom" 15 | ], 16 | "noUnusedLocals": true, 17 | "noImplicitReturns": true, 18 | "noImplicitThis": true, 19 | "alwaysStrict": true, 20 | "noUnusedParameters": false, 21 | "pretty": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "allowUnreachableCode": false, 24 | "typeRoots": [ 25 | "node_modules/@types" 26 | ], 27 | "types": ["jasmine", "miniprogram-api-typings"] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /demo/mobilenet/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@tensorflow/tfjs-backend-cpu@2.8.6": 6 | version "2.8.6" 7 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-2.8.6.tgz#ef60c3294a04c8c600abb4b438263c06d9b7b7bd" 8 | integrity sha512-x9WTTE9p3Pon2D0d6HH1UCIJsU1w3v9sF3vxJcp+YStrjDefWoW5pwxHCckEKTRra7GWg3CwMKK3Si2dat4H1A== 9 | dependencies: 10 | "@types/seedrandom" "2.4.27" 11 | seedrandom "2.4.3" 12 | 13 | "@tensorflow/tfjs-backend-wasm@2.8.6": 14 | version "2.8.6" 15 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-wasm/-/tfjs-backend-wasm-2.8.6.tgz#472cc0308e8bb2609fc5d869b553aedba25c2995" 16 | integrity sha512-uJQ8e5akjuesOiQGzfOmnlFt4+hppnBGL6Oa+fCeDQFCtfIG4uNoN1ItSA7u/h5TYsjlE5VDZwJQMkO/JAHnTQ== 17 | dependencies: 18 | "@tensorflow/tfjs-backend-cpu" "2.8.6" 19 | "@types/emscripten" "~0.0.34" 20 | 21 | "@tensorflow/tfjs-converter@2.8.6": 22 | version "2.8.6" 23 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-2.8.6.tgz#6182d302ae883e0c45f47674a78bdd33e23db3a6" 24 | integrity sha512-Uv4YC66qjVC9UwBxz0IeLZ8KS2CReh63WlGRtHcSwDEYiwsa7cvp9H6lFSSPT7kiJmrK6JtHeJGIVcTuNnSt9w== 25 | 26 | "@tensorflow/tfjs-core@2.8.6": 27 | version "2.8.6" 28 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-2.8.6.tgz#d5e9d5fc1d1a83e3fbf80942f3154300a9f82494" 29 | integrity sha512-jS28M1POUOjnWgx3jp1v5D45DUQE8USsAHHkL/01z75KnYCAAmgqJSH4YKLiYACg3eBLWXH/KTcSc6dHAX7Kfg== 30 | dependencies: 31 | "@types/offscreencanvas" "~2019.3.0" 32 | "@types/seedrandom" "2.4.27" 33 | "@types/webgl-ext" "0.0.30" 34 | node-fetch "~2.6.1" 35 | seedrandom "2.4.3" 36 | 37 | "@types/emscripten@~0.0.34": 38 | version "0.0.34" 39 | resolved "https://registry.yarnpkg.com/@types/emscripten/-/emscripten-0.0.34.tgz#12b4a344274fb102ff2f6c877b37587bc3e46008" 40 | integrity sha512-QSb9ojDincskc+uKMI0KXp8e1NALFINCrMlp8VGKGcTSxeEyRTTKyjWw75NYrCZHUsVEEEpr1tYHpbtaC++/sQ== 41 | 42 | "@types/offscreencanvas@~2019.3.0": 43 | version "2019.3.0" 44 | resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" 45 | integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== 46 | 47 | "@types/seedrandom@2.4.27": 48 | version "2.4.27" 49 | resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41" 50 | integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE= 51 | 52 | "@types/webgl-ext@0.0.30": 53 | version "0.0.30" 54 | resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" 55 | integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg== 56 | 57 | fetch-wechat@0.0.3: 58 | version "0.0.3" 59 | resolved "https://registry.yarnpkg.com/fetch-wechat/-/fetch-wechat-0.0.3.tgz#2789c332a24bf9f4114b580c02d2934646e74f12" 60 | integrity sha512-uEvCnHSaM53AjYmtiU8n7dB3RO4JX4RBxF3lXNf+cBHr5NRclmTAr5XiRtLL2lSdBcRwvLixhnjwCRJhyEKZgQ== 61 | 62 | miniprogram-api-typings@^2.8.1: 63 | version "2.8.1" 64 | resolved "https://registry.yarnpkg.com/miniprogram-api-typings/-/miniprogram-api-typings-2.8.1.tgz#a56eec32eb03fc1517d27d04f7371745598798cc" 65 | integrity sha512-SWM+lobQTOlbBnoW1fZN7Fmp7WcMEQfwxcBmQuuz92xoPXS3smALFecOcJ/Kwaup86SpR7ETYaZnMj6d30pdfQ== 66 | 67 | node-fetch@~2.6.1: 68 | version "2.6.1" 69 | resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" 70 | integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== 71 | 72 | seedrandom@2.4.3: 73 | version "2.4.3" 74 | resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc" 75 | integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw= 76 | 77 | typescript@^3.3.3333: 78 | version "3.5.1" 79 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.1.tgz#ba72a6a600b2158139c5dd8850f700e231464202" 80 | integrity sha512-64HkdiRv1yYZsSe4xC1WVgamNigVYjlssIoaH2HcZF0+ijsk5YK2g0G34w9wJkze8+5ow4STd22AynfO6ZYYLw== 81 | -------------------------------------------------------------------------------- /demo/posenet/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | const fetchWechat = require('fetch-wechat'); 19 | import * as tf from '@tensorflow/tfjs-core'; 20 | import * as webgl from '@tensorflow/tfjs-backend-webgl'; 21 | const plugin = requirePlugin('tfjsPlugin'); 22 | const ENABLE_DEBUG = true; 23 | // app.js 24 | App({ 25 | onLaunch: function() { 26 | plugin.configPlugin( 27 | { 28 | fetchFunc: fetchWechat.fetchFunc(), 29 | tf, 30 | webgl, 31 | canvas: wx.createOffscreenCanvas() 32 | }, 33 | ENABLE_DEBUG); 34 | } 35 | }) 36 | -------------------------------------------------------------------------------- /demo/posenet/app.json: -------------------------------------------------------------------------------- 1 | { 2 | "pages": [ 3 | "pages/index/index" 4 | ], 5 | "plugins": { 6 | "tfjsPlugin": { 7 | "version": "dev", 8 | "provider": "wx6afed118d9e81df9" 9 | } 10 | }, 11 | "sitemapLocation": "sitemap.json" 12 | } 13 | -------------------------------------------------------------------------------- /demo/posenet/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tfjs-wechat-lugin-example", 3 | "version": "0.0.1", 4 | "scripts": { 5 | "build": "tsc" 6 | }, 7 | "main": "dist/index.js", 8 | "license": "Apache-2.0", 9 | "devDependencies": { 10 | "typescript": "^3.3.3333" 11 | }, 12 | "dependencies": { 13 | "@tensorflow-models/posenet": "2.2.2", 14 | "@tensorflow/tfjs-backend-webgl": "3.5.0", 15 | "@tensorflow/tfjs-converter": "3.5.0", 16 | "@tensorflow/tfjs-core": "3.5.0", 17 | "fetch-wechat": "0.0.3" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /demo/posenet/pages/index/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import * as posenet from '@tensorflow-models/posenet'; 19 | import {detectPoseInRealTime, drawPoses} from '../../posenet/posenet'; 20 | const CANVAS_ID = 'image'; 21 | const POSENET_URL = 22 | 'https://www.gstaticcnapps.cn/tfjs-models/savedmodel/posenet/mobilenet/float/050/model-stride16.json'; 23 | Page({ 24 | data: {result: ''}, 25 | posenetModel: undefined, 26 | canvas: undefined, 27 | poses: undefined, 28 | ctx: undefined, 29 | posenet() { 30 | if (this.posenetModel == null) { 31 | this.setData({result: 'loading posenet model...'}); 32 | posenet 33 | .load({ 34 | architecture: 'MobileNetV1', 35 | outputStride: 16, 36 | inputResolution: 193, 37 | multiplier: 0.5, 38 | modelUrl: POSENET_URL 39 | }) 40 | .then((model) => { 41 | this.posenetModel = model; 42 | this.setData({result: 'model loaded.'}); 43 | }); 44 | } 45 | }, 46 | executePosenet(frame) { 47 | if (this.posenetModel) { 48 | const start = Date.now(); 49 | detectPoseInRealTime(frame, this.posenetModel, false) 50 | .then((poses) => { 51 | this.poses = poses; 52 | drawPoses(this); 53 | const result = `${Date.now() - start}ms`; 54 | this.setData({result}); 55 | }) 56 | .catch((err) => { 57 | console.log(err, err.stack); 58 | }); 59 | } 60 | }, 61 | async onReady() { 62 | console.log('create canvas context for #image...'); 63 | setTimeout(() => { 64 | this.ctx = wx.createCanvasContext(CANVAS_ID); 65 | console.log('ctx', this.ctx); 66 | }, 500); 67 | 68 | this.posenet(); 69 | 70 | // Start the camera API to feed the captured images to the models. 71 | // @ts-ignore the ts definition for this method is worng. 72 | const context = wx.createCameraContext(this); 73 | let count = 0; 74 | const listener = (context as any).onCameraFrame((frame) => { 75 | count++; 76 | if (count === 3) { 77 | this.executePosenet(frame); 78 | count = 0; 79 | } 80 | }); 81 | listener.start(); 82 | }, 83 | onUnload() { 84 | if (this.posenetModel) { 85 | this.posenetModel.dispose(); 86 | } 87 | } 88 | }); 89 | -------------------------------------------------------------------------------- /demo/posenet/pages/index/index.wxml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | {{result}} 12 | 13 | -------------------------------------------------------------------------------- /demo/posenet/pages/index/index.wxss: -------------------------------------------------------------------------------- 1 | /* pages/index/index.wxss */ 2 | .buttons { 3 | display: flex; 4 | flex-direction: row; 5 | align-items: stretch; 6 | } 7 | 8 | .buttons .button { 9 | width: 100px; 10 | height: 36px; 11 | font-size: 15px; 12 | } 13 | 14 | .image-canvas { 15 | width: 224px; 16 | height: 224px; 17 | position: absolute; 18 | top: -300px; 19 | } 20 | .container { 21 | height: 100%; 22 | display: flex; 23 | flex-direction: column; 24 | align-items: center; 25 | justify-content: space-between; 26 | padding: 20rpx 0; 27 | box-sizing: border-box; 28 | position: relative; 29 | } -------------------------------------------------------------------------------- /demo/posenet/posenet/posenet.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2018 Google Inc. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | import * as tf from '@tensorflow/tfjs-core'; 18 | 19 | import {drawKeypoints, drawSkeleton} from './util'; 20 | 21 | export const videoWidth = 288; 22 | export const videoHeight = 352; 23 | export interface Point { 24 | x: number; 25 | y: number; 26 | } 27 | export interface CanvasNode { 28 | width: number; 29 | height: number; 30 | getContext: Function; 31 | createImage: Function; 32 | } 33 | 34 | /** 35 | * Feeds an image to posenet to estimate poses - this is where the magic 36 | * happens. This function loops with a requestAnimationFrame method. 37 | */ 38 | export async function detectPoseInRealTime(image, net, mirror) { 39 | const video: tf.Tensor = tf.tidy(() => { 40 | const temp = tf.tensor(new Uint8Array(image.data), [image.height, image.width, 4]); 41 | return tf.slice(temp, [0, 0, 0], [-1, -1, 3]); 42 | }); 43 | 44 | // since images are being fed from a webcam 45 | const flipHorizontal = mirror; 46 | 47 | const pose = await net.estimateSinglePose( 48 | video, {flipHorizontal}); 49 | video.dispose(); 50 | return [pose]; 51 | } 52 | 53 | export function drawPoses(page) { 54 | if (page.poses == null || page.ctx == null) return; 55 | const ctx = page.ctx; 56 | const poses = page.poses; 57 | const minPoseConfidence = 0.3; 58 | const minPartConfidence = 0.3; 59 | // For each pose (i.e. person) detected in an image, loop through the poses 60 | // and draw the resulting skeleton and keypoints if over certain confidence 61 | // scores 62 | poses.forEach(({score, keypoints}) => { 63 | if (score >= minPoseConfidence) { 64 | drawKeypoints(keypoints, minPartConfidence, ctx); 65 | drawSkeleton(keypoints, minPartConfidence, ctx); 66 | } 67 | }); 68 | ctx.draw(); 69 | return poses; 70 | } 71 | -------------------------------------------------------------------------------- /demo/posenet/posenet/util.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2018 Google Inc. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | import * as posenet from '@tensorflow-models/posenet'; 18 | 19 | const color = 'aqua'; 20 | const boundingBoxColor = 'red'; 21 | const lineWidth = 2; 22 | 23 | function toTuple({ y, x }: { [key: string]: number }): [number, number] { 24 | return [y, x]; 25 | } 26 | 27 | export function drawPoint( 28 | ctx: WechatMiniprogram.CanvasContext, y: number, x: number, r: number, 29 | color: string) { 30 | ctx.beginPath(); 31 | ctx.arc(x, y, r, 0, 2 * Math.PI); 32 | ctx.fillStyle = color; 33 | ctx.fill(); 34 | } 35 | 36 | /** 37 | * Draws a line on a canvas, i.e. a joint 38 | */ 39 | export function drawSegment( 40 | [ay, ax]: [number, number], [by, bx]: [number, number], color: string, 41 | scale: number, ctx: WechatMiniprogram.CanvasContext) { 42 | ctx.beginPath(); 43 | ctx.moveTo(ax * scale, ay * scale); 44 | ctx.lineTo(bx * scale, by * scale); 45 | ctx.lineWidth = lineWidth; 46 | ctx.strokeStyle = color; 47 | ctx.stroke(); 48 | } 49 | 50 | /** 51 | * Draws a pose skeleton by looking up all adjacent keypoints/joints 52 | */ 53 | // tslint:disable-next-line:no-any 54 | export function drawSkeleton( 55 | keypoints: any, minConfidence: number, ctx: WechatMiniprogram.CanvasContext, 56 | scale = 1) { 57 | const adjacentKeyPoints = 58 | posenet.getAdjacentKeyPoints(keypoints, minConfidence); 59 | 60 | // tslint:disable-next-line:no-any 61 | adjacentKeyPoints.forEach((keypoints: any) => { 62 | drawSegment( 63 | toTuple(keypoints[0].position), toTuple(keypoints[1].position), color, 64 | scale, ctx); 65 | }); 66 | } 67 | 68 | /** 69 | * Draw pose keypoints onto a canvas 70 | */ 71 | // tslint:disable-next-line:no-any 72 | export function drawKeypoints( 73 | keypoints: any, minConfidence: number, ctx: WechatMiniprogram.CanvasContext, 74 | scale = 1) { 75 | for (let i = 0; i < keypoints.length; i++) { 76 | const keypoint = keypoints[i]; 77 | 78 | if (keypoint.score < minConfidence) { 79 | continue; 80 | } 81 | 82 | const { y, x } = keypoint.position; 83 | drawPoint(ctx, y * scale, x * scale, 3, color); 84 | } 85 | } 86 | 87 | /** 88 | * Draw the bounding box of a pose. For example, for a whole person standing 89 | * in an image, the bounding box will begin at the nose and extend to one of 90 | * ankles 91 | */ 92 | // tslint:disable-next-line:no-any 93 | export function drawBoundingBox( 94 | keypoints: any, ctx: WechatMiniprogram.CanvasContext) { 95 | const boundingBox = posenet.getBoundingBox(keypoints); 96 | 97 | ctx.rect( 98 | boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX, 99 | boundingBox.maxY - boundingBox.minY); 100 | 101 | ctx.strokeStyle = boundingBoxColor; 102 | ctx.stroke(); 103 | } 104 | -------------------------------------------------------------------------------- /demo/posenet/sitemap.json: -------------------------------------------------------------------------------- 1 | { 2 | "desc": "关于本文件的更多信息，请参考文档 https://developers.weixin.qq.com/miniprogram/dev/framework/sitemap.html", 3 | "rules": [{ 4 | "action": "allow", 5 | "page": "*" 6 | }] 7 | } -------------------------------------------------------------------------------- /demo/posenet/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "moduleResolution": "node", 5 | "noImplicitAny": false, 6 | "sourceMap": true, 7 | "removeComments": false, 8 | "preserveConstEnums": true, 9 | "declaration": true, 10 | "target": "es5", 11 | "skipLibCheck": true, 12 | "lib": [ 13 | "es2015", 14 | "dom" 15 | ], 16 | "noUnusedLocals": true, 17 | "noImplicitReturns": true, 18 | "noImplicitThis": true, 19 | "alwaysStrict": true, 20 | "noUnusedParameters": false, 21 | "pretty": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "allowUnreachableCode": false, 24 | "typeRoots": [ 25 | "plugin/node_modules", 26 | "plugin/typings", 27 | "node_modules/@types" 28 | ], 29 | "types": ["jasmine", "miniprogram-api-typings"], 30 | "outDir": "../dist" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /demo/posenet/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@tensorflow-models/posenet@2.2.2": 6 | version "2.2.2" 7 | resolved "https://registry.yarnpkg.com/@tensorflow-models/posenet/-/posenet-2.2.2.tgz#2abcfa33b43892135d232c9c4408a38cc8f74ba6" 8 | integrity sha512-0SXIksRet/IdX7WVH+JSD6W3upkGHix1hwtd3xykIoIMGR7zQ4SC5+wZcNt9ofASyxNYQoI+tUULUo4LNw0c3w== 9 | 10 | "@tensorflow/tfjs-backend-cpu@3.5.0": 11 | version "3.5.0" 12 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.5.0.tgz#6058fcefa28931f18d774278f53f0f3176e0f229" 13 | integrity sha512-aFSbjVLBKV0OpvE4QGSpIvtxhxpqyz09WhnuMZMhxjxCrjrPYbX7gEjXIHvgF8dEs6SX19XUaChgIftyKd2YIQ== 14 | dependencies: 15 | "@types/seedrandom" "2.4.27" 16 | seedrandom "2.4.3" 17 | 18 | "@tensorflow/tfjs-backend-webgl@3.5.0": 19 | version "3.5.0" 20 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.5.0.tgz#55160168fd7e21c26f8f84cb06e699ae5c2a41b5" 21 | integrity sha512-zeuOmfi0wbrZwjUp6M4hsctQvUuQVmO18A8V68xp+u6QC4jp3P5hNdjF7iOiMW/3iNVylYht8MbHYjQLyJAQdw== 22 | dependencies: 23 | "@tensorflow/tfjs-backend-cpu" "3.5.0" 24 | "@types/offscreencanvas" "~2019.3.0" 25 | "@types/seedrandom" "2.4.27" 26 | "@types/webgl-ext" "0.0.30" 27 | "@types/webgl2" "0.0.5" 28 | seedrandom "2.4.3" 29 | 30 | "@tensorflow/tfjs-converter@3.5.0": 31 | version "3.5.0" 32 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-3.5.0.tgz#6625723e347a452c227469a92541930d5b4aa12f" 33 | integrity sha512-eAx1nEaREzPfcIOnyp1PUvQdjob9MtCJM/syh4v05FBV7UrUNpxNMliA4JgjiFvdnhrM4e06/CeqMA8/AVw2wg== 34 | 35 | "@tensorflow/tfjs-core@3.5.0": 36 | version "3.5.0" 37 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.5.0.tgz#e1ff14f2121208242a192e935f52e86b19be279e" 38 | integrity sha512-NOs9hY9nj/iPaz2cSrWpq8G07i96/ot+1l1RoBoA1s1cZakb43pLhntm1Mj4vhSxp7dbc/BSs+t6/S+45OZTUA== 39 | dependencies: 40 | "@types/offscreencanvas" "~2019.3.0" 41 | "@types/seedrandom" "2.4.27" 42 | "@types/webgl-ext" "0.0.30" 43 | node-fetch "~2.6.1" 44 | seedrandom "2.4.3" 45 | 46 | "@types/offscreencanvas@~2019.3.0": 47 | version "2019.3.0" 48 | resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" 49 | integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q== 50 | 51 | "@types/seedrandom@2.4.27": 52 | version "2.4.27" 53 | resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41" 54 | integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE= 55 | 56 | "@types/webgl-ext@0.0.30": 57 | version "0.0.30" 58 | resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" 59 | integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg== 60 | 61 | "@types/webgl2@0.0.5": 62 | version "0.0.5" 63 | resolved "https://registry.yarnpkg.com/@types/webgl2/-/webgl2-0.0.5.tgz#dd925e20ab8ace80eb4b1e46fda5b109c508fb0d" 64 | integrity sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow== 65 | 66 | fetch-wechat@0.0.3: 67 | version "0.0.3" 68 | resolved "https://registry.yarnpkg.com/fetch-wechat/-/fetch-wechat-0.0.3.tgz#2789c332a24bf9f4114b580c02d2934646e74f12" 69 | integrity sha512-uEvCnHSaM53AjYmtiU8n7dB3RO4JX4RBxF3lXNf+cBHr5NRclmTAr5XiRtLL2lSdBcRwvLixhnjwCRJhyEKZgQ== 70 | 71 | node-fetch@~2.6.1: 72 | version "2.6.1" 73 | resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" 74 | integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== 75 | 76 | seedrandom@2.4.3: 77 | version "2.4.3" 78 | resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc" 79 | integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw= 80 | 81 | typescript@^3.3.3333: 82 | version "3.9.7" 83 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa" 84 | integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw== 85 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /doc/setting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tensorflow/tfjs-wechat/61902723d83697eb56e99744bbedb54a6aabe59c/doc/setting.png -------------------------------------------------------------------------------- /karma.conf.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | path = require('path'); 18 | 19 | const karmaTypescriptConfig = { 20 | tsconfig: 'tsconfig.json', 21 | // Disable coverage reports and instrumentation by default for tests 22 | coverageOptions: {instrumentation: false}, 23 | reports: {}, 24 | bundlerOptions: {sourceMap: true} 25 | }; 26 | module.exports = function(config) { 27 | config.set({ 28 | frameworks: ['jasmine', 'karma-typescript'], 29 | files: [ 30 | 'node_modules/miniprogram-simulate/build.js', 31 | 'src/plugin/utils/**/*.ts', 32 | 'src/plugin/test/**/*.ts', 33 | ], 34 | preprocessors: { 35 | 'src/plugin/**/*.ts': 'karma-typescript' 36 | }, 37 | karmaTypescriptConfig, 38 | reporters: ['progress'], 39 | autoWatch: true, 40 | browsers: ['Chrome'], 41 | browserStack: { 42 | username: process.env.BROWSERSTACK_USERNAME, 43 | accessKey: process.env.BROWSERSTACK_KEY 44 | }, 45 | reportSlowerThan: 500, 46 | browserNoActivityTimeout: 30000, 47 | customLaunchers: { 48 | bs_chrome_mac: { 49 | base: 'BrowserStack', 50 | browser: 'chrome', 51 | browser_version: 'latest', 52 | os: 'OS X', 53 | os_version: 'Sierra' 54 | } 55 | }, 56 | client: { 57 | args: ['--grep', config.grep || ''] 58 | } 59 | }); 60 | }; 61 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tfjs-wechat", 3 | "version": "0.2.0", 4 | "scripts": { 5 | "build": "rimraf dist/ && tsc && ncp src/plugin dist --filter='^(((?!node_modules)(?!typings).)(?!\\.ts$))*$'", 6 | "build-demo": "cd demo && rimraf dist/ && cd posenet && tsc && cd .. && ncp posenet dist --filter='^(((?!node_modules)(?!typings).)(?!\\.ts$))*$'", 7 | "build-mobilenet": "cd demo && rimraf dist/ && cd mobilenet && tsc && cd .. && ncp mobilenet dist --filter='^(((?!node_modules)(?!typings).)(?!\\.ts$))*$'", 8 | "test": "karma start", 9 | "test-ci": "yarn lint && yarn run-browserstack", 10 | "run-browserstack": "karma start --singleRun --browsers='bs_chrome_mac' --reporters='dots,karma-typescript,BrowserStack'", 11 | "lint": "tslint -p . -t verbose" 12 | }, 13 | "main": "dist/index.js", 14 | "license": "Apache-2.0", 15 | "dependencies": { 16 | "abab": "2.0.0", 17 | "text-encoder": "0.0.4" 18 | }, 19 | "devDependencies": { 20 | "@tensorflow/tfjs-core": "3.5.0", 21 | "@tensorflow/tfjs-backend-webgl": "3.5.0", 22 | "@types/jasmine": "~2.8.6", 23 | "clang-format": "1.2.4", 24 | "jasmine-core": "~3.1.0", 25 | "karma": "~6.1.0", 26 | "karma-browserstack-launcher": "~1.4.0", 27 | "karma-chrome-launcher": "~2.2.0", 28 | "karma-commonjs": "1.0.0", 29 | "karma-dirname-preprocessor": "0.0.4", 30 | "karma-filemap-preprocessor": "^0.1.0", 31 | "karma-firefox-launcher": "~1.1.0", 32 | "karma-jasmine": "~2.0.1", 33 | "karma-typescript": "~5.4.0", 34 | "karma-typescript-es6-transform": "~5.0.2", 35 | "miniprogram-api-typings": "^2.10.3-1", 36 | "miniprogram-simulate": "^1.1.6", 37 | "ncp": "2.0.0", 38 | "path": "0.12.7", 39 | "rimraf": "~2.6.2", 40 | "tslint": "~5.8.0", 41 | "tslint-no-circular-imports": "~0.5.0", 42 | "typescript": "^3.3.3333", 43 | "yalc": "^1.0.0-pre.50" 44 | }, 45 | "resolutions": { 46 | "**/**/set-value": "3.0.1", 47 | "**/**/https-proxy-agent": "2.2.3" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /project.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packOptions": { 3 | "ignore": [ 4 | { 5 | "type": "folder", 6 | "value": ".vscode" 7 | }, 8 | { 9 | "type": "folder", 10 | "value": "node_modules" 11 | }, 12 | { 13 | "type": "folder", 14 | "value": "src" 15 | }, 16 | { 17 | "type": "folder", 18 | "value": "dist/plugin/test" 19 | } 20 | ] 21 | }, 22 | "miniprogramRoot": "demo/dist/", 23 | "pluginRoot": "dist/", 24 | "compileType": "plugin", 25 | "setting": { 26 | "urlCheck": false, 27 | "es6": true, 28 | "enhance": false, 29 | "postcss": true, 30 | "minified": true, 31 | "newFeature": true, 32 | "nodeModules": true, 33 | "autoAudits": false, 34 | "scopeDataCheck": false, 35 | "checkInvalidKey": true, 36 | "checkSiteMap": true, 37 | "uploadWithSourceMap": true, 38 | "useMultiFrameRuntime": true, 39 | "useApiHook": true, 40 | "useApiHostProcess": true, 41 | "babelSetting": { 42 | "ignore": [], 43 | "disablePlugins": [], 44 | "outputPath": "" 45 | }, 46 | "bundle": false, 47 | "useIsolateContext": true, 48 | "useCompilerModule": false, 49 | "userConfirmedUseCompilerModuleSwitch": false, 50 | "userConfirmedBundleSwitch": false, 51 | "packNpmManually": false, 52 | "packNpmRelationList": [], 53 | "minifyWXSS": true 54 | }, 55 | "appid": "wx6afed118d9e81df9", 56 | "projectname": "TensorFlowJS", 57 | "scripts": { 58 | "beforeCompile": "", 59 | "beforePreview": "", 60 | "beforeUpload": "" 61 | }, 62 | "libVersion": "2.16.1", 63 | "simulatorType": "wechat", 64 | "simulatorPluginLibVersion": {}, 65 | "condition": {} 66 | } -------------------------------------------------------------------------------- /src/plugin/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | package-lock.json 3 | npm-debug.log 4 | yarn-error.log 5 | .DS_Store 6 | .idea/ 7 | *.tgz 8 | .cache 9 | package/ 10 | 11 | bazel-* 12 | 13 | **.yalc 14 | **yalc.lock 15 | -------------------------------------------------------------------------------- /src/plugin/api/config.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | import { setupWechatPlatform, SystemConfig } from '../utils/wechat_platform'; 18 | 19 | /** 20 | * @param config SystemConfig object that contains the Tensorflow.js runtime, 21 | * fetch polyfill and WeChat offline canvas. 22 | * @param debug enable debug logging for the plugin. 23 | */ 24 | export function configPlugin(config: SystemConfig, debug?: boolean) { 25 | setupWechatPlatform(config, debug); 26 | } 27 | -------------------------------------------------------------------------------- /src/plugin/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | export { configPlugin } from './api/config'; 18 | export { fileStorageIO } from './utils/file_storage'; 19 | export { localStorageIO } from './utils/local_storage'; 20 | export { SystemConfig } from './utils/wechat_platform'; 21 | -------------------------------------------------------------------------------- /src/plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tfjs-wechat", 3 | "version": "0.2.0", 4 | "scripts": { 5 | "build": "tsc" 6 | }, 7 | "main": "dist/index.js", 8 | "license": "Apache-2.0", 9 | "dependencies": { 10 | "abab": "2.0.0", 11 | "text-encoder": "0.0.4", 12 | "base64-js": "1.3.1" 13 | }, 14 | "devDependencies": { 15 | "@tensorflow/tfjs-core": "3.5.0", 16 | "@tensorflow/tfjs-backend-webgl": "3.5.0", 17 | "miniprogram-api-typings": "^3.3.2", 18 | "typescript": "^3.3.3333" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/plugin/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "index.js" 3 | } 4 | -------------------------------------------------------------------------------- /src/plugin/test/file_storage_test.ts: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs-core'; 2 | import {fileStorageIO} from '../utils/file_storage'; 3 | 4 | class FakeFileSystemManager { 5 | private _storage: {}; 6 | constructor(storage) { 7 | this._storage = storage; 8 | } 9 | 10 | unlink({filePath, success, fail}) { 11 | const res = delete this._storage[filePath]; 12 | success({data: res}); 13 | } 14 | 15 | readFile({filePath, encoding, success, fail}) { 16 | success({data: this._storage[filePath]}); 17 | } 18 | 19 | writeFile({filePath, data, encoding, success, fail}) { 20 | this._storage[filePath] = data; 21 | success({data: this._storage[filePath]}); 22 | } 23 | 24 | access({path, success, fail}) { 25 | if (this._storage[path]) { 26 | success(); 27 | } else { 28 | fail(); 29 | } 30 | } 31 | 32 | mkdir({dirPath, recursive, success, fail}) { 33 | success(); 34 | } 35 | } 36 | 37 | describe('fileStorageIO', () => { 38 | // Test data. 39 | const modelTopology1: {} = { 40 | 'class_name': 'Sequential', 41 | 'keras_version': '2.1.4', 42 | 'config': [{ 43 | 'class_name': 'Dense', 44 | 'config': { 45 | 'kernel_initializer': { 46 | 'class_name': 'VarianceScaling', 47 | 'config': { 48 | 'distribution': 'uniform', 49 | 'scale': 1.0, 50 | 'seed': null, 51 | 'mode': 'fan_avg' 52 | } 53 | }, 54 | 'name': 'dense', 55 | 'kernel_constraint': null, 56 | 'bias_regularizer': null, 57 | 'bias_constraint': null, 58 | 'dtype': 'float32', 59 | 'activation': 'linear', 60 | 'trainable': true, 61 | 'kernel_regularizer': null, 62 | 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 63 | 'units': 1, 64 | 'batch_input_shape': [null, 3], 65 | 'use_bias': true, 66 | 'activity_regularizer': null 67 | } 68 | }], 69 | 'backend': 'tensorflow' 70 | }; 71 | 72 | const weightSpecs1: tf.io.WeightsManifestEntry[] = [ 73 | { 74 | name: 'dense/kernel', 75 | shape: [3, 1], 76 | dtype: 'float32', 77 | }, 78 | { 79 | name: 'dense/bias', 80 | shape: [1], 81 | dtype: 'float32', 82 | } 83 | ]; 84 | const weightData1 = new ArrayBuffer(16); 85 | 86 | const artifacts1: tf.io.ModelArtifacts = { 87 | modelTopology: modelTopology1, 88 | weightSpecs: weightSpecs1, 89 | weightData: weightData1, 90 | format: 'layers-model', 91 | generatedBy: 'TensorFlow.js v0.0.0', 92 | convertedBy: null 93 | }; 94 | 95 | const artifactsV0: tf.io.ModelArtifacts = { 96 | modelTopology: modelTopology1, 97 | weightSpecs: weightSpecs1, 98 | weightData: weightData1 99 | }; 100 | 101 | let storage = {}; 102 | // tslint:disable-next-line:no-any 103 | let fileSystemManager: any; 104 | 105 | beforeEach(() => { 106 | storage = {}; 107 | fileSystemManager = new FakeFileSystemManager(storage); 108 | if (!wx.getFileSystemManager) { 109 | wx.getFileSystemManager = () => 110 | null; // miniprogram-simulate don't have it 111 | } 112 | }); 113 | 114 | it('constructs an IOHandler', async () => { 115 | const handler = fileStorageIO('foo', fileSystemManager); 116 | expect(typeof handler.load).toBe('function'); 117 | expect(typeof handler.save).toBe('function'); 118 | }); 119 | 120 | it('save returns a SaveResult', async () => { 121 | const handler = fileStorageIO('FooModel', fileSystemManager); 122 | const result = await handler.save(artifacts1); 123 | expect(result.modelArtifactsInfo).toBeDefined(); 124 | expect(result.modelArtifactsInfo).not.toBeNull(); 125 | 126 | expect(result.modelArtifactsInfo.modelTopologyType).toBe('JSON'); 127 | expect(result.modelArtifactsInfo.weightDataBytes).toBe(16); 128 | }); 129 | 130 | it('Save-load round trip succeeds', async () => { 131 | const handler = fileStorageIO('FooModel', fileSystemManager); 132 | await handler.save(artifacts1); 133 | 134 | const handler2 = fileStorageIO('FooModel', fileSystemManager); 135 | const loaded = await handler2.load(); 136 | 137 | expect(loaded.modelTopology).toEqual(modelTopology1); 138 | expect(loaded.weightSpecs).toEqual(weightSpecs1); 139 | expect(loaded.weightData).toEqual(weightData1); 140 | expect(loaded.format).toEqual('layers-model'); 141 | expect(loaded.generatedBy).toEqual('TensorFlow.js v0.0.0'); 142 | expect(loaded.convertedBy).toEqual(null); 143 | }); 144 | 145 | it('Save-load round trip succeeds: v0 format', async () => { 146 | const handler1 = fileStorageIO('FooModel', fileSystemManager); 147 | await handler1.save(artifactsV0); 148 | 149 | const handler2 = fileStorageIO('FooModel', fileSystemManager); 150 | const loaded = await handler2.load(); 151 | 152 | expect(loaded.modelTopology).toEqual(modelTopology1); 153 | expect(loaded.weightSpecs).toEqual(weightSpecs1); 154 | expect(loaded.weightData).toEqual(weightData1); 155 | expect(loaded.format).toEqual(undefined); 156 | expect(loaded.generatedBy).toEqual(undefined); 157 | expect(loaded.convertedBy).toEqual(undefined); 158 | }); 159 | }); 160 | -------------------------------------------------------------------------------- /src/plugin/test/local_storage_test.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2020 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import * as tf from '@tensorflow/tfjs-core'; 19 | import { localStorageIO } from '../utils/local_storage'; 20 | 21 | describe('localStorageIO', () => { 22 | // Test data. 23 | const modelTopology1: {} = { 24 | 'class_name': 'Sequential', 25 | 'keras_version': '2.1.4', 26 | 'config': [{ 27 | 'class_name': 'Dense', 28 | 'config': { 29 | 'kernel_initializer': { 30 | 'class_name': 'VarianceScaling', 31 | 'config': { 32 | 'distribution': 'uniform', 33 | 'scale': 1.0, 34 | 'seed': null, 35 | 'mode': 'fan_avg' 36 | } 37 | }, 38 | 'name': 'dense', 39 | 'kernel_constraint': null, 40 | 'bias_regularizer': null, 41 | 'bias_constraint': null, 42 | 'dtype': 'float32', 43 | 'activation': 'linear', 44 | 'trainable': true, 45 | 'kernel_regularizer': null, 46 | 'bias_initializer': { 'class_name': 'Zeros', 'config': {} }, 47 | 'units': 1, 48 | 'batch_input_shape': [null, 3], 49 | 'use_bias': true, 50 | 'activity_regularizer': null 51 | } 52 | }], 53 | 'backend': 'tensorflow' 54 | }; 55 | 56 | const weightSpecs1: tf.io.WeightsManifestEntry[] = [ 57 | { 58 | name: 'dense/kernel', 59 | shape: [3, 1], 60 | dtype: 'float32', 61 | }, 62 | { 63 | name: 'dense/bias', 64 | shape: [1], 65 | dtype: 'float32', 66 | } 67 | ]; 68 | const weightData1 = new ArrayBuffer(16); 69 | 70 | const artifacts1: tf.io.ModelArtifacts = { 71 | modelTopology: modelTopology1, 72 | weightSpecs: weightSpecs1, 73 | weightData: weightData1, 74 | format: 'layers-model', 75 | generatedBy: 'TensorFlow.js v0.0.0', 76 | convertedBy: null 77 | }; 78 | 79 | const artifactsV0: tf.io.ModelArtifacts = { 80 | modelTopology: modelTopology1, 81 | weightSpecs: weightSpecs1, 82 | weightData: weightData1 83 | }; 84 | 85 | let storage: {}; 86 | 87 | beforeEach(() => { 88 | storage = {}; 89 | spyOn(wx, 'getStorage').and.callFake(({ key, success, fail }) => { 90 | success({ data: storage[key] }); 91 | }); 92 | spyOn(wx, 'setStorageSync').and.callFake((key, value) => { 93 | storage[key] = value; 94 | }); 95 | spyOn(wx, 'removeStorageSync').and.callFake((key) => { 96 | delete storage[key]; 97 | }); 98 | }); 99 | 100 | it('constructs an IOHandler', async () => { 101 | const handler = localStorageIO('foo'); 102 | expect(typeof handler.load).toBe('function'); 103 | expect(typeof handler.save).toBe('function'); 104 | }); 105 | 106 | it('save returns a SaveResult', async () => { 107 | const handler = localStorageIO('FooModel'); 108 | const result = await handler.save(artifacts1); 109 | expect(result.modelArtifactsInfo).toBeDefined(); 110 | expect(result.modelArtifactsInfo).not.toBeNull(); 111 | 112 | expect(result.modelArtifactsInfo.modelTopologyType).toBe('JSON'); 113 | expect(result.modelArtifactsInfo.weightDataBytes).toBe(16); 114 | }); 115 | 116 | it('Save-load round trip succeeds', async () => { 117 | const handler = localStorageIO('FooModel'); 118 | await handler.save(artifacts1); 119 | 120 | const handler2 = localStorageIO('FooModel'); 121 | const loaded = await handler2.load(); 122 | 123 | expect(loaded.modelTopology).toEqual(modelTopology1); 124 | expect(loaded.weightSpecs).toEqual(weightSpecs1); 125 | expect(loaded.weightData).toEqual(weightData1); 126 | expect(loaded.format).toEqual('layers-model'); 127 | expect(loaded.generatedBy).toEqual('TensorFlow.js v0.0.0'); 128 | expect(loaded.convertedBy).toEqual(null); 129 | }); 130 | 131 | it('Save-load round trip succeeds: v0 format', async () => { 132 | const handler1 = localStorageIO('FooModel'); 133 | await handler1.save(artifactsV0); 134 | 135 | const handler2 = localStorageIO('FooModel'); 136 | const loaded = await handler2.load(); 137 | 138 | expect(loaded.modelTopology).toEqual(modelTopology1); 139 | expect(loaded.weightSpecs).toEqual(weightSpecs1); 140 | expect(loaded.weightData).toEqual(weightData1); 141 | expect(loaded.format).toEqual(undefined); 142 | expect(loaded.generatedBy).toEqual(undefined); 143 | expect(loaded.convertedBy).toEqual(undefined); 144 | }); 145 | }); 146 | -------------------------------------------------------------------------------- /src/plugin/test/wechat_platform_test.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import { Platform } from '@tensorflow/tfjs-core/dist/platforms/platform'; 19 | 20 | import { setupWechatPlatform, WECHAT_WEBGL_BACKEND_NAME } from '../utils/wechat_platform'; 21 | 22 | const fetchFunc = () => { }; 23 | let backends: { [key: string]: {} } = {}; 24 | let platformName: string; 25 | let platform: Platform; 26 | let currentBackend: string; 27 | // tslint:disable-next-line:no-any 28 | let global: any = {}; 29 | const kernelFunc = () => true; 30 | const kernelName = 'div'; 31 | const tf = { 32 | getKernelsForBackend: 33 | (backend) => [{ kernelName, backendName: backend, kernelFunc }], 34 | registerKernel: (kernel) => true, 35 | getBackend: () => currentBackend, 36 | findBackend: (name: string) => backends[name], 37 | registerBackend: (name: string, callback: {}, priority: number) => { 38 | backends[name] = callback; 39 | }, 40 | setBackend: (name: string) => currentBackend = name, 41 | ENV: { 42 | global, 43 | set: () => { }, 44 | setPlatform: (name, p) => { 45 | platformName = name; 46 | platform = p; 47 | } 48 | } 49 | }; 50 | const webgl = { 51 | setWebGLContext: () => { }, 52 | }; 53 | const gl = {}; 54 | const canvas = { 55 | getContext: () => gl 56 | }; 57 | const backendName = 'test-webgl'; 58 | const config = { 59 | fetchFunc, 60 | tf, 61 | webgl, 62 | canvas 63 | }; 64 | 65 | function clearState() { 66 | platform = undefined; 67 | platformName = undefined; 68 | backends = {}; 69 | currentBackend = undefined; 70 | global = {}; 71 | } 72 | 73 | describe('setupWechatPlatform', () => { 74 | beforeEach(() => { 75 | backends = {}; 76 | }); 77 | 78 | afterEach(() => clearState()); 79 | 80 | it('should register platform', () => { 81 | setupWechatPlatform(config); 82 | expect(platformName).toEqual('wechat'); 83 | expect(platform).toBeDefined(); 84 | }); 85 | 86 | it('should polyfill now', () => { 87 | setupWechatPlatform(config); 88 | const earlier = Date.now(); 89 | const result = platform.now(); 90 | expect(result).toBeGreaterThanOrEqual(earlier); 91 | }); 92 | 93 | it('encodeUTF8 single string', () => { 94 | setupWechatPlatform(config); 95 | const bytes = platform.encode('hello', 'utf-8'); 96 | expect(bytes.length).toBe(5); 97 | expect(bytes).toEqual(new Uint8Array([104, 101, 108, 108, 111])); 98 | }); 99 | 100 | it('encodeUTF8 two strings delimited', () => { 101 | setupWechatPlatform(config); 102 | const bytes = platform.encode('hello\x00world', 'utf-8'); 103 | expect(bytes.length).toBe(11); 104 | expect(bytes).toEqual( 105 | new Uint8Array([104, 101, 108, 108, 111, 0, 119, 111, 114, 108, 100])); 106 | }); 107 | 108 | it('encodeUTF8 cyrillic', () => { 109 | setupWechatPlatform(config); 110 | const bytes = platform.encode('Здраво', 'utf-8'); 111 | expect(bytes.length).toBe(12); 112 | expect(bytes).toEqual(new Uint8Array( 113 | [208, 151, 208, 180, 209, 128, 208, 176, 208, 178, 208, 190])); 114 | }); 115 | 116 | it('decode single string', () => { 117 | setupWechatPlatform(config); 118 | const s = 119 | platform.decode(new Uint8Array([104, 101, 108, 108, 111]), 'utf-8'); 120 | expect(s.length).toBe(5); 121 | expect(s).toEqual('hello'); 122 | }); 123 | 124 | it('decode two strings delimited', () => { 125 | setupWechatPlatform(config); 126 | const s = platform.decode( 127 | new Uint8Array([104, 101, 108, 108, 111, 0, 119, 111, 114, 108, 100]), 128 | 'utf-8'); 129 | expect(s.length).toBe(11); 130 | expect(s).toEqual('hello\x00world'); 131 | }); 132 | 133 | it('decode cyrillic', () => { 134 | setupWechatPlatform(config); 135 | const s = platform.decode( 136 | new Uint8Array( 137 | [208, 151, 208, 180, 209, 128, 208, 176, 208, 178, 208, 190]), 138 | 'utf-8'); 139 | expect(s.length).toBe(6); 140 | expect(s).toEqual('Здраво'); 141 | }); 142 | 143 | it('should polyfill fetch', () => { 144 | spyOn(config, 'fetchFunc'); 145 | setupWechatPlatform(config); 146 | platform.fetch('url', {}); 147 | expect(config.fetchFunc).toHaveBeenCalledWith('url', {}); 148 | }); 149 | 150 | it('should polyfill btoa', () => { 151 | setupWechatPlatform(config); 152 | expect(tf.ENV.global.btoa).toBeDefined(); 153 | }); 154 | 155 | it('should polyfill atob', () => { 156 | setupWechatPlatform(config); 157 | expect(tf.ENV.global.atob).toBeDefined(); 158 | }); 159 | 160 | it('should set tf backend to wechat-webgl', () => { 161 | setupWechatPlatform(config); 162 | expect(tf.getBackend()).toEqual(WECHAT_WEBGL_BACKEND_NAME); 163 | }); 164 | 165 | it('should set tf backend to test-webgl', () => { 166 | const configWithBackendName = { fetchFunc, tf, webgl, canvas, backendName }; 167 | setupWechatPlatform(configWithBackendName); 168 | expect(tf.getBackend()).toEqual(backendName); 169 | }); 170 | 171 | it('should set the WEBGL context', () => { 172 | spyOn(webgl, 'setWebGLContext'); 173 | setupWechatPlatform(config); 174 | expect(webgl.setWebGLContext).toHaveBeenCalledWith(1, gl); 175 | }); 176 | 177 | it('should register kernel for new backend', () => { 178 | spyOn(tf, 'registerKernel'); 179 | setupWechatPlatform(config); 180 | expect(tf.registerKernel) 181 | .toHaveBeenCalledWith( 182 | { kernelName, backendName: WECHAT_WEBGL_BACKEND_NAME, kernelFunc }); 183 | }); 184 | }); 185 | -------------------------------------------------------------------------------- /src/plugin/typings/abab.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'abab'; 2 | -------------------------------------------------------------------------------- /src/plugin/utils/file_storage.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Save and load model into miniprogram file system 3 | * https://developers.weixin.qq.com/miniprogram/dev/api/file/wx.getFileSystemManager.html 4 | */ 5 | import {io} from '@tensorflow/tfjs-core'; 6 | import {getModelArtifactsInfoForJSON} from './model_artifacts'; 7 | 8 | type StoragePaths = { 9 | info: string, 10 | modelArtifactsWithoutWeights: string, 11 | weightData: string, 12 | }; 13 | 14 | const PATH_SEPARATOR = '/'; 15 | const PATH_PREFIX = 'tensorflowjs_models'; 16 | const INFO_SUFFIX = 'info.json'; 17 | const MODEL_SUFFIX = 'model_without_weight.json'; 18 | const WEIGHT_DATA_SUFFIX = 'weight_data'; 19 | 20 | // get user path 21 | // https://developers.weixin.qq.com/miniprogram/dev/api/base/env/env.html 22 | function getUserDataPath() { 23 | if (!wx.env) { 24 | // only for tests. If not, test failed in miniprogram-simulate compiler 25 | wx.env = { 26 | USER_DATA_PATH: 27 | 'http://usr', // value of wx.env.USER_DATA_PATH in simulate 28 | }; 29 | } 30 | return wx.env.USER_DATA_PATH; 31 | } 32 | const MODEL_PATH = [getUserDataPath(), PATH_PREFIX].join(PATH_SEPARATOR); 33 | 34 | function getModelPaths(prefix: string): StoragePaths { 35 | return { 36 | info: [MODEL_PATH, `${prefix}_${INFO_SUFFIX}`].join(PATH_SEPARATOR), 37 | modelArtifactsWithoutWeights: 38 | [MODEL_PATH, `${prefix}_${MODEL_SUFFIX}`].join(PATH_SEPARATOR), 39 | weightData: 40 | [MODEL_PATH, `${prefix}_${WEIGHT_DATA_SUFFIX}`].join(PATH_SEPARATOR), 41 | }; 42 | } 43 | 44 | /** 45 | * Make remove file 46 | * This function will ignore removed error (file not existed) 47 | * https://developers.weixin.qq.com/miniprogram/dev/api/file/FileSystemManager.unlink.html 48 | * @param fsm the file system manager 49 | * @param filePath the file path to be removed 50 | */ 51 | function removeFile( 52 | fsm: WechatMiniprogram.FileSystemManager, 53 | filePath: string, 54 | ): Promise { 56 | return new Promise((resolve, reject) => { 57 | fsm.unlink({ 58 | filePath, 59 | success: (res) => { 60 | resolve(res); 61 | }, 62 | fail: (res) => { 63 | // ignore error as we don't care if file is removed failed 64 | resolve(res); 65 | } 66 | }); 67 | }); 68 | } 69 | 70 | /** 71 | * Read file 72 | * https://developers.weixin.qq.com/miniprogram/dev/api/file/FileSystemManager.readFile.html 73 | * @param fsm the file system manager 74 | * @param filePath the file path 75 | * @param encoding the encoding, default reture ArrayBuffer if undefined 76 | */ 77 | function readFile( 78 | fsm: WechatMiniprogram.FileSystemManager, 79 | filePath: string, 80 | encoding?: 'utf-8', 81 | ): Promise { 82 | return new Promise((resolve, reject) => { 83 | fsm.readFile({ 84 | filePath, 85 | encoding, 86 | success: (res) => { 87 | if (res.data) { 88 | resolve(res.data); 89 | return; 90 | } 91 | reject(new Error('Empty File')); 92 | }, 93 | fail: (res) => { 94 | reject(new Error(res.errMsg)); 95 | } 96 | }); 97 | }); 98 | } 99 | 100 | /** 101 | * Write file 102 | * https://developers.weixin.qq.com/miniprogram/dev/api/file/FileSystemManager.writeFile.html 103 | * @param fsm the file system manager 104 | * @param filePath the file path 105 | * @param data data to save 106 | * @param encoding encoding 107 | */ 108 | function writeFile( 109 | fsm: WechatMiniprogram.FileSystemManager, filePath: string, 110 | data: string|ArrayBuffer, encoding: 'binary'|'utf-8' = 'binary'): 111 | Promise { 112 | return new Promise((resolve, reject) => { 113 | removeFile(fsm, filePath).then(() => { 114 | fsm.writeFile({ 115 | filePath, 116 | data, 117 | encoding, 118 | success: (res) => { 119 | resolve(res); 120 | }, 121 | fail: (res) => { 122 | reject(new Error(res.errMsg)); 123 | } 124 | }); 125 | }); 126 | }); 127 | } 128 | 129 | /** 130 | * Create folder 131 | * https://developers.weixin.qq.com/miniprogram/dev/api/file/FileSystemManager.mkdir.html 132 | * @param fsm the file system manager 133 | * @param dirPath the dir path 134 | */ 135 | function mkdir(fsm: WechatMiniprogram.FileSystemManager, dirPath: string): 136 | Promise { 137 | return new Promise((resolve, reject) => { 138 | fsm.access({ 139 | path: dirPath, 140 | success: () => { 141 | resolve(); 142 | }, 143 | fail: () => { 144 | fsm.mkdir({ 145 | dirPath, 146 | recursive: true, 147 | success: (res) => { 148 | resolve(res); 149 | }, 150 | fail: (res) => { 151 | reject(new Error(res.errMsg)); 152 | } 153 | }); 154 | } 155 | }); 156 | }); 157 | } 158 | 159 | class FileStorageHandler implements io.IOHandler { 160 | protected readonly paths: StoragePaths; 161 | 162 | constructor( 163 | private prefix: string, 164 | private fsm: WechatMiniprogram.FileSystemManager) { 165 | if (prefix == null || !prefix) { 166 | throw new Error('prefix must not be null, undefined or empty.'); 167 | } 168 | this.paths = getModelPaths(this.prefix); 169 | } 170 | 171 | /** 172 | * Save model artifacts to File 173 | * 174 | * @param modelArtifacts The model artifacts to be stored. 175 | * @returns An instance of SaveResult. 176 | */ 177 | async save(modelArtifacts: io.ModelArtifacts): Promise { 178 | if (modelArtifacts.modelTopology instanceof ArrayBuffer) { 179 | throw new Error( 180 | 'AsyncStorageHandler.save() does not support saving model topology ' + 181 | 'in binary format.'); 182 | } else { 183 | // We save three items separately for each model, 184 | // a ModelArtifactsInfo, a ModelArtifacts without weights 185 | // and the model weights. 186 | const modelArtifactsInfo = getModelArtifactsInfoForJSON(modelArtifacts); 187 | const {weightData, ...modelArtifactsWithoutWeights} = modelArtifacts; 188 | 189 | try { 190 | await mkdir(this.fsm, MODEL_PATH); 191 | await writeFile( 192 | this.fsm, this.paths.info, JSON.stringify(modelArtifactsInfo), 193 | 'utf-8'); 194 | await writeFile( 195 | this.fsm, this.paths.modelArtifactsWithoutWeights, 196 | JSON.stringify(modelArtifactsWithoutWeights), 'utf-8'); 197 | await writeFile(this.fsm, this.paths.weightData, weightData); 198 | return {modelArtifactsInfo}; 199 | } catch (err) { 200 | // If saving failed, clean up all items saved so far. 201 | await removeFile(this.fsm, this.paths.info); 202 | await removeFile(this.fsm, this.paths.modelArtifactsWithoutWeights); 203 | await removeFile(this.fsm, this.paths.weightData); 204 | 205 | throw new Error(err); 206 | } 207 | } 208 | } 209 | 210 | /** 211 | * Load a model from files. 212 | * 213 | * @returns The loaded model (if loading succeeds). 214 | */ 215 | async load(): Promise { 216 | const info = JSON.parse( 217 | (await readFile(this.fsm, this.paths.info, 'utf-8')) as string); 218 | if (info == null) { 219 | throw new Error( 220 | `In file storage, there is no model with name '${this.prefix}'`); 221 | } 222 | 223 | if (info.modelTopologyType !== 'JSON') { 224 | throw new Error( 225 | 'fileStorage does not support loading non-JSON model ' + 226 | 'topology yet.'); 227 | } 228 | 229 | const modelArtifacts = JSON.parse( 230 | (await readFile( 231 | this.fsm, this.paths.modelArtifactsWithoutWeights, 'utf-8')) as 232 | string); 233 | 234 | // load weight data 235 | modelArtifacts.weightData = await readFile(this.fsm, this.paths.weightData); 236 | return modelArtifacts; 237 | } 238 | } 239 | 240 | /** 241 | * Factory function for FileStorage IOHandler. 242 | * 243 | * This `IOHandler` supports both `save` and `load`. 244 | * 245 | * For each model's saved artifacts, three items are saved to user files. 246 | * - `${PATH_PREFIX}/${prefix}/info.json`: Contains meta-info about the 247 | * model, such as date saved, type of the topology, size in bytes, etc. 248 | * - `${PATH_PREFIX}/${prefix}/model_without_weight.json`: The topology, 249 | * weights_specs and all other information about the model except for the 250 | * weights. 251 | * - `${PATH_PREFIX}/${prefix}/weight_data`: Concatenated binary 252 | * weight values, stored as binary. 253 | * 254 | * @param prefix A unique identifier for the model to be saved. Must be a 255 | * non-empty string. 256 | * @param fileManager WeChat fileManager. 257 | * @returns An instance of `IOHandler` 258 | */ 259 | export function fileStorageIO( 260 | prefix: string, 261 | fileManager: WechatMiniprogram.FileSystemManager): io.IOHandler { 262 | return new FileStorageHandler(prefix, fileManager); 263 | } 264 | -------------------------------------------------------------------------------- /src/plugin/utils/local_storage.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2020 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import { io } from '@tensorflow/tfjs-core'; 19 | import { fromByteArray, toByteArray } from 'base64-js'; 20 | import { getModelArtifactsInfoForJSON } from './model_artifacts'; 21 | 22 | type StorageKeys = { 23 | info: string, 24 | modelArtifactsWithoutWeights: string, 25 | weightData: string, 26 | weightDataChunkSize: string 27 | }; 28 | 29 | const PATH_SEPARATOR = '/'; 30 | const PATH_PREFIX = 'tensorflowjs_models'; 31 | const INFO_SUFFIX = 'info'; 32 | const MODEL_SUFFIX = 'model_without_weight'; 33 | const WEIGHT_DATA_SUFFIX = 'weight_data'; 34 | const WEIGHT_DATA_SIZE_SUFFIX = 'weight_data_size'; 35 | 36 | function getModelKeys(path: string): StorageKeys { 37 | return { 38 | info: [PATH_PREFIX, path, INFO_SUFFIX].join(PATH_SEPARATOR), 39 | modelArtifactsWithoutWeights: 40 | [PATH_PREFIX, path, MODEL_SUFFIX].join(PATH_SEPARATOR), 41 | weightData: [PATH_PREFIX, path, WEIGHT_DATA_SUFFIX].join(PATH_SEPARATOR), 42 | weightDataChunkSize: 43 | [PATH_PREFIX, path, WEIGHT_DATA_SIZE_SUFFIX].join(PATH_SEPARATOR) 44 | }; 45 | } 46 | 47 | /** 48 | * Split the string into array of substrings with the same size. 49 | * @param str the str to be splitted 50 | * @param size 51 | */ 52 | function splitString(str: string, size: number): string[] { 53 | return str.match(new RegExp(`.{1,${size}}`, 'g')); 54 | } 55 | 56 | // tslint:disable-next-line:no-any 57 | function getStorage(key: string): Promise { 58 | return new Promise((resolve, reject) => { 59 | wx.getStorage( 60 | { key, success: (res) => resolve(res.data), fail: (res) => reject(res) }); 61 | }); 62 | } 63 | class LocalStorageHandler implements io.IOHandler { 64 | protected readonly keys: StorageKeys; 65 | 66 | constructor(protected readonly modelPath: string) { 67 | if (modelPath == null || !modelPath) { 68 | throw new Error('modelPath must not be null, undefined or empty.'); 69 | } 70 | this.keys = getModelKeys(this.modelPath); 71 | } 72 | 73 | /** 74 | * Save model artifacts to AsyncStorage 75 | * 76 | * @param modelArtifacts The model artifacts to be stored. 77 | * @returns An instance of SaveResult. 78 | */ 79 | async save(modelArtifacts: io.ModelArtifacts): Promise { 80 | if (modelArtifacts.modelTopology instanceof ArrayBuffer) { 81 | throw new Error( 82 | 'AsyncStorageHandler.save() does not support saving model topology ' + 83 | 'in binary format.'); 84 | } else { 85 | // We save three items separately for each model, 86 | // a ModelArtifactsInfo, a ModelArtifacts without weights 87 | // and the model weights. 88 | const modelArtifactsInfo: io.ModelArtifactsInfo = 89 | getModelArtifactsInfoForJSON(modelArtifacts); 90 | const { weightData, ...modelArtifactsWithoutWeights } = modelArtifacts; 91 | const weights = splitString(fromByteArray(new Uint8Array(weightData)), 92 | 800 * 1024); 93 | 94 | try { 95 | wx.setStorageSync(this.keys.info, JSON.stringify(modelArtifactsInfo)); 96 | wx.setStorageSync( 97 | this.keys.modelArtifactsWithoutWeights, 98 | JSON.stringify(modelArtifactsWithoutWeights)); 99 | // split the weight string into 10M chunk, size local storage has a 100 | // size limit. 101 | wx.setStorageSync(this.keys.weightDataChunkSize, weights.length); 102 | weights.forEach((value, index) => { 103 | wx.setStorageSync( 104 | `${this.keys.weightData}:${index}`, value); 105 | }); 106 | return { modelArtifactsInfo }; 107 | } catch (err) { 108 | // If saving failed, clean up all items saved so far. 109 | wx.removeStorageSync(this.keys.info); 110 | weights.forEach((value, index) => { 111 | wx.removeStorageSync(`${this.keys.weightData}:${index}`); 112 | }); 113 | wx.removeStorageSync(this.keys.weightDataChunkSize); 114 | wx.removeStorageSync(this.keys.modelArtifactsWithoutWeights); 115 | 116 | throw new Error( 117 | `Failed to save model '${this.modelPath}' to local storage. 118 | Error info ${err}`); 119 | } 120 | } 121 | } 122 | 123 | /** 124 | * Load a model from local storage. 125 | * 126 | * See the documentation to `browserLocalStorage` for details on the saved 127 | * artifacts. 128 | * 129 | * @returns The loaded model (if loading succeeds). 130 | */ 131 | async load(): Promise { 132 | const info = 133 | JSON.parse(await getStorage(this.keys.info)) as io.ModelArtifactsInfo; 134 | if (info == null) { 135 | throw new Error( 136 | `In local storage, there is no model with name '${this.modelPath}'`); 137 | } 138 | 139 | if (info.modelTopologyType !== 'JSON') { 140 | throw new Error( 141 | 'BrowserLocalStorage does not support loading non-JSON model ' + 142 | 'topology yet.'); 143 | } 144 | 145 | const modelArtifacts: io.ModelArtifacts = 146 | JSON.parse(await getStorage(this.keys.modelArtifactsWithoutWeights)); 147 | 148 | // Load weight data. 149 | const weightDataSize = await getStorage(this.keys.weightDataChunkSize); 150 | let weightDataBase64 = ''; 151 | for (let i = 0; i < weightDataSize; i++) { 152 | const partialData = await getStorage(`${this.keys.weightData}:${i}`); 153 | if (partialData == null) { 154 | throw new Error( 155 | `In local storage, the binary weight values of model ` + 156 | `'${this.modelPath}' are missing.`); 157 | } 158 | weightDataBase64 += partialData; 159 | } 160 | modelArtifacts.weightData = toByteArray(weightDataBase64).buffer; 161 | 162 | return modelArtifacts; 163 | } 164 | } 165 | 166 | /** 167 | * Factory function for AsyncStorage IOHandler. 168 | * 169 | * This `IOHandler` supports both `save` and `load`. 170 | * 171 | * For each model's saved artifacts, three items are saved to async storage. 172 | * - `${PATH_PREFIX}/${modelPath}/info`: Contains meta-info about the 173 | * model, such as date saved, type of the topology, size in bytes, etc. 174 | * - `${PATH_PREFIX}/${modelPath}/model_without_weight`: The topology, 175 | * weights_specs and all other information about the model except for the 176 | * weights. 177 | * - `${PATH_PREFIX}/${modelPath}/weight_data`: Concatenated binary 178 | * weight values, stored as a base64-encoded string. 179 | * 180 | * @param modelPath A unique identifier for the model to be saved. Must be a 181 | * non-empty string. 182 | * @returns An instance of `IOHandler` 183 | */ 184 | export function localStorageIO(modelPath: string): io.IOHandler { 185 | return new LocalStorageHandler(modelPath); 186 | } 187 | -------------------------------------------------------------------------------- /src/plugin/utils/model_artifacts.ts: -------------------------------------------------------------------------------- 1 | import { io } from '@tensorflow/tfjs-core'; 2 | 3 | /** 4 | * Populate ModelArtifactsInfo fields for a model with JSON topology. 5 | * @param modelArtifacts 6 | * @returns A ModelArtifactsInfo object. 7 | */ 8 | export function getModelArtifactsInfoForJSON(modelArtifacts: io.ModelArtifacts): 9 | io.ModelArtifactsInfo { 10 | if (modelArtifacts.modelTopology instanceof ArrayBuffer) { 11 | throw new Error('Expected JSON model topology, received ArrayBuffer.'); 12 | } 13 | 14 | return { 15 | dateSaved: new Date(), 16 | // TODO followup on removing this from the the interface 17 | modelTopologyType: 'JSON', 18 | weightDataBytes: modelArtifacts.weightData == null ? 19 | 0 : 20 | modelArtifacts.weightData.byteLength, 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /src/plugin/utils/wechat_platform.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2019 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | import * as webgl_backend from '@tensorflow/tfjs-backend-webgl'; 19 | import * as tfjs from '@tensorflow/tfjs-core'; 20 | import { atob, btoa } from 'abab'; 21 | import { TextDecoder, TextEncoder } from 'text-encoder'; 22 | 23 | export interface SystemConfig { 24 | /** 25 | * A function used to override the `window.fetch` function. 26 | */ 27 | fetchFunc: Function; 28 | /** 29 | * TensorFlow.js exported root object. 30 | */ 31 | // tslint:disable-next-line:no-any 32 | tf: any; 33 | /** 34 | * TensorFlow.js webgl backend object. 35 | */ 36 | // tslint:disable-next-line:no-any 37 | webgl: any; 38 | /** 39 | * The WeChat offline canvas, can be created by calling 40 | * wx.createOfflineCanvas() 41 | */ 42 | // tslint:disable-next-line:no-any 43 | canvas: any; 44 | /** 45 | * Optional name of wechat webgl backend. 46 | */ 47 | // tslint:disable-next-line:no-any 48 | backendName?: string; 49 | } 50 | 51 | export let systemFetchFunc: Function; 52 | 53 | // Implement the WeChat Platform for TFJS 54 | export class PlatformWeChat implements tfjs.Platform { 55 | constructor(fetchFunc: Function) { 56 | systemFetchFunc = fetchFunc; 57 | } 58 | fetch(path: string, requestInits?: RequestInit): Promise { 59 | return systemFetchFunc(path, requestInits); 60 | } 61 | now(): number { 62 | return Date.now(); 63 | } 64 | encode(text: string, encoding: string): Uint8Array { 65 | if (encoding !== 'utf-8' && encoding !== 'utf8') { 66 | throw new Error( 67 | `Browser's encoder only supports utf-8, but got ${encoding}`); 68 | } 69 | return new TextEncoder(encoding).encode(text); 70 | } 71 | decode(bytes: Uint8Array, encoding: string): string { 72 | return new TextDecoder(encoding).decode(bytes); 73 | } 74 | } 75 | 76 | export const WECHAT_WEBGL_BACKEND_NAME = 'wechat-webgl'; 77 | 78 | /** 79 | * Setup the fetch polyfill and WebGL backend for WeChat. 80 | * @param config: SystemConfig object contains Tensorflow.js runtime, fetch 81 | * polyfill and WeChat offline canvas. 82 | * @param debug: flag to enable/disable debugging. 83 | */ 84 | export function setupWechatPlatform(config: SystemConfig, debug = false): void { 85 | const tf = config.tf as typeof tfjs; 86 | const backendName = config.backendName || WECHAT_WEBGL_BACKEND_NAME; 87 | if (debug) { 88 | console.log(tf); 89 | } 90 | // Skip initialization if the backend has been set. 91 | if (tf.getBackend() === backendName) { 92 | return; 93 | } 94 | const webgl = config.webgl as typeof webgl_backend; 95 | tf.ENV.setPlatform('wechat', new PlatformWeChat(config.fetchFunc)); 96 | setBase64Methods(tf); 97 | if (config.webgl && config.canvas) { 98 | initWebGL(tf, webgl, config.canvas, backendName, debug); 99 | } else { 100 | console.log( 101 | 'webgl backend is not initialized, ' + 102 | 'please inject webgl backend and the offscreen canvas.'); 103 | } 104 | } 105 | 106 | /** 107 | * Polyfill btoa and atob method on the global scope which will be used by 108 | * model parser. 109 | */ 110 | export function setBase64Methods(tf: typeof tfjs) { 111 | tf.ENV.global.btoa = btoa; 112 | tf.ENV.global.atob = atob; 113 | } 114 | /** 115 | * Initialize webgl backend using the WebGLRenderingContext from the webgl 116 | * canvas node. 117 | * @param canvas: webgl canvas node container return from node selector. 118 | * @param platform: platform name where the mini app is running (ios, android, 119 | * devtool). 120 | * @param debug: enable/disable debug logging. 121 | */ 122 | const BACKEND_PRIORITY = 2; 123 | export function initWebGL( 124 | // tslint:disable-next-line:no-any 125 | tf: typeof tfjs, webgl: typeof webgl_backend, canvas: any, 126 | backendName = WECHAT_WEBGL_BACKEND_NAME, debug = false): void { 127 | if (tf.findBackend(backendName) == null) { 128 | const WEBGL_ATTRIBUTES = { 129 | alpha: false, 130 | antialias: false, 131 | premultipliedAlpha: false, 132 | preserveDrawingBuffer: false, 133 | depth: false, 134 | stencil: false, 135 | failIfMajorPerformanceCaveat: true 136 | }; 137 | const gl = canvas.getContext('webgl', WEBGL_ATTRIBUTES); 138 | if (debug) { 139 | console.log('start backend registration'); 140 | } 141 | webgl.setWebGLContext(1, gl); 142 | tf.ENV.set('WEBGL_VERSION', 1); 143 | try { 144 | tf.registerBackend(backendName, () => { 145 | const context = new webgl.GPGPUContext(gl); 146 | return new webgl.MathBackendWebGL(context); 147 | }, BACKEND_PRIORITY); 148 | 149 | // Register all the webgl kernels on the rn-webgl backend 150 | const kernels = tf.getKernelsForBackend('webgl'); 151 | kernels.forEach(kernelConfig => { 152 | const newKernelConfig = Object.assign({}, kernelConfig, { backendName }); 153 | tf.registerKernel(newKernelConfig); 154 | }); 155 | } catch (e) { 156 | throw (new Error(`Failed to register Webgl backend: ${e.message}`)); 157 | } 158 | } 159 | tf.setBackend(backendName); 160 | if (debug) { 161 | console.log('current backend = ', tf.getBackend()); 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /src/plugin/yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@tensorflow/tfjs-backend-cpu@3.5.0": 6 | version "3.5.0" 7 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.5.0.tgz#6058fcefa28931f18d774278f53f0f3176e0f229" 8 | integrity sha512-aFSbjVLBKV0OpvE4QGSpIvtxhxpqyz09WhnuMZMhxjxCrjrPYbX7gEjXIHvgF8dEs6SX19XUaChgIftyKd2YIQ== 9 | dependencies: 10 | "@types/seedrandom" "2.4.27" 11 | seedrandom "2.4.3" 12 | 13 | "@tensorflow/tfjs-backend-webgl@3.5.0": 14 | version "3.5.0" 15 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.5.0.tgz#55160168fd7e21c26f8f84cb06e699ae5c2a41b5" 16 | integrity sha512-zeuOmfi0wbrZwjUp6M4hsctQvUuQVmO18A8V68xp+u6QC4jp3P5hNdjF7iOiMW/3iNVylYht8MbHYjQLyJAQdw== 17 | dependencies: 18 | "@tensorflow/tfjs-backend-cpu" "3.5.0" 19 | "@types/offscreencanvas" "~2019.3.0" 20 | "@types/seedrandom" "2.4.27" 21 | "@types/webgl-ext" "0.0.30" 22 | "@types/webgl2" "0.0.5" 23 | seedrandom "2.4.3" 24 | 25 | "@tensorflow/tfjs-core@3.5.0": 26 | version "3.5.0" 27 | resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-3.5.0.tgz#e1ff14f2121208242a192e935f52e86b19be279e" 28 | integrity sha512-NOs9hY9nj/iPaz2cSrWpq8G07i96/ot+1l1RoBoA1s1cZakb43pLhntm1Mj4vhSxp7dbc/BSs+t6/S+45OZTUA== 29 | dependencies: 30 | "@types/offscreencanvas" "~2019.3.0" 31 | "@types/seedrandom" "2.4.27" 32 | "@types/webgl-ext" "0.0.30" 33 | node-fetch "~2.6.1" 34 | seedrandom "2.4.3" 35 | 36 | "@types/offscreencanvas@~2019.3.0": 37 | version "2019.3.0" 38 | resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553" 39 | 40 | "@types/seedrandom@2.4.27": 41 | version "2.4.27" 42 | resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41" 43 | 44 | "@types/webgl-ext@0.0.30": 45 | version "0.0.30" 46 | resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d" 47 | 48 | "@types/webgl2@0.0.5": 49 | version "0.0.5" 50 | resolved "https://registry.yarnpkg.com/@types/webgl2/-/webgl2-0.0.5.tgz#dd925e20ab8ace80eb4b1e46fda5b109c508fb0d" 51 | integrity sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow== 52 | 53 | abab@2.0.0: 54 | version "2.0.0" 55 | resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.0.tgz#aba0ab4c5eee2d4c79d3487d85450fb2376ebb0f" 56 | 57 | base64-js@1.3.1: 58 | version "1.3.1" 59 | resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" 60 | integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== 61 | 62 | miniprogram-api-typings@^3.3.2: 63 | version "3.3.2" 64 | resolved "https://registry.yarnpkg.com/miniprogram-api-typings/-/miniprogram-api-typings-3.3.2.tgz#1e6aa65817e6534850141cd2af23eeff6ba61b4e" 65 | integrity sha512-i8yxwPOGYZ33J/VoJR/mA0Ps2JKuUE6wsn8pdHqKpUrskZuxNsjVf8Z5TPLBisYis3drLdJQ5Gjon8WIwXswYg== 66 | 67 | node-fetch@~2.6.1: 68 | version "2.6.1" 69 | resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" 70 | integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== 71 | 72 | seedrandom@2.4.3: 73 | version "2.4.3" 74 | resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc" 75 | 76 | text-encoder@0.0.4: 77 | version "0.0.4" 78 | resolved "https://registry.yarnpkg.com/text-encoder/-/text-encoder-0.0.4.tgz#fbee4c6fb251dd02e2f40f9983ddd68c5c166e2c" 79 | 80 | typescript@^3.3.3333: 81 | version "3.3.4000" 82 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.3.4000.tgz#76b0f89cfdbf97827e1112d64f283f1151d6adf0" 83 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "moduleResolution": "node", 5 | "noImplicitAny": false, 6 | "sourceMap": true, 7 | "removeComments": false, 8 | "preserveConstEnums": true, 9 | "declaration": true, 10 | "target": "es5", 11 | "skipLibCheck": true, 12 | "lib": [ 13 | "es2015", 14 | "dom" 15 | ], 16 | "noUnusedLocals": true, 17 | "noImplicitReturns": true, 18 | "noImplicitThis": true, 19 | "alwaysStrict": true, 20 | "noUnusedParameters": false, 21 | "pretty": true, 22 | "noFallthroughCasesInSwitch": true, 23 | "allowUnreachableCode": false, 24 | "typeRoots": [ 25 | "plugin/node_modules", 26 | "plugin/typings", 27 | "node_modules/@types" 28 | ], 29 | "types": ["jasmine", "miniprogram-api-typings"], 30 | "outDir": "dist" 31 | }, 32 | "include": [ 33 | "src/plugin" 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["tslint-no-circular-imports"], 3 | "linterOptions": { 4 | "exclude": [ 5 | "src/data/compiled_api.ts" 6 | ] 7 | }, 8 | "rules": { 9 | "array-type": [true, "array-simple"], 10 | "arrow-return-shorthand": true, 11 | "ban": [true, 12 | ["fit"], 13 | ["fdescribe"], 14 | ["xit"], 15 | ["xdescribe"], 16 | ["fitAsync"], 17 | ["xitAsync"], 18 | ["fitFakeAsync"], 19 | ["xitFakeAsync"] 20 | ], 21 | "ban-types": [true, 22 | ["Object", "Use {} instead."], 23 | ["String", "Use 'string' instead."], 24 | ["Number", "Use 'number' instead."], 25 | ["Boolean", "Use 'boolean' instead."] 26 | ], 27 | "class-name": true, 28 | "interface-name": [true, "never-prefix"], 29 | "jsdoc-format": true, 30 | "forin": false, 31 | "label-position": true, 32 | "max-line-length": { 33 | "options": {"limit": 80, "ignore-pattern": "^import |^export "} 34 | }, 35 | "new-parens": true, 36 | "no-angle-bracket-type-assertion": true, 37 | "no-any": true, 38 | "no-construct": true, 39 | "no-consecutive-blank-lines": true, 40 | "no-debugger": true, 41 | "no-default-export": true, 42 | "no-inferrable-types": true, 43 | "no-namespace": [true, "allow-declarations"], 44 | "no-reference": true, 45 | "no-require-imports": true, 46 | "no-string-throw": true, 47 | "no-unused-expression": true, 48 | "no-unused-variable": [true, {"ignore-pattern": "^_"}], 49 | "no-var-keyword": true, 50 | "object-literal-shorthand": true, 51 | "only-arrow-functions": [true, "allow-declarations", "allow-named-functions"], 52 | "prefer-const": true, 53 | "radix": true, 54 | "restrict-plus-operands": true, 55 | "semicolon": [true, "always", "ignore-bound-class-methods"], 56 | "switch-default": true, 57 | "triple-equals": [true, "allow-null-check"], 58 | "use-isnan": true, 59 | "variable-name": [ 60 | true, 61 | "check-format", 62 | "ban-keywords", 63 | "allow-leading-underscore", 64 | "allow-trailing-underscore" 65 | ] 66 | } 67 | } 68 | --------------------------------------------------------------------------------