├── samples ├── flow.png ├── yokoi.jpg └── airplane.jpg ├── .github └── workflows │ ├── release.yml │ └── test.yml ├── README.md ├── package.json ├── .gitignore ├── tensorflow.html ├── postinstall.js ├── icons └── icon.svg ├── test └── tensorflow_spec.js ├── locales ├── ja │ └── tensorflow.html └── en-US │ └── tensorflow.html ├── LICENSE ├── examples └── image-recognition.json └── tensorflow.js /samples/flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kazuhitoyokoi/node-red-contrib-tensorflow/HEAD/samples/flow.png -------------------------------------------------------------------------------- /samples/yokoi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kazuhitoyokoi/node-red-contrib-tensorflow/HEAD/samples/yokoi.jpg -------------------------------------------------------------------------------- /samples/airplane.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kazuhitoyokoi/node-red-contrib-tensorflow/HEAD/samples/airplane.jpg -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release Node-RED node 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | release: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: actions/setup-node@v3 11 | with: 12 | registry-url: https://registry.npmjs.org/ 13 | - run: npm publish 14 | env: 15 | NODE_AUTH_TOKEN: ${{secrets.npm_token}} 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Node-RED node 2 | on: 3 | push: 4 | branches: '*' 5 | pull_request: 6 | branches: '*' 7 | schedule: 8 | - cron: '0 0 * * *' 9 | jobs: 10 | build: 11 | strategy: 12 | matrix: 13 | os: [ ubuntu-latest, windows-latest, macos-latest ] 14 | runs-on: ${{ matrix.os }} 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-node@v3 18 | with: 19 | node-version: lts/* 20 | - run: npm install 21 | - run: npm test 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | node-red-contrib-tensorflow 2 | ================ 3 | 4 | Node-RED nodes for pre-trained TensorFlow models 5 | 6 | 7 | 8 | Example of object detection 9 | 10 | ## Install 11 | 12 | To install the stable version use the `Menu - Manage palette - Install` 13 | option and search for `node-red-contrib-tensorflow`, or run the following 14 | command in your Node-RED user directory, typically `~/.node-red` 15 | 16 | npm install node-red-contrib-tensorflow 17 | 18 | ## TensorFlow Models 19 | - [Object Detection](https://github.com/tensorflow/tfjs-models/tree/master/coco-ssd) -- The node to identify objects in an image 20 | - [MediaPipe Handpose](https://github.com/tensorflow/tfjs-models/tree/master/handpose) -- The node to detect fingers in a hand 21 | - [MobileNet](https://github.com/tensorflow/tfjs-models/tree/master/mobilenet) -- The node to classify images with MobileNet 22 | - [PoseNet Model](https://github.com/tensorflow/tfjs-models/tree/master/posenet) -- The node to estimate human pose 23 | 24 | ## How to use 25 | https://www.linux.com/news/using-tensorflow-js-and-node-red-with-image-recognition-applications/ 26 | 27 | ## Known issue 28 | - Conflict with other TensorFlow.js modules 29 | 30 | See the details: https://github.com/kazuhitoyokoi/node-red-contrib-tensorflow/issues/3 31 | 32 | ## License 33 | 34 | Apache-2.0 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-red-contrib-tensorflow", 3 | "version": "0.4.2", 4 | "description": "TensorFlow.js nodes using pre-trained models", 5 | "main": "index.js", 6 | "scripts": { 7 | "postinstall": "node postinstall.js", 8 | "test": "mocha \"test/**/*_spec.js\" --timeout 300000", 9 | "coverage": "nyc npm test" 10 | }, 11 | "engines": { 12 | "node": ">=16.0.0" 13 | }, 14 | "node-red": { 15 | "version": ">=3.0.0", 16 | "nodes": { 17 | "node": "tensorflow.js" 18 | } 19 | }, 20 | "keywords": [ 21 | "node-red", 22 | "tensorflow" 23 | ], 24 | "author": "kazuhitoyokoi", 25 | "license": "Apache-2.0", 26 | "repository": { 27 | "type": "git", 28 | "url": "git+https://github.com/kazuhitoyokoi/node-red-contrib-tensorflow.git" 29 | }, 30 | "bugs": { 31 | "url": "https://github.com/kazuhitoyokoi/node-red-contrib-tensorflow/issues" 32 | }, 33 | "homepage": "https://github.com/kazuhitoyokoi/node-red-contrib-tensorflow#readme", 34 | "dependencies": { 35 | "@tensorflow-models/coco-ssd": "2.2.2", 36 | "@tensorflow-models/handpose": "0.0.7", 37 | "@tensorflow-models/mobilenet": "2.1.0", 38 | "@tensorflow-models/posenet": "2.2.2", 39 | "@tensorflow/tfjs-node": "4.1.0", 40 | "jimp": "0.16.2", 41 | "pureimage": "0.2.1", 42 | "request": "2.88.2", 43 | "request-promise": "4.2.6", 44 | "stream-buffers": "3.0.2" 45 | }, 46 | "devDependencies": { 47 | "mocha": "10.2.0", 48 | "node-red": "3.0.2", 49 | "node-red-node-test-helper": "0.3.0" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | models/ 3 | 4 | # Logs 5 | logs 6 | *.log 7 | npm-debug.log* 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | 12 | # Diagnostic reports (https://nodejs.org/api/report.html) 13 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 14 | 15 | # Runtime data 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | 21 | # Directory for instrumented libs generated by jscoverage/JSCover 22 | lib-cov 23 | 24 | # Coverage directory used by tools like istanbul 25 | coverage 26 | *.lcov 27 | 28 | # nyc test coverage 29 | .nyc_output 30 | 31 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 32 | .grunt 33 | 34 | # Bower dependency directory (https://bower.io/) 35 | bower_components 36 | 37 | # node-waf configuration 38 | .lock-wscript 39 | 40 | # Compiled binary addons (https://nodejs.org/api/addons.html) 41 | build/Release 42 | 43 | # Dependency directories 44 | node_modules/ 45 | jspm_packages/ 46 | 47 | # TypeScript v1 declaration files 48 | typings/ 49 | 50 | # TypeScript cache 51 | *.tsbuildinfo 52 | 53 | # Optional npm cache directory 54 | .npm 55 | 56 | # Optional eslint cache 57 | .eslintcache 58 | 59 | # Microbundle cache 60 | .rpt2_cache/ 61 | .rts2_cache_cjs/ 62 | .rts2_cache_es/ 63 | .rts2_cache_umd/ 64 | 65 | # Optional REPL history 66 | .node_repl_history 67 | 68 | # Output of 'npm pack' 69 | *.tgz 70 | 71 | # Yarn Integrity file 72 | .yarn-integrity 73 | 74 | # dotenv environment variables file 75 | .env 76 | .env.test 77 | 78 | # parcel-bundler cache (https://parceljs.org/) 79 | .cache 80 | 81 | # Next.js build output 82 | .next 83 | 84 | # Nuxt.js build / generate output 85 | .nuxt 86 | dist 87 | 88 | # Gatsby files 89 | .cache/ 90 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 91 | # https://nextjs.org/blog/next-9-1#public-directory-support 92 | # public 93 | 94 | # vuepress build output 95 | .vuepress/dist 96 | 97 | # Serverless directories 98 | .serverless/ 99 | 100 | # FuseBox cache 101 | .fusebox/ 102 | 103 | # DynamoDB Local files 104 | .dynamodb/ 105 | 106 | # TernJS port file 107 | .tern-port 108 | -------------------------------------------------------------------------------- /tensorflow.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 23 | 24 | 39 | 40 | 46 | 47 | 62 | 63 | 69 | 70 | 85 | 86 | 92 | -------------------------------------------------------------------------------- /postinstall.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var fs = require('fs'); 3 | var child_process = require('child_process'); 4 | var rp = require('request-promise'); 5 | 6 | var models = [{ 7 | name: 'mobilenet', 8 | urls: ['https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/model.json', 9 | 'https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/group1-shard1of5', 10 | 'https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/group1-shard2of5', 11 | 'https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/group1-shard3of5', 12 | 'https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/group1-shard4of5', 13 | 'https://storage.googleapis.com/tfhub-tfjs-modules/google/imagenet/mobilenet_v1_100_224/classification/1/group1-shard5of5'] 14 | }, { 15 | name: 'coco-ssd', 16 | urls: ['https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/model.json', 17 | 'https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/group1-shard1of5', 18 | 'https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/group1-shard2of5', 19 | 'https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/group1-shard3of5', 20 | 'https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/group1-shard4of5', 21 | 'https://storage.googleapis.com/tfjs-models/savedmodel/ssdlite_mobilenet_v2/group1-shard5of5'] 22 | }, { 23 | name: 'posenet', 24 | urls: ['https://storage.googleapis.com/tfjs-models/savedmodel/posenet/mobilenet/float/075/model-stride16.json', 25 | 'https://storage.googleapis.com/tfjs-models/savedmodel/posenet/mobilenet/float/075/group1-shard1of2.bin', 26 | 'https://storage.googleapis.com/tfjs-models/savedmodel/posenet/mobilenet/float/075/group1-shard2of2.bin'] 27 | }]; 28 | 29 | (async () => { 30 | try { fs.mkdirSync(__dirname + '/models'); } catch (e) {} 31 | for (var i = 0; i < models.length; i++) { 32 | var model = models[i]; 33 | try { fs.mkdirSync(__dirname + '/models/' + model.name); } catch (e) {} 34 | for (var j = 0; j < model.urls.length; j++) { 35 | var flag = true; 36 | for (var k = 0; flag && k < 8; k++) { 37 | try { 38 | console.log('Downloading: ' + model.name + ', ' + model.urls[j]); 39 | var response = await rp.get({ url: model.urls[j], encoding: null }); 40 | var file = model.urls[j].split('/').slice(-1)[0]; 41 | fs.writeFileSync(__dirname + '/models/' + model.name + '/' + file, Buffer.from(response)); 42 | console.log('Downloaded: ' + model.name + ', ' + model.urls[j]); 43 | flag = false; 44 | } catch (err) { 45 | console.log(err); 46 | } 47 | } 48 | } 49 | } 50 | })(); 51 | 52 | var modelfile = '/proc/device-tree/model'; 53 | if (fs.existsSync(modelfile) && fs.readFileSync(modelfile).toString().startsWith('Raspberry Pi')) { 54 | var cmd = 'npm rebuild @tensorflow/tfjs-node --build-from-source'; 55 | var spawn = child_process.spawnSync(cmd, { cwd: path.resolve(__dirname, '..', '..'), shell: true }); 56 | console.log(spawn.stderr.toString() + '\n----\n' + spawn.stdout.toString()); 57 | } 58 | -------------------------------------------------------------------------------- /icons/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 17 | 19 | 20 | 22 | image/svg+xml 23 | 25 | WhiteWhite Icon 26 | 27 | 28 | 29 | 49 | 51 | 53 | 55 | 59 | 60 | 62 | 66 | 67 | 68 | WhiteWhite Icon 70 | 73 | 77 | 81 | 88 | 89 | 93 | 97 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /test/tensorflow_spec.js: -------------------------------------------------------------------------------- 1 | var tensorflowNode = require('../tensorflow.js'); 2 | var fs = require('fs'); 3 | var helper = require('node-red-node-test-helper'); 4 | 5 | describe('tensorflow nodes', function () { 6 | afterEach(function () { 7 | helper.unload(); 8 | }); 9 | 10 | describe('cocossd node', function () { 11 | it('should be loaded', function (done) { 12 | var flow = [{ id: 'n1', type: 'cocossd', name: 'test name' }]; 13 | helper.load(tensorflowNode, flow, function () { 14 | var n1 = helper.getNode('n1'); 15 | n1.should.have.property('name', 'test name'); 16 | done(); 17 | }); 18 | }); 19 | 20 | it('should have result', function (done) { 21 | var flow = [{ id: 'n1', type: 'cocossd', wires: [['n2']] }, 22 | { id: 'n2', type: 'helper' }]; 23 | helper.load(tensorflowNode, flow, function () { 24 | var n2 = helper.getNode('n2'); 25 | var n1 = helper.getNode('n1'); 26 | n2.on("input", function (msg) { 27 | console.log('msg.payload = ' + JSON.stringify(msg.payload)); 28 | msg.should.have.property('payload', 'person'); 29 | done(); 30 | }); 31 | setTimeout(function () { 32 | n1.receive({ payload: fs.readFileSync('./samples/yokoi.jpg') }); 33 | }, 30000); 34 | }); 35 | }); 36 | }); 37 | 38 | describe('handpose node', function () { 39 | it('should be loaded', function (done) { 40 | var flow = [{ id: 'n1', type: 'handpose', name: 'test name' }]; 41 | helper.load(tensorflowNode, flow, function () { 42 | var n1 = helper.getNode('n1'); 43 | n1.should.have.property('name', 'test name'); 44 | done(); 45 | }); 46 | }); 47 | }); 48 | 49 | describe('mobilenet node', function () { 50 | it('should be loaded', function (done) { 51 | var flow = [{ id: 'n1', type: 'mobilenet', name: 'test name' }]; 52 | helper.load(tensorflowNode, flow, function () { 53 | var n1 = helper.getNode('n1'); 54 | n1.should.have.property('name', 'test name'); 55 | done(); 56 | }); 57 | }); 58 | 59 | it('should have result', function (done) { 60 | var flow = [{ id: 'n1', type: 'mobilenet', wires: [['n2']] }, 61 | { id: 'n2', type: 'helper' }]; 62 | helper.load(tensorflowNode, flow, function () { 63 | var n2 = helper.getNode('n2'); 64 | var n1 = helper.getNode('n1'); 65 | n2.on("input", function (msg) { 66 | console.log('msg.payload = ' + JSON.stringify(msg.payload)); 67 | msg.should.have.property('payload', ['suit', 'suit of clothes']); 68 | done(); 69 | }); 70 | setTimeout(function () { 71 | n1.receive({ payload: fs.readFileSync('./samples/yokoi.jpg') }); 72 | }, 30000); 73 | }); 74 | }); 75 | }); 76 | 77 | describe('posenet node', function () { 78 | it('should be loaded', function (done) { 79 | var flow = [{ id: 'n1', type: 'posenet', name: 'test name' }]; 80 | helper.load(tensorflowNode, flow, function () { 81 | var n1 = helper.getNode('n1'); 82 | n1.should.have.property('name', 'test name'); 83 | done(); 84 | }); 85 | }); 86 | 87 | it('should have result', function (done) { 88 | var flow = [{ id: 'n1', type: 'posenet', wires: [['n2']] }, 89 | { id: 'n2', type: 'helper' }]; 90 | helper.load(tensorflowNode, flow, function () { 91 | var n2 = helper.getNode('n2'); 92 | var n1 = helper.getNode('n1'); 93 | n2.on("input", function (msg) { 94 | console.log('msg.payload = ' + JSON.stringify(msg.payload)); 95 | msg.should.have.property('payload', { 96 | nose: { x: 279.8517126239227, y: 315.6404763307089 }, 97 | leftEye: { x: 339.40865489202713, y: 260.4071526731498 }, 98 | rightEye: { x: 237.3576722497606, y: 255.88562608414585 }, 99 | leftEar: { x: 433.2652916815494, y: 308.92149127411005 }, 100 | rightShoulder: { x: 66.64864482953854, y: 589.8049531335496 } 101 | }); 102 | done(); 103 | }); 104 | setTimeout(function () { 105 | n1.receive({ payload: fs.readFileSync('./samples/yokoi.jpg') }); 106 | }, 30000); 107 | }); 108 | }); 109 | }); 110 | }); 111 | -------------------------------------------------------------------------------- /locales/ja/tensorflow.html: -------------------------------------------------------------------------------- 1 | 32 | 33 | 60 | 61 | 89 | 90 | 121 | -------------------------------------------------------------------------------- /locales/en-US/tensorflow.html: -------------------------------------------------------------------------------- 1 | 32 | 33 | 62 | 63 | 91 | 92 | 123 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /examples/image-recognition.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "1899229c3cb17e3b", 4 | "type": "cocossd", 5 | "z": "ce6f81f482665217", 6 | "name": "", 7 | "x": 140, 8 | "y": 420, 9 | "wires": [ 10 | [ 11 | "22409a0694cfe84f", 12 | "42c42a4bfcd7694b" 13 | ] 14 | ] 15 | }, 16 | { 17 | "id": "1728a1ee6946cc72", 18 | "type": "posenet", 19 | "z": "ce6f81f482665217", 20 | "name": "", 21 | "x": 140, 22 | "y": 640, 23 | "wires": [ 24 | [ 25 | "8459ccb4fddf804c", 26 | "2d136cd00a5a69bf" 27 | ] 28 | ] 29 | }, 30 | { 31 | "id": "8459ccb4fddf804c", 32 | "type": "image", 33 | "z": "ce6f81f482665217", 34 | "name": "", 35 | "width": 160, 36 | "data": "annotatedInput", 37 | "dataType": "msg", 38 | "thumbnail": false, 39 | "active": true, 40 | "pass": true, 41 | "outputs": 1, 42 | "x": 380, 43 | "y": 640, 44 | "wires": [ 45 | [ 46 | "1ee2b229a9c39eb7" 47 | ] 48 | ] 49 | }, 50 | { 51 | "id": "22409a0694cfe84f", 52 | "type": "image", 53 | "z": "ce6f81f482665217", 54 | "name": "", 55 | "width": 160, 56 | "data": "annotatedInput", 57 | "dataType": "msg", 58 | "thumbnail": false, 59 | "active": true, 60 | "pass": true, 61 | "outputs": 1, 62 | "x": 380, 63 | "y": 420, 64 | "wires": [ 65 | [ 66 | "35c3466d388d4f95" 67 | ] 68 | ] 69 | }, 70 | { 71 | "id": "30536dbce1855454", 72 | "type": "http request", 73 | "z": "ce6f81f482665217", 74 | "name": "get person image", 75 | "method": "GET", 76 | "ret": "bin", 77 | "paytoqs": "ignore", 78 | "url": "https://raw.githubusercontent.com/kazuhitoyokoi/node-red-contrib-tensorflow/main/samples/yokoi.jpg", 79 | "tls": "", 80 | "persist": false, 81 | "proxy": "", 82 | "insecureHTTPParser": false, 83 | "authType": "", 84 | "senderr": false, 85 | "headers": [], 86 | "x": 170, 87 | "y": 200, 88 | "wires": [ 89 | [ 90 | "a0e8f16c06baf803" 91 | ] 92 | ] 93 | }, 94 | { 95 | "id": "ca2cf6a2ca223d47", 96 | "type": "image", 97 | "z": "ce6f81f482665217", 98 | "name": "", 99 | "width": 160, 100 | "data": "payload", 101 | "dataType": "msg", 102 | "thumbnail": false, 103 | "active": true, 104 | "pass": true, 105 | "outputs": 1, 106 | "x": 620, 107 | "y": 200, 108 | "wires": [ 109 | [ 110 | "1899229c3cb17e3b" 111 | ] 112 | ] 113 | }, 114 | { 115 | "id": "f70c4cfecae94dd4", 116 | "type": "handpose", 117 | "z": "ce6f81f482665217", 118 | "name": "", 119 | "x": 140, 120 | "y": 1040, 121 | "wires": [ 122 | [ 123 | "b116717891b0c559", 124 | "24caf1574254b6ae" 125 | ] 126 | ] 127 | }, 128 | { 129 | "id": "1fdbd18d7446813c", 130 | "type": "mobilenet", 131 | "z": "ce6f81f482665217", 132 | "name": "", 133 | "x": 140, 134 | "y": 1220, 135 | "wires": [ 136 | [ 137 | "82bd91adfdddf1fd" 138 | ] 139 | ] 140 | }, 141 | { 142 | "id": "2d136cd00a5a69bf", 143 | "type": "debug", 144 | "z": "ce6f81f482665217", 145 | "name": "result", 146 | "active": true, 147 | "tosidebar": true, 148 | "console": false, 149 | "tostatus": true, 150 | "complete": "payload", 151 | "targetType": "msg", 152 | "statusVal": "payload", 153 | "statusType": "auto", 154 | "x": 210, 155 | "y": 720, 156 | "wires": [] 157 | }, 158 | { 159 | "id": "82bd91adfdddf1fd", 160 | "type": "debug", 161 | "z": "ce6f81f482665217", 162 | "name": "result", 163 | "active": true, 164 | "tosidebar": true, 165 | "console": false, 166 | "tostatus": true, 167 | "complete": "payload", 168 | "targetType": "msg", 169 | "statusVal": "payload", 170 | "statusType": "auto", 171 | "x": 290, 172 | "y": 1220, 173 | "wires": [] 174 | }, 175 | { 176 | "id": "1ee2b229a9c39eb7", 177 | "type": "http request", 178 | "z": "ce6f81f482665217", 179 | "name": "get hand image", 180 | "method": "GET", 181 | "ret": "bin", 182 | "paytoqs": "ignore", 183 | "url": "https://upload.wikimedia.org/wikipedia/commons/9/93/Wijsvinger.jpg", 184 | "tls": "", 185 | "persist": false, 186 | "proxy": "", 187 | "insecureHTTPParser": false, 188 | "authType": "", 189 | "senderr": false, 190 | "headers": [], 191 | "x": 160, 192 | "y": 860, 193 | "wires": [ 194 | [ 195 | "9a1d126f633b380a" 196 | ] 197 | ] 198 | }, 199 | { 200 | "id": "b116717891b0c559", 201 | "type": "image", 202 | "z": "ce6f81f482665217", 203 | "name": "", 204 | "width": 160, 205 | "data": "annotatedInput", 206 | "dataType": "msg", 207 | "thumbnail": false, 208 | "active": true, 209 | "pass": true, 210 | "outputs": 1, 211 | "x": 380, 212 | "y": 1040, 213 | "wires": [ 214 | [ 215 | "9c1147eade443f6f" 216 | ] 217 | ] 218 | }, 219 | { 220 | "id": "c8a30cb66cd51f0b", 221 | "type": "image", 222 | "z": "ce6f81f482665217", 223 | "name": "", 224 | "width": 160, 225 | "data": "payload", 226 | "dataType": "msg", 227 | "thumbnail": false, 228 | "active": true, 229 | "pass": true, 230 | "outputs": 1, 231 | "x": 600, 232 | "y": 860, 233 | "wires": [ 234 | [ 235 | "f70c4cfecae94dd4" 236 | ] 237 | ] 238 | }, 239 | { 240 | "id": "3c6d6d8d771aee0c", 241 | "type": "status", 242 | "z": "ce6f81f482665217", 243 | "name": "get cocossd status", 244 | "scope": [ 245 | "1899229c3cb17e3b" 246 | ], 247 | "x": 130, 248 | "y": 120, 249 | "wires": [ 250 | [ 251 | "c30776cb9b7e5ab6" 252 | ] 253 | ] 254 | }, 255 | { 256 | "id": "c30776cb9b7e5ab6", 257 | "type": "switch", 258 | "z": "ce6f81f482665217", 259 | "name": "model is loaded?", 260 | "property": "status.text", 261 | "propertyType": "msg", 262 | "rules": [ 263 | { 264 | "t": "eq", 265 | "v": "model loaded", 266 | "vt": "str" 267 | } 268 | ], 269 | "checkall": "true", 270 | "repair": false, 271 | "outputs": 1, 272 | "x": 350, 273 | "y": 120, 274 | "wires": [ 275 | [ 276 | "e59e34991eb806d4" 277 | ] 278 | ] 279 | }, 280 | { 281 | "id": "a0e8f16c06baf803", 282 | "type": "change", 283 | "z": "ce6f81f482665217", 284 | "name": "backup original image", 285 | "rules": [ 286 | { 287 | "t": "set", 288 | "p": "input", 289 | "pt": "msg", 290 | "to": "payload", 291 | "tot": "msg" 292 | } 293 | ], 294 | "action": "", 295 | "property": "", 296 | "from": "", 297 | "to": "", 298 | "reg": false, 299 | "x": 400, 300 | "y": 200, 301 | "wires": [ 302 | [ 303 | "ca2cf6a2ca223d47" 304 | ] 305 | ] 306 | }, 307 | { 308 | "id": "9a1d126f633b380a", 309 | "type": "change", 310 | "z": "ce6f81f482665217", 311 | "name": "backup original image", 312 | "rules": [ 313 | { 314 | "t": "set", 315 | "p": "input", 316 | "pt": "msg", 317 | "to": "payload", 318 | "tot": "msg" 319 | } 320 | ], 321 | "action": "", 322 | "property": "", 323 | "from": "", 324 | "to": "", 325 | "reg": false, 326 | "x": 380, 327 | "y": 860, 328 | "wires": [ 329 | [ 330 | "c8a30cb66cd51f0b" 331 | ] 332 | ] 333 | }, 334 | { 335 | "id": "35c3466d388d4f95", 336 | "type": "change", 337 | "z": "ce6f81f482665217", 338 | "name": "recover original image", 339 | "rules": [ 340 | { 341 | "t": "set", 342 | "p": "payload", 343 | "pt": "msg", 344 | "to": "input", 345 | "tot": "msg" 346 | } 347 | ], 348 | "action": "", 349 | "property": "", 350 | "from": "", 351 | "to": "", 352 | "reg": false, 353 | "x": 600, 354 | "y": 420, 355 | "wires": [ 356 | [ 357 | "1728a1ee6946cc72" 358 | ] 359 | ] 360 | }, 361 | { 362 | "id": "9c1147eade443f6f", 363 | "type": "change", 364 | "z": "ce6f81f482665217", 365 | "name": "recover original image", 366 | "rules": [ 367 | { 368 | "t": "set", 369 | "p": "payload", 370 | "pt": "msg", 371 | "to": "input", 372 | "tot": "msg" 373 | } 374 | ], 375 | "action": "", 376 | "property": "", 377 | "from": "", 378 | "to": "", 379 | "reg": false, 380 | "x": 600, 381 | "y": 1040, 382 | "wires": [ 383 | [ 384 | "1fdbd18d7446813c" 385 | ] 386 | ] 387 | }, 388 | { 389 | "id": "42c42a4bfcd7694b", 390 | "type": "debug", 391 | "z": "ce6f81f482665217", 392 | "name": "result", 393 | "active": true, 394 | "tosidebar": true, 395 | "console": false, 396 | "tostatus": true, 397 | "complete": "payload", 398 | "targetType": "msg", 399 | "statusVal": "payload", 400 | "statusType": "auto", 401 | "x": 210, 402 | "y": 500, 403 | "wires": [] 404 | }, 405 | { 406 | "id": "24caf1574254b6ae", 407 | "type": "debug", 408 | "z": "ce6f81f482665217", 409 | "name": "result", 410 | "active": true, 411 | "tosidebar": true, 412 | "console": false, 413 | "tostatus": true, 414 | "complete": "payload", 415 | "targetType": "msg", 416 | "statusVal": "payload", 417 | "statusType": "auto", 418 | "x": 210, 419 | "y": 1120, 420 | "wires": [] 421 | }, 422 | { 423 | "id": "e59e34991eb806d4", 424 | "type": "delay", 425 | "z": "ce6f81f482665217", 426 | "name": "wait for loading other models", 427 | "pauseType": "delay", 428 | "timeout": "10", 429 | "timeoutUnits": "seconds", 430 | "rate": "1", 431 | "nbRateUnits": "1", 432 | "rateUnits": "second", 433 | "randomFirst": "1", 434 | "randomLast": "5", 435 | "randomUnits": "seconds", 436 | "drop": false, 437 | "allowrate": false, 438 | "outputs": 1, 439 | "x": 600, 440 | "y": 120, 441 | "wires": [ 442 | [ 443 | "30536dbce1855454" 444 | ] 445 | ] 446 | }, 447 | { 448 | "id": "89b5218177b2a48c", 449 | "type": "comment", 450 | "z": "ce6f81f482665217", 451 | "name": "Install `node-red-contrib-image-output` module before trying this flow", 452 | "info": "", 453 | "x": 290, 454 | "y": 60, 455 | "wires": [] 456 | } 457 | ] -------------------------------------------------------------------------------- /tensorflow.js: -------------------------------------------------------------------------------- 1 | var jimp = require('jimp'); 2 | var streamBuffers = require('stream-buffers'); 3 | var pureimage = require('pureimage'); 4 | var tf = require('@tensorflow/tfjs-node'); 5 | var cocossd = require('@tensorflow-models/coco-ssd'); 6 | var handpose = require('@tensorflow-models/handpose'); 7 | var mobilenet = require('@tensorflow-models/mobilenet'); 8 | var posenet = require('@tensorflow-models/posenet'); 9 | 10 | module.exports = function (RED) { 11 | tf.setBackend('cpu'); 12 | 13 | RED.httpAdmin.get("/models/:dir/:name", function (req, res) { 14 | var options = { root: __dirname + '/models/' + req.params.dir, dotfiles: 'deny' }; 15 | res.sendFile(req.params.name, options); 16 | }); 17 | 18 | function CocossdNode(config) { 19 | RED.nodes.createNode(this, config); 20 | var node = this; 21 | var modelCocossd; 22 | 23 | setTimeout(function () { 24 | node.status({ fill: "green", shape: 'ring', text: 'loading model...' }); 25 | cocossd.load({ 26 | modelUrl: 'http://localhost:' + RED.settings.uiPort + '/models/coco-ssd/model.json' 27 | }).then(function (model) { 28 | modelCocossd = model; 29 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 30 | }).catch(function (error) { 31 | cocossd.load().then(function (model2) { 32 | modelCocossd = model2; 33 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 34 | }).catch(function (error2) { 35 | node.error(error); 36 | node.error(error2); 37 | node.status({ fill: 'red', shape: 'ring', text: 'fail to load model' }); 38 | }); 39 | }); 40 | }, 1000); 41 | 42 | node.on('input', function (msg) { 43 | node.status({ fill: "green", shape: 'dot', text: 'analyzing...' }); 44 | jimp.read(msg.payload).then(function (data) { 45 | return data.getBufferAsync(jimp.MIME_PNG); 46 | }).then(function (buffer) { 47 | var rsb = new streamBuffers.ReadableStreamBuffer(); 48 | rsb.put(buffer); 49 | pureimage.decodePNGFromStream(rsb).then(function (image) { 50 | var cv = pureimage.make(image.width, image.height); 51 | cv.getContext('2d').drawImage(image, 0, 0); 52 | if (modelCocossd) { 53 | modelCocossd.detect(cv).then(function (result) { 54 | msg.details = result; 55 | if (0 < result.length) { 56 | msg.payload = result[0].class 57 | var cv2 = pureimage.make(image.width, image.height); 58 | var ctx = cv2.getContext('2d'); 59 | ctx.drawImage(image, 0, 0); 60 | ctx.strokeStyle = 'rgb(255, 111, 0)'; 61 | ctx.strokeRect(result[0].bbox[0], result[0].bbox[1], result[0].bbox[2], result[0].bbox[3]); 62 | ctx.fillStyle = 'rgba(255, 111, 0, 0.5)'; 63 | ctx.fillRect(result[0].bbox[0], result[0].bbox[1], result[0].bbox[2], result[0].bbox[3]); 64 | var wsb = new streamBuffers.WritableStreamBuffer({ initialSize: 1, incrementAmount: 1 }); 65 | pureimage.encodePNGToStream(cv2, wsb).then(function () { 66 | msg.annotatedInput = wsb.getContents(); 67 | node.send(msg); 68 | node.status({}); 69 | }).catch(function (error) { 70 | node.error(error, msg); 71 | node.status({ fill: 'red', shape: 'ring', text: error }); 72 | }); 73 | } else { 74 | msg.annotatedInput = msg.payload; 75 | msg.payload = null; 76 | node.send(msg); 77 | node.status({}); 78 | } 79 | }, function (error) { 80 | node.error(error, msg); 81 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 82 | }); 83 | } else { 84 | node.status({ fill: 'red', shape: 'ring', text: 'model is not loaded' }); 85 | } 86 | }); 87 | }).catch(function (error) { 88 | node.error(error, msg); 89 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 90 | }); 91 | }); 92 | } 93 | 94 | RED.nodes.registerType("cocossd", CocossdNode); 95 | 96 | function HandposeNode(config) { 97 | RED.nodes.createNode(this, config); 98 | var node = this; 99 | var modelHandpose; 100 | 101 | setTimeout(function () { 102 | node.status({ fill: "green", shape: 'ring', text: 'loading model...' }); 103 | handpose.load().then(function (model) { 104 | modelHandpose = model; 105 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 106 | }).catch(function (error) { 107 | node.error(error); 108 | node.status({ fill: 'red', shape: 'ring', text: 'fail to load model' }); 109 | }); 110 | }, 1000); 111 | 112 | node.on('input', function (msg) { 113 | node.status({ fill: "green", shape: 'dot', text: 'analyzing...' }); 114 | jimp.read(msg.payload).then(function (data) { 115 | return data.getBufferAsync(jimp.MIME_PNG); 116 | }).then(function (buffer) { 117 | var rsb = new streamBuffers.ReadableStreamBuffer(); 118 | rsb.put(buffer); 119 | pureimage.decodePNGFromStream(rsb).then(function (image) { 120 | var cv = pureimage.make(image.width, image.height); 121 | cv.getContext('2d').drawImage(image, 0, 0); 122 | if (modelHandpose) { 123 | modelHandpose.estimateHands(cv).then(function (result) { 124 | msg.details = result; 125 | var position = {}; 126 | if (0 < result.length) { 127 | position['palmBase'] = { x: result[0].landmarks[0][0], 128 | y: result[0].landmarks[0][1] }; 129 | position['thumb'] = { x: result[0].landmarks[4][0], 130 | y: result[0].landmarks[4][1] }; 131 | position['indexFinger'] = { x: result[0].landmarks[8][0], 132 | y: result[0].landmarks[8][1] }; 133 | position['middleFinger'] = { x: result[0].landmarks[12][0], 134 | y: result[0].landmarks[12][1] }; 135 | position['ringFinger'] = { x: result[0].landmarks[16][0], 136 | y: result[0].landmarks[16][1] }; 137 | position['pinky'] = { x: result[0].landmarks[20][0], 138 | y: result[0].landmarks[20][1] }; 139 | } 140 | if (0 < Object.keys(position).length) { 141 | msg.payload = position; 142 | var cv2 = pureimage.make(image.width, image.height); 143 | var ctx = cv2.getContext('2d'); 144 | ctx.drawImage(image, 0, 0); 145 | ctx.strokeStyle = 'rgb(255, 111, 0)'; 146 | ctx.lineWidth = 6; 147 | try { ctx.drawLine({ start: position.palmBase, end: position.thumb }); } catch (e) {} 148 | try { ctx.drawLine({ start: position.palmBase, end: position.indexFinger }); } catch (e) {} 149 | try { ctx.drawLine({ start: position.palmBase, end: position.middleFinger }); } catch (e) {} 150 | try { ctx.drawLine({ start: position.palmBase, end: position.ringFinger }); } catch (e) {} 151 | try { ctx.drawLine({ start: position.palmBase, end: position.pinky }); } catch (e) {} 152 | 153 | var wsb = new streamBuffers.WritableStreamBuffer({ initialSize: 1, incrementAmount: 1 }); 154 | pureimage.encodePNGToStream(cv2, wsb).then(function () { 155 | msg.annotatedInput = wsb.getContents(); 156 | node.send(msg); 157 | node.status({}); 158 | }).catch(function (error) { 159 | node.error(error, msg); 160 | node.status({ fill: 'red', shape: 'ring', text: error }); 161 | }); 162 | } else { 163 | msg.annotatedInput = msg.payload; 164 | msg.payload = null; 165 | node.send(msg); 166 | node.status({}); 167 | } 168 | }, function (error) { 169 | node.error(error, msg); 170 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 171 | }); 172 | } else { 173 | node.status({ fill: 'red', shape: 'ring', text: 'model is not loaded' }); 174 | } 175 | }); 176 | }).catch(function (error) { 177 | node.error(error, msg); 178 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 179 | }); 180 | }); 181 | } 182 | 183 | RED.nodes.registerType("handpose", HandposeNode); 184 | 185 | function MobilenetNode(config) { 186 | RED.nodes.createNode(this, config); 187 | var node = this; 188 | var modelMobilenet; 189 | 190 | setTimeout(function () { 191 | node.status({ fill: "green", shape: 'ring', text: 'loading model...' }); 192 | mobilenet.load({ 193 | version: 1, 194 | alpha: 1.0, 195 | modelUrl: 'http://localhost:' + RED.settings.uiPort + '/models/mobilenet/model.json', 196 | inputRange: [0, 1] 197 | }).then(function (model) { 198 | modelMobilenet = model; 199 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 200 | }).catch(function (error) { 201 | mobilenet.load().then(function (model2) { 202 | modelMobilenet = model2; 203 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 204 | }).catch(function (error2) { 205 | node.error(error); 206 | node.error(error2); 207 | node.status({ fill: 'red', shape: 'ring', text: 'fail to load model' }); 208 | }); 209 | }); 210 | }, 1000); 211 | 212 | node.on('input', function (msg) { 213 | node.status({ fill: "green", shape: 'dot', text: 'analyzing...' }); 214 | jimp.read(msg.payload).then(function (data) { 215 | return data.getBufferAsync(jimp.MIME_PNG); 216 | }).then(function (buffer) { 217 | var rsb = new streamBuffers.ReadableStreamBuffer(); 218 | rsb.put(buffer); 219 | pureimage.decodePNGFromStream(rsb).then(function (image) { 220 | var cv = pureimage.make(image.width, image.height); 221 | cv.getContext('2d').drawImage(image, 0, 0); 222 | if (modelMobilenet) { 223 | modelMobilenet.classify(cv).then(function (result) { 224 | if (result && 0 < result.length && result[0].className) { 225 | msg.payload = result[0].className.split(', '); 226 | } else { 227 | msg.payload = null; 228 | } 229 | msg.details = result; 230 | node.send(msg); 231 | node.status({}); 232 | }, function (error) { 233 | node.error(error, msg); 234 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 235 | }); 236 | } else { 237 | node.status({ fill: 'red', shape: 'ring', text: 'model is not loaded' }); 238 | } 239 | }); 240 | }).catch(function (error) { 241 | node.error(error, msg); 242 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 243 | }); 244 | }); 245 | } 246 | 247 | RED.nodes.registerType("mobilenet", MobilenetNode); 248 | 249 | function PosenetNode(config) { 250 | RED.nodes.createNode(this, config); 251 | var node = this; 252 | var modelPosenet; 253 | 254 | setTimeout(function () { 255 | node.status({ fill: "green", shape: 'ring', text: 'loading model...' }); 256 | 257 | posenet.load({ 258 | modelUrl: 'http://localhost:' + RED.settings.uiPort + '/models/posenet/model-stride16.json' 259 | }).then(function (model) { 260 | modelPosenet = model; 261 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 262 | }).catch(function (error) { 263 | posenet.load().then(function (model2) { 264 | modelPosenet = model2; 265 | node.status({ fill: "green", shape: 'ring', text: 'model loaded' }); 266 | }).catch(function (error2) { 267 | node.error(error); 268 | node.error(error2); 269 | node.status({ fill: 'red', shape: 'ring', text: 'fail to load model' }); 270 | }); 271 | }); 272 | }, 1000); 273 | 274 | node.on('input', function (msg) { 275 | node.status({ fill: "green", shape: 'dot', text: 'analyzing...' }); 276 | jimp.read(msg.payload).then(function (data) { 277 | return data.getBufferAsync(jimp.MIME_PNG); 278 | }).then(function (buffer) { 279 | var rsb = new streamBuffers.ReadableStreamBuffer(); 280 | rsb.put(buffer); 281 | pureimage.decodePNGFromStream(rsb).then(function (image) { 282 | var cv = pureimage.make(image.width, image.height); 283 | cv.getContext('2d').drawImage(image, 0, 0); 284 | if (modelPosenet) { 285 | modelPosenet.estimateSinglePose(cv).then(function (result) { 286 | msg.details = result; 287 | var position = {}; 288 | for (var i = 0; i < result.keypoints.length; i++) { 289 | if (result.keypoints[i].score > 0.6) { 290 | position[result.keypoints[i].part] = result.keypoints[i].position; 291 | } 292 | } 293 | if (position.leftShoulder && position.rightShoulder) { 294 | position['center'] = { x: (position.leftShoulder.x + position.rightShoulder.x)/2, 295 | y: (position.leftShoulder.y + position.rightShoulder.y)/2 }; 296 | } 297 | if (0 < Object.keys(position).length) { 298 | msg.payload = position; 299 | var cv2 = pureimage.make(image.width, image.height); 300 | var ctx = cv2.getContext('2d'); 301 | ctx.drawImage(image, 0, 0); 302 | ctx.strokeStyle = 'rgb(255, 111, 0)'; 303 | ctx.lineWidth = 10; 304 | 305 | try { ctx.drawLine({ start: position.nose, end: position.leftEye }); } catch (e) {} 306 | try { ctx.drawLine({ start: position.leftEye, end: position.leftEar }); } catch (e) {} 307 | try { ctx.drawLine({ start: position.nose, end: position.rightEye }); } catch (e) {} 308 | try { ctx.drawLine({ start: position.rightEye, end: position.rightEar }); } catch (e) {} 309 | try { ctx.drawLine({ start: position.nose, end: position.center }); } catch (e) {} 310 | try { ctx.drawLine({ start: position.leftShoulder, end: position.rightShoulder }); } catch (e) {} 311 | try { ctx.drawLine({ start: position.leftShoulder, end: position.leftElbow }); } catch (e) {} 312 | try { ctx.drawLine({ start: position.leftElbow, end: position.leftWrist }); } catch (e) {} 313 | try { ctx.drawLine({ start: position.rightShoulder, end: position.rightElbow }); } catch (e) {} 314 | try { ctx.drawLine({ start: position.rightElbow, end: position.rightWrist }); } catch (e) {} 315 | try { ctx.drawLine({ start: position.center, end: position.leftHip }); } catch (e) {} 316 | try { ctx.drawLine({ start: position.leftHip, end: position.leftKnee }); } catch (e) {} 317 | try { ctx.drawLine({ start: position.leftKnee, end: position.leftAnkle }); } catch (e) {} 318 | try { ctx.drawLine({ start: position.center, end: position.rightHip }); } catch (e) {} 319 | try { ctx.drawLine({ start: position.rightHip, end: position.rightKnee }); } catch (e) {} 320 | try { ctx.drawLine({ start: position.rightKnee, end: position.rightAnkle }); } catch (e) {} 321 | 322 | var wsb = new streamBuffers.WritableStreamBuffer({ initialSize: 1, incrementAmount: 1 }); 323 | pureimage.encodePNGToStream(cv2, wsb).then(function () { 324 | msg.annotatedInput = wsb.getContents(); 325 | node.send(msg); 326 | node.status({}); 327 | }).catch(function (error) { 328 | node.error(error, msg); 329 | node.status({ fill: 'red', shape: 'ring', text: error }); 330 | }); 331 | } else { 332 | msg.annotatedInput = msg.payload; 333 | msg.payload = null; 334 | node.send(msg); 335 | node.status({}); 336 | } 337 | }, function (error) { 338 | node.error(error, msg); 339 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 340 | }); 341 | } else { 342 | node.status({ fill: 'red', shape: 'ring', text: 'model is not loaded' }); 343 | } 344 | }); 345 | }).catch(function (error) { 346 | node.error(error, msg); 347 | node.status({ fill: 'red', shape: 'ring', text: 'error' }); 348 | }); 349 | }); 350 | } 351 | 352 | RED.nodes.registerType("posenet", PosenetNode); 353 | }; 354 | --------------------------------------------------------------------------------