├── public ├── favicon.ico ├── assets │ └── star.gif ├── style.css └── index.html ├── README.md ├── .gitignore ├── client ├── app.js ├── history.js ├── index.js └── components │ ├── utils.js │ └── Camera.js ├── .prettierrc.yml ├── .editorconfig ├── webpack.config.js ├── LICENSE ├── server └── index.js ├── .babelrc └── package.json /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kirstenlindsmith/PoseNet_React/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /public/assets/star.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kirstenlindsmith/PoseNet_React/HEAD/public/assets/star.gif -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | A basic adaptation of PoseNet to work with React.js! 2 | 3 | Basic scripts to get up and running: 4 | - npm install 5 | - npm run start-dev -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | public/bundle.js 3 | public/bundle.js.map 4 | npm-debug.log 5 | yarn-error.log 6 | .eslintrc.json 7 | .prettierignore 8 | .travis.yml 9 | package-lock.json 10 | secrets.js 11 | -------------------------------------------------------------------------------- /client/app.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import Camera from './components/Camera.js' 3 | 4 | const App = () => { 5 | return ( 6 |
7 | 8 |
9 | ) 10 | } 11 | 12 | export default App 13 | -------------------------------------------------------------------------------- /client/history.js: -------------------------------------------------------------------------------- 1 | import createHistory from 'history/createBrowserHistory' 2 | import createMemoryHistory from 'history/createMemoryHistory' 3 | 4 | const history = 5 | process.env.NODE_ENV === 'test' ? createMemoryHistory() : createHistory() 6 | 7 | export default history 8 | -------------------------------------------------------------------------------- /.prettierrc.yml: -------------------------------------------------------------------------------- 1 | # printWidth: 80 # 80 2 | # tabWidth: 2 # 2 3 | # useTabs: false # false 4 | semi: false # true 5 | singleQuote: true # false 6 | # trailingComma: none # none | es5 | all 7 | bracketSpacing: false # true 8 | # jsxBracketSameLine: false # false 9 | # arrowParens: avoid # avoid | always 10 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [*.{yml,yaml}] 16 | indent_style = space 17 | -------------------------------------------------------------------------------- /public/style.css: -------------------------------------------------------------------------------- 1 | .webcam { 2 | margin: 0 auto; 3 | margin-left: auto; 4 | margin-right: auto; 5 | padding: 6px; 6 | } 7 | 8 | #videoNoShow { 9 | transform: scaleX(-1); 10 | -moz-transform: scaleX(-1); 11 | -o-transform: scaleX(-1); 12 | -webkit-transform: scaleX(-1); 13 | display: none !important; 14 | } 15 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | PoseNet in React.js 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /client/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom' 3 | import {Router} from 'react-router-dom' 4 | import history from './history' 5 | import App from './app' 6 | 7 | // // establishes socket connection 8 | // import './socket' 9 | 10 | ReactDOM.render( 11 | 12 | 13 | , 14 | document.getElementById('app') 15 | ) 16 | 17 | // ReactDOM.render(

HELLO

, document.getElementById('app')) 18 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const isDev = process.env.NODE_ENV === 'development' 2 | 3 | module.exports = { 4 | mode: isDev ? 'development' : 'production', 5 | entry: [ 6 | '@babel/polyfill', // enables async-await 7 | './client/index.js' 8 | ], 9 | output: { 10 | path: __dirname, 11 | filename: './public/bundle.js' 12 | }, 13 | resolve: { 14 | extensions: ['.js', '.jsx'] 15 | }, 16 | devtool: 'source-map', 17 | watchOptions: { 18 | ignored: /node_modules/ 19 | }, 20 | module: { 21 | rules: [ 22 | { 23 | test: /\.jsx?$/, 24 | exclude: /node_modules/, 25 | loader: 'babel-loader' 26 | } 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /server/index.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const express = require('express') 3 | const morgan = require('morgan') 4 | const compression = require('compression') 5 | const PORT = process.env.PORT || 1337 6 | const app = express() 7 | 8 | module.exports = app 9 | 10 | if (process.env.NODE_ENV !== 'production') require('../secrets') 11 | 12 | const createApp = () => { 13 | // logging middleware 14 | app.use(morgan('dev')) 15 | 16 | // body parsing middleware 17 | app.use(express.json()) 18 | app.use(express.urlencoded({extended: true})) 19 | 20 | // compression middleware 21 | app.use(compression()) 22 | 23 | // auth and api routes 24 | // app.use('/api', require('./api')) 25 | 26 | // static file-serving middleware 27 | app.use(express.static(path.join(__dirname, '..', 'public'))) 28 | 29 | // any remaining requests with an extension (.js, .css, etc.) send 404 30 | app.use((req, res, next) => { 31 | if (path.extname(req.path).length) { 32 | const err = new Error('Not found') 33 | err.status = 404 34 | next(err) 35 | } else { 36 | next() 37 | } 38 | }) 39 | 40 | // sends index.html 41 | app.use('*', (req, res) => { 42 | res.sendFile(path.join(__dirname, '..', 'public/index.html')) 43 | }) 44 | 45 | // error handling endware 46 | app.use((err, req, res, next) => { 47 | console.error(err) 48 | console.error(err.stack) 49 | res.status(err.status || 500).send(err.message || 'Internal server error.') 50 | }) 51 | } 52 | 53 | const startListening = () => { 54 | // start listening (and create a 'server' object representing our server) 55 | app.listen(PORT, () => console.log(`I'm listening on port ${PORT}!`)) 56 | } 57 | 58 | async function bootApp() { 59 | await createApp() 60 | await startListening() 61 | } 62 | 63 | if (require.main === module) { 64 | bootApp() 65 | } else { 66 | createApp() 67 | } 68 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/react", 4 | "@babel/env" 5 | /* 6 | Babel uses these "presets" to know how to transpile your Javascript code. Here's what we're saying with these: 7 | 8 | 'react': teaches Babel to recognize JSX - a must have for React! 9 | 10 | 'env': teaches Babel to transpile Javascript . This preset is highly configurable, and you can reduce the size of your bundle by limiting the number of features you transpile. Learn more here: https://github.com/babel/babel-preset-env 11 | */ 12 | ], 13 | "plugins": [ 14 | /* 15 | These plugins teach Babel to recognize EcmaScript language features that have reached "stage 2" in the process of approval for inclusion in the official EcmaScript specification (called the "TC39 process"). There are 5 stages in the process, starting at 0 (basically a brand new proposal) going up to 4 (finished and ready for inclusion). Read more about it here: http://2ality.com/2015/11/tc39-process.html. Using new language features before they're officially part of EcmaScript is fun, but it also carries a risk: sometimes proposed features can change substantially (or be rejected entirely) before finally being included in the language, so if you jump on the bandwagon too early, you risk having your code be dependent on defunct/nonstandard syntax! "Stage 2" is a fairly safe place to start - after stage 2, the feature is well on its way to official inclusion and only minor changes are expected. 16 | */ 17 | "@babel/plugin-syntax-dynamic-import", 18 | "@babel/plugin-syntax-import-meta", 19 | "@babel/plugin-proposal-class-properties", 20 | "@babel/plugin-proposal-json-strings", 21 | [ 22 | "@babel/plugin-proposal-decorators", 23 | { 24 | "legacy": true 25 | } 26 | ], 27 | "@babel/plugin-proposal-function-sent", 28 | "@babel/plugin-proposal-export-namespace-from", 29 | "@babel/plugin-proposal-numeric-separator", 30 | "@babel/plugin-proposal-throw-expressions" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /client/components/utils.js: -------------------------------------------------------------------------------- 1 | import * as posenet from '@tensorflow-models/posenet' 2 | 3 | const pointRadius = 3 4 | 5 | export const config = { 6 | videoWidth: 900, 7 | videoHeight: 700, 8 | flipHorizontal: true, 9 | algorithm: 'single-pose', 10 | showVideo: true, 11 | showSkeleton: true, 12 | showPoints: true, 13 | minPoseConfidence: 0.1, 14 | minPartConfidence: 0.5, 15 | maxPoseDetections: 2, 16 | nmsRadius: 20, 17 | outputStride: 16, 18 | imageScaleFactor: 0.5, 19 | skeletonColor: '#ffadea', 20 | skeletonLineWidth: 6, 21 | loadingText: 'Loading...please be patient...' 22 | } 23 | 24 | function toTuple({x, y}) { 25 | return [x, y] 26 | } 27 | 28 | export function drawKeyPoints( 29 | keypoints, 30 | minConfidence, 31 | skeletonColor, 32 | canvasContext, 33 | scale = 1 34 | ) { 35 | keypoints.forEach(keypoint => { 36 | if (keypoint.score >= minConfidence) { 37 | const {x, y} = keypoint.position 38 | canvasContext.beginPath() 39 | canvasContext.arc(x * scale, y * scale, pointRadius, 0, 2 * Math.PI) 40 | canvasContext.fillStyle = skeletonColor 41 | canvasContext.fill() 42 | } 43 | }) 44 | } 45 | 46 | function drawSegment( 47 | [firstX, firstY], 48 | [nextX, nextY], 49 | color, 50 | lineWidth, 51 | scale, 52 | canvasContext 53 | ) { 54 | canvasContext.beginPath() 55 | canvasContext.moveTo(firstX * scale, firstY * scale) 56 | canvasContext.lineTo(nextX * scale, nextY * scale) 57 | canvasContext.lineWidth = lineWidth 58 | canvasContext.strokeStyle = color 59 | canvasContext.stroke() 60 | } 61 | 62 | export function drawSkeleton( 63 | keypoints, 64 | minConfidence, 65 | color, 66 | lineWidth, 67 | canvasContext, 68 | scale = 1 69 | ) { 70 | const adjacentKeyPoints = posenet.getAdjacentKeyPoints( 71 | keypoints, 72 | minConfidence 73 | ) 74 | 75 | adjacentKeyPoints.forEach(keypoints => { 76 | drawSegment( 77 | toTuple(keypoints[0].position), 78 | toTuple(keypoints[1].position), 79 | color, 80 | lineWidth, 81 | scale, 82 | canvasContext 83 | ) 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "PoseNetReact", 3 | "version": "1.0.0", 4 | "description": "Basic boilerplate for using PoseNet in React.js", 5 | "engines": { 6 | "node": ">= 7.0.0" 7 | }, 8 | "main": "index.js", 9 | "scripts": { 10 | "build-client": "webpack", 11 | "build-client-watch": "webpack -w", 12 | "deploy": "script/deploy", 13 | "heroku-token": "script/encrypt-heroku-auth-token", 14 | "lint": "eslint ./ --ignore-path .gitignore", 15 | "lint-fix": "npm run lint -- --fix", 16 | "precommit": "lint-staged", 17 | "prepare": "if [ -d .git ]; then npm-merge-driver install; fi", 18 | "prettify": "prettier --write \"**/*.{js,jsx,json,css,scss,md}\"", 19 | "postinstall": "touch secrets.js", 20 | "seed": "node script/seed.js", 21 | "start": "node server", 22 | "start-dev": "NODE_ENV='development' npm run build-client-watch & NODE_ENV='development' npm run start-server", 23 | "start-server": "nodemon server -e html,js,scss --ignore public --ignore client", 24 | "test": "NODE_ENV='test' mocha \"./server/**/*.spec.js\" \"./client/**/*.spec.js\" \"./script/**/*.spec.js\" --require @babel/polyfill --require @babel/register" 25 | }, 26 | "lint-staged": { 27 | "*.{js,jsx}": [ 28 | "prettier --write", 29 | "npm run lint-fix", 30 | "git add" 31 | ], 32 | "*.{css,scss,json,md}": [ 33 | "prettier --write", 34 | "git add" 35 | ] 36 | }, 37 | "author": "Kirsten Lindsimth", 38 | "license": "MIT", 39 | "dependencies": { 40 | "@tensorflow-models/posenet": "^1.0.0", 41 | "@tensorflow/tfjs": "^1.0.1", 42 | "axios": "^0.15.3", 43 | "compression": "^1.7.3", 44 | "connect-session-sequelize": "^4.1.0", 45 | "dat.gui": "^0.7.5", 46 | "express": "^4.16.3", 47 | "express-session": "^1.15.1", 48 | "fs": "0.0.1-security", 49 | "history": "^4.6.3", 50 | "morgan": "^1.9.1", 51 | "passport": "^0.3.2", 52 | "passport-google-oauth": "^1.0.0", 53 | "pg": "^6.1.2", 54 | "pg-hstore": "^2.3.2", 55 | "prop-types": "^15.6.2", 56 | "react": "^16.4.2", 57 | "react-dom": "^16.4.2", 58 | "react-redux": "^5.0.2", 59 | "react-router-dom": "^4.3.1", 60 | "redux": "^3.6.0", 61 | "redux-logger": "^2.8.1", 62 | "redux-thunk": "^2.3.0", 63 | "sequelize": "^4.38.0", 64 | "socket.io": "^2.1.1", 65 | "stats.js": "^0.17.0" 66 | }, 67 | "devDependencies": { 68 | "@babel/core": "^7.3.4", 69 | "@babel/plugin-proposal-class-properties": "^7.3.4", 70 | "@babel/plugin-proposal-decorators": "7.0.0-beta.54", 71 | "@babel/plugin-proposal-export-namespace-from": "7.0.0-beta.54", 72 | "@babel/plugin-proposal-function-sent": "7.0.0-beta.54", 73 | "@babel/plugin-proposal-json-strings": "7.0.0-beta.54", 74 | "@babel/plugin-proposal-numeric-separator": "7.0.0-beta.54", 75 | "@babel/plugin-proposal-throw-expressions": "7.0.0-beta.54", 76 | "@babel/plugin-syntax-dynamic-import": "7.0.0-beta.54", 77 | "@babel/plugin-syntax-import-meta": "7.0.0-beta.54", 78 | "@babel/polyfill": "^7.0.0-beta.55", 79 | "@babel/preset-env": "^7.3.4", 80 | "@babel/preset-react": "^7.0.0-beta.55", 81 | "@babel/register": "^7.0.0-beta.55", 82 | "axios-mock-adapter": "^1.15.0", 83 | "babel-eslint": "^8.2.6", 84 | "babel-loader": "^8.0.0-beta.4", 85 | "chai": "^3.5.0", 86 | "enzyme": "^3.9.0", 87 | "enzyme-adapter-react-16": "^1.11.2", 88 | "eslint": "^4.19.1", 89 | "eslint-config-fullstack": "^5.1.0", 90 | "eslint-config-prettier": "^2.9.0", 91 | "eslint-plugin-react": "^7.10.0", 92 | "husky": "^0.14.3", 93 | "lint-staged": "^7.2.0", 94 | "mocha": "^5.2.0", 95 | "nodemon": "^1.18.3", 96 | "npm-merge-driver": "^2.3.5", 97 | "prettier": "1.11.1", 98 | "react-test-renderer": "^16.4.2", 99 | "redux-devtools-extension": "^2.13.5", 100 | "redux-mock-store": "^1.5.3", 101 | "supertest": "^3.1.0", 102 | "webpack": "^4.16.4", 103 | "webpack-cli": "^3.1.0" 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /client/components/Camera.js: -------------------------------------------------------------------------------- 1 | import {drawKeyPoints, drawSkeleton} from './utils' 2 | import React, {Component} from 'react' 3 | import * as posenet from '@tensorflow-models/posenet' 4 | 5 | class PoseNet extends Component { 6 | static defaultProps = { 7 | videoWidth: 900, 8 | videoHeight: 700, 9 | flipHorizontal: true, 10 | algorithm: 'single-pose', 11 | showVideo: true, 12 | showSkeleton: true, 13 | showPoints: true, 14 | minPoseConfidence: 0.1, 15 | minPartConfidence: 0.5, 16 | maxPoseDetections: 2, 17 | nmsRadius: 20, 18 | outputStride: 16, 19 | imageScaleFactor: 0.5, 20 | skeletonColor: '#ffadea', 21 | skeletonLineWidth: 6, 22 | loadingText: 'Loading...please be patient...' 23 | } 24 | 25 | constructor(props) { 26 | super(props, PoseNet.defaultProps) 27 | } 28 | 29 | getCanvas = elem => { 30 | this.canvas = elem 31 | } 32 | 33 | getVideo = elem => { 34 | this.video = elem 35 | } 36 | 37 | async componentDidMount() { 38 | try { 39 | await this.setupCamera() 40 | } catch (error) { 41 | throw new Error( 42 | 'This browser does not support video capture, or this device does not have a camera' 43 | ) 44 | } 45 | 46 | try { 47 | this.posenet = await posenet.load() 48 | } catch (error) { 49 | throw new Error('PoseNet failed to load') 50 | } finally { 51 | setTimeout(() => { 52 | this.setState({loading: false}) 53 | }, 200) 54 | } 55 | 56 | this.detectPose() 57 | } 58 | 59 | async setupCamera() { 60 | if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { 61 | throw new Error( 62 | 'Browser API navigator.mediaDevices.getUserMedia not available' 63 | ) 64 | } 65 | const {videoWidth, videoHeight} = this.props 66 | const video = this.video 67 | video.width = videoWidth 68 | video.height = videoHeight 69 | 70 | const stream = await navigator.mediaDevices.getUserMedia({ 71 | audio: false, 72 | video: { 73 | facingMode: 'user', 74 | width: videoWidth, 75 | height: videoHeight 76 | } 77 | }) 78 | 79 | video.srcObject = stream 80 | 81 | return new Promise(resolve => { 82 | video.onloadedmetadata = () => { 83 | video.play() 84 | resolve(video) 85 | } 86 | }) 87 | } 88 | 89 | detectPose() { 90 | const {videoWidth, videoHeight} = this.props 91 | const canvas = this.canvas 92 | const canvasContext = canvas.getContext('2d') 93 | 94 | canvas.width = videoWidth 95 | canvas.height = videoHeight 96 | 97 | this.poseDetectionFrame(canvasContext) 98 | } 99 | 100 | poseDetectionFrame(canvasContext) { 101 | const { 102 | algorithm, 103 | imageScaleFactor, 104 | flipHorizontal, 105 | outputStride, 106 | minPoseConfidence, 107 | minPartConfidence, 108 | maxPoseDetections, 109 | nmsRadius, 110 | videoWidth, 111 | videoHeight, 112 | showVideo, 113 | showPoints, 114 | showSkeleton, 115 | skeletonColor, 116 | skeletonLineWidth 117 | } = this.props 118 | 119 | const posenetModel = this.posenet 120 | const video = this.video 121 | 122 | const findPoseDetectionFrame = async () => { 123 | let poses = [] 124 | 125 | switch (algorithm) { 126 | case 'multi-pose': { 127 | poses = await posenetModel.estimateMultiplePoses( 128 | video, 129 | imageScaleFactor, 130 | flipHorizontal, 131 | outputStride, 132 | maxPoseDetections, 133 | minPartConfidence, 134 | nmsRadius 135 | ) 136 | break 137 | } 138 | case 'single-pose': { 139 | const pose = await posenetModel.estimateSinglePose( 140 | video, 141 | imageScaleFactor, 142 | flipHorizontal, 143 | outputStride 144 | ) 145 | poses.push(pose) 146 | break 147 | } 148 | } 149 | 150 | canvasContext.clearRect(0, 0, videoWidth, videoHeight) 151 | 152 | if (showVideo) { 153 | canvasContext.save() 154 | canvasContext.scale(-1, 1) 155 | canvasContext.translate(-videoWidth, 0) 156 | canvasContext.drawImage(video, 0, 0, videoWidth, videoHeight) 157 | canvasContext.restore() 158 | } 159 | 160 | poses.forEach(({score, keypoints}) => { 161 | if (score >= minPoseConfidence) { 162 | if (showPoints) { 163 | drawKeyPoints( 164 | keypoints, 165 | minPartConfidence, 166 | skeletonColor, 167 | canvasContext 168 | ) 169 | } 170 | if (showSkeleton) { 171 | drawSkeleton( 172 | keypoints, 173 | minPartConfidence, 174 | skeletonColor, 175 | skeletonLineWidth, 176 | canvasContext 177 | ) 178 | } 179 | } 180 | }) 181 | requestAnimationFrame(findPoseDetectionFrame) 182 | } 183 | findPoseDetectionFrame() 184 | } 185 | 186 | render() { 187 | return ( 188 |
189 |
190 |
193 |
194 | ) 195 | } 196 | } 197 | 198 | export default PoseNet 199 | --------------------------------------------------------------------------------