├── threeJs
├── changeXY.js
├── noseModel.js
└── nose.js
├── .gitignore
├── README.md
├── package.json
├── webpack.config.js
├── poseNet.js
├── index.html
├── main.css
└── index.js
/threeJs/changeXY.js:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | bundle.js
3 | bundle.js.map
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | #Description
3 | This application allows a user to interact with a 3D rendering simply using motion! The user must grant the application web cam access. Button's on the application allow the user to change the 3D object.
4 |
5 | #To Run Application
6 | 1. Clone & download repo
7 | 2. npm i
8 | 3. npm run start-dev
9 | 4. copy path of the index html onto your browser
10 | 5. HAVE FUN!
11 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "hackathonidea2",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "start-dev": "webpack -w",
8 | "test": "echo \"Error: no test specified\" && exit 1"
9 | },
10 | "author": "",
11 | "license": "ISC",
12 | "dependencies": {
13 | "ml5": "^0.1.2",
14 | "three": "^0.98.0",
15 | "webpack": "^4.25.1"
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | entry: './index.js', // assumes your entry point is the index.js in the root of your project folder
3 | mode: 'development',
4 | output: {
5 | path: __dirname, // assumes your bundle.js will also be in the root of your project folder
6 | filename: 'bundle.js'
7 | },
8 | devtool: 'source-maps',
9 | module: {
10 | rules: [
11 | {
12 | test: /\.js$/,
13 | exclude: /node_modules/,
14 |
15 | }
16 | ]
17 | }
18 | }
--------------------------------------------------------------------------------
/poseNet.js:
--------------------------------------------------------------------------------
1 |
2 | import * as ml5 from 'ml5'
3 |
4 |
5 | let video = document.getElementsByTagName('video')
6 | let poseNet = ml5.poseNet(video, modelReady)
7 |
8 | poseNet.on('pose', function(results) {
9 | let poses = results;
10 | let isLoaded = true
11 | console.log(poses)
12 | console.log(isLoaded)
13 |
14 | });
15 |
16 | function modelReady() {
17 | let status = document.getElementById('status')
18 | status.innerHTML = 'Model loaded'
19 |
20 | }
21 |
22 |
23 |
24 | export default poseNet;
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | My first ml5 posenet app
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/threeJs/noseModel.js:
--------------------------------------------------------------------------------
1 | import * as THREE from 'three'
2 |
3 | const scene = new THREE.Scene();
4 | const camera = new THREE.PerspectiveCamera( 75, window.innerWidth / window.innerHeight, 0.1, 1000 );
5 |
6 | const renderer = new THREE.WebGLRenderer();
7 | renderer.setSize( window.innerWidth , window.innerHeight );
8 | document.body.appendChild( renderer.domElement );
9 |
10 | const geometry = new THREE.BoxGeometry( .01, .01, .01 );
11 | const material = new THREE.MeshBasicMaterial( { color: 400020 } );
12 | const cubeTwo = new THREE.Mesh( geometry, material );
13 |
14 | scene.add(cubeTwo);
15 |
16 | const render = function (){
17 | requestAnimationFrame(render)
18 |
19 | renderer.render(scene, camera)
20 | }
21 |
22 |
--------------------------------------------------------------------------------
/threeJs/nose.js:
--------------------------------------------------------------------------------
1 | function loopThroughPoses (poses, nose, leftEye, rightEye){
2 |
3 | for (let i = 0; i < poses.length; i++) {
4 | // For each pose detected, loop through all the keypoints
5 | let pose = poses[i].pose;
6 | for (let j = 0; j < pose.keypoints.length; j++) {
7 | // A keypoint is an object describing a body part (like rightArm or leftShoulder)
8 | let keypoint = pose.keypoints[j];
9 |
10 | // Only draw an ellipse is the pose probability is bigger than 0.2
11 | if (keypoint.score > 0.2 ) {
12 | if (keypoint.part === 'nose'){
13 | nose.x = keypoint.position.x
14 | nose.y = keypoint.position.y
15 | }
16 | }
17 | }
18 | }
19 | }
20 | export default loopThroughPoses;
21 |
--------------------------------------------------------------------------------
/main.css:
--------------------------------------------------------------------------------
1 | body{
2 | display: flex;
3 | flex-direction: column;
4 | align-content: flex-start;
5 | background:black;
6 |
7 |
8 | }
9 | .buttons{
10 | margin-bottom:4%;
11 | display: flex;
12 | justify-content: center;
13 | height: 3em;
14 | }
15 | button{
16 | font-size: 12px;
17 | font-weight: bold;
18 | font-family: Helvetica;
19 |
20 | width: 11.5%;
21 | color:white;
22 | background: transparent;
23 |
24 | border-radius: 6px;
25 | margin-left:2%;
26 | }
27 | button:hover {
28 | background-color: white;
29 | color: black;
30 | border: 4px solid white;
31 | }
32 | #video{
33 | margin: auto;
34 | }
35 | video{
36 | filter: drop-shadow(0 0 0.75rem white);
37 |
38 | }
39 | canvas{
40 | margin: auto;
41 | filter: drop-shadow(0 0 0.75rem white);
42 | }
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | import * as ml5 from 'ml5'
2 | import * as THREE from 'three'
3 | import loopThroughPoses from './threeJs/nose'
4 |
5 | let video = document.createElement('video')
6 | let vidDiv = document.getElementById('video')
7 | let leftEyeButton = document.getElementById('leftEye')
8 | let rightEyeButton = document.getElementById('rightEye')
9 | let noseButton = document.getElementById('nose')
10 | let mouthButton = document.getElementById('mouth')
11 | video.setAttribute('width', 255);
12 | video.setAttribute('height', 255);
13 | video.autoplay = true
14 | vidDiv.appendChild(video)
15 |
16 | // get the users webcam stream to render in the video
17 | navigator.mediaDevices.getUserMedia({ video: true, audio: false })
18 | .then(function(stream) {
19 | video.srcObject = stream;
20 | // video.hiddend();
21 | })
22 | .catch(function(err) {
23 | console.log("An error occurred! " + err);
24 | });
25 |
26 | let options = {
27 | flipHorizontal: true,
28 | minConfidence: 0.5
29 | }
30 |
31 | let poseNet = ml5.poseNet(video, options, modelReady)
32 |
33 | //three.js code
34 | const scene = new THREE.Scene();
35 | scene.background = new THREE.Color( 0xf0f0f0 );
36 | // Create a basic perspective camera
37 | const camera = new THREE.PerspectiveCamera( 75, (window.innerWidth/2) /(window.innerHeight/2), 0.1, 1000 );
38 | camera.position.z = 20;
39 | let i = 0;
40 | camera.position.set( 0, 7, 15 );
41 | camera.lookAt( scene.position );
42 |
43 | // Create a renderer with Antialiasing
44 | const renderer = new THREE.WebGLRenderer({antialias:true});
45 |
46 | // Configure renderer clear color
47 | renderer.setClearColor("#2E2B40");
48 |
49 | // Configure renderer size
50 | renderer.setSize(window.innerWidth/2, window.innerHeight/2);
51 |
52 | // Append Renderer to DOM
53 | document.body.appendChild( renderer.domElement );
54 |
55 |
56 | let newSphereGeo = false
57 | let rightEyeBool = false
58 | let noseBool = false
59 | let mouthBool = false
60 |
61 | let geometry = new THREE.BoxGeometry( 1, 1, 1 );
62 | let sphereGeo = new THREE.SphereGeometry(1, 50, 50, 0, Math.PI * 2, 0, Math.PI * 2);
63 | let mouthGeo = new THREE.TorusGeometry( 1, 0.5, 6, 100 );
64 | let halfMouth = new THREE.BoxGeometry( 5, 1, 1 );
65 | let material = new THREE.MeshPhongMaterial( { color: "0x2194ce" } );
66 |
67 | let cube01 = new THREE.Mesh( geometry, material );
68 | let cube02 = new THREE.Mesh( sphereGeo, material );
69 | let cube03 = new THREE.Mesh( sphereGeo, material );
70 | let oMouth = new THREE.Mesh( mouthGeo, material );
71 | // let halfMouthObj = new THREE.Mesh( halfMouth, material );
72 |
73 |
74 | let light = new THREE.PointLight( 0xFFFF00 );
75 | light.position.set( -10, 0, 10 );
76 |
77 | function createHemisphereLight() {
78 | return new THREE.HemisphereLight(0x303F9F, 0x000000, 1);
79 | }
80 | var loader = new THREE.TextureLoader();
81 | var groundTexture = loader.load( 'https://img.freepik.com/free-photo/white-marble-texture-with-natural-pattern-for-background-or-design-art-work_24076-186.jpg?size=338&ext=jpg' );
82 | groundTexture.wrapS = groundTexture.wrapT = THREE.RepeatWrapping;
83 | groundTexture.repeat.set( 25, 25 );
84 | groundTexture.anisotropy = 16;
85 | var groundMaterial = new THREE.MeshLambertMaterial( { map: groundTexture } );
86 | var mesh = new THREE.Mesh( new THREE.PlaneBufferGeometry( 20000, 20000 ), groundMaterial );
87 | mesh.position.y = - 250;
88 | mesh.rotation.x = - Math.PI / 2;
89 | mesh.receiveShadow = true;
90 | scene.add( mesh );
91 |
92 | scene.add(light, cube01, cube02, cube03, oMouth, createHemisphereLight());
93 |
94 | // Render Loop
95 |
96 | let lastXPosition = 100;
97 | let lastYPosition = 100;
98 | let changeX = 1;
99 | let changeY = 1;
100 |
101 | const changeYXPosition = (faceObj, shape, leftEyeShape, rightEyeShape) => {
102 |
103 |
104 | changeX = faceObj.x - lastXPosition
105 | changeY = faceObj.y - lastYPosition
106 |
107 | console.log('changes x,y', changeX, changeY)
108 | shape.position.x += (changeX * 0.20)
109 | shape.position.y += -(changeY * 0.20)
110 | rightEyeShape.position.x = shape.position.x + 3
111 | rightEyeShape.position.y = shape.position.y + 4
112 | leftEyeShape.position.x = shape.position.x - 3
113 | leftEyeShape.position.y = shape.position.y + 4
114 | oMouth.position.x = shape.position.x
115 | oMouth.position.y = shape.position.y - 4
116 |
117 | console.log(`shape position x, y`, shape.position.x, shape.position.y)
118 | lastXPosition = faceObj.x
119 | lastYPosition = faceObj.y
120 | console.log('lastpositions', lastXPosition, lastYPosition)
121 | }
122 | // import changeYXPosition from './threeJs/changeXY'
123 |
124 | const render = function (aNose, shape, aRightEye, aLeftEye) {
125 | newSphereGeo ? aRightEye.geometry = geometry : aRightEye.geometry = sphereGeo
126 | rightEyeBool ? aLeftEye.geometry = geometry : aLeftEye.geometry = sphereGeo
127 | noseBool ? cube01.geometry = geometry : cube01.geometry = sphereGeo
128 | mouthBool ? oMouth.geometry = mouthGeo : oMouth.geometry = halfMouth
129 | // if (newSphereGeo) {
130 | // aRightEye.geometry = geometry
131 | // } else {
132 | // aRightEye.geometry = sphereGeo
133 | // }
134 | console.log(newSphereGeo)
135 | changeYXPosition(aNose, shape, aRightEye, aLeftEye)
136 | aRightEye.rotation.x += 0.1
137 | aLeftEye.rotation.x -= 0.1
138 |
139 |
140 | renderer.render(scene, camera);
141 |
142 | };
143 |
144 | let nose = {}
145 | let rightEye = {}
146 | let leftEye = {}
147 |
148 | poseNet.on('pose', function(results) {
149 | let poses = results;
150 | loopThroughPoses(poses, nose, rightEye, leftEye)
151 | let estimatedNose = {
152 | x: nose.x,
153 | y: nose.y
154 | }
155 | if (estimatedNose.x && estimatedNose.y){
156 | console.log("On POSE", estimatedNose.x)
157 | render(estimatedNose, cube01, cube02, cube03)
158 | }
159 | });
160 |
161 | function modelReady() {
162 |
163 | console.log("model Loaded")
164 |
165 | }
166 | // render()
167 | leftEyeButton.addEventListener('click', function(){
168 | if (newSphereGeo) {
169 | newSphereGeo = false
170 | } else {
171 | newSphereGeo = true
172 | }
173 | });
174 |
175 | rightEyeButton.addEventListener('click', function(){
176 | if (rightEyeBool) {
177 | rightEyeBool = false
178 | } else {
179 | rightEyeBool = true
180 | }
181 | });
182 |
183 | noseButton.addEventListener('click', function(){
184 | if (noseBool) {
185 | noseBool = false
186 | } else {
187 | noseBool = true
188 | }
189 | });
190 | mouthButton.addEventListener('click', function(){
191 | if (mouthBool) {
192 | mouthBool = false
193 | } else {
194 | mouthBool = true
195 | }
196 | });
197 | window.addEventListener( 'resize', onWindowResize, false );
198 |
199 | function onWindowResize() {
200 |
201 | camera.aspect = (window.innerWidth/2) / (window.innerHeight/2);
202 | camera.updateProjectionMatrix();
203 | renderer.setSize( window.innerWidth/2, window.innerHeight/2 );
204 | // video.setAttribute('width', window.innerWidth/2);
205 | // video.setAttribute('height', window.innerWidth/2);
206 | }
207 |
208 |
--------------------------------------------------------------------------------