├── ARButton.js
├── demo.gif
├── index.html
├── main.css
├── readme.md
└── three.module.js


/ARButton.js:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * @author mrdoob / http://mrdoob.com
  3 |  * @author Mugen87 / https://github.com/Mugen87
  4 |  */
  5 | 
  6 | var ARButton = {
  7 | 
  8 | 	createButton: function ( renderer ) {
  9 | 
 10 | 		function showStartAR( /*device*/ ) {
 11 | 
 12 | 			var currentSession = null;
 13 | 
 14 | 			function onSessionStarted( session ) {
 15 | 
 16 | 				session.addEventListener( 'end', onSessionEnded );
 17 | 
 18 | 				/*
 19 | 				session.updateWorldTrackingState( {
 20 | 					'planeDetectionState': { 'enabled': true }
 21 | 				} );
 22 | 				*/
 23 | 
 24 | 				renderer.xr.setReferenceSpaceType( 'local' );
 25 | 				renderer.xr.setSession( session );
 26 | 				button.textContent = 'STOP AR';
 27 | 
 28 | 				currentSession = session;
 29 | 
 30 | 			}
 31 | 
 32 | 			function onSessionEnded( /*event*/ ) {
 33 | 
 34 | 				currentSession.removeEventListener( 'end', onSessionEnded );
 35 | 
 36 | 				button.textContent = 'START AR';
 37 | 
 38 | 				currentSession = null;
 39 | 
 40 | 			}
 41 | 
 42 | 			//
 43 | 
 44 | 			button.style.display = '';
 45 | 
 46 | 			button.style.cursor = 'pointer';
 47 | 			button.style.left = 'calc(50% - 50px)';
 48 | 			button.style.width = '100px';
 49 | 
 50 | 			button.textContent = 'START AR';
 51 | 
 52 | 			button.onmouseenter = function () {
 53 | 
 54 | 				button.style.opacity = '1.0';
 55 | 
 56 | 			};
 57 | 
 58 | 			button.onmouseleave = function () {
 59 | 
 60 | 				button.style.opacity = '0.5';
 61 | 
 62 | 			};
 63 | 
 64 | 			button.onclick = function () {
 65 | 
 66 | 				if ( currentSession === null ) {
 67 | 
 68 | 					navigator.xr.requestSession( 'immersive-ar' ).then( onSessionStarted );
 69 | 
 70 | 				} else {
 71 | 
 72 | 					currentSession.end();
 73 | 
 74 | 				}
 75 | 
 76 | 			};
 77 | 
 78 | 		}
 79 | 
 80 | 		function disableButton() {
 81 | 
 82 | 			button.style.display = '';
 83 | 
 84 | 			button.style.cursor = 'auto';
 85 | 			button.style.left = 'calc(50% - 75px)';
 86 | 			button.style.width = '150px';
 87 | 
 88 | 			button.onmouseenter = null;
 89 | 			button.onmouseleave = null;
 90 | 
 91 | 			button.onclick = null;
 92 | 
 93 | 		}
 94 | 
 95 | 		function showARNotSupported() {
 96 | 
 97 | 			disableButton();
 98 | 
 99 | 			button.textContent = 'AR NOT SUPPORTED';
100 | 
101 | 		}
102 | 
103 | 		function stylizeElement( element ) {
104 | 
105 | 			element.style.position = 'absolute';
106 | 			element.style.bottom = '20px';
107 | 			element.style.padding = '12px 6px';
108 | 			element.style.border = '1px solid #fff';
109 | 			element.style.borderRadius = '4px';
110 | 			element.style.background = 'rgba(0,0,0,0.1)';
111 | 			element.style.color = '#fff';
112 | 			element.style.font = 'normal 13px sans-serif';
113 | 			element.style.textAlign = 'center';
114 | 			element.style.opacity = '0.5';
115 | 			element.style.outline = 'none';
116 | 			element.style.zIndex = '999';
117 | 
118 | 		}
119 | 
120 | 		if ( 'xr' in navigator ) {
121 | 
122 | 			var button = document.createElement( 'button' );
123 | 			button.style.display = 'none';
124 | 
125 | 			stylizeElement( button );
126 | 
127 | 			navigator.xr.isSessionSupported( 'immersive-ar' ).then( function ( supported ) {
128 | 
129 | 				supported ? showStartAR() : showARNotSupported();
130 | 
131 | 			} ).catch( showARNotSupported );
132 | 
133 | 			return button;
134 | 
135 | 		} else {
136 | 
137 | 			var message = document.createElement( 'a' );
138 | 			message.href = 'https://immersiveweb.dev/';
139 | 
140 | 			if ( window.isSecureContext === false ) {
141 | 
142 | 				message.innerHTML = 'WEBXR NEEDS HTTPS'; // TODO Improve message
143 | 
144 | 			} else {
145 | 
146 | 				message.innerHTML = 'WEBXR NOT AVAILABLE';
147 | 
148 | 			}
149 | 
150 | 			message.style.left = 'calc(50% - 90px)';
151 | 			message.style.width = '180px';
152 | 			message.style.textDecoration = 'none';
153 | 
154 | 			stylizeElement( message );
155 | 
156 | 			return message;
157 | 
158 | 		}
159 | 
160 | 	}
161 | 
162 | };
163 | 
164 | export { ARButton };
165 | 


--------------------------------------------------------------------------------
/demo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nosy-b/holography/a5ea0985018fbdaded2a7392aaa6fd9c05565e4b/demo.gif


--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | 	<head>
  4 | 		<title>Holograms</title>
  5 | 		<meta charset="utf-8">
  6 | 		<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
  7 |         <link type="text/css" rel="stylesheet" href="main.css">
  8 |         
  9 |         <!-- Load TensorFlow.js -->
 10 |         <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@1.2"></script>
 11 |         <!-- Load BodyPix -->
 12 |         <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/body-pix@2.0"></script>
 13 | 
 14 |     </head>
 15 |     
 16 |     
 17 |    
 18 | 
 19 |   
 20 |     <body>  
 21 | 
 22 |         <div class="containerV">
 23 |             <video video autoplay playsinline class="hidden">
 24 |             </video>
 25 |             <div class="red-square">
 26 |                 <div class="overlay">
 27 |                     <label for="videoSource">Choose your camera: </label><select id="videoSource"></select>
 28 |                     <p>When in AR mode, create an hologram by looking at a person/face/photo and tap on the screen, it should appear gently. (Beware of camera framing, not too close)</p>
 29 |                     <div class="linkG"><p><b><a href="https://github.com/nosy-b/holography">https://github.com/nosy-b/holography</a></b></p></div>
 30 |                 </div>
 31 |             </div>
 32 |            
 33 |         </div>
 34 |         <div id="container"></div>
 35 | 
 36 |         <script type="x-shader/x-vertex" id="vertexshaderParticle">
 37 |             uniform sampler2D originalImage;
 38 |             uniform sampler2D depthMap;
 39 |             uniform float time;
 40 |             uniform vec3 userPos;
 41 |             uniform float width;
 42 |             uniform float height;
 43 |             uniform int optionMode;
 44 |             attribute vec2 uvs;
 45 |             varying vec2 vUv;
 46 |             varying float vDepth;
 47 |             float distancePoint = 3.;
 48 |             float localRadius = 0.3;
 49 |             const float M_PI = 3.1415926535897932384626433832795;
 50 | 
 51 |             float luminance(float r, float g, float b){
 52 |                 return  (r * 0.3) + (g * 0.59) + (b * 0.11);
 53 |             }
 54 | 
 55 |             void main() {
 56 |                 
 57 |                 vUv = uvs;
 58 |                 float ratio = height/width;
 59 |                 gl_PointSize = 19.;
 60 |                 vec4 dMap = texture2D( depthMap, uvs) ;
 61 |                 vec4 img = texture2D( originalImage, uvs) ;
 62 |                 float lumi = luminance(img.r, img.g, img.b);
 63 |                 vec3 pos = distancePoint * vec3(uvs.x - 0.5, (uvs.y - 0.5) * ratio - 0.25, 1. + lumi / 40.); 
 64 |                 pos.xyz *= 0.65;  // Reduce long portrait sample (todo automatic)
 65 |                 vDepth = dMap.r; 
 66 | 
 67 |                 vec3 originalPhotoPos = vec3(position.x, position.y - 2., position.z + 6.); // Starting position
 68 |                 vec3 interpolatedPos = mix(originalPhotoPos, pos, min( /*position.x*/ + (lumi + 0.2) / 10. * time * 15., 1.));
 69 | 
 70 |                 // Particles Continuous animation
 71 |                 // Depends on the distance to the viewer
 72 |                 // vec3 camPos = viewMatrix[3].xyz;
 73 |                 // float distToCam = distance(interpolatedPos, camPos /*cameraPosition*/ ) * 1.;
 74 |                 // interpolatedPos += distToCam / 5.;
 75 |                 interpolatedPos += (position / 60.) * (sin(position.y * time * 20.) * 2. )  / time; 
 76 |                 // interpolatedPos += (position / 60.) * (position.y + sin(time * 0.2 * M_PI ) ); 
 77 |                  gl_PointSize +=  4. * sin(uvs.y + time * 0.5 * M_PI) ; 
 78 |                 
 79 | 
 80 |                 if(optionMode == 1){  // Mode for particles to circle at bottom
 81 |                     if(uvs.y < 0.1) {
 82 |                         float alpha = time * 6. + position.x * 10.;
 83 |                         float x = (localRadius + position.z / 5.) * cos(alpha);
 84 |                         float z = (localRadius + position.z / 5.) * sin(alpha);
 85 |                         interpolatedPos += vec3(x, interpolatedPos.y/8. + 0., z + 0.);
 86 |                         // gl_PointSize = 8.;
 87 |                     }  
 88 |                 }  
 89 |                  
 90 |                 gl_Position = projectionMatrix * modelViewMatrix * vec4(interpolatedPos, 1.0 );
 91 |             }
 92 | 
 93 |         </script>
 94 | 
 95 |         <script type="x-shader/x-fragment" id="fragmentshaderParticle">
 96 |             #extension GL_EXT_shader_texture_lod : enable
 97 |             #extension GL_OES_standard_derivatives : enable
 98 | 
 99 |             uniform float time;
100 |             uniform sampler2D originalImage;
101 |             uniform sampler2D depthMap;
102 |             varying vec2 vUv;
103 |             varying float vDepth;
104 | 
105 | 
106 |             void main() {
107 | 
108 |                 vec4 pointSpriteTexture = texture2D( originalImage, vUv );
109 |                 vec4 earthMap = texture2D( depthMap, vUv ) ;
110 |                 gl_FragColor =  pointSpriteTexture;
111 |                 // gl_FragColor.a = vDepth * 0.7;
112 |                 // if(vDepth < 0.5) gl_FragColor.a = 0.05 * (4. - time *2.);   // To make original pic with people out disappear
113 |                 float r = 0.0, delta = 0.0, alpha = 1.0;
114 |                 vec2 cxy = 2.0 * gl_PointCoord - 1.0;
115 |                 r = dot(cxy, cxy);
116 |                 // #ifdef GL_OES_standard_derivatives
117 |                     delta = fwidth(r);
118 |                     alpha = 1.0 - smoothstep(1.0 - delta, 1.0 + delta, r);
119 |                 // #endif
120 |                 
121 |                 gl_FragColor *= alpha;
122 |   
123 | 
124 |             }
125 | 
126 |         </script>
127 | 
128 |         <script type="module">
129 | 
130 |             import * as THREE from './three.module.js';
131 |             import { ARButton } from './ARButton.js';
132 |             //import { OrbitControls } from './controls/OrbitControls.js';
133 |             var renderer, scene, camera, controls;
134 |             var time = 0;
135 |             var particles;
136 |             var texture;
137 |             var shaderMaterialParticles;
138 |             var textureSegmentation;
139 |             var textureName = "im_7.jpg"; 
140 |             var depthMapName = "depthMapFriends.png";
141 |             var subsamplingFactor = 1;
142 | 
143 |             var textureOriginal = new THREE.Texture(); // new THREE.TextureLoader().load( textureName );
144 |             var textureDepthMap = new THREE.Texture(); // new THREE.TextureLoader().load( depthMapName);
145 | 
146 |             // Photo
147 |             var constraints;
148 |             var imageCapture;
149 |             var mediaStream;
150 |             
151 |             var controller;
152 | 
153 |             var grabFrameButton = document.querySelector('button#grabFrame');
154 |             var videoSelect = document.querySelector('select#videoSource');
155 |             var video = document.querySelector('video');
156 |             //grabFrameButton.onclick = grabFrame;
157 |             videoSelect.onchange = getStream;
158 | 
159 |             var net; // Model for inference
160 | 
161 |             async function loadModel(){
162 |                 // Load the model
163 |                 net = await bodyPix.load(
164 |                     {
165 |                         architecture: 'MobileNetV1',
166 |                         outputStride: 16,
167 |                         multiplier: 0.75,
168 |                         quantBytes: 2,
169 |                         internalResolution: 'medium',
170 |                         segmentationThreshold: 0.7
171 |                         }
172 |                     );
173 |             }
174 |             loadModel();
175 | 
176 |             // Get a list of available media input (and output) devices
177 |             // then get a MediaStream for the currently selected input device
178 |             navigator.mediaDevices.enumerateDevices()
179 |                 .then(gotDevices)
180 |                 .catch(error => {
181 |                     console.log('enumerateDevices() error: ', error);
182 |                 })
183 |                 .then(getStream);
184 | 
185 |             // From the list of media devices available, set up the camera source <select>,
186 |             // then get a video stream from the default camera source.
187 |             function gotDevices(deviceInfos) {
188 |                 for (var i = 0; i !== deviceInfos.length; ++i) {
189 |                     var deviceInfo = deviceInfos[i];
190 |                     console.log('Found media input or output device: ', deviceInfo);
191 |                     var option = document.createElement('option');
192 |                     option.value = deviceInfo.deviceId;
193 |                     if (deviceInfo.kind === 'videoinput') {
194 |                     option.text = deviceInfo.label || 'Camera ' + (videoSelect.length + 1);
195 |                     videoSelect.appendChild(option);
196 |                     }
197 |                 }
198 |             }
199 | 
200 |             // Get a video stream from the currently selected camera source.
201 |             function getStream() {
202 |             if (mediaStream) {
203 |                 mediaStream.getTracks().forEach(track => {
204 |                 track.stop();
205 |                 });
206 |             }
207 |             var videoSource = videoSelect.value;
208 |             constraints = {
209 |                 video: {deviceId: videoSource ? {exact: videoSource} : undefined}
210 |             };
211 |             navigator.mediaDevices.getUserMedia(constraints)
212 |                 .then(gotStream)
213 |                 .catch(error => {
214 |                 console.log('getUserMedia error: ', error);
215 |                 });
216 |             }
217 | 
218 |             // Display the stream from the currently selected camera source, and then
219 |             // create an ImageCapture object, using the video from the stream.
220 |             function gotStream(stream) {
221 |                 console.log('getUserMedia() got stream: ', stream);
222 |                 mediaStream = stream;
223 |                 video.srcObject = stream;
224 |                 video.classList.remove('hidden');
225 |                 imageCapture = new ImageCapture(stream.getVideoTracks()[0]);
226 |                 getCapabilities();
227 |             }
228 | 
229 |             // Get the PhotoCapabilities for the currently selected camera source.
230 |             function getCapabilities() {
231 |             imageCapture.getPhotoCapabilities().then(function(capabilities) {
232 |                 console.log('Camera capabilities:', capabilities);
233 |                 if (capabilities.zoom.max > 0) {
234 |                 zoomInput.min = capabilities.zoom.min;
235 |                 zoomInput.max = capabilities.zoom.max;
236 |                 zoomInput.value = capabilities.zoom.current;
237 |                 zoomInput.classList.remove('hidden');
238 |                 }
239 |             }).catch(function(error) {
240 |                 console.log('getCapabilities() error: ', error);
241 |             });
242 |             }
243 | 
244 | 
245 |             // Get an ImageBitmap from the currently selected camera source and
246 |             // display this with a canvas element.
247 |             function grabFrame() {
248 | 
249 |                 imageCapture.grabFrame().then(function(imageBitmap) {
250 |                     console.log('Grabbed frame:', imageBitmap);
251 |                     var canvas = document.createElement('canvas');
252 |                     var ctx = canvas.getContext('2d');
253 |                     
254 |                     canvas.width = imageBitmap.width;
255 |                     canvas.height = imageBitmap.height;
256 |                     canvas.getContext('2d').drawImage(imageBitmap, 0, 0);
257 |                     textureOriginal = new THREE.Texture(canvas);
258 |                     textureOriginal.needsUpdate = true;
259 |                     
260 |                     // particles.material.uniforms.originalImage.value = texture;
261 | 
262 |                     // Then we create particles and launch the segmentation
263 |                     video.hidden = true;
264 |                     // createParticles(textureOriginal);
265 |                     loadAndPredict(textureOriginal.image);
266 |                     
267 |                    
268 |                 }).catch(function(error) {
269 |                     console.log('grabFrame() error: ', error);
270 |                 });
271 |             }
272 | 
273 | 
274 |             function init() {
275 | 
276 |                 camera = new THREE.PerspectiveCamera( 70, window.innerWidth / window.innerHeight, 0.1, 500 );
277 |                 //camera.position.z = 2;
278 |              //   camera.position.copy(new THREE.Vector3(0.6,0.7,0.8));
279 |                 scene = new THREE.Scene();
280 | 
281 |                 renderer = new THREE.WebGLRenderer({antialias:true, alpha:true});
282 |                 renderer.xr.enabled = true;
283 |                 renderer.setPixelRatio( window.devicePixelRatio );
284 |                 renderer.setSize( window.innerWidth, window.innerHeight );
285 | 
286 |                 var container = document.getElementById( 'container' );
287 |                 container.appendChild( renderer.domElement );
288 |                 document.body.appendChild( ARButton.createButton( renderer ) );
289 |                 
290 |                 var uniforms = {
291 |                     optionMode: {value: 1},
292 |                     userPos: {value: camera.position},
293 |                     width: {value: 1},
294 |                     height: {value: 1},
295 |                     time: {value: time},
296 |                     originalImage: { value: textureOriginal },
297 |                     depthMap: { value: textureDepthMap}
298 | 				};
299 | 
300 | 				shaderMaterialParticles = new THREE.ShaderMaterial( {
301 | 
302 | 					uniforms: uniforms,
303 | 					vertexShader: document.getElementById( 'vertexshaderParticle' ).textContent,
304 | 					fragmentShader: document.getElementById( 'fragmentshaderParticle' ).textContent,
305 |                     //blending: THREE.AdditiveBlending,
306 | 					depthTest: false,
307 | 					transparent: true,
308 | 					vertexColors: true
309 |                 } );
310 |               
311 |                 var geometryC = new THREE.BoxGeometry( 0.1,0.1,0.1 );
312 |                 var materialC = new THREE.MeshBasicMaterial( {color: 0x00ff00, side: THREE.DoubleSide, transparent:true, opacity:0.3} );
313 |                 var cube = new THREE.Mesh( geometryC, materialC );
314 |                // scene.add( cube );
315 | 
316 |                 //loadTextures();
317 | 
318 | 
319 |                 // grabFrame and show extracte silouhette
320 |                 function onSelect() {
321 |                     // time = 0;
322 |                     grabFrame();
323 |                     /*
324 |                     var geometry = new THREE.CylinderBufferGeometry( 0, 0.05, 0.2, 32 ).rotateX( Math.PI / 2 );
325 |                     var material = new THREE.MeshBasicMaterial( { color: 0xffffff * Math.random() } );
326 |                     var mesh = new THREE.Mesh( geometry, material );
327 |                     mesh.position.set( 0, 0, - 0.3 ).applyMatrix4( controller.matrixWorld );
328 |                     mesh.quaternion.setFromRotationMatrix( controller.matrixWorld );
329 |                     scene.add( mesh );
330 |                     */
331 |                     
332 |                 }
333 | 
334 |                 controller = renderer.xr.getController( 0 );
335 | 				controller.addEventListener( 'select', onSelect );
336 | 				scene.add( controller );
337 |                 
338 |             }
339 | 
340 | 
341 | 
342 |             init();
343 | 
344 | 
345 |             function loadTextures(){
346 | 
347 |                 var loader = new THREE.TextureLoader();
348 |                 // load a resource
349 |                 loader.load(
350 |                     // resource URL
351 |                     textureName,
352 | 
353 |                     // onLoad callback
354 |                     function ( texture ) {
355 |                         // in this example we create the material when the texture is loaded
356 |                         var material = new THREE.MeshBasicMaterial( {
357 |                             map: texture
358 |                         } );
359 | 
360 |                         //createParticles(texture);
361 |                         loadAndPredict(texture.image);
362 | 
363 |                     },
364 | 
365 |                     // onProgress callback currently not supported
366 |                     undefined,
367 | 
368 |                     // onError callback
369 |                     function ( err ) {
370 |                         console.error( 'An error happened. Fuckkk not agaiiin!' );
371 |                     }
372 |                 );
373 |  
374 | 
375 |             }
376 | 
377 |        /*         
378 |             // Create particles uniformly for an image
379 |             function createParticles(tex){
380 | 
381 |                 console.log(tex.image.width);
382 |                 if(tex.image.width > 1920 || tex.image.height > 1920)
383 |                     subsamplingFactor = 4;
384 |                 var w = tex.image.width / subsamplingFactor;
385 |                 var h = tex.image.height / subsamplingFactor;
386 |                 var geometry = new THREE.BufferGeometry();
387 |                 var positions = [];
388 |                 var uvs = [];
389 |                 var colors = [];
390 |                 var color = new THREE.Color();
391 | 
392 |                 for ( var i = 0; i < w; ++i){
393 | 
394 |                     for ( var j = 0; j < h; ++j){
395 |                        
396 |                         positions.push(Math.random(), Math.random(), Math.random());
397 |                         uvs.push(i/w, j/h);
398 |                         
399 |                     }
400 |                 }
401 | 
402 |                 geometry.setAttribute( 'position', new THREE.Float32BufferAttribute( positions, 3 ) );
403 |                 geometry.setAttribute( 'uvs', new THREE.Float32BufferAttribute( uvs, 2 ) );
404 | 
405 |                 particles = new THREE.Points( geometry, shaderMaterialParticles );
406 |                 particles.material.uniforms.originalImage.value = tex;
407 |                 particles.position.z = -4;
408 |                // particles.position.y -= 0.5;
409 |                 particles.frustumCulled = false;
410 |                 particles.material.uniforms.width.value = w ;
411 |                 particles.material.uniforms.height.value = h ;
412 |                 scene.add( particles );
413 |                 console.log(particles);
414 |                                    
415 |                 animate();
416 | 
417 |             }
418 |         */
419 | 
420 | 
421 |             function animate() {
422 | 
423 |                 renderer.setAnimationLoop( render );
424 |             }
425 | 
426 | 
427 | 
428 | 			function render() {
429 |                 time += 0.01;
430 |                 //if(time > 10) time = 0;
431 |                 particles.material.uniforms.time.value = time ;
432 |                // particles.material.uniforms.userPos.value = camera.position; // Normally passed by default...
433 |                 renderer.render( scene, camera );
434 |                 // console.log(camera);
435 |             }
436 | 
437 | 
438 | 
439 |             // Part TensorFlow https://github.com/tensorflow/tfjs-models/tree/master/body-pix
440 |             async function loadAndPredict(img) {
441 |                 if(net == null){
442 |                     console.log("loading model");
443 |                     net = await bodyPix.load(
444 |                     {
445 |                         architecture: 'MobileNetV1',
446 |                         outputStride: 16,
447 |                         multiplier: 0.75,
448 |                         quantBytes: 2,
449 |                         internalResolution: 'medium',
450 |                         segmentationThreshold: 0.7
451 |                         }
452 |                     );
453 |                 }
454 |                 /**
455 |                  * One of (see documentation below):
456 |                  *   - net.segmentPerson
457 |                  *   - net.segmentPersonParts
458 |                  *   - net.segmentMultiPerson
459 |                  *   - net.segmentMultiPersonParts
460 |                  * See documentation below for details on each method.
461 |                  */
462 |                 var segmentation = await net.segmentPerson(img); // net.segmentPerson(img);
463 |                 console.log(segmentation);
464 | 
465 |                 createTextureAndParticlesFromSegmentation(segmentation);
466 |             }
467 | 
468 | /*
469 |             // Create Data texture using the segmentation array result
470 |             function createTextureFromSegmentation(seg){
471 |                 var size = seg.width * seg.height;
472 |                 var data = new Uint8Array( 3 * size );
473 | 
474 |                 for ( var i = 0; i < size; i ++ ) {
475 |                     var stride = i * 3;
476 |                     data[ stride ] = seg.data[i] * 255;
477 |                     data[ stride + 1 ] = seg.data[i];
478 |                     data[ stride + 2 ] = seg.data[i];
479 |                 }
480 |                 textureSegmentation = new THREE.DataTexture( data, seg.width, seg.height, THREE.RGBFormat );
481 |                 textureSegmentation.flipY = true;
482 |                 time = 0;
483 |                 particles.material.uniforms.time.value = time ;
484 |                 particles.material.uniforms.depthMap.value = textureSegmentation;
485 |             }
486 | */
487 | 
488 |             // Create Data texture using the segmentation array result AND THE PARTICLES (only on people)
489 |             function createTextureAndParticlesFromSegmentation(seg){
490 |                 
491 | 
492 |                 if(particles) scene.remove(particles);
493 | 
494 |                 var size = seg.width * seg.height;
495 |                 var data = new Uint8Array( 3 * size );
496 |                 subsamplingFactor = 4;
497 |                 var w = seg.width;
498 |                 var h = seg.height;
499 |                 var geometry = new THREE.BufferGeometry();
500 |                 var positions = [];
501 |                 var uvs = [];
502 |                 var u,v;
503 |                 var nbParticles = 0;
504 |                 var nAdded = 0;
505 | 
506 |                 for ( var i = 0; i < size; i ++ ) {
507 | 
508 |                     // Texture part
509 |                     var stride = i * 3;
510 |                     data[ stride ] = seg.data[i] * 255;
511 |                     data[ stride + 1 ] = seg.data[i];
512 |                     data[ stride + 2 ] = seg.data[i];
513 | 
514 |                     // Particles part. We test if People or not to create a particle
515 |                     if(seg.data[i] > 0){
516 |                         if(nAdded % subsamplingFactor == 0){
517 |                             nbParticles++;
518 |                             u = (i % w ) / w; 
519 |                             v = Math.floor(i / w) / h;
520 |                             positions.push(Math.random(), Math.random(), Math.random());
521 |                             uvs.push(u, 1. - v);  // flip y
522 |                         }
523 |                         nAdded++;
524 |                     }
525 |                 }
526 | 
527 | 
528 |                 // Part particles
529 |                 geometry.setAttribute( 'position', new THREE.Float32BufferAttribute( positions, 3 ) );
530 |                 geometry.setAttribute( 'uvs', new THREE.Float32BufferAttribute( uvs, 2 ) );
531 | 
532 |                 particles = new THREE.Points( geometry, shaderMaterialParticles );
533 |                 particles.material.uniforms.originalImage.value = textureOriginal;
534 |                 particles.position.z = -4;
535 |                 // particles.position.y -= 0.5;
536 |                 particles.frustumCulled = false;
537 |                 particles.material.uniforms.width.value = w ;
538 |                 particles.material.uniforms.height.value = h ;
539 | 
540 |                 // Part texture
541 |                 textureSegmentation = new THREE.DataTexture( data, seg.width, seg.height, THREE.RGBFormat );
542 |                 textureSegmentation.flipY = true;
543 |                 time = 0;
544 |                 particles.material.uniforms.time.value = time ;
545 |                 particles.material.uniforms.depthMap.value = textureSegmentation;
546 | 
547 |                 
548 |                 scene.add( particles );
549 |                 console.log("Particles count: ", nbParticles);
550 |                                 
551 |                 animate();
552 | 
553 |             }
554 |           
555 | 
556 |         </script>
557 | 
558 | 
559 |     </body>
560 | </html>


--------------------------------------------------------------------------------
/main.css:
--------------------------------------------------------------------------------
 1 | body {
 2 | 	margin: 0;
 3 | 	background-color: #000;
 4 | 	color: #fff;
 5 | 	font-family: Monospace;
 6 | 	font-size: 13px;
 7 | 	line-height: 24px;
 8 | 	overscroll-behavior: none;
 9 | 	overflow:hidden;
10 | }
11 | 
12 | a {
13 | 	color: rgb(240, 205, 140);
14 | 	text-decoration: none;
15 | }
16 | 
17 | a:hover {
18 | 	text-decoration: underline;
19 | }
20 | 
21 | button {
22 | 	cursor: pointer;
23 | 	text-transform: uppercase;
24 | }
25 | 
26 | canvas {
27 | 	display: block;
28 | }
29 | .containerV { position:relative; }
30 | .containerV video {
31 |     position:relative;
32 |     z-index:0;
33 | }
34 | .overlay {
35 | 
36 | 	z-index:1;
37 | 	color: white;
38 | 	text-align: center;
39 | 	margin: 12px;	
40 | 	margin-top: 42px;
41 | }
42 | .red-square {
43 | 	background-color: #FF4136;
44 | 	width: 300px;
45 | 	height: 300px;
46 | 	position: absolute;
47 | 	left: 50%;
48 | 	top: 50%;
49 | 	transform: translate(-50%, -50%);
50 |   }
51 | #info {
52 | 	position: absolute;
53 | 	top: 0px;
54 | 	width: 100%;
55 | 	padding: 10px;
56 | 	box-sizing: border-box;
57 | 	text-align: center;
58 | 	-moz-user-select: none;
59 | 	-webkit-user-select: none;
60 | 	-ms-user-select: none;
61 | 	user-select: none;
62 | 	pointer-events: none;
63 | 	z-index: 1; /* TODO Solve this in HTML */
64 | }
65 | 
66 | .linkG {
67 | 	font-size:xx-small;
68 | 	color: white;
69 | }
70 | 
71 | #infoSeismic {
72 | 	position: absolute;
73 | 	top: 0px;
74 | 	width: 280px;
75 | 	padding: 10px;
76 | 	box-sizing: border-box;
77 | 	text-align: center;
78 | 	-moz-user-select: none;
79 | 	-webkit-user-select: none;
80 | 	-ms-user-select: none;
81 | 	user-select: none;
82 | 	pointer-events: none;
83 | 	z-index: 1; /* TODO Solve this in HTML */
84 | 	background-color: rgba(35, 4, 83, 0.6);
85 | }
86 | 
87 | a, button, input, select {
88 | 	pointer-events: auto;
89 | }
90 | 
91 | .dg.ac {
92 | 	-moz-user-select: none;
93 | 	-webkit-user-select: none;
94 | 	-ms-user-select: none;
95 | 	user-select: none;
96 | 	z-index: 2 !important; /* TODO Solve this in HTML */
97 | }
98 | 
99 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | # Holography
 2 | 
 3 | Simple demo of WebXR and Deep Learning using Tensorflow to create holograms on-the-fly. 
 4 | It works on Android with the last Chrome (>= r81)
 5 | 
 6 | ! REMARK! : I discovered that the possibility to take pictures WHILE in AR Context just works on LG phones surprisingly (https://developers.google.com/ar/discover/supported-devices) as they use the wide angle camera for AR (the phone I used by chance)!! So to use this demo with other phones, take the picture (using usual browser get media api) but before launching the AR Context!
 7 | 
 8 | ![alt text](https://raw.githubusercontent.com/nosy-b/holography/master/demo.gif "Holography")
 9 | 
10 | DEMO here: http://nosy-b.github.io/holography
11 | 
12 | ## How to
13 | 
14 | - First be sure to have an android phone with chrome >=r81
15 | - Choose your camera feed (ideally back camera)
16 | - Tap on start AR (the position of your phone at that moment will be the center of the frame)
17 | - Move a bit around if you want to help the space detection
18 | - Look at a human (real or a photo) and tap on the screen
19 | - It should appear coming from your phone to the direction you were looking at when clicked on Start (initial position)
20 | 
21 | ## Description
22 | 
23 | Small code that create particles for people extracted from camera feed using Bodypix model
24 | Feel free to use this dirty little code for your own experiments
25 | I put demos on my twitter sometimes  https://twitter.com/AlexandreDevaux
26 | 
27 | ## Getting Started
28 | 
29 | ### Dependencies
30 | 
31 | * ThreeJS, TensorFlowJS
32 | 
33 | 
34 | ## Help
35 | 
36 | One thing to be careful about is not to frame too close the subject you want to make a hologram of. The camera frame is smaller than what your screen shows. (A good todo would be to show camera frame while in AR)
37 | 
38 | ## Authors
39 | 
40 | Alexandre Devaux 
41 | [@AlexandreDevaux](https://twitter.com/AlexandreDevaux)
42 | 
43 | ## Version History
44 | 
45 | 
46 | 
47 | ## License
48 | 
49 | Check the license of THREEJS and Tensorflow
50 | 
51 | 


--------------------------------------------------------------------------------