├── .prettierignore ├── .vscode └── extensions.json ├── public ├── favicon.ico ├── favicon-16x16.png ├── favicon-32x32.png ├── apple-touch-icon.png ├── android-chrome-192x192.png ├── android-chrome-512x512.png ├── assets │ ├── images │ │ ├── static_1.png │ │ ├── static_2.png │ │ ├── static_3.png │ │ ├── static_4.png │ │ ├── static_5.png │ │ ├── static_6.png │ │ ├── static_7.png │ │ ├── static_stripe_1.png │ │ ├── static_stripe_2.png │ │ ├── static_stripe_3.png │ │ ├── static_stripe_4.png │ │ └── static_stripe_5.png │ └── videos │ │ ├── h_video.mp4 │ │ ├── s_video.mp4 │ │ ├── texture_anim_1.mp4 │ │ ├── texture_anim_2.mp4 │ │ ├── texture_anim_3.mp4 │ │ ├── texture_anim_4.mp4 │ │ └── texture_anim_5.mp4 └── site.webmanifest ├── .prettierrc.yml ├── bubble-figure-demo.jpg ├── src ├── styles │ └── style.css └── js │ ├── textures │ ├── index.ts │ ├── static.ts │ └── video.ts │ ├── utils │ ├── maths.ts │ ├── browser.ts │ └── three.ts │ ├── index.ts │ ├── bubbleFigure │ ├── head.ts │ ├── physicalBody.ts │ ├── index.ts │ ├── keypoints.ts │ ├── body.ts │ └── alignPose.ts │ ├── media.ts │ ├── cinematography.ts │ ├── bodyDetection.ts │ ├── parameters.ts │ ├── physics.ts │ ├── shapes │ ├── basic.ts │ └── falling.ts │ └── controls.ts ├── .gitignore ├── tsconfig.json ├── package.json ├── LICENSE ├── .github └── workflows │ └── static.yml ├── README.md └── index.html /.prettierignore: -------------------------------------------------------------------------------- 1 | .git 2 | node_modules 3 | dist 4 | .idea 5 | .vscode 6 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["svelte.svelte-vscode"] 3 | } 4 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /.prettierrc.yml: -------------------------------------------------------------------------------- 1 | printWidth: 120 2 | singleQuote: true 3 | tabWidth: 2 4 | endOfLine: 'auto' 5 | -------------------------------------------------------------------------------- /bubble-figure-demo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/bubble-figure-demo.jpg -------------------------------------------------------------------------------- /public/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/favicon-16x16.png -------------------------------------------------------------------------------- /public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/favicon-32x32.png -------------------------------------------------------------------------------- /public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/apple-touch-icon.png -------------------------------------------------------------------------------- /public/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/android-chrome-192x192.png -------------------------------------------------------------------------------- /public/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/android-chrome-512x512.png -------------------------------------------------------------------------------- /public/assets/images/static_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_1.png -------------------------------------------------------------------------------- /public/assets/images/static_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_2.png -------------------------------------------------------------------------------- /public/assets/images/static_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_3.png -------------------------------------------------------------------------------- /public/assets/images/static_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_4.png -------------------------------------------------------------------------------- /public/assets/images/static_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_5.png -------------------------------------------------------------------------------- /public/assets/images/static_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_6.png -------------------------------------------------------------------------------- /public/assets/images/static_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_7.png -------------------------------------------------------------------------------- /public/assets/videos/h_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/h_video.mp4 -------------------------------------------------------------------------------- /public/assets/videos/s_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/s_video.mp4 -------------------------------------------------------------------------------- /public/assets/images/static_stripe_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_stripe_1.png -------------------------------------------------------------------------------- /public/assets/images/static_stripe_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_stripe_2.png -------------------------------------------------------------------------------- /public/assets/images/static_stripe_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_stripe_3.png -------------------------------------------------------------------------------- /public/assets/images/static_stripe_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_stripe_4.png -------------------------------------------------------------------------------- /public/assets/images/static_stripe_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/images/static_stripe_5.png -------------------------------------------------------------------------------- /public/assets/videos/texture_anim_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/texture_anim_1.mp4 -------------------------------------------------------------------------------- /public/assets/videos/texture_anim_2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/texture_anim_2.mp4 -------------------------------------------------------------------------------- /public/assets/videos/texture_anim_3.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/texture_anim_3.mp4 -------------------------------------------------------------------------------- /public/assets/videos/texture_anim_4.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/texture_anim_4.mp4 -------------------------------------------------------------------------------- /public/assets/videos/texture_anim_5.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wunderdogsw/go-23-app/HEAD/public/assets/videos/texture_anim_5.mp4 -------------------------------------------------------------------------------- /src/styles/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | 6 | .hidden { 7 | display: none; 8 | } 9 | 10 | input { 11 | width: 3.125rem; 12 | margin: 0.1875rem; 13 | } 14 | -------------------------------------------------------------------------------- /public/site.webmanifest: -------------------------------------------------------------------------------- 1 | {"name":"go-23-app","short_name":"go-23-app","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /src/js/textures/index.ts: -------------------------------------------------------------------------------- 1 | import THREE from 'three'; 2 | 3 | import { getRandomItem } from '../utils/maths'; 4 | import { COLOR_STATIC_TEXTURES } from './static'; 5 | import { COLOR_VIDEO_TEXTURES } from './video'; 6 | 7 | const COLOR_TEXTURES = [...COLOR_VIDEO_TEXTURES, ...COLOR_STATIC_TEXTURES]; 8 | 9 | type GetRandomColorTextureReturnType = THREE.VideoTexture | THREE.Texture; 10 | export function getRandomColorTexture(): GetRandomColorTextureReturnType { 11 | return getRandomItem(COLOR_TEXTURES) as GetRandomColorTextureReturnType; 12 | } 13 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "useDefineForClassFields": true, 5 | "module": "ESNext", 6 | "lib": ["ESNext", "DOM", "dom.iterable"], 7 | "moduleResolution": "Node", 8 | "strict": true, 9 | "resolveJsonModule": true, 10 | "isolatedModules": true, 11 | "esModuleInterop": true, 12 | "noEmit": true, 13 | "noUnusedLocals": true, 14 | "noUnusedParameters": true, 15 | "noImplicitReturns": true, 16 | "skipLibCheck": true, 17 | "allowJs": true 18 | }, 19 | "include": ["src/**/*"] 20 | } 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "go-23-app", 3 | "private": true, 4 | "version": "1.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "tsc && vite build", 9 | "preview": "vite preview", 10 | "prettier:check": "prettier --check .", 11 | "prettier:fix": "prettier --write ." 12 | }, 13 | "devDependencies": { 14 | "@types/three": "^0.150.1", 15 | "prettier": "2.8.7", 16 | "ts-migrate": "^0.1.35", 17 | "typescript": "^4.9.3", 18 | "vite": "^4.2.0" 19 | }, 20 | "dependencies": { 21 | "@tensorflow-models/pose-detection": "^2.1.0", 22 | "cannon-es": "^0.20.0", 23 | "hotkeys-js": "^3.10.2", 24 | "three": "^0.151.3" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/js/textures/static.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | const COLOR_IMAGES_PATH = 'assets/images/'; 4 | 5 | const COLOR_IMAGE_FILENAMES = [ 6 | 'static_1.png', 7 | 'static_2.png', 8 | 'static_3.png', 9 | 'static_4.png', 10 | 'static_5.png', 11 | 'static_6.png', 12 | 'static_7.png', 13 | 'static_stripe_1.png', 14 | 'static_stripe_2.png', 15 | 'static_stripe_3.png', 16 | 'static_stripe_4.png', 17 | 'static_stripe_5.png', 18 | ]; 19 | 20 | export const COLOR_STATIC_TEXTURES = createStaticTextures(COLOR_IMAGE_FILENAMES); 21 | 22 | function createStaticTextures(paths: string[]): THREE.Texture[] { 23 | return paths.map((path) => createStaticTexture(path)); 24 | } 25 | 26 | function createStaticTexture(filename: string, rotation = -1.57, x = 0.5, y = 0.5): THREE.Texture { 27 | const fullPath = `${COLOR_IMAGES_PATH}${filename}`; 28 | const texture = new THREE.TextureLoader().load(fullPath); 29 | 30 | texture.rotation = rotation; 31 | texture.center.set(x, y); 32 | 33 | return texture; 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Wunderdog 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/js/utils/maths.ts: -------------------------------------------------------------------------------- 1 | // lazy source: ChatGPT 2 | export function getRandomInt(min: number, max: number): number { 3 | return Math.floor(Math.random() * (max - min + 1)) + min; 4 | } 5 | 6 | export function getRandomFloat(min: number, max: number): number { 7 | return Math.random() * (max - min) + min; 8 | } 9 | 10 | export function getRandomItem(array: ItemType[] = []): ItemType | undefined { 11 | if (!array.length) { 12 | return undefined; 13 | } 14 | const index = getRandomInt(0, array.length - 1); 15 | return array[index]; 16 | } 17 | 18 | export function getSum( 19 | values: ValueType[], 20 | getValue = (value: ValueType): number => value as number 21 | ): number { 22 | let sum = 0; 23 | 24 | for (let i = 0; i < values.length; i++) { 25 | const value = getValue(values[i]); 26 | sum += value; 27 | } 28 | 29 | return sum; 30 | } 31 | 32 | export function getAverage(values: ValueType[]): number { 33 | const sum = getSum(values); 34 | return sum / values.length; 35 | } 36 | 37 | export function isStringNumeric(str: string): boolean { 38 | return !isNaN(parseFloat(str)); 39 | } 40 | -------------------------------------------------------------------------------- /src/js/index.ts: -------------------------------------------------------------------------------- 1 | import { initBodyDetection } from './bodyDetection'; 2 | import { resetBubbleFigure, updateBubbleFigure } from './bubbleFigure/index'; 3 | import { BUBBLE_BODY_MATERIAL } from './bubbleFigure/physicalBody'; 4 | import { clearScene, initCinematography, renderScene, updateCamera } from './cinematography'; 5 | import { initControls } from './controls'; 6 | import { initParameters } from './parameters'; 7 | import { addCollidingContactMaterial, initWorld, worldStep } from './physics'; 8 | import { SHAPE_BODY_MATERIAL, resetShapes, updateShapes } from './shapes/falling'; 9 | 10 | const render = async function () { 11 | requestAnimationFrame(render); 12 | worldStep(); 13 | updateShapes(); 14 | await updateBubbleFigure(); 15 | renderScene(); 16 | }; 17 | 18 | async function start() { 19 | initParameters(); 20 | initCinematography(); 21 | initWorld(); 22 | await initControls({ onSubmit: updateParameters }); 23 | resetShapes(); 24 | addCollidingContactMaterial(BUBBLE_BODY_MATERIAL, SHAPE_BODY_MATERIAL); 25 | render(); 26 | await initBodyDetection(); 27 | } 28 | 29 | function updateParameters() { 30 | clearScene(); 31 | resetBubbleFigure(); 32 | updateCamera(); 33 | resetShapes(); 34 | } 35 | 36 | start(); 37 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/head.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | import { createBody } from '../physics'; 4 | import { createBubble } from '../shapes/basic'; 5 | import { getRandomFloat, getRandomInt } from '../utils/maths'; 6 | import { BUBBLE_BODY_MATERIAL } from './physicalBody'; 7 | 8 | export function createBubbleHead(radius = 1.2, numSpheres = 50): THREE.Group { 9 | const group = new THREE.Group(); 10 | group.name = 'HEAD'; 11 | group.visible = false; 12 | group.userData.radius = radius; 13 | 14 | for (let i = 0; i < numSpheres; i++) { 15 | const bubble = createHeadBubble(radius); 16 | group.add(bubble); 17 | } 18 | 19 | return group; 20 | } 21 | 22 | function createHeadBubble(radius: number): THREE.Mesh { 23 | const randomRadius = getRandomFloat(0.1, 0.4); 24 | 25 | const bubble = createBubble({ radius: randomRadius, offset: 0 }); 26 | const angle1 = getRandomInt(0, 50); 27 | const angle2 = getRandomInt(0, 50); 28 | 29 | const x = radius * Math.sin(angle1) * Math.cos(angle2); 30 | const y = radius * Math.sin(angle1) * Math.sin(angle2); 31 | const z = radius * getRandomFloat(0, 0.5); 32 | 33 | bubble.position.set(x, y, z); 34 | bubble.userData.body = createBody(bubble, BUBBLE_BODY_MATERIAL, 0); 35 | 36 | return bubble; 37 | } 38 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/physicalBody.ts: -------------------------------------------------------------------------------- 1 | import * as CANNON from 'cannon-es'; 2 | import * as THREE from 'three'; 3 | 4 | import { getWorld } from '../physics'; 5 | 6 | export const BUBBLE_BODY_MATERIAL = new CANNON.Material('bubbleMaterial'); 7 | 8 | export function alignMeshPhysicalBody(mesh: THREE.Mesh) { 9 | alignMeshPhysicalBodyTrajectory(mesh); 10 | alignMeshPhysicalBodyVisibility(mesh); 11 | } 12 | 13 | function alignMeshPhysicalBodyTrajectory(mesh: THREE.Mesh) { 14 | const body = mesh.userData?.body; 15 | if (!body) { 16 | return; 17 | } 18 | 19 | let target = new THREE.Vector3(); 20 | mesh.getWorldPosition(target); 21 | target.z = 0; 22 | 23 | body.position.copy(target); 24 | body.quaternion.copy(mesh.quaternion); 25 | } 26 | 27 | function alignMeshPhysicalBodyVisibility(mesh: THREE.Mesh) { 28 | const body = mesh.userData?.body; 29 | if (!body) { 30 | return; 31 | } 32 | 33 | const isMeshVisible = mesh.visible && (!mesh.parent || mesh.parent.visible); 34 | const isBodyInWorld = mesh.userData?.bodyInWorld; 35 | const includeInWorld = isMeshVisible && !isBodyInWorld; 36 | 37 | if (includeInWorld) { 38 | getWorld().addBody(body); 39 | } else { 40 | getWorld().removeBody(body); 41 | } 42 | 43 | mesh.userData.bodyInWorld = includeInWorld; 44 | } 45 | -------------------------------------------------------------------------------- /.github/workflows/static.yml: -------------------------------------------------------------------------------- 1 | # reference: https://github.com/sitek94/vite-deploy-demo 2 | name: Deploy 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | build: 11 | name: Build 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout repo 16 | uses: actions/checkout@v2 17 | 18 | - name: Setup Node 19 | uses: actions/setup-node@v1 20 | with: 21 | node-version: 16 22 | 23 | - name: Install dependencies 24 | uses: bahmutov/npm-install@v1 25 | 26 | - name: Build project 27 | run: npm run build 28 | 29 | - name: Upload production-ready build files 30 | uses: actions/upload-artifact@v2 31 | with: 32 | name: production-files 33 | path: ./dist 34 | 35 | deploy: 36 | name: Deploy 37 | needs: build 38 | runs-on: ubuntu-latest 39 | if: github.ref == 'refs/heads/main' 40 | 41 | steps: 42 | - name: Download artifact 43 | uses: actions/download-artifact@v2 44 | with: 45 | name: production-files 46 | path: ./dist 47 | 48 | - name: Deploy to GitHub Pages 49 | uses: peaceiris/actions-gh-pages@v3 50 | with: 51 | github_token: ${{ secrets.GITHUB_TOKEN }} 52 | publish_dir: ./dist 53 | -------------------------------------------------------------------------------- /src/js/media.ts: -------------------------------------------------------------------------------- 1 | import { getParameters } from './parameters'; 2 | 3 | export const VIDEO_SIZE = { 4 | width: 640, 5 | height: 480, 6 | }; 7 | 8 | export function getCameraVideoElement( 9 | deviceId: string, 10 | width = VIDEO_SIZE.width, 11 | height = VIDEO_SIZE.height 12 | ): Promise { 13 | return new Promise((resolve, reject) => { 14 | if (!navigator?.mediaDevices) { 15 | reject('No media devices'); 16 | } 17 | 18 | navigator.mediaDevices 19 | .getUserMedia({ video: { deviceId, width, height } }) 20 | .then((stream) => { 21 | const video = document.createElement('video'); 22 | video.srcObject = stream; 23 | video.onloadedmetadata = () => { 24 | video.play(); 25 | resolve(video); 26 | }; 27 | }) 28 | .catch(reject); 29 | }); 30 | } 31 | 32 | export async function getVideoInputDevices() { 33 | if (!navigator?.mediaDevices) { 34 | return []; 35 | } 36 | 37 | const devices = await navigator.mediaDevices.enumerateDevices(); 38 | return devices.filter(({ kind }) => kind === 'videoinput'); 39 | } 40 | 41 | export async function getSelectedVideoInputDeviceId() { 42 | const videoInputDevices = await getVideoInputDevices(); 43 | const defaultInputDeviceId = videoInputDevices[0]?.deviceId ?? null; 44 | const { videoDeviceId } = getParameters(); 45 | return videoDeviceId ?? defaultInputDeviceId; 46 | } 47 | -------------------------------------------------------------------------------- /src/js/utils/browser.ts: -------------------------------------------------------------------------------- 1 | import { isStringNumeric } from './maths'; 2 | 3 | export function getQueryStringValue(key: string): string | null { 4 | return new URLSearchParams(window.location.search).get(key); 5 | } 6 | 7 | export function createSelectOption(label: string, value: string, selectedValue: string): HTMLOptionElement { 8 | const option = document.createElement('option'); 9 | option.value = value; 10 | option.textContent = label; 11 | 12 | if (value === selectedValue) { 13 | option.selected = true; 14 | } 15 | 16 | return option; 17 | } 18 | 19 | export function setInputValueByName(name: string, value: unknown) { 20 | const elements = document.getElementsByName(name); 21 | if (!elements.length) { 22 | throw Error(`Didn't find input name ${name}`); 23 | } 24 | 25 | elements.forEach((element) => { 26 | (element as HTMLInputElement).value = value as string; 27 | }); 28 | } 29 | 30 | type ConvertFormToJsonReturnType = Record; 31 | export function convertFormToJson(form: HTMLFormElement): ConvertFormToJsonReturnType { 32 | const formData = new FormData(form); 33 | const entries = formData.entries(); 34 | const json: ConvertFormToJsonReturnType = {}; 35 | 36 | for (const [key, value] of entries) { 37 | const valueString = value.toString(); 38 | const isNumeric = isStringNumeric(valueString); 39 | json[key] = isNumeric ? parseFloat(valueString) : valueString; 40 | } 41 | 42 | return json; 43 | } 44 | -------------------------------------------------------------------------------- /src/js/textures/video.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | const COLOR_VIDEOS_PATH = 'assets/videos/'; 4 | 5 | const COLOR_VIDEO_FILENAMES = [ 6 | 'h_video.mp4', 7 | 's_video.mp4', 8 | 'texture_anim_1.mp4', 9 | 'texture_anim_2.mp4', 10 | 'texture_anim_3.mp4', 11 | 'texture_anim_4.mp4', 12 | 'texture_anim_5.mp4', 13 | ]; 14 | 15 | export const COLOR_VIDEO_TEXTURES = createVideoTextures(COLOR_VIDEO_FILENAMES); 16 | 17 | function createVideoElement(filename: string): HTMLVideoElement { 18 | const video = document.createElement('video'); 19 | const src = `${COLOR_VIDEOS_PATH}${filename}`; 20 | const attributes = { 21 | id: filename, 22 | src, 23 | autoplay: true, 24 | muted: true, 25 | loop: true, 26 | type: 'video/mp4', 27 | crossorigin: 'anonymous', 28 | playsinline: true, 29 | 'webkit-playsinline': true, 30 | style: 'display: none', 31 | }; 32 | 33 | Object.entries(attributes).forEach(([key, value]) => { 34 | video.setAttribute(key, value.toString()); 35 | }); 36 | 37 | video.muted = true; 38 | video.play(); 39 | return video; 40 | } 41 | 42 | function createVideoTexture(video: HTMLVideoElement, rotation = -1.57, x = 0.5, y = 0.5): THREE.VideoTexture { 43 | const texture = new THREE.VideoTexture(video); 44 | 45 | texture.rotation = rotation; 46 | texture.center.set(x, y); 47 | 48 | return texture; 49 | } 50 | 51 | function createVideoTextures(paths: string[]): THREE.VideoTexture[] { 52 | return paths.map((path) => { 53 | const video = createVideoElement(path); 54 | return createVideoTexture(video); 55 | }); 56 | } 57 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/index.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | import { detectPoses } from '../bodyDetection'; 4 | import { getScene } from '../cinematography'; 5 | import { getWorld } from '../physics'; 6 | import { disposeGroup } from '../utils/three'; 7 | import { alignBubbleFigurePose } from './alignPose'; 8 | import { createBubbleBody } from './body'; 9 | import { createBubbleHead } from './head'; 10 | 11 | let bubbleFigure: THREE.Group | null = null; 12 | 13 | export async function updateBubbleFigure() { 14 | const { poses, posesLost, posesFound } = await detectPoses(); 15 | 16 | if (posesLost) { 17 | disposeBubbleFigure(); 18 | } else if (posesFound) { 19 | createBubbleFigure(); 20 | } 21 | 22 | if (!poses.length || !bubbleFigure) { 23 | return; 24 | } 25 | 26 | alignBubbleFigurePose(bubbleFigure, poses[0]); 27 | } 28 | 29 | export function resetBubbleFigure() { 30 | disposeBubbleFigure(); 31 | createBubbleFigure(); 32 | } 33 | 34 | function createBubbleFigure() { 35 | bubbleFigure = new THREE.Group(); 36 | const head = createBubbleHead(); 37 | const body = createBubbleBody(); 38 | 39 | bubbleFigure.name = 'FIGURE'; 40 | bubbleFigure.add(head); 41 | bubbleFigure.add(body); 42 | 43 | getScene().add(bubbleFigure); 44 | } 45 | 46 | function disposeBubbleFigure() { 47 | if (!bubbleFigure) { 48 | return; 49 | } 50 | 51 | getScene().remove(bubbleFigure); 52 | disposeGroup(bubbleFigure, (mesh: THREE.Mesh) => { 53 | if (!mesh.userData?.body) { 54 | return; 55 | } 56 | 57 | getWorld().removeBody(mesh.userData.body); 58 | }); 59 | 60 | bubbleFigure = null; 61 | } 62 | -------------------------------------------------------------------------------- /src/js/cinematography.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | import { getParameters } from './parameters'; 4 | import { visibleHeightAtZDepth, visibleWidthAtZDepth } from './utils/three'; 5 | 6 | let camera: THREE.PerspectiveCamera; 7 | let renderer: THREE.WebGLRenderer; 8 | let scene: THREE.Scene; 9 | 10 | export function initCinematography() { 11 | scene = new THREE.Scene(); 12 | addLightingToScene(); 13 | initCamera(); 14 | initRenderer(); 15 | } 16 | 17 | export function getScene(): THREE.Scene { 18 | return scene; 19 | } 20 | 21 | export function renderScene() { 22 | renderer.render(scene, camera); 23 | } 24 | 25 | export function clearScene() { 26 | scene.clear(); 27 | addLightingToScene(); 28 | } 29 | 30 | export function updateCamera() { 31 | const { cameraZ, cameraZoom } = getParameters(); 32 | 33 | camera.position.z = cameraZ; 34 | camera.zoom = cameraZoom / 100; 35 | camera.updateProjectionMatrix(); 36 | } 37 | 38 | function initCamera() { 39 | camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000); 40 | updateCamera(); 41 | setSceneSize(camera); 42 | } 43 | 44 | function initRenderer() { 45 | renderer = new THREE.WebGLRenderer({ antialias: true }); 46 | document.body.appendChild(renderer.domElement); 47 | 48 | renderer.shadowMap.enabled = true; 49 | renderer.setClearColor('#000000'); 50 | renderer.setSize(window.innerWidth, window.innerHeight); 51 | } 52 | 53 | function addLightingToScene() { 54 | const ambientLight = new THREE.AmbientLight(0xffffff, 1); 55 | scene.add(ambientLight); 56 | } 57 | 58 | function setSceneSize(camera: THREE.PerspectiveCamera) { 59 | scene.userData.height = visibleHeightAtZDepth(camera); 60 | scene.userData.width = visibleWidthAtZDepth(camera, scene.userData.height); 61 | } 62 | -------------------------------------------------------------------------------- /src/js/bodyDetection.ts: -------------------------------------------------------------------------------- 1 | import { Keypoint, Pose, PoseDetector } from '@tensorflow-models/pose-detection'; 2 | 3 | import { getCameraVideoElement, getSelectedVideoInputDeviceId } from './media'; 4 | import { getParameters } from './parameters'; 5 | import { getSum } from './utils/maths'; 6 | 7 | // importing pose detection libraries locally doesn't work 8 | // @ts-ignore 9 | const { poseDetection } = window; 10 | 11 | let video: HTMLVideoElement; 12 | let detector: PoseDetector; 13 | let hasPoses = false; 14 | 15 | async function getDetector(): Promise { 16 | const model = poseDetection.SupportedModels.BlazePose; 17 | const detectorConfig = { 18 | runtime: 'mediapipe', 19 | solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/pose', 20 | }; 21 | return await poseDetection.createDetector(model, detectorConfig); 22 | } 23 | 24 | export async function initBodyDetection() { 25 | try { 26 | const videoInputDeviceId = await getSelectedVideoInputDeviceId(); 27 | video = await getCameraVideoElement(videoInputDeviceId); 28 | detector = await getDetector(); 29 | } catch (error) { 30 | console.error(error); 31 | } 32 | } 33 | 34 | async function getPoses(): Promise { 35 | if (!(detector && video)) { 36 | return []; 37 | } 38 | 39 | const poses = await detector.estimatePoses(video, {}); 40 | 41 | if (!poses?.length) { 42 | return []; 43 | } 44 | 45 | const { keypoints } = poses[0]; 46 | const scoreSum = getSum(keypoints, (keypoint) => keypoint?.score ?? 0); 47 | 48 | const { minPosesScore } = getParameters(); 49 | 50 | if (scoreSum < minPosesScore) { 51 | return []; 52 | } 53 | 54 | return poses; 55 | } 56 | 57 | type DetectPosesReturnType = { 58 | poses: Pose[]; 59 | posesLost: boolean; 60 | posesFound: boolean; 61 | }; 62 | export async function detectPoses(): Promise { 63 | const poses = await getPoses(); 64 | const posesExist = !!poses.length; 65 | 66 | const posesLost = !posesExist && hasPoses; 67 | const posesFound = posesExist && !hasPoses; 68 | 69 | hasPoses = posesExist; 70 | 71 | return { poses, posesLost, posesFound }; 72 | } 73 | -------------------------------------------------------------------------------- /src/js/parameters.ts: -------------------------------------------------------------------------------- 1 | const PARAMETERS_LOCAL_STORAGE_KEY = 'parameters'; 2 | 3 | type ParametersType = { 4 | torsoOffsetPercentage: number; 5 | torsoThickCount: number; 6 | torsoThickRadius: number; 7 | torsoMediumCount: number; 8 | torsoMediumRadius: number; 9 | torsoSmallCount: number; 10 | torsoSmallRadius: number; 11 | limbsOffsetPercentage: number; 12 | limbsThickCount: number; 13 | limbsThickRadius: number; 14 | limbsMediumCount: number; 15 | limbsMediumRadius: number; 16 | limbsSmallCount: number; 17 | limbsSmallRadius: number; 18 | cameraZ: number; 19 | cameraZoom: number; 20 | amountShapes: number; 21 | minPosesScore: number; 22 | videoDeviceId: string; 23 | }; 24 | const PARAMETERS_DEFAULT_VALUES: ParametersType = { 25 | torsoOffsetPercentage: 3, 26 | torsoThickCount: 120, 27 | torsoThickRadius: 0.4, 28 | torsoMediumCount: 8, 29 | torsoMediumRadius: 0.2, 30 | torsoSmallCount: 15, 31 | torsoSmallRadius: 0.1, 32 | limbsOffsetPercentage: 1, 33 | limbsThickCount: 60, 34 | limbsThickRadius: 0.4, 35 | limbsMediumCount: 8, 36 | limbsMediumRadius: 0.2, 37 | limbsSmallCount: 15, 38 | limbsSmallRadius: 0.05, 39 | cameraZ: 6, 40 | cameraZoom: 100, 41 | amountShapes: 3, 42 | minPosesScore: 20, 43 | videoDeviceId: '', 44 | }; 45 | 46 | const EMPTY_JSON = '{}'; 47 | 48 | export function initParameters() { 49 | const parameters = window.localStorage.getItem(PARAMETERS_LOCAL_STORAGE_KEY); 50 | const hasParameters = !!parameters; 51 | if (hasParameters) { 52 | return; 53 | } 54 | 55 | setDefaultParameters(); 56 | } 57 | 58 | export function setDefaultParameters() { 59 | const defaultValues = JSON.stringify(PARAMETERS_DEFAULT_VALUES); 60 | window.localStorage.setItem(PARAMETERS_LOCAL_STORAGE_KEY, defaultValues); 61 | } 62 | 63 | export function getParameters(): ParametersType { 64 | const json = window.localStorage.getItem(PARAMETERS_LOCAL_STORAGE_KEY) ?? EMPTY_JSON; 65 | return JSON.parse(json); 66 | } 67 | 68 | export function setParameters(parameters: object) { 69 | const json = JSON.stringify(parameters); 70 | window.localStorage.setItem(PARAMETERS_LOCAL_STORAGE_KEY, json); 71 | } 72 | -------------------------------------------------------------------------------- /src/js/physics.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | import * as CANNON from 'cannon-es'; 3 | 4 | const COLLIDING_CONTACT_MATERIAL_OPTIONS = { 5 | friction: 0.0, 6 | restitution: 1.0, 7 | }; 8 | 9 | let world: CANNON.World; 10 | 11 | export function initWorld() { 12 | world = new CANNON.World(); 13 | world.gravity.set(0, 0, 0); 14 | } 15 | 16 | export function getWorld(): CANNON.World { 17 | return world; 18 | } 19 | 20 | export function worldStep() { 21 | world.step(1 / 60); 22 | } 23 | 24 | export function addCollidingContactMaterial( 25 | material1: CANNON.Material, 26 | material2: CANNON.Material, 27 | options: CANNON.ContactMaterialOptions = COLLIDING_CONTACT_MATERIAL_OPTIONS 28 | ) { 29 | const contactMaterial = new CANNON.ContactMaterial(material1, material2, options); 30 | world.addContactMaterial(contactMaterial); 31 | } 32 | 33 | export function createBody(mesh: THREE.Mesh, material: CANNON.Material, mass = 0): CANNON.Body { 34 | const shape = createCannonBodyFromMesh(mesh); 35 | 36 | const { x, y, z } = mesh.position; 37 | const position = new CANNON.Vec3(x, y, z); 38 | 39 | return new CANNON.Body({ 40 | mass, 41 | shape, 42 | position, 43 | material, 44 | }); 45 | } 46 | 47 | function createCannonBodyFromMesh(mesh: THREE.Mesh) { 48 | const { geometry } = mesh; 49 | 50 | if (geometry instanceof THREE.SphereGeometry) { 51 | const { radius } = geometry.parameters; 52 | return new CANNON.Sphere(radius); 53 | } else if (geometry instanceof THREE.ConeGeometry) { 54 | const { radius, height, radialSegments } = geometry.parameters; 55 | return new CANNON.Cylinder(0.0001, radius, height, radialSegments); 56 | } else if (geometry instanceof THREE.CylinderGeometry) { 57 | const { radiusTop, radiusBottom, height, radialSegments } = geometry.parameters; 58 | return new CANNON.Cylinder(radiusTop, radiusBottom, height, radialSegments); 59 | } 60 | 61 | // Trimesh as fallback. However, collision detection is not supported 62 | const { position } = geometry.attributes; 63 | // @ts-ignore 64 | const vertices = position.array ?? []; 65 | const indices = Object.keys(vertices).map(Number); 66 | 67 | return new CANNON.Trimesh(vertices, indices); 68 | } 69 | -------------------------------------------------------------------------------- /src/js/shapes/basic.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | import { createRandomEuler } from '../utils/three'; 3 | import { getRandomColorTexture } from '../textures'; 4 | import { getRandomFloat } from '../utils/maths'; 5 | 6 | export function createCone(texture: THREE.Texture, radius = 0.8, height = 2, segments = 32): THREE.Mesh { 7 | const geometry = new THREE.ConeGeometry(radius, height, segments); 8 | const material = new THREE.MeshStandardMaterial({ 9 | map: texture, 10 | }); 11 | const cone = new THREE.Mesh(geometry, material); 12 | cone.castShadow = true; 13 | cone.receiveShadow = true; 14 | return cone; 15 | } 16 | 17 | export function createCylinder( 18 | texture: THREE.Texture, 19 | radiusTop = 0.3, 20 | radiusBottom = 0.3, 21 | height = 3, 22 | radialSegments = 20 23 | ): THREE.Mesh { 24 | const geometry = new THREE.CylinderGeometry(radiusTop, radiusBottom, height, radialSegments); 25 | const material = new THREE.MeshStandardMaterial({ 26 | map: texture, 27 | }); 28 | const cylinder = new THREE.Mesh(geometry, material); 29 | 30 | cylinder.castShadow = true; 31 | cylinder.receiveShadow = true; 32 | return cylinder; 33 | } 34 | 35 | export function createSphere(texture: THREE.Texture, radius = 0.6): THREE.Mesh { 36 | const geometry = new THREE.SphereGeometry(radius); 37 | const material = new THREE.MeshStandardMaterial({ 38 | map: texture, 39 | }); 40 | const sphere = new THREE.Mesh(geometry, material); 41 | 42 | sphere.castShadow = true; 43 | sphere.receiveShadow = true; 44 | return sphere; 45 | } 46 | 47 | export function createBubble({ 48 | radius = 0.4, 49 | x = 0, 50 | y = 0, 51 | z = 0, 52 | offset = 0.5, 53 | rotation = createRandomEuler(), 54 | texture = getRandomColorTexture(), 55 | } = {}): THREE.Mesh { 56 | const bubble = createSphere(texture, radius); 57 | 58 | bubble.position.set(x, y, z); 59 | bubble.userData.rotation = rotation; 60 | // Custom property to draw the bubble always with the same offset. 61 | bubble.userData.offset = createRandomOffsetVector(offset); 62 | 63 | return bubble; 64 | } 65 | 66 | const createRandomOffsetVector = (offset: number): THREE.Vector3 => { 67 | const min = -offset / 2; 68 | const max = offset / 2; 69 | 70 | const x = getRandomFloat(min, max); 71 | const y = getRandomFloat(min, max); 72 | const z = getRandomFloat(min, max); 73 | 74 | return new THREE.Vector3(x, y, z); 75 | }; 76 | -------------------------------------------------------------------------------- /src/js/controls.ts: -------------------------------------------------------------------------------- 1 | import hotkeys from 'hotkeys-js'; 2 | 3 | import { getSelectedVideoInputDeviceId, getVideoInputDevices } from './media'; 4 | import { getParameters, setDefaultParameters, setParameters } from './parameters'; 5 | import { convertFormToJson, createSelectOption, getQueryStringValue, setInputValueByName } from './utils/browser'; 6 | 7 | type OnSubmitType = () => void; 8 | type InitControlsParamsType = { 9 | onSubmit: OnSubmitType; 10 | }; 11 | export async function initControls({ onSubmit }: InitControlsParamsType) { 12 | initInputValues(); 13 | await initVideoInput(); 14 | initEventHandlers(onSubmit); 15 | initToggle(); 16 | } 17 | 18 | function initInputValues() { 19 | const parameters = getParameters(); 20 | Object.entries(parameters).forEach(([name, value]) => setInputValueByName(name, value)); 21 | } 22 | 23 | async function initVideoInput() { 24 | const [videoInputControl] = document.getElementsByName('videoDeviceId'); 25 | if (!videoInputControl) { 26 | throw Error(`Haven't found video control input name videoDeviceId`); 27 | } 28 | 29 | const videoInputDevices = await getVideoInputDevices(); 30 | const selectedVideoDeviceId = await getSelectedVideoInputDeviceId(); 31 | 32 | videoInputDevices.forEach(({ deviceId, label }) => { 33 | const option = createSelectOption(label, deviceId, selectedVideoDeviceId); 34 | videoInputControl.appendChild(option); 35 | }); 36 | } 37 | 38 | function initEventHandlers(onSubmit: OnSubmitType) { 39 | const controls = document.getElementById('controls'); 40 | const resetButton = document.getElementById('reset'); 41 | 42 | if (!controls || !resetButton) { 43 | throw Error('Form controls and reset button not found :('); 44 | } 45 | 46 | controls.onsubmit = (event) => submitControlsForm(event, onSubmit); 47 | resetButton.onclick = () => resetInputValues(onSubmit); 48 | } 49 | 50 | function submitControlsForm(event: SubmitEvent, onSubmit: OnSubmitType) { 51 | event.preventDefault(); 52 | 53 | const parameters = convertFormToJson(event.target as HTMLFormElement); 54 | setParameters(parameters); 55 | onSubmit(); 56 | } 57 | 58 | function resetInputValues(onSubmit: OnSubmitType) { 59 | setDefaultParameters(); 60 | initInputValues(); 61 | onSubmit(); 62 | } 63 | 64 | function initToggle() { 65 | const controlsQueryString = getQueryStringValue('controls'); 66 | if (!!controlsQueryString) { 67 | toggleControls(); 68 | } 69 | 70 | hotkeys('ctrl+k, command+k', () => toggleControls()); 71 | } 72 | 73 | function toggleControls() { 74 | const controls = document.getElementById('controls'); 75 | if (!controls) { 76 | throw Error('Form controls not found'); 77 | } 78 | 79 | controls.classList.toggle('hidden'); 80 | } 81 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/keypoints.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | import { Keypoint } from '@tensorflow-models/pose-detection'; 3 | 4 | import { getAverage } from '../utils/maths'; 5 | import { getObjectX, getObjectY } from '../utils/three'; 6 | 7 | export type KeypointsMapType = Map; 8 | 9 | export function createPoseKeypointsMap(keypoints: Keypoint[]): KeypointsMapType { 10 | const keypointsMap = new Map(); 11 | 12 | if (!keypoints.length) { 13 | return keypointsMap; 14 | } 15 | 16 | for (let i = 0; i < keypoints.length; i++) { 17 | const keypoint = keypoints[i]; 18 | keypointsMap.set(keypoint.name, keypoint); 19 | } 20 | 21 | addExtraKeypointsToMap(keypointsMap); 22 | 23 | return keypointsMap; 24 | } 25 | 26 | export function createVectorByKeypointName(keypointsMap: KeypointsMapType, name: string): THREE.Vector3 | null { 27 | const keypoint = keypointsMap.get(name); 28 | if (!keypoint) { 29 | return null; 30 | } 31 | 32 | return createVectorByKeypoint(keypoint); 33 | } 34 | 35 | function addExtraKeypointsToMap(keypointsMap: KeypointsMapType) { 36 | const neck = createAverageKeypoint({ 37 | keypointsMap, 38 | name: 'neck', 39 | startKeypointName: 'left_shoulder', 40 | endKeypointName: 'right_shoulder', 41 | }); 42 | neck && keypointsMap.set(neck.name, neck); 43 | 44 | const stomach = createAverageKeypoint({ 45 | keypointsMap, 46 | name: 'stomach', 47 | startKeypointName: 'left_hip', 48 | endKeypointName: 'right_hip', 49 | }); 50 | stomach && keypointsMap.set(stomach.name, stomach); 51 | } 52 | 53 | type CreateAverageKeypointParamsType = { 54 | name: string; 55 | keypointsMap: KeypointsMapType; 56 | startKeypointName: string; 57 | endKeypointName: string; 58 | }; 59 | function createAverageKeypoint({ 60 | name, 61 | keypointsMap, 62 | startKeypointName, 63 | endKeypointName, 64 | }: CreateAverageKeypointParamsType): Required | null { 65 | const startKeypoint = keypointsMap.get(startKeypointName); 66 | const endKeypoint = keypointsMap.get(endKeypointName); 67 | 68 | if (!startKeypoint || !endKeypoint) { 69 | return null; 70 | } 71 | 72 | const x = getAverage([startKeypoint.x, endKeypoint.x]); 73 | const y = getAverage([startKeypoint.y, endKeypoint.y]); 74 | const z = startKeypoint.z && endKeypoint.z ? getAverage([startKeypoint.z, endKeypoint.z]) : 0; 75 | const score = startKeypoint.score && endKeypoint.score ? getAverage([startKeypoint.score, endKeypoint.score]) : 0; 76 | 77 | return { name, x, y, z, score }; 78 | } 79 | 80 | function createVectorByKeypoint(keypoint: Keypoint): THREE.Vector3 { 81 | const objectX = getObjectX(keypoint.x); 82 | const objectY = getObjectY(keypoint.y); 83 | return new THREE.Vector3(objectX, objectY, 0); 84 | } 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The Bubble Figure 2 | 3 | Bubble Figure Demo 4 | 5 | ## What is this? 6 | 7 | The Bubble Figure is an interactive art installation that runs in the browser. When a person is recognized in the webcam 8 | it makes them appear as a colorful bubble figure. They can then interact with various shapes that fall from above, for 9 | example, by hitting them. Simple and fun. 10 | 11 | ## Where can I try it? 12 | 13 | [Bubble Figure demo on GitHub pages](https://wunderdogsw.github.io/go-23-app/). Please make sure to step back from the webcam so that your pose is detected. 14 | 15 | ## Who was it made for? 16 | 17 | The installation was made for [Grand One](https://grandone.fi), 2023 edition. These are annual design awards that 18 | take place in Helsinki, Finland. Please see [event photos](https://www.paavopykalainen.com/2023/Grand-One-2023/n-FrhNhf/). 19 | 20 | ## Why was it built? 21 | 22 | Since [Wunderdog](https://www.wunderdog.fi) took part in Grand One 2023, we wanted to experiment, have fun, and show we 23 | could do something beyond our regular day-to-day work. 24 | 25 | We decided to build an interactive experience using the event's visual language of basic colorful shapes: spheres, cones 26 | and cylinders. After brainstorming various concepts, and taking the schedule and resources into account, we settled on 27 | The Bubble Figure. 28 | 29 | ## How does it work? 30 | 31 | The Bubble Figure uses [TensorFlow.js](https://github.com/tensorflow/tfjs-models) pose detection to recognize a person 32 | via the webcam. Pose keypoints (e.g. left shoulder, right wrist) are then used to render the figure via [Three.js](https://threejs.org). 33 | 34 | The figure is randomized every time a new person is detected. The shapes falling from above are also randomized, while 35 | using certain geometries and textures to fit the Grand One visual language. 36 | 37 | To simulate physics, [cannon-es](https://pmndrs.github.io/cannon-es/) is used. 38 | 39 | ## How can I run it? 40 | 41 | The project is managed via [npm](https://www.npmjs.com) and [Vite](https://vitejs.dev). Unfortunately the pose detection 42 | libraries don't work as npm imports, so they are loaded via a CDN. 43 | 44 | In the browser, hit `Command + K` on Mac or `Ctrl + K` on Windows to adjust various parameters. Clicking Apply will 45 | save the parameters in the browser's local storage. 46 | 47 | Please use Google Chrome, other browsers are not officially supported. 48 | 49 | ### Setup 50 | 51 | `npm install` 52 | 53 | ### Development 54 | 55 | `npm run dev` 56 | 57 | ### Build 58 | 59 | `npm run build` 60 | 61 | ### Preview 62 | 63 | `npm run preview` 64 | 65 | ## How could I contribute? 66 | 67 | At the moment, we are not accepting any further contributions. For any questions or comments, please [contact Wunderdog](https://www.wunderdog.fi/contact). 68 | -------------------------------------------------------------------------------- /src/js/utils/three.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | import { getScene } from '../cinematography'; 4 | import { VIDEO_SIZE } from '../media'; 5 | import { getRandomInt } from './maths'; 6 | 7 | type VisibleBoundingBoxReturnType = { 8 | left: number; 9 | right: number; 10 | top: number; 11 | bottom: number; 12 | }; 13 | export function visibleBoundingBox(): VisibleBoundingBoxReturnType { 14 | const { width, height } = getScene().userData; 15 | 16 | const left = -width / 2; 17 | const right = width / 2; 18 | const top = height / 2; 19 | const bottom = -height / 2; 20 | 21 | return { left, right, top, bottom }; 22 | } 23 | 24 | export function getObjectX(videoX: number): number { 25 | const { width: sceneWidth } = getScene().userData; 26 | const { width: videoWidth } = VIDEO_SIZE; 27 | // this calculation flips the x coordinate for a mirror effect 28 | return ((videoWidth - videoX) / videoWidth - 0.5) * sceneWidth; 29 | } 30 | 31 | export function getObjectY(videoY: number): number { 32 | const { height: sceneHeight } = getScene().userData; 33 | const { height: videoHeight } = VIDEO_SIZE; 34 | return (0.5 - videoY / videoHeight) * sceneHeight; 35 | } 36 | 37 | export function getVectorsRadiansAngle(startVector: THREE.Vector3, endVector: THREE.Vector3): number { 38 | // the vector angleTo function doesn't seem to produce the desired result 39 | const deltaX = endVector.x - startVector.x; 40 | const deltaY = endVector.y - startVector.y; 41 | return Math.atan2(deltaY, deltaX); 42 | } 43 | 44 | export function createRandomEuler(): THREE.Euler { 45 | const x = getRandomRadiansAngle(); 46 | const y = getRandomRadiansAngle(); 47 | const z = getRandomRadiansAngle(); 48 | return new THREE.Euler(x, y, z); 49 | } 50 | 51 | export function disposeMesh(mesh: THREE.Mesh) { 52 | if (mesh.material instanceof THREE.Material) { 53 | mesh.material.dispose(); 54 | } 55 | mesh.geometry.dispose(); 56 | } 57 | 58 | export function disposeGroup(group: THREE.Group, onMeshDisposedCallback: (mesh: THREE.Mesh) => void) { 59 | group.traverse((object) => { 60 | if (!(object instanceof THREE.Mesh)) { 61 | return; 62 | } 63 | 64 | disposeMesh(object); 65 | !!onMeshDisposedCallback && onMeshDisposedCallback(object); 66 | }); 67 | } 68 | 69 | // reference: https://codepen.io/discoverthreejs/pen/VbWLeM 70 | export function visibleHeightAtZDepth(camera: THREE.PerspectiveCamera, depth = 0): number { 71 | const cameraZ = camera.position.z; 72 | const compensatedDepth = depth < cameraZ ? depth - cameraZ : depth + cameraZ; 73 | const verticalFOVRadians = (camera.fov * Math.PI) / 180; 74 | 75 | return 2 * Math.tan(verticalFOVRadians / 2) * Math.abs(compensatedDepth); 76 | } 77 | 78 | export function visibleWidthAtZDepth(camera: THREE.PerspectiveCamera, visibleHeight: number): number { 79 | return visibleHeight * camera.aspect; 80 | } 81 | 82 | function getRandomRadiansAngle(): number { 83 | const degrees = getRandomInt(0, 359); 84 | return THREE.MathUtils.degToRad(degrees); 85 | } 86 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/body.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | 3 | import { getParameters } from '../parameters'; 4 | import { createBody } from '../physics'; 5 | import { createBubble } from '../shapes/basic'; 6 | import { BUBBLE_BODY_MATERIAL } from './physicalBody'; 7 | 8 | export function createBubbleBody(): THREE.Group { 9 | const body = new THREE.Group(); 10 | body.name = 'BODY'; 11 | body.add(createBubbleTorso()); 12 | body.add(createLimbs()); 13 | 14 | return body; 15 | } 16 | 17 | function createBubbleTorso(): THREE.Group { 18 | const { 19 | torsoThickRadius, 20 | torsoThickCount, 21 | torsoOffsetPercentage, 22 | torsoMediumRadius, 23 | torsoMediumCount, 24 | torsoSmallRadius, 25 | torsoSmallCount, 26 | } = getParameters(); 27 | 28 | const userData = { startKeypointName: 'neck', endKeypointName: 'stomach' }; 29 | 30 | const thickBubbles = createBubblesGroup(torsoThickRadius, torsoThickCount, torsoOffsetPercentage, userData); 31 | const middleBubbles = createBubblesGroup(torsoMediumRadius, torsoMediumCount, torsoOffsetPercentage, userData); 32 | const smallBubbles = createBubblesGroup(torsoSmallRadius, torsoSmallCount, torsoOffsetPercentage, userData); 33 | 34 | const torso = new THREE.Group(); 35 | torso.name = 'TORSO'; 36 | 37 | torso.add(thickBubbles); 38 | torso.add(middleBubbles); 39 | torso.add(smallBubbles); 40 | 41 | return torso; 42 | } 43 | 44 | function createLimbs(): THREE.Group { 45 | const LINES_KEYPOINTS = [ 46 | ['left_elbow', 'left_shoulder'], 47 | ['left_wrist', 'left_elbow'], 48 | ['stomach', 'left_knee'], 49 | ['neck', 'right_shoulder'], 50 | ['right_elbow', 'right_wrist'], 51 | ['stomach', 'right_knee'], 52 | ['left_knee', 'left_foot_index'], 53 | ['right_knee', 'right_foot_index'], 54 | ['right_shoulder', 'right_elbow'], 55 | ['left_shoulder', 'neck'], 56 | ]; 57 | 58 | const { 59 | limbsThickRadius, 60 | limbsThickCount, 61 | limbsOffsetPercentage, 62 | limbsMediumRadius, 63 | limbsMediumCount, 64 | limbsSmallRadius, 65 | limbsSmallCount, 66 | } = getParameters(); 67 | 68 | const limbs = new THREE.Group(); 69 | limbs.name = 'LIMBS'; 70 | 71 | for (let [startKeypointName, endKeypointName] of LINES_KEYPOINTS) { 72 | const userData = { startKeypointName, endKeypointName }; 73 | 74 | const thickBubbles = createBubblesGroup(limbsThickRadius, limbsThickCount, limbsOffsetPercentage, userData); 75 | const middleBubbles = createBubblesGroup(limbsMediumRadius, limbsMediumCount, limbsOffsetPercentage, userData); 76 | const smallBubbles = createBubblesGroup(limbsSmallRadius, limbsSmallCount, limbsOffsetPercentage, userData); 77 | 78 | limbs.add(thickBubbles); 79 | limbs.add(middleBubbles); 80 | limbs.add(smallBubbles); 81 | } 82 | 83 | return limbs; 84 | } 85 | 86 | function createBubblesGroup(radius = 0.2, numberOfBubbles = 5, offset = 0, userData = {}): THREE.Group { 87 | const group = new THREE.Group(); 88 | group.visible = false; 89 | group.userData = userData; 90 | 91 | for (let i = 0; i < numberOfBubbles; i++) { 92 | const x = i * radius * 2; 93 | const bubble = createBubble({ x, radius, offset }); 94 | bubble.userData.body = createBody(bubble, BUBBLE_BODY_MATERIAL, 0); 95 | group.add(bubble); 96 | } 97 | 98 | return group; 99 | } 100 | -------------------------------------------------------------------------------- /src/js/bubbleFigure/alignPose.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | import { Pose } from '@tensorflow-models/pose-detection'; 3 | 4 | import { getVectorsRadiansAngle } from '../utils/three'; 5 | import { createPoseKeypointsMap, createVectorByKeypointName, KeypointsMapType } from './keypoints'; 6 | import { alignMeshPhysicalBody } from './physicalBody'; 7 | 8 | export function alignBubbleFigurePose(figure: THREE.Group, pose: Pose) { 9 | const { keypoints } = pose; 10 | const keypointsMap = createPoseKeypointsMap(keypoints); 11 | 12 | if (!keypointsMap.size) { 13 | return; 14 | } 15 | 16 | alignBubbleHead(figure, keypointsMap); 17 | alignBubbleBody(figure, keypointsMap); 18 | } 19 | 20 | function alignBubbleHead(figure: THREE.Group, keypointsMap: KeypointsMapType) { 21 | const head = figure.getObjectByName('HEAD'); 22 | if (!head) { 23 | return; 24 | } 25 | 26 | const leftOuterEyeVector = createVectorByKeypointName(keypointsMap, 'left_eye_outer'); 27 | const rightOuterEyeVector = createVectorByKeypointName(keypointsMap, 'right_eye_outer'); 28 | const noseVector = createVectorByKeypointName(keypointsMap, 'nose'); 29 | const neckVector = createVectorByKeypointName(keypointsMap, 'neck'); 30 | 31 | if (!(leftOuterEyeVector && rightOuterEyeVector && neckVector && noseVector)) { 32 | head.visible = false; 33 | return; 34 | } 35 | 36 | const y = neckVector.y + head.userData.radius * 2; 37 | head.position.set(noseVector.x, y, noseVector.z); 38 | 39 | const headAngle = getVectorsRadiansAngle(leftOuterEyeVector, rightOuterEyeVector); 40 | 41 | for (let i = 0; i < head.children.length; i++) { 42 | const bubble = head.children[i]; 43 | if (!(bubble instanceof THREE.Mesh)) { 44 | continue; 45 | } 46 | 47 | bubble.rotation.z = bubble.userData.rotation.z + headAngle; 48 | alignMeshPhysicalBody(bubble); 49 | } 50 | 51 | head.visible = true; 52 | } 53 | 54 | function alignBubbleBody(figure: THREE.Group, keypointsMap: KeypointsMapType) { 55 | const body = figure.getObjectByName('BODY'); 56 | if (!body) { 57 | return; 58 | } 59 | 60 | body.traverse((object) => { 61 | const isGroup = object instanceof THREE.Group; 62 | isGroup && alignBubbleLine(keypointsMap, object); 63 | }); 64 | } 65 | 66 | function alignBubbleLine(keypointsMap: KeypointsMapType, group: THREE.Group) { 67 | const { userData } = group; 68 | 69 | // since the entire body is traversed, some groups don't need to be drawn 70 | if (!userData?.startKeypointName || !userData?.endKeypointName) { 71 | return; 72 | } 73 | 74 | const startVector = createVectorByKeypointName(keypointsMap, userData.startKeypointName); 75 | const endVector = createVectorByKeypointName(keypointsMap, userData.endKeypointName); 76 | 77 | if (!startVector || !endVector) { 78 | group.visible = false; 79 | return; 80 | } 81 | 82 | const direction = endVector.clone().sub(startVector); 83 | const angle = getVectorsRadiansAngle(endVector, startVector); 84 | 85 | for (let i = 0; i < group.children.length; i++) { 86 | const bubble = group.children[i]; 87 | if (!(bubble instanceof THREE.Mesh)) { 88 | continue; 89 | } 90 | 91 | const scalar = i / group.children.length; 92 | const position = startVector.clone().add(direction.clone().multiplyScalar(scalar)); 93 | 94 | position.add(bubble.userData.offset); 95 | bubble.position.copy(position); 96 | bubble.rotation.z = bubble.userData.rotation.z + angle; 97 | 98 | alignMeshPhysicalBody(bubble); 99 | } 100 | 101 | group.visible = true; 102 | } 103 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | The Bubble Figure | Wunderdog 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /src/js/shapes/falling.ts: -------------------------------------------------------------------------------- 1 | import * as CANNON from 'cannon-es'; 2 | import * as THREE from 'three'; 3 | 4 | import { getScene } from '../cinematography'; 5 | import { getParameters } from '../parameters'; 6 | import { addCollidingContactMaterial, createBody, getWorld } from '../physics'; 7 | import { getRandomColorTexture } from '../textures'; 8 | import { getRandomFloat, getRandomItem } from '../utils/maths'; 9 | import { disposeMesh, getVectorsRadiansAngle, visibleBoundingBox } from '../utils/three'; 10 | import { createCone, createCylinder, createSphere } from './basic'; 11 | 12 | export const SHAPE_BODY_MATERIAL = new CANNON.Material('shapeMaterial'); 13 | 14 | const SHAPE_POSITION_DEPTH = 0; 15 | const SHAPE_FACTORIES = [createSphere, createCylinder, createCone]; 16 | const MOVE_SPEED_RANGE = { 17 | min: 2, 18 | max: 7, 19 | }; 20 | const ROTATION_RANGE = { 21 | min: -5, 22 | max: 5, 23 | }; 24 | const VISIBLE_AREA_MARGIN = 5; 25 | const SHAPE_BODY_MASS = 1; 26 | 27 | let visualArea: THREE.Box3; 28 | let visualAreaWithMargin: THREE.Box3; 29 | let shapes: THREE.Mesh[] = []; 30 | 31 | export function resetShapes() { 32 | clearShapes(); 33 | setupVisibleArea(); 34 | setupWorld(); 35 | 36 | const { amountShapes } = getParameters(); 37 | 38 | for (let i = 0; i < amountShapes; i++) { 39 | const shape = createNewShape(); 40 | shapes.push(shape); 41 | } 42 | } 43 | 44 | export function updateShapes() { 45 | for (let i = 0; i < shapes.length; i++) { 46 | const shape = shapes[i]; 47 | applyTrajectory(shape); 48 | 49 | if (!isShapeVisible(shape)) { 50 | disposeShape(shape); 51 | shapes[i] = createNewShape(); 52 | } 53 | } 54 | } 55 | 56 | function createNewShape(): THREE.Mesh { 57 | const texture = getRandomColorTexture(); 58 | const createShape = getRandomItem(SHAPE_FACTORIES); 59 | 60 | if (!createShape) { 61 | throw Error('No shape factory found'); 62 | } 63 | 64 | const shape = createShape(texture); 65 | const body = createBody(shape, SHAPE_BODY_MATERIAL, SHAPE_BODY_MASS); 66 | shape.userData = { body, trajectory: null }; 67 | 68 | applyTrajectory(shape); 69 | 70 | getScene().add(shape); 71 | getWorld().addBody(shape.userData.body); 72 | 73 | return shape; 74 | } 75 | 76 | function disposeShape(shape: THREE.Mesh) { 77 | getScene().remove(shape); 78 | getWorld().removeBody(shape.userData.body); 79 | disposeMesh(shape); 80 | } 81 | 82 | function setupWorld() { 83 | // For keeping shape position z fixed 84 | getWorld().addEventListener('postStep', keepFixedDepth); 85 | 86 | addCollidingContactMaterial(SHAPE_BODY_MATERIAL, SHAPE_BODY_MATERIAL); 87 | } 88 | 89 | function setupVisibleArea() { 90 | visualArea = createVisibleAreaBox(); 91 | visualAreaWithMargin = createVisibleAreaBox(VISIBLE_AREA_MARGIN); 92 | } 93 | 94 | function createVisibleAreaBox(margin = 0): THREE.Box3 { 95 | const { top, right, bottom, left } = visibleBoundingBox(); 96 | 97 | const bottomLeft = new THREE.Vector3(left - margin, bottom - margin, SHAPE_POSITION_DEPTH); 98 | const topRight = new THREE.Vector3(right + margin, top + margin, SHAPE_POSITION_DEPTH); 99 | 100 | return new THREE.Box3(bottomLeft, topRight); 101 | } 102 | 103 | function applyTrajectory(shape: THREE.Mesh) { 104 | if (!shape.userData.trajectory) { 105 | shape.userData.trajectory = generateTrajectory(); 106 | 107 | updateBodyByTrajectory(shape); 108 | } 109 | 110 | updateShapeByBody(shape); 111 | } 112 | 113 | function isShapeVisible(shape: THREE.Mesh) { 114 | if (!shape) { 115 | return false; 116 | } 117 | return visualAreaWithMargin.containsPoint(shape.position); 118 | } 119 | 120 | function updateBodyByTrajectory(shape: THREE.Mesh) { 121 | const { rotation, velocity, start } = shape.userData.trajectory; 122 | shape.userData.body.position.copy(start); 123 | shape.userData.body.velocity.x = velocity.x; 124 | shape.userData.body.velocity.y = velocity.y; 125 | shape.userData.body.angularVelocity.copy(rotation); 126 | shape.userData.body.angularVelocity.normalize(); 127 | } 128 | 129 | function updateShapeByBody(shape: THREE.Mesh) { 130 | shape.position.copy(shape.userData.body.position); 131 | shape.quaternion.copy(shape.userData.body.quaternion); 132 | } 133 | 134 | type GenerateTrajectoryReturnType = { start: THREE.Vector3; rotation: THREE.Vector3; velocity: THREE.Vector3 }; 135 | function generateTrajectory(): GenerateTrajectoryReturnType { 136 | const startVector = new THREE.Vector3( 137 | getRandomFloat(visualArea.min.x, visualArea.max.x), 138 | visualAreaWithMargin.max.y, 139 | SHAPE_POSITION_DEPTH 140 | ); 141 | 142 | const velocity = generateTrajectoryVelocity(startVector); 143 | 144 | const rotation = new THREE.Vector3( 145 | getRandomFloat(ROTATION_RANGE.min, ROTATION_RANGE.max), 146 | getRandomFloat(ROTATION_RANGE.min, ROTATION_RANGE.max), 147 | getRandomFloat(ROTATION_RANGE.min, ROTATION_RANGE.max) 148 | ); 149 | 150 | return { 151 | start: startVector, 152 | rotation, 153 | velocity, 154 | }; 155 | } 156 | 157 | function generateTrajectoryVelocity(startVector: THREE.Vector3): THREE.Vector3 { 158 | const endDirection = new THREE.Vector3( 159 | getRandomFloat(visualArea.min.x, visualArea.max.x), 160 | visualAreaWithMargin.min.y, 161 | startVector.z 162 | ); 163 | 164 | const speed = getRandomFloat(MOVE_SPEED_RANGE.min, MOVE_SPEED_RANGE.max); 165 | const angle = getVectorsRadiansAngle(startVector, endDirection); 166 | 167 | return calculateVelocity(angle, speed, startVector.z); 168 | } 169 | 170 | function calculateVelocity(angleRadians: number, speed: number, depth: number): THREE.Vector3 { 171 | const x = speed * Math.cos(angleRadians); 172 | const y = speed * Math.sin(angleRadians); 173 | return new THREE.Vector3(x, y, depth); 174 | } 175 | 176 | function clearShapes() { 177 | for (let shape of shapes) { 178 | disposeShape(shape); 179 | } 180 | 181 | shapes = []; 182 | getWorld().removeEventListener('postStep', keepFixedDepth); 183 | } 184 | 185 | function keepFixedDepth() { 186 | for (let shape of shapes) { 187 | shape.userData.body.position.z = SHAPE_POSITION_DEPTH; 188 | shape.position.copy(shape.userData.body.position); 189 | } 190 | } 191 | --------------------------------------------------------------------------------