Hand tracking demo

├── .gitignore ├── .nvmrc ├── .prettierrc ├── README.md ├── netlify.toml ├── package.json ├── public └── index.html ├── src ├── Axes.tsx ├── Button.tsx ├── Crate.tsx ├── DefaultHandControllers.tsx ├── Grab.tsx ├── HandModel.ts ├── Level.tsx ├── Roboto.json ├── Text.tsx ├── index.tsx ├── poses │ ├── default.ts │ ├── idle.ts │ └── pinch.ts ├── react-app-env.d.ts ├── store.tsx └── styles.css ├── tsconfig.json └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .vscode 3 | .yalc 4 | yalc.lock -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v12.18.3 -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 2, 4 | "useTabs": false, 5 | "semi": false, 6 | "singleQuote": true, 7 | "trailingComma": "none", 8 | "bracketSpacing": true, 9 | "jsxBracketSameLine": true 10 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # react-xr-default-hands 2 | 3 | Hands, but by default, in react-xr with additional hand tracking support for grabbing geometries! 4 | Try it out: https://hands.dries.io 5 | More info: https://twitter.com/CROEWENS/status/1411746127418888193 6 | 7 | ![](https://thumbs.gfycat.com/ImpassionedInferiorJay-size_restricted.gif) 8 | 9 | 10 | This was a prototype that I built for leveraging hand tracking in my WebXR game called [Plockle](https://plockle.com). 11 | ![](https://thumbs.gfycat.com/BeautifulThornyGalah-size_restricted.gif) 12 | 13 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [[redirects]] 2 | from = "/*" 3 | to = "/index.html" 4 | status = 200 -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-xr-default-hands", 3 | "version": "1.0.0", 4 | "description": "", 5 | "keywords": [], 6 | "main": "lib/index.html", 7 | "dependencies": { 8 | "@react-spring/three": "^9.2.3", 9 | "@react-three/drei": "^6.2.0", 10 | "@react-three/fiber": "^7.0.1", 11 | "@react-three/xr": "^3.1.2", 12 | "@types/react": "^17.0.13", 13 | "@types/react-dom": "^17.0.8", 14 | "@types/react-router-dom": "^5.1.7", 15 | "@types/three": "0.129.1", 16 | "immer": "^9.0.3", 17 | "parcel": "^2.0.0-beta.3.1", 18 | "react": "^18.0.0-alpha-ed6c091fe-20210701", 19 | "react-dom": "^18.0.0-alpha-ed6c091fe-20210701", 20 | "react-merge-refs": "^1.1.0", 21 | "react-router-dom": "^5.2.0", 22 | "rimraf": "^3.0.2", 23 | "three": "0.129.0", 24 | "typescript": "^4.3.2", 25 | "zustand": "^3.5.2" 26 | }, 27 | "devDependencies": {}, 28 | "scripts": { 29 | "start": "yarn clean:cache && parcel public/index.html", 30 | "start:https": "yarn start --https", 31 | "build": "parcel build public/index.html", 32 | "clean": "rimraf node_modules .cache dist", 33 | "clean:cache": "rimraf .parcel-cache" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Hand tracking demo 8 | 9 | 10 | 11 | 14 |

15 | 16 | 17 | -------------------------------------------------------------------------------- /src/Axes.tsx: -------------------------------------------------------------------------------- 1 | import { useFrame } from '@react-three/fiber' 2 | import { XRController } from '@react-three/xr' 3 | import React, { useRef } from 'react' 4 | import { BufferGeometry, LineBasicMaterial, Mesh, MeshBasicMaterial, Object3D, SphereGeometry, Vector3 } from 'three' 5 | 6 | import { HandModel } from './HandModel' 7 | 8 | interface Props { 9 | controller: XRController 10 | model: HandModel 11 | } 12 | 13 | export function Axes({ controller, model }: Props) { 14 | useFrame(() => { 15 | if (!model || model?.bones.length === 0) { 16 | return 17 | } 18 | 19 | const indexTip = model!.bones.find((bone) => (bone as any).jointName === 'index-finger-tip')! as Object3D 20 | const thumbTip = model!.bones.find((bone) => (bone as any).jointName === 'thumb-tip')! as Object3D 21 | 22 | const indexJoint = model!.bones.find((bone) => (bone as any).jointName === 'index-finger-phalanx-proximal')! as Object3D 23 | const thumbJoint = model!.bones.find((bone) => (bone as any).jointName === 'thumb-phalanx-proximal')! as Object3D 24 | 25 | const position: Vector3 = indexTip.getWorldPosition(new Vector3()).add(thumbTip.getWorldPosition(new Vector3())).multiplyScalar(0.5) 26 | 27 | const indexKnuckle = model!.bones.find((bone) => (bone as any).jointName === 'index-finger-metacarpal')! as Object3D 28 | const pinkyKnuckle = model!.bones.find((bone) => (bone as any).jointName === 'pinky-finger-metacarpal')! as Object3D 29 | 30 | indexTipRef.current?.position.copy(indexTip.getWorldPosition(new Vector3())) 31 | indexJointRef.current?.position.copy(indexJoint.getWorldPosition(new Vector3())) 32 | thumbTipRef.current?.position.copy(thumbTip.getWorldPosition(new Vector3())) 33 | thumbJointRef.current?.position.copy(thumbJoint.getWorldPosition(new Vector3())) 34 | indexKnuckleRef.current?.position.copy(indexKnuckle.getWorldPosition(new Vector3())) 35 | pinkyKnuckleRef.current?.position.copy(pinkyKnuckle.getWorldPosition(new Vector3())) 36 | positionRef.current?.position.copy(position.clone()) 37 | 38 | const z = thumbJoint.getWorldPosition(new Vector3()).sub(indexJoint.getWorldPosition(new Vector3())).normalize() 39 | 40 | const zPoints: Vector3[] = [position.clone(), position.clone().add(z)] 41 | const zGeom = new BufferGeometry().setFromPoints(zPoints) 42 | zRef.current.geometry = zGeom 43 | 44 | const y = indexKnuckle.getWorldPosition(new Vector3()).sub(pinkyKnuckle.getWorldPosition(new Vector3())).normalize() 45 | 46 | // const yPoints: Vector3[] = [ 47 | // applyControllerOffsetRotation(position.clone().sub(y.clone().divideScalar(2))), 48 | // applyControllerOffsetRotation(position.clone().add(y).sub(y.clone().divideScalar(2))) 49 | // ] 50 | // const yGeom = new BufferGeometry().setFromPoints(yPoints) 51 | // yRef.current.geometry = yGeom 52 | 53 | const x = new Vector3().crossVectors(z, y).negate() 54 | 55 | const xPoints: Vector3[] = [position.clone(), position.clone().add(x.clone())] 56 | const xGeom = new BufferGeometry().setFromPoints(xPoints) 57 | xRef.current.geometry = xGeom 58 | 59 | const y2 = new Vector3().crossVectors(x, z).negate() 60 | 61 | const y2Points: Vector3[] = [position.clone(), position.clone().add(y2.clone())] 62 | const y2Geom = new BufferGeometry().setFromPoints(y2Points) 63 | y2Ref.current.geometry = y2Geom 64 | 65 | const distance = indexTip.getWorldPosition(new Vector3()).sub(thumbTip.getWorldPosition(new Vector3())) 66 | 67 | thumbTipCollidingRef.current?.position.copy( 68 | thumbTip.getWorldPosition(new Vector3()).add(new Vector3().copy(distance.clone().divideScalar(20))) 69 | ) 70 | 71 | indexTipCollidingRef.current?.position.copy( 72 | indexTip.getWorldPosition(new Vector3()).sub(new Vector3().copy(distance.clone().divideScalar(20))) 73 | ) 74 | }) 75 | 76 | const thumbTipRef = useRef(null) 77 | const thumbJointRef = useRef(null) 78 | const thumbTipCollidingRef = useRef(null) 79 | const indexTipRef = useRef(null) 80 | const indexJointRef = useRef(null) 81 | const indexTipCollidingRef = useRef(null) 82 | const indexKnuckleRef = useRef(null) 83 | const pinkyKnuckleRef = useRef(null) 84 | const positionRef = useRef(null) 85 | 86 | const zRef = useRef() 87 | const yRef = useRef() 88 | const y2Ref = useRef() 89 | const xRef = useRef() 90 | 91 | return ( 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | {/* @ts-ignore */} 104 | 105 | {/* @ts-ignore 106 | */} 107 | {/* @ts-ignore */} 108 | 109 | {/* @ts-ignore */} 110 | 111 | 112 | ) 113 | } 114 | -------------------------------------------------------------------------------- /src/Button.tsx: -------------------------------------------------------------------------------- 1 | import { RoundedBox } from '@react-three/drei' 2 | import { Interactive } from '@react-three/xr' 3 | import React, { Suspense, useState } from 'react' 4 | import { a, useSpring } from '@react-spring/three' 5 | import Text from './Text' 6 | 7 | export function Button({ children, label, onClick, highlight = true, fontSize = 0.07, args, ...rest }: any) { 8 | const [hovered, setHovered] = useState(false) 9 | const [style] = useSpring( 10 | { 11 | scale: hovered ? 1.05 : 1, 12 | color: hovered ? '#66f' : highlight ? '#f3f3f3' : '#6e6e6e' 13 | }, 14 | [hovered, highlight] 15 | ) 16 | 17 | return ( 18 | setHovered(true)} onBlur={() => setHovered(false)} onSelect={onClick ? onClick : () => null}> 19 | 20 | 21 | 22 | 23 | 24 | {label || children} 25 | 26 | 27 | 28 | 29 | 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /src/Crate.tsx: -------------------------------------------------------------------------------- 1 | import { useGLTF } from '@react-three/drei' 2 | import React, { useEffect, useRef, useState } from 'react' 3 | import { Box3, Matrix3, Mesh, MeshStandardMaterial, Object3D, Vector3 } from 'three' 4 | import { OBB } from 'three/examples/jsm/math/OBB' 5 | 6 | import { Grab } from './Grab' 7 | import { useStore } from './store' 8 | 9 | export default function Crate(props: any) { 10 | const group = useRef() 11 | const { nodes, materials } = useGLTF('https://vazxmixjsiawhamofees.supabase.co/storage/v1/object/public/models/crate/model.gltf') as any 12 | 13 | const materialsRef = useRef(materials) 14 | const [isGrabbed, setGrabbed] = useState(false) 15 | 16 | const set = useStore((store) => store.set) 17 | 18 | useEffect(() => { 19 | materialsRef.current = Object.values(materials).reduce((object: object, material: MeshStandardMaterial) => { 20 | object[material.name] = material.clone() 21 | material.metalness = 0.5 22 | return object 23 | }, {}) 24 | }, [materials]) 25 | 26 | useEffect(() => { 27 | if (isGrabbed) { 28 | Object.values(materialsRef.current).forEach((material: MeshStandardMaterial) => { 29 | material.metalness = 0.1 30 | }) 31 | } else { 32 | Object.values(materialsRef.current).forEach((material: MeshStandardMaterial) => { 33 | material.metalness = 0.5 34 | }) 35 | } 36 | }, [isGrabbed]) 37 | 38 | return ( 39 | { 41 | setGrabbed(isGrabbed) 42 | if (isGrabbed) { 43 | set((store) => { 44 | store.hands.interacting!.current[controller.inputSource.handedness] = group.current 45 | }) 46 | } 47 | }} 48 | callback={({ controller, model }) => { 49 | // do initial position check (if futher, don't check for collisions) 50 | const position = model.getHandPosition() 51 | const cratePosition = group.current.getWorldPosition(new Vector3()) 52 | 53 | // calculate based on bounding box 54 | // for now hardcodedackage 55 | if (position.distanceTo(cratePosition) > 0.2) { 56 | // console.log('IGNORED') 57 | return 58 | } 59 | 60 | let mesh: Mesh | undefined = undefined 61 | group.current!.traverse((object) => { 62 | if (!mesh && object instanceof Mesh && object.geometry) { 63 | mesh = object 64 | } 65 | }) 66 | if (!mesh) { 67 | return 68 | } 69 | 70 | const obb = new OBB( 71 | new Vector3().setFromMatrixPosition(group.current!.matrixWorld), 72 | ((mesh as Mesh).geometry!.boundingBox as Box3).getSize(new Vector3()).multiply(group.current!.scale).divideScalar(2), 73 | new Matrix3().setFromMatrix4(group.current!.matrixWorld.clone().makeScale(1, 1, 1)) 74 | ) 75 | 76 | const matrix = model!.getHandRotationMatrix() 77 | 78 | const indexTip = model!.bones.find((bone) => (bone as any).jointName === 'index-finger-tip')! as Object3D 79 | const thumbTip = model!.bones.find((bone) => (bone as any).jointName === 'thumb-tip')! as Object3D 80 | 81 | const thumbOBB = new OBB( 82 | indexTip.getWorldPosition(new Vector3()), 83 | new Vector3(0.05, 0.05, 0.05).divideScalar(2), 84 | new Matrix3().setFromMatrix4(matrix) 85 | ) 86 | const indexOBB = new OBB( 87 | thumbTip.getWorldPosition(new Vector3()), 88 | new Vector3(0.05, 0.05, 0.05).divideScalar(2), 89 | new Matrix3().setFromMatrix4(matrix) 90 | ) 91 | 92 | return obb.intersectsOBB(thumbOBB, Number.EPSILON) && obb.intersectsOBB(indexOBB, Number.EPSILON) 93 | }} 94 | ref={group} 95 | {...props} 96 | dispose={null}> 97 | 98 | 99 | 100 | 101 | 102 | ) 103 | } 104 | 105 | useGLTF.preload('https://vazxmixjsiawhamofees.supabase.co/storage/v1/object/public/models/crate/model.gltf') 106 | -------------------------------------------------------------------------------- /src/DefaultHandControllers.tsx: -------------------------------------------------------------------------------- 1 | import { useFrame } from '@react-three/fiber' 2 | import { useXR, useXREvent, XREvent } from '@react-three/xr' 3 | import React, { useEffect, useRef, useState } from 'react' 4 | import { BoxBufferGeometry, Color, Intersection, Mesh, MeshBasicMaterial, XRHandedness } from 'three' 5 | 6 | import { Axes } from './Axes' 7 | import { HandModel } from './HandModel' 8 | import { useStore } from './store' 9 | 10 | enum HandAction { 11 | 'release', 12 | 'grab' 13 | } 14 | 15 | export function DefaultHandControllers({ modelPaths }: { modelPaths?: { [key in XRHandedness]?: string } }) { 16 | const { controllers, isHandTracking, isPresenting, hoverState } = useXR() 17 | 18 | const models = useStore((store) => store.hands.models) 19 | const set = useStore((store) => store.set) 20 | 21 | const handActions = useRef<{ [key in XRHandedness]?: { date: number; distance: number; action: HandAction }[] }>({ left: [], right: [] }) 22 | 23 | const [pinched, setPinched] = useState<{ [key in XRHandedness]?: boolean }>({ left: false, right: false }) 24 | const [rays] = React.useState(new Map()) 25 | 26 | const modelsRef = useRef<{ [key in XRHandedness]?: HandModel }>({ 27 | left: undefined, 28 | right: undefined 29 | }) 30 | 31 | const interactingRef = useRef<{ [key in XRHandedness]?: HandModel }>({ 32 | left: undefined, 33 | right: undefined 34 | }) 35 | 36 | useEffect(() => { 37 | set((store) => { 38 | store.hands.models = modelsRef 39 | store.hands.interacting = interactingRef 40 | }) 41 | }, []) 42 | 43 | useEffect(() => { 44 | // handle cleanups 45 | if (models?.current) { 46 | controllers.forEach((c) => { 47 | let model = models?.current[c.inputSource.handedness] 48 | if (!model) { 49 | const model = new HandModel(c.controller, c.inputSource, modelPaths) 50 | models!.current[c.inputSource.handedness] = model 51 | 52 | const ray = new Mesh() 53 | ray.rotation.set(Math.PI / 2, 0, 0) 54 | ray.material = new MeshBasicMaterial({ color: new Color(0xffffff), opacity: 0.8, transparent: true }) 55 | ray.geometry = new BoxBufferGeometry(0.002, 1, 0.002) 56 | 57 | rays.set(c.controller.id, ray) 58 | c.controller.add(ray) 59 | } 60 | }) 61 | } 62 | }, [controllers]) 63 | 64 | useEffect(() => { 65 | // fix this firing twice when going in vr mode 66 | // this doesn't do anything && Object.values(models!.current).filter((model) => !!model).length === controllers.length 67 | if (isPresenting) { 68 | controllers.forEach((c, index) => { 69 | let model = models!.current[c.inputSource.handedness] 70 | if (model) { 71 | if (isHandTracking) { 72 | model.load(c.hand, c.inputSource, true) 73 | } else { 74 | model.load(c.controller, c.inputSource, false) 75 | } 76 | models!.current[c.inputSource.handedness] = model 77 | } 78 | }) 79 | } 80 | }, [controllers, isHandTracking, models]) 81 | 82 | useFrame(() => { 83 | controllers.map((c, index) => { 84 | if (isHandTracking) { 85 | const model = models?.current[c.inputSource.handedness] 86 | if (model && !model?.loading) { 87 | const distance = model!.getThumbIndexDistance() 88 | 89 | // get todo actions 90 | const actions = handActions.current[c.inputSource.handedness]!.filter(({ date }) => Date.now() > date) 91 | // remove from initial list 92 | actions.forEach((x) => 93 | handActions.current[c.inputSource.handedness]!.splice(handActions.current[c.inputSource.handedness]!.indexOf(x), 1) 94 | ) 95 | 96 | // mutates the actions, but we don't need those anymore 97 | // otherwise .slice(-1) 98 | const action = actions.pop() 99 | 100 | const isPinched = pinched[c.inputSource.handedness] 101 | handActions.current[c.inputSource.handedness]!.push({ 102 | date: Date.now() + 200, 103 | action: isPinched ? HandAction.release : HandAction.grab, 104 | distance: distance + (isPinched ? 0.05 : -0.05) 105 | }) 106 | 107 | // might be that we still push a "grab", even though we should wait for a release 108 | if (!isPinched && ((action?.action === HandAction.grab && action.distance > distance) || distance < 0.01)) { 109 | c.controller.dispatchEvent({ type: 'selectstart', fake: true }) 110 | setPinched({ ...pinched, [c.inputSource.handedness]: true }) 111 | } 112 | 113 | if (isPinched && ((action?.action === HandAction.release && action.distance < distance) || distance > 0.1)) { 114 | c.controller.dispatchEvent({ type: 'selectend', fake: true }) 115 | setPinched({ ...pinched, [c.inputSource.handedness]: false }) 116 | } 117 | } 118 | } 119 | 120 | const ray = rays.get(c.controller.id) 121 | if (!ray) return 122 | 123 | const intersection: Intersection = hoverState[c.inputSource.handedness].values().next().value 124 | if (!intersection || c.inputSource.handedness === 'none') { 125 | ray.visible = false 126 | return 127 | } 128 | 129 | const rayLength = intersection?.distance ?? 5 130 | 131 | // Tiny offset to clip ray on AR devices 132 | // that don't have handedness set to 'none' 133 | const offset = -0.01 134 | ray.visible = true 135 | ray.scale.y = rayLength + offset 136 | ray.position.z = -rayLength / 2 - offset 137 | }) 138 | }) 139 | 140 | useXREvent('selectstart', (e: XREvent) => { 141 | if (!isHandTracking) { 142 | const model = models?.current[e.controller.inputSource.handedness] 143 | if (model) { 144 | model.setPose('pinch') 145 | } 146 | } 147 | }) 148 | 149 | useXREvent('selectend', (e: XREvent) => { 150 | if (!isHandTracking) { 151 | const model = models?.current[e.controller.inputSource.handedness] 152 | if (model) { 153 | model.setPose('idle') 154 | } 155 | } 156 | }) 157 | 158 | return ( 159 | <> 160 | {process.env.NODE_ENV === 'development' && 161 | controllers.map((c, index) => 162 | models?.current[c.inputSource.handedness] ? : null 163 | )} 164 | 165 | ) 166 | } 167 | -------------------------------------------------------------------------------- /src/Grab.tsx: -------------------------------------------------------------------------------- 1 | import { useFrame } from '@react-three/fiber' 2 | import { useXR, useXREvent, XRController, XREvent } from '@react-three/xr' 3 | import React, { forwardRef, ReactNode, useRef } from 'react' 4 | import mergeRefs from 'react-merge-refs' 5 | import { Box3, Matrix3, Matrix4, Mesh, Object3D, Quaternion, Vector3 } from 'three' 6 | import { OBB } from 'three/examples/jsm/math/OBB' 7 | 8 | import { HandModel } from './HandModel' 9 | import { useStore } from './store' 10 | 11 | export const Grab = forwardRef( 12 | ( 13 | { 14 | children, 15 | disabled = false, 16 | onChange, 17 | // instead of checking for a generic way here, do the calculation in the Interactable itself. 18 | callback, 19 | ...props 20 | }: { 21 | children: ReactNode 22 | disabled?: boolean 23 | onChange: ({ isGrabbed, controller }: { isGrabbed: boolean; controller: XRController }) => void 24 | callback: ({ controller, model }: { controller: XRController; model: HandModel }) => boolean 25 | }, 26 | passedRef 27 | ) => { 28 | const grabbingController = useRef() 29 | const ref = useRef() 30 | const previousTransform = useRef(undefined) 31 | const { isHandTracking } = useXR() 32 | 33 | const set = useStore((store) => store.set) 34 | 35 | const interacting = useStore((store) => ({ 36 | left: store.hands.interacting?.current.left, 37 | right: store.hands.interacting?.current.right 38 | })) 39 | 40 | const models = useStore((store) => store.hands.models) 41 | 42 | useXREvent('selectend', (e: XREvent) => { 43 | if ( 44 | e.controller === grabbingController.current && 45 | interacting[e.controller.inputSource.handedness] && 46 | ((isHandTracking && e.originalEvent.fake) || !isHandTracking) 47 | ) { 48 | set((store) => { 49 | store.hands.interacting!.current[grabbingController.current!.inputSource.handedness] = undefined 50 | }) 51 | grabbingController.current = undefined 52 | previousTransform.current = undefined 53 | onChange({ isGrabbed: false, controller: e.controller }) 54 | } 55 | }) 56 | 57 | useXREvent('selectstart', (e: XREvent) => { 58 | // if the controller is already interacting, don't do anything 59 | // if hand tracking is enabled, but it's not a fake event, don't do anything 60 | if ((disabled && interacting[e.controller.inputSource.handedness]) || (isHandTracking && !e.originalEvent.fake)) { 61 | return 62 | } 63 | 64 | const model = models?.current[e.controller.inputSource.handedness] 65 | 66 | if (!model) { 67 | return 68 | } 69 | 70 | let colliding = false 71 | 72 | if (!callback) { 73 | // NOT USED FOR NOW HERE, WE USE THE CALLBACK FOR TESTING 74 | let mesh: Mesh | undefined = undefined 75 | ref.current!.traverse((object) => { 76 | if (!mesh && object instanceof Mesh && object.geometry) { 77 | mesh = object 78 | } 79 | }) 80 | if (!mesh) { 81 | return 82 | } 83 | 84 | const obb = new OBB( 85 | new Vector3().setFromMatrixPosition(ref.current!.matrixWorld), 86 | ((mesh as Mesh).geometry!.boundingBox as Box3).getSize(new Vector3()).multiply(ref.current!.scale).divideScalar(2), 87 | new Matrix3().setFromMatrix4(ref.current!.matrixWorld.clone().makeScale(1, 1, 1)) 88 | ) 89 | 90 | const matrix = model!.getHandRotationMatrix() 91 | 92 | const indexTip = model!.bones.find((bone) => (bone as any).jointName === 'index-finger-tip')! as Object3D 93 | const thumbTip = model!.bones.find((bone) => (bone as any).jointName === 'thumb-tip')! as Object3D 94 | 95 | const thumbOBB = new OBB( 96 | indexTip.getWorldPosition(new Vector3()), 97 | new Vector3(0.05, 0.05, 0.05).divideScalar(2), 98 | new Matrix3().setFromMatrix4(matrix) 99 | ) 100 | const indexOBB = new OBB( 101 | thumbTip.getWorldPosition(new Vector3()), 102 | new Vector3(0.05, 0.05, 0.05).divideScalar(2), 103 | new Matrix3().setFromMatrix4(matrix) 104 | ) 105 | 106 | colliding = obb.intersectsOBB(thumbOBB, Number.EPSILON) && obb.intersectsOBB(indexOBB, Number.EPSILON) 107 | } else { 108 | colliding = callback({ controller: e.controller, model: model }) 109 | } 110 | 111 | if (colliding) { 112 | grabbingController.current = e.controller 113 | const transform = model.getHandTransform() 114 | previousTransform.current = transform.clone() 115 | onChange({ isGrabbed: true, controller: e.controller }) 116 | } 117 | }) 118 | 119 | useFrame(() => { 120 | if (!grabbingController.current || !previousTransform.current || !ref.current) { 121 | return 122 | } 123 | 124 | const model = models?.current[grabbingController.current.inputSource.handedness] 125 | 126 | if (!model) { 127 | return 128 | } 129 | 130 | let transform = model.getHandTransform() 131 | 132 | // apply previous transform 133 | ref.current!.applyMatrix4(previousTransform.current.clone().invert()) 134 | 135 | if (isHandTracking) { 136 | // get quaternion from previous matrix 137 | const previousQuaternion = new Quaternion() 138 | previousTransform.current.decompose(new Vector3(), previousQuaternion, new Vector3(1, 1, 1)) 139 | 140 | // get quaternion from current matrix 141 | const currentQuaternion = new Quaternion() 142 | transform.decompose(new Vector3(), currentQuaternion, new Vector3(1, 1, 1)) 143 | 144 | // slerp to current quaternion 145 | previousQuaternion.slerp(currentQuaternion, 0.1) 146 | 147 | const position = model.getHandPosition() 148 | transform = new Matrix4().compose(position, previousQuaternion, new Vector3(1, 1, 1)) 149 | } 150 | 151 | ref.current!.applyMatrix4(transform) 152 | 153 | ref.current!.updateWorldMatrix(false, true) 154 | previousTransform.current = transform.clone() 155 | }) 156 | 157 | return ( 158 | 163 | > 164 | {children} 165 | 166 | ) 167 | } 168 | ) 169 | -------------------------------------------------------------------------------- /src/HandModel.ts: -------------------------------------------------------------------------------- 1 | import { Euler, Group, Matrix4, Mesh, Object3D, Quaternion, Vector3, XRHandedness, XRInputSource } from 'three' 2 | import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader' 3 | 4 | import { defaultPose } from './poses/default' 5 | import { idlePose } from './poses/idle' 6 | import { pinchPose } from './poses/pinch' 7 | 8 | const DEFAULT_HAND_PROFILE_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles/generic-hand/' 9 | 10 | const XRHandJoints = [ 11 | 'wrist', 12 | 'thumb-metacarpal', 13 | 'thumb-phalanx-proximal', 14 | 'thumb-phalanx-distal', 15 | 'thumb-tip', 16 | 'index-finger-metacarpal', 17 | 'index-finger-phalanx-proximal', 18 | 'index-finger-phalanx-intermediate', 19 | 'index-finger-phalanx-distal', 20 | 'index-finger-tip', 21 | 'middle-finger-metacarpal', 22 | 'middle-finger-phalanx-proximal', 23 | 'middle-finger-phalanx-intermediate', 24 | 'middle-finger-phalanx-distal', 25 | 'middle-finger-tip', 26 | 'ring-finger-metacarpal', 27 | 'ring-finger-phalanx-proximal', 28 | 'ring-finger-phalanx-intermediate', 29 | 'ring-finger-phalanx-distal', 30 | 'ring-finger-tip', 31 | 'pinky-finger-metacarpal', 32 | 'pinky-finger-phalanx-proximal', 33 | 'pinky-finger-phalanx-intermediate', 34 | 'pinky-finger-phalanx-distal', 35 | 'pinky-finger-tip' 36 | ] 37 | 38 | export type XRPose = 'pinch' | 'idle' | 'default' 39 | 40 | const poses: { [key in XRPose]: object } = { 41 | idle: idlePose, 42 | pinch: pinchPose, 43 | default: defaultPose 44 | } 45 | 46 | class HandModel extends Object3D { 47 | controller: Group 48 | bones: Object3D[] = [] 49 | 50 | inputSource: XRInputSource 51 | path: string 52 | 53 | model: Object3D 54 | isHandTracking: boolean 55 | 56 | modelPaths?: { [key in XRHandedness]?: string } 57 | 58 | loading: boolean 59 | loaded: boolean 60 | 61 | constructor(controller: Group, inputSource: XRInputSource, modelPaths?: { [key in XRHandedness]?: string }) { 62 | super() 63 | 64 | this.controller = controller 65 | this.inputSource = inputSource 66 | this.modelPaths = modelPaths 67 | } 68 | 69 | load(controller: Group, inputSource: XRInputSource, isHandTracking: boolean) { 70 | this.controller.remove(this) 71 | 72 | this.controller = controller 73 | this.inputSource = inputSource 74 | this.isHandTracking = isHandTracking 75 | 76 | this.loading = true 77 | this.loaded = false 78 | const loader = new GLTFLoader() 79 | // loader.setPath(this.modelPath ?? DEFAULT_HAND_PROFILE_PATH) 80 | const fileHandedness = isHandTracking ? this.inputSource.handedness : 'right' 81 | loader.load((this.modelPaths && this.modelPaths[fileHandedness]) ?? `${DEFAULT_HAND_PROFILE_PATH}${fileHandedness}.glb`, (gltf) => { 82 | this.model = gltf.scene.children[0] 83 | 84 | // clearing everything first 85 | super.clear() 86 | super.add(this.model) 87 | 88 | const mesh = this.model.getObjectByProperty('type', 'SkinnedMesh')! as Mesh 89 | mesh.frustumCulled = false 90 | mesh.castShadow = true 91 | mesh.receiveShadow = true 92 | ;(mesh.material as any).side = 0 // Workaround: force FrontSide = 0 93 | 94 | this.bones = [] 95 | XRHandJoints.forEach((jointName: string) => { 96 | const bone = this.model.getObjectByName(jointName) 97 | if (bone !== undefined) { 98 | ;(bone as any).jointName = jointName 99 | } else { 100 | console.log(`Couldn't find ${jointName} in ${this.inputSource.handedness} hand mesh`) 101 | } 102 | this.bones.push(bone!) 103 | }) 104 | 105 | if (!isHandTracking) { 106 | this.setPose('idle') 107 | this.model.setRotationFromEuler(new Euler(Math.PI / 2, -Math.PI / 2, 0)) 108 | 109 | // hand position offset 110 | this.model.position.sub(new Vector3(-0.02, 0.05, -0.12)) 111 | 112 | // only mirror the left one (this is also the right model here) 113 | if (this.inputSource.handedness === 'left') { 114 | this.model.applyMatrix4(new Matrix4().makeScale(-1, 1, 1)) 115 | } 116 | } 117 | 118 | this.loading = false 119 | this.loaded = true 120 | this.controller.add(this) 121 | }) 122 | } 123 | 124 | updateMatrixWorld(force: boolean) { 125 | super.updateMatrixWorld(force) 126 | 127 | for (let i = 0; i < this.bones.length; i++) { 128 | const bone = this.bones[i] 129 | if (bone) { 130 | const XRJoint = ((this.controller as any)?.joints || [])[(bone as any).jointName] 131 | if (XRJoint?.visible) { 132 | const position = XRJoint.position 133 | bone.position.copy(position) 134 | bone.quaternion.copy(XRJoint.quaternion) 135 | } 136 | } 137 | } 138 | } 139 | 140 | setPose(poseType: XRPose = 'idle') { 141 | const pose = poses[poseType] 142 | for (let i = 0; i < this.bones.length; i++) { 143 | const bone = this.bones[i] 144 | if (bone) { 145 | const joint = pose[(bone as any).jointName] 146 | const position = joint.position 147 | bone.position.copy(new Vector3().fromArray(position)) 148 | bone.quaternion.copy(new Quaternion().fromArray(joint.quaternion)) 149 | } 150 | } 151 | } 152 | 153 | // should not be used anymore 154 | getThumbIndexDistance() { 155 | const indexTip = this!.bones.find((bone) => (bone as any).jointName === 'index-finger-tip')! as Object3D 156 | const thumbTip = this!.bones.find((bone) => (bone as any).jointName === 'thumb-tip')! as Object3D 157 | 158 | return indexTip.getWorldPosition(new Vector3()).distanceTo(thumbTip.getWorldPosition(new Vector3())) 159 | } 160 | 161 | getHandTransform() { 162 | const quaternion = new Quaternion() 163 | this.getHandRotationMatrix().decompose(new Vector3(), quaternion, new Vector3()) 164 | const position = this.getHandPosition() 165 | 166 | return new Matrix4().compose(position, quaternion, new Vector3(1, 1, 1)) 167 | } 168 | 169 | getHandRotationMatrix() { 170 | const indexTip = this!.bones.find((bone) => (bone as any).jointName === 'index-finger-phalanx-proximal')! as Object3D 171 | const thumbTip = this!.bones.find((bone) => (bone as any).jointName === 'thumb-phalanx-proximal')! as Object3D 172 | const indexKnuckle = this!.bones.find((bone) => (bone as any).jointName === 'index-finger-metacarpal')! as Object3D 173 | const pinkyKnuckle = this!.bones.find((bone) => (bone as any).jointName === 'pinky-finger-metacarpal')! as Object3D 174 | 175 | const z = thumbTip.getWorldPosition(new Vector3()).sub(indexTip.getWorldPosition(new Vector3())).normalize() 176 | 177 | const y = indexKnuckle.getWorldPosition(new Vector3()).sub(pinkyKnuckle.getWorldPosition(new Vector3())).normalize() 178 | 179 | const x = new Vector3().crossVectors(z, y).negate() 180 | 181 | const y2 = new Vector3().crossVectors(x, z).negate() 182 | 183 | return new Matrix4().makeBasis(x, y2, z) 184 | } 185 | 186 | getHandPosition() { 187 | const indexTip = this!.bones.find((bone) => (bone as any).jointName === 'index-finger-tip')! as Object3D 188 | const thumbTip = this!.bones.find((bone) => (bone as any).jointName === 'thumb-tip')! as Object3D 189 | 190 | const position: Vector3 = indexTip.getWorldPosition(new Vector3()).add(thumbTip.getWorldPosition(new Vector3())).multiplyScalar(0.5) 191 | 192 | return position 193 | } 194 | } 195 | 196 | export { HandModel } 197 | -------------------------------------------------------------------------------- /src/Level.tsx: -------------------------------------------------------------------------------- 1 | import { XRInteractionEvent } from '@react-three/xr' 2 | import React, { ReactElement, Suspense, useMemo, useState } from 'react' 3 | import { Vector3 } from 'three' 4 | import { Button } from './Button' 5 | import Crate from './Crate' 6 | import { useStore } from './store' 7 | 8 | interface Props {} 9 | 10 | export default function Level({}: Props): ReactElement { 11 | const [shouldReload, setReload] = useState(false) 12 | const interacting = useStore((store) => ({ 13 | left: store.hands.interacting?.current.left, 14 | right: store.hands.interacting?.current.right 15 | })) 16 | 17 | const blocks = useMemo(() => { 18 | return [...Array(10)].map((_) => { 19 | const angle = Math.random() * Math.PI * 2 20 | return ( 21 | 22 | 23 | 24 | ) 25 | }) 26 | }, [shouldReload]) 27 | 28 | return ( 29 | <> 30 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | {blocks} 50 | 51 | ) 52 | } 53 | -------------------------------------------------------------------------------- /src/Text.tsx: -------------------------------------------------------------------------------- 1 | import React, { forwardRef, useLayoutEffect, useRef, useMemo } from 'react' 2 | import { useLoader } from '@react-three/fiber' 3 | 4 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 5 | // @ts-ignore 6 | import robotoFont from 'url:/src/Roboto.json' 7 | import { DoubleSide, Font, FontLoader, Mesh, MeshBasicMaterial, ShapeGeometry, Vector3 } from 'three' 8 | 9 | const Text = forwardRef(({ children, vAlign = 'center', hAlign = 'center', size = 1, color = '#000000', ...props }, ref) => { 10 | const font = useLoader(FontLoader, robotoFont) as Font 11 | 12 | const matLite = new MeshBasicMaterial({ 13 | color: color, 14 | side: DoubleSide 15 | }) 16 | 17 | const geometry = useMemo(() => { 18 | if (font) { 19 | const shapes = font.generateShapes(children, 1) 20 | const geometry = new ShapeGeometry(shapes) 21 | geometry.computeBoundingBox() 22 | return geometry 23 | } 24 | }, [font]) 25 | 26 | const mesh = useRef() 27 | 28 | useLayoutEffect(() => { 29 | const size = new Vector3() 30 | mesh.current.geometry.computeBoundingBox() 31 | mesh.current.geometry.boundingBox.getSize(size) 32 | mesh.current.position.x = hAlign === 'center' ? -size.x / 2 : hAlign === 'right' ? 0 : -size.x 33 | mesh.current.position.y = vAlign === 'center' ? -size.y / 2 : vAlign === 'top' ? 0 : -size.y 34 | }, [children]) 35 | 36 | return ( 37 | 38 | 39 | 40 | ) 41 | }) 42 | 43 | export default Text 44 | -------------------------------------------------------------------------------- /src/index.tsx: -------------------------------------------------------------------------------- 1 | import './styles.css' 2 | 3 | import { OrbitControls } from '@react-three/drei' 4 | import { VRCanvas } from '@react-three/xr' 5 | import React from 'react' 6 | import ReactDOM from 'react-dom' 7 | import { VRButton } from 'three/examples/jsm/webxr/VRButton' 8 | 9 | import { DefaultHandControllers } from './DefaultHandControllers' 10 | import Level from './Level' 11 | 12 | // Oculus Browser with #webxr-hands flag enabled 13 | function HandControllersExample() { 14 | return ( 15 | { 18 | args.gl.setClearColor('grey') 19 | void document.body.appendChild(VRButton.createButton(args.gl)) 20 | }}> 21 | 22 | 23 | 24 | 25 | 26 | 27 | ) 28 | } 29 | 30 | ;(ReactDOM as any).createRoot(document.getElementById('root')).render() 31 | -------------------------------------------------------------------------------- /src/poses/default.ts: -------------------------------------------------------------------------------- 1 | export const defaultPose = { 2 | wrist: { 3 | position: [1.8626447606528984e-10, -8.940696516468449e-10, 5.96046212386625e-10], 4 | quaternion: [-0.7071068286895752, 5.760118781950041e-8, 5.760119137221409e-8, 0.7071068286895752] 5 | }, 6 | 'thumb-metacarpal': { 7 | position: [-0.028028501197695732, -0.0359584279358387, 0.019157717004418373], 8 | quaternion: [-0.4763745069503784, 0.7100346684455872, 0.18028733134269714, 0.48622485995292664] 9 | }, 10 | 'thumb-phalanx-proximal': { 11 | position: [-0.044955722987651825, -0.05921865254640579, 0.03430764749646187], 12 | quaternion: [-0.47673991322517395, 0.6406403183937073, 0.11477915197610855, 0.5908678770065308] 13 | }, 14 | 'thumb-phalanx-distal': { 15 | position: [-0.06691253185272217, -0.08312328159809113, 0.0437118336558342], 16 | quaternion: [-0.39142173528671265, 0.7194157242774963, 0.09644205868244171, 0.5656226277351379] 17 | }, 18 | 'thumb-tip': { 19 | position: [-0.07789862900972366, -0.09167803823947906, 0.04891208931803703], 20 | quaternion: [-0.3947904706001282, 0.7221444249153137, 0.09097205102443695, 0.5606889724731445] 21 | }, 22 | 'index-finger-metacarpal': { 23 | position: [-0.015884488821029663, -0.029212743043899536, 0.009419834241271019], 24 | quaternion: [-0.77494215965271, 0.05332411825656891, -0.03195803984999657, 0.6289674043655396] 25 | }, 26 | 'index-finger-phalanx-proximal': { 27 | position: [-0.023550676181912422, -0.08854873478412628, 0.007316457573324442], 28 | quaternion: [-0.7392873764038086, 0.01683184690773487, 0.028036130592226982, 0.6725956797599792] 29 | }, 30 | 'index-finger-phalanx-intermediate': { 31 | position: [-0.022021925076842308, -0.13375452160835266, 0.01055047009140253], 32 | quaternion: [-0.7388502955436707, -0.01528934109956026, 0.020967017859220505, 0.673369824886322] 33 | }, 34 | 'index-finger-phalanx-distal': { 35 | position: [-0.02069162018597126, -0.15792004764080048, 0.01276903785765171], 36 | quaternion: [-0.686407744884491, -0.04156109690666199, 0.031442154198884964, 0.7253471612930298] 37 | }, 38 | 'index-finger-tip': { 39 | position: [-0.019425027072429657, -0.16941747069358826, 0.012159150093793869], 40 | quaternion: [-0.6862807869911194, -0.049977876245975494, 0.03852125257253647, 0.7245944738388062] 41 | }, 42 | 'middle-finger-metacarpal': { 43 | position: [-0.0032591435592621565, -0.029106352478265762, 0.009419835172593594], 44 | quaternion: [-0.7821785807609558, -0.023043932393193245, -0.06304748356342316, 0.6194276809692383] 45 | }, 46 | 'middle-finger-phalanx-proximal': { 47 | position: [-0.001725903362967074, -0.09164661914110184, 0.002543156733736396], 48 | quaternion: [-0.7427287697792053, -0.04151846468448639, 0.028317611664533615, 0.6677038669586182] 49 | }, 50 | 'middle-finger-phalanx-intermediate': { 51 | position: [0.002640301128849387, -0.13811546564102173, 0.007021433673799038], 52 | quaternion: [-0.7380437850952148, -0.06258413195610046, 0.036574337631464005, 0.6708479523658752] 53 | }, 54 | 'middle-finger-phalanx-distal': { 55 | position: [0.005685935262590647, -0.1653556078672409, 0.009792773053050041], 56 | quaternion: [-0.6704923510551453, -0.0809938833117485, 0.01419301237910986, 0.7373456954956055] 57 | }, 58 | 'middle-finger-tip': { 59 | position: [0.007302379701286554, -0.17740768194198608, 0.008763404563069344], 60 | quaternion: [-0.6713148355484009, -0.07821036130189896, 0.008088618516921997, 0.7369899153709412] 61 | }, 62 | 'ring-finger-metacarpal': { 63 | position: [0.008265813812613487, -0.026990527287125587, 0.009419836103916168], 64 | quaternion: [-0.779512882232666, -0.12313608825206757, -0.04890383780002594, 0.6122137308120728] 65 | }, 66 | 'ring-finger-phalanx-proximal': { 67 | position: [0.017465244978666306, -0.0846937820315361, 0.006529302801936865], 68 | quaternion: [-0.7372061014175415, -0.11990050226449966, 0.046976324170827866, 0.6632829904556274] 69 | }, 70 | 'ring-finger-phalanx-intermediate': { 71 | position: [0.02676110528409481, -0.1263144463300705, 0.010884810239076614], 72 | quaternion: [-0.7334646582603455, -0.1584361046552658, 0.03868652135133743, 0.6598719954490662] 73 | }, 74 | 'ring-finger-phalanx-distal': { 75 | position: [0.03319219499826431, -0.15188787877559662, 0.01416861079633236], 76 | quaternion: [-0.7179667949676514, -0.13197831809520721, 0.006449223030358553, 0.683420717716217] 77 | }, 78 | 'ring-finger-tip': { 79 | position: [0.03552267327904701, -0.16357211768627167, 0.014942227862775326], 80 | quaternion: [-0.7187318205833435, -0.13417485356330872, 0.003463511122390628, 0.6822094321250916] 81 | }, 82 | 'pinky-finger-metacarpal': { 83 | position: [0.022998575121164322, -0.034073565155267715, 0.009419831447303295], 84 | quaternion: [-0.6950896382331848, -0.24571159482002258, -0.052773747593164444, 0.6735659837722778] 85 | }, 86 | 'pinky-finger-phalanx-proximal': { 87 | position: [0.03505408763885498, -0.07389584183692932, 0.013691178523004055], 88 | quaternion: [-0.7371914982795715, -0.16642968356609344, -0.004589088726788759, 0.6548503041267395] 89 | }, 90 | 'pinky-finger-phalanx-intermediate': { 91 | position: [0.042367469519376755, -0.10740291327238083, 0.01811724156141281], 92 | quaternion: [-0.7181382179260254, -0.22876721620559692, 0.011543449014425278, 0.6571223735809326] 93 | }, 94 | 'pinky-finger-phalanx-distal': { 95 | position: [0.04895488917827606, -0.12640321254730225, 0.0209718719124794], 96 | quaternion: [-0.7028482556343079, -0.1945386528968811, -0.017829783260822296, 0.6839891672134399] 97 | }, 98 | 'pinky-finger-tip': { 99 | position: [0.05193105340003967, -0.13790695369243622, 0.02176165021955967], 100 | quaternion: [-0.7031674981117249, -0.2156880795955658, 0.0012595904991030693, 0.6775193810462952] 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/poses/idle.ts: -------------------------------------------------------------------------------- 1 | export const idlePose = { 2 | wrist: { position: [0, 0, 0], quaternion: [-0.6661913377195962, -0.01640709815701443, -0.11120849576590103, 0.7372601841595182] }, 3 | 'thumb-metacarpal': { 4 | position: [-0.03472973685711622, -0.03378564119338989, 0.010930486023426056], 5 | quaternion: [-0.4370977050405571, 0.801618309803501, 0.07092227508523968, 0.40165123245521117] 6 | }, 7 | 'thumb-phalanx-proximal': { 8 | position: [-0.053650377318263054, -0.048898518085479736, 0.03262612223625183], 9 | quaternion: [-0.5233785648562362, 0.7282073040614634, 0.1202576486209243, 0.4258251966578869] 10 | }, 11 | 'thumb-phalanx-distal': { 12 | position: [-0.07035422325134277, -0.06988000869750977, 0.05318659543991089], 13 | quaternion: [-0.4581473272467597, 0.7965817964345437, 0.10612693530136055, 0.37986253004120674] 14 | }, 15 | 'thumb-tip': { 16 | position: [-0.0838727355003357, -0.08245950937271118, 0.0694720670580864], 17 | quaternion: [-0.4581473272467597, 0.7965817964345437, 0.10612693530136055, 0.37986253004120674] 18 | }, 19 | 'index-finger-metacarpal': { 20 | position: [-0.022320279851555824, -0.03902029991149902, 0.001892339438199997], 21 | quaternion: [-0.7246411098667119, 0.07379692097282581, -0.16971426364871023, 0.6638119802024115] 22 | }, 23 | 'index-finger-phalanx-proximal': { 24 | position: [-0.03621675749309361, -0.09194290637969971, -0.0076351650059223175], 25 | quaternion: [-0.7178758851519482, 0.014103382112460663, -0.09023996284710636, 0.6901536475565782] 26 | }, 27 | 'index-finger-phalanx-intermediate': { 28 | position: [-0.04186903452500701, -0.12942814826965332, -0.0064560286700725555], 29 | quaternion: [-0.7210158724051519, -0.009055942274381036, -0.10289564726845996, 0.6851763184922297] 30 | }, 31 | 'index-finger-phalanx-distal': { 32 | position: [-0.04517357889562845, -0.15348654985427856, -0.0054865144193172455], 33 | quaternion: [-0.7015695377374556, -0.042174841107665895, -0.09454715252033762, 0.7050406388618262] 34 | }, 35 | 'index-finger-tip': { 36 | position: [-0.04632382467389107, -0.17581260204315186, -0.006704933941364288], 37 | quaternion: [-0.7015695377374556, -0.042174841107665895, -0.09454715252033762, 0.7050406388618262] 38 | }, 39 | 'middle-finger-metacarpal': { 40 | position: [-0.009995488449931145, -0.04066729545593262, 0.004085216671228409], 41 | quaternion: [-0.7433134312652817, -0.03421360938765304, -0.16345737120332623, 0.6477625024903804] 42 | }, 43 | 'middle-finger-phalanx-proximal': { 44 | position: [-0.014013143256306648, -0.09428155422210693, -0.008505694568157196], 45 | quaternion: [-0.9808402755032382, -0.015176837171140472, -0.09547985644838834, 0.1691319442825862] 46 | }, 47 | 'middle-finger-phalanx-intermediate': { 48 | position: [-0.021833037957549095, -0.10864841938018799, 0.031182728707790375], 49 | quaternion: [0.7920805673660862, -0.04819466602845819, 0.08346967527489871, 0.6027590416384704] 50 | }, 51 | 'middle-finger-phalanx-distal': { 52 | position: [-0.0238752830773592, -0.08212053775787354, 0.03832988440990448], 53 | quaternion: [0.2946729689061379, 0.0006527793449358052, 0.0037696326864468167, 0.955590500760958] 54 | }, 55 | 'middle-finger-tip': { 56 | position: [-0.02366100437939167, -0.06711900234222412, 0.018341291695833206], 57 | quaternion: [0.2946729689061379, 0.0006527793449358052, 0.0037696326864468167, 0.955590500760958] 58 | }, 59 | 'ring-finger-metacarpal': { 60 | position: [0.001606617122888565, -0.03937530517578125, 0.006404057145118713], 61 | quaternion: [-0.7528343283461107, -0.1418028902789975, -0.15518802882148325, 0.6237379979468028] 62 | }, 63 | 'ring-finger-phalanx-proximal': { 64 | position: [0.004814211279153824, -0.09050309658050537, -0.0005191080272197723], 65 | quaternion: [-0.9866094475388417, -0.05959877380566362, -0.13640739820695835, 0.06665437646825109] 66 | }, 67 | 'ring-finger-phalanx-intermediate': { 68 | position: [-0.0053722187876701355, -0.09626609086990356, 0.03667929023504257], 69 | quaternion: [0.6924812721369185, -0.02458229946205448, 0.16012588303971034, 0.7030114507414129] 70 | }, 71 | 'ring-finger-phalanx-distal': { 72 | position: [-0.010346893221139908, -0.07018381357192993, 0.03562350571155548], 73 | quaternion: [0.35032013919195115, -0.05025873404845777, 0.14068244212261186, 0.9246395569122036] 74 | }, 75 | 'ring-finger-tip': { 76 | position: [-0.010712355375289917, -0.05287259817123413, 0.018459565937519073], 77 | quaternion: [0.35032013919195115, -0.05025873404845777, 0.14068244212261186, 0.9246395569122036] 78 | }, 79 | 'pinky-finger-metacarpal': { 80 | position: [0.016442324966192245, -0.03768932819366455, 0.00937221571803093], 81 | quaternion: [-0.6706119869110772, -0.2552411067241566, -0.1670991033321349, 0.6761726333673979] 82 | }, 83 | 'pinky-finger-phalanx-proximal': { 84 | position: [0.02196628227829933, -0.08298414945602417, 0.010729294270277023], 85 | quaternion: [0.9705382983826922, 0.055846077967221575, 0.23396139656455833, 0.014095810216820616] 86 | }, 87 | 'pinky-finger-phalanx-intermediate': { 88 | position: [0.007966633886098862, -0.08294636011123657, 0.03807436674833298], 89 | quaternion: [0.6245149964487291, -0.04919814445092069, 0.19564827203562896, 0.7545079956121755] 90 | }, 91 | 'pinky-finger-phalanx-distal': { 92 | position: [0.004511065781116486, -0.06341385841369629, 0.03370495140552521], 93 | quaternion: [0.4009713821417304, -0.06275414592364197, 0.20712684434689518, 0.8901586028476283] 94 | }, 95 | 'pinky-finger-tip': { 96 | position: [0.002585865557193756, -0.046552300453186035, 0.01977252960205078], 97 | quaternion: [0.4009713821417304, -0.06275414592364197, 0.20712684434689518, 0.8901586028476283] 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/poses/pinch.ts: -------------------------------------------------------------------------------- 1 | export const pinchPose = { 2 | wrist: { position: [0, 0, 0], quaternion: [-0.6527377146928867, -0.04197306512862318, -0.14750412810399346, 0.7418990967870879] }, 3 | 'thumb-metacarpal': { 4 | position: [-0.028708115220069885, -0.03802686929702759, 0.01521763950586319], 5 | quaternion: [-0.5018827760314641, 0.7401433178094579, 0.26408695476159333, 0.361330359297011] 6 | }, 7 | 'thumb-phalanx-proximal': { 8 | position: [-0.03747981786727905, -0.06252908706665039, 0.03470578417181969], 9 | quaternion: [-0.5550449762706562, 0.6602166569844041, 0.29175448580945107, 0.41342273784482036] 10 | }, 11 | 'thumb-phalanx-distal': { 12 | position: [-0.0449826717376709, -0.09105652570724487, 0.05119417980313301], 13 | quaternion: [-0.47353951037964503, 0.7562219659595647, 0.22196308825783814, 0.39321884207232527] 14 | }, 15 | 'thumb-tip': { 16 | position: [-0.05551515519618988, -0.10836756229400635, 0.06518009305000305], 17 | quaternion: [-0.47353951037964503, 0.7562219659595647, 0.22196308825783814, 0.39321884207232527] 18 | }, 19 | 'index-finger-metacarpal': { 20 | position: [-0.02306044101715088, -0.03862184286117554, -0.0009634285233914852], 21 | quaternion: [-0.7085028717326327, 0.05255272987000617, -0.20691323120441896, 0.6726431491380622] 22 | }, 23 | 'index-finger-phalanx-proximal': { 24 | position: [-0.036951854825019836, -0.0910634994506836, -0.01286252960562706], 25 | quaternion: [-0.9520349028684798, -0.01887202901320798, -0.0894724011539764, 0.2920069856576565] 26 | }, 27 | 'index-finger-phalanx-intermediate': { 28 | position: [-0.04299519956111908, -0.11227923631668091, 0.017989562824368477], 29 | quaternion: [-0.9948080696577799, -0.018279256564374204, -0.09906207423043162, 0.014473381511597695] 30 | }, 31 | 'index-finger-phalanx-distal': { 32 | position: [-0.04777246713638306, -0.11306709051132202, 0.04180603101849556], 33 | quaternion: [-0.9966383793141129, -0.0348255636574372, -0.07342604494118556, 0.010379639607186953] 34 | }, 35 | 'index-finger-tip': { 36 | position: [-0.05066494643688202, -0.11464649438858032, 0.06395097076892853], 37 | quaternion: [-0.9966383793141129, -0.0348255636574372, -0.07342604494118556, 0.010379639607186953] 38 | }, 39 | 'middle-finger-metacarpal': { 40 | position: [-0.011009499430656433, -0.04054677486419678, 0.0022756149992346764], 41 | quaternion: [-0.728576836221424, -0.05452649690398996, -0.20251115292559713, 0.6520673951346632] 42 | }, 43 | 'middle-finger-phalanx-proximal': { 44 | position: [-0.014801859855651855, -0.0938032865524292, -0.011814166558906436], 45 | quaternion: [-0.9643559144702265, 0.002296662407013733, -0.14345090442071629, 0.22233810647066332] 46 | }, 47 | 'middle-finger-phalanx-intermediate': { 48 | position: [-0.02672255039215088, -0.11218321323394775, 0.025101996958255768], 49 | quaternion: [0.829592712909983, -0.09380243778711712, 0.1057267691567219, 0.5401841201252863] 50 | }, 51 | 'middle-finger-phalanx-distal': { 52 | position: [-0.02876339852809906, -0.08694499731063843, 0.03595779836177826], 53 | quaternion: [0.36512814687348283, -0.047265183334492235, 0.0021122995466034376, 0.9297542562396359] 54 | }, 55 | 'middle-finger-tip': { 56 | position: [-0.026344195008277893, -0.06916528940200806, 0.018560543656349182], 57 | quaternion: [0.36512814687348283, -0.047265183334492235, 0.0021122995466034376, 0.9297542562396359] 58 | }, 59 | 'ring-finger-metacarpal': { 60 | position: [0.0003604888916015625, -0.03952282667160034, 0.0056415945291519165], 61 | quaternion: [-0.7396808097017777, -0.16093438809609775, -0.1956158263891162, 0.6234636083631186] 62 | }, 63 | 'ring-finger-phalanx-proximal': { 64 | position: [0.0033015459775924683, -0.09054785966873169, -0.0021084961481392384], 65 | quaternion: [-0.9716843359574324, -0.05038339502943255, -0.17894891706806382, 0.14583672322305824] 66 | }, 67 | 'ring-finger-phalanx-intermediate': { 68 | position: [-0.00968681275844574, -0.10230308771133423, 0.03273133561015129], 69 | quaternion: [0.7759395377553959, -0.05574459995025362, 0.18814084493382177, 0.5995109638629774] 70 | }, 71 | 'ring-finger-phalanx-distal': { 72 | position: [-0.015669360756874084, -0.07702267169952393, 0.03832182288169861], 73 | quaternion: [0.32587486330368326, -0.10939771910707881, 0.1317591102661254, 0.9297726869417349] 74 | }, 75 | 'ring-finger-tip': { 76 | position: [-0.013075575232505798, -0.060324668884277344, 0.020747322589159012], 77 | quaternion: [0.32587486330368326, -0.10939771910707881, 0.1317591102661254, 0.9297726869417349] 78 | }, 79 | 'pinky-finger-metacarpal': { 80 | position: [0.01489970088005066, -0.038179636001586914, 0.00994948297739029], 81 | quaternion: [-0.6574740246753815, -0.27719103523257715, -0.2055514915428402, 0.6698071522373896] 82 | }, 83 | 'pinky-finger-phalanx-proximal': { 84 | position: [0.01950991153717041, -0.08358919620513916, 0.010780317708849907], 85 | quaternion: [-0.954898987936288, -0.09855212426684874, -0.26847438050731093, 0.07985554866106723] 86 | }, 87 | 'pinky-finger-phalanx-intermediate': { 88 | position: [0.0042420923709869385, -0.0898999571800232, 0.036680374294519424], 89 | quaternion: [0.7371802888124096, -0.028210442936276114, 0.26473157891790616, 0.6210367008635028] 90 | }, 91 | 'pinky-finger-phalanx-distal': { 92 | position: [-0.0029739439487457275, -0.07099878787994385, 0.03847714141011238], 93 | quaternion: [0.38578432275097324, -0.11353125177035334, 0.2603789668506576, 0.8777721257893837] 94 | }, 95 | 'pinky-finger-tip': { 96 | position: [-0.0038779377937316895, -0.05425739288330078, 0.024298351258039474], 97 | quaternion: [0.38578432275097324, -0.11353125177035334, 0.2603789668506576, 0.8777721257893837] 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /src/store.tsx: -------------------------------------------------------------------------------- 1 | import produce, { setAutoFreeze } from 'immer' 2 | import { MutableRefObject } from 'react' 3 | import { Object3D, XRHandedness } from 'three' 4 | import createStore, { State as ZustandState, StateCreator } from 'zustand' 5 | 6 | import { HandModel } from './HandModel' 7 | 8 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 9 | // @ts-ignore 10 | const immer = 11 | (config: StateCreator void) => void>): StateCreator => 12 | (set, get, api) => 13 | config((fn) => set(produce(fn) as (state: T) => T), get, api) 14 | 15 | setAutoFreeze(false) 16 | 17 | export type State = { 18 | hands: { 19 | models?: MutableRefObject<{ [key in XRHandedness]?: HandModel }> 20 | interacting?: MutableRefObject<{ [key in XRHandedness]?: Object3D }> 21 | } 22 | set: (fn: (state: State) => void | State) => void 23 | } 24 | 25 | export const useStore = createStore( 26 | immer((set, get, api) => { 27 | return { 28 | hands: { 29 | models: undefined, 30 | interacting: undefined 31 | }, 32 | set: (fn: (state: State) => State) => { 33 | set(fn) 34 | } 35 | } 36 | }) 37 | ) 38 | -------------------------------------------------------------------------------- /src/styles.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | html, 6 | body, 7 | #root { 8 | width: 100%; 9 | height: 100%; 10 | margin: 0; 11 | padding: 0; 12 | } 13 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["DOM", "ESNext"], 4 | "typeRoots": [ 5 | "node_modules/@types", 6 | "src/utils/types" 7 | ] , 8 | "target": "esnext", 9 | "module": "commonjs", 10 | "noImplicitAny": false, 11 | "jsx": "react-jsx", 12 | "strict": false, 13 | "esModuleInterop": true, 14 | "outDir": "dist", 15 | "removeComments": true 16 | } 17 | } 18 | 19 | --------------------------------------------------------------------------------