;
13 | ```
14 |
15 | ## Parameters
16 |
17 | | Parameter | Type | Description |
18 | | --- | --- | --- |
19 | | camera | THREE.Camera | |
20 | | cameraOffset | THREE.Object3D | |
21 | | avatar | [Avatar](./three-avatar.avatar.md) | |
22 | | isFirstPerson | boolean | First person view or 3rd person view. |
23 |
24 | **Returns:**
25 |
26 | Promise<void>
27 |
28 |
--------------------------------------------------------------------------------
/docs/three-avatar.simpleboundingboxcollider._constructor_.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [SimpleBoundingBoxCollider](./three-avatar.simpleboundingboxcollider.md) > [(constructor)](./three-avatar.simpleboundingboxcollider._constructor_.md)
4 |
5 | ## SimpleBoundingBoxCollider.(constructor)
6 |
7 | Constructs a new instance of the `SimpleBoundingBoxCollider` class
8 |
9 | **Signature:**
10 |
11 | ```typescript
12 | constructor(moveTarget: THREE.Object3D, getBoxes: () => THREE.Box3[] | undefined);
13 | ```
14 |
15 | ## Parameters
16 |
17 | | Parameter | Type | Description |
18 | | --- | --- | --- |
19 | | moveTarget | THREE.Object3D | Objects to move. |
20 | | getBoxes | () => THREE.Box3\[\] \| undefined | Get a list of bounding boxes of collisionable objects. |
21 |
22 |
--------------------------------------------------------------------------------
/docs/three-avatar.simpleboundingboxcollider.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [SimpleBoundingBoxCollider](./three-avatar.simpleboundingboxcollider.md)
4 |
5 | ## SimpleBoundingBoxCollider class
6 |
7 | Avatar extension to determine collision using [BoundingBox](https://threejs.org/docs/?q=Box3#api/en/math/Box3.setFromObject).
8 |
9 | **Signature:**
10 |
11 | ```typescript
12 | export declare class SimpleBoundingBoxCollider
13 | ```
14 |
15 | ## Constructors
16 |
17 | | Constructor | Modifiers | Description |
18 | | --- | --- | --- |
19 | | [(constructor)(moveTarget, getBoxes)](./three-avatar.simpleboundingboxcollider._constructor_.md) | | Constructs a new instance of the SimpleBoundingBoxCollider class |
20 |
21 | ## Methods
22 |
23 | | Method | Modifiers | Description |
24 | | --- | --- | --- |
25 | | [moveTo(x, y, z)](./three-avatar.simpleboundingboxcollider.moveto.md) | | |
26 | | [setup(avatar)](./three-avatar.simpleboundingboxcollider.setup.md) | | |
27 |
28 |
--------------------------------------------------------------------------------
/docs/three-avatar.simpleboundingboxcollider.moveto.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [SimpleBoundingBoxCollider](./three-avatar.simpleboundingboxcollider.md) > [moveTo](./three-avatar.simpleboundingboxcollider.moveto.md)
4 |
5 | ## SimpleBoundingBoxCollider.moveTo() method
6 |
7 | **Signature:**
8 |
9 | ```typescript
10 | moveTo(x: number, y: number, z: number): void;
11 | ```
12 |
13 | ## Parameters
14 |
15 | | Parameter | Type | Description |
16 | | --- | --- | --- |
17 | | x | number | |
18 | | y | number | |
19 | | z | number | |
20 |
21 | **Returns:**
22 |
23 | void
24 |
25 |
--------------------------------------------------------------------------------
/docs/three-avatar.simpleboundingboxcollider.setup.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [SimpleBoundingBoxCollider](./three-avatar.simpleboundingboxcollider.md) > [setup](./three-avatar.simpleboundingboxcollider.setup.md)
4 |
5 | ## SimpleBoundingBoxCollider.setup() method
6 |
7 | **Signature:**
8 |
9 | ```typescript
10 | setup(avatar: Avatar): void;
11 | ```
12 |
13 | ## Parameters
14 |
15 | | Parameter | Type | Description |
16 | | --- | --- | --- |
17 | | avatar | [Avatar](./three-avatar.avatar.md) | |
18 |
19 | **Returns:**
20 |
21 | void
22 |
23 |
--------------------------------------------------------------------------------
/docs/three-avatar.vector3tupple.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [Vector3Tupple](./three-avatar.vector3tupple.md)
4 |
5 | ## Vector3Tupple type
6 |
7 | x,y,z
8 |
9 | **Signature:**
10 |
11 | ```typescript
12 | export type Vector3Tupple = [number, number, number];
13 | ```
14 |
--------------------------------------------------------------------------------
/docs/three-avatar.vrhandgetter.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [VRHandGetter](./three-avatar.vrhandgetter.md)
4 |
5 | ## VRHandGetter type
6 |
7 | Get XR controller objects. Use to get the position of the IK hand.
8 |
9 | **Signature:**
10 |
11 | ```typescript
12 | export type VRHandGetter = {
13 | left?: () => THREE.Object3D | undefined;
14 | right?: () => THREE.Object3D | undefined;
15 | };
16 | ```
17 |
--------------------------------------------------------------------------------
/docs/three-avatar.wristrotationoffsetset.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | [Home](./index.md) > [@verseengine/three-avatar](./three-avatar.md) > [WristRotationOffsetSet](./three-avatar.wristrotationoffsetset.md)
4 |
5 | ## WristRotationOffsetSet type
6 |
7 | Offset of rotation angle between the XR controller and the avatar's wrist.
8 |
9 | **Signature:**
10 |
11 | ```typescript
12 | export type WristRotationOffsetSet = {
13 | left: Vector3Tupple;
14 | right: Vector3Tupple;
15 | };
16 | ```
17 | **References:** [Vector3Tupple](./three-avatar.vector3tupple.md)
18 |
19 |
--------------------------------------------------------------------------------
/example/asset/animation/Breakdance Uprock Var 1.fbx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/animation/Breakdance Uprock Var 1.fbx
--------------------------------------------------------------------------------
/example/asset/animation/idle.fbx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/animation/idle.fbx
--------------------------------------------------------------------------------
/example/asset/animation/walk.fbx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/animation/walk.fbx
--------------------------------------------------------------------------------
/example/asset/avatar-example/rpm.glb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/avatar-example/rpm.glb
--------------------------------------------------------------------------------
/example/asset/avatar-example/vrm-v0.vrm:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/avatar-example/vrm-v0.vrm
--------------------------------------------------------------------------------
/example/asset/avatar-example/vrm-v1.vrm:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/VerseEngine/three-avatar/f33eedd77b301a66bf134e4635b553dda2930175/example/asset/avatar-example/vrm-v1.vrm
--------------------------------------------------------------------------------
/example/avatar.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
54 |
58 |
69 |
203 |
204 |
205 |
206 |
209 |
212 |
213 |
214 |
Drag avatar file here (glb, vrm).
215 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
--------------------------------------------------------------------------------
/example/main.js:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import {
3 | createAvatarIK,
4 | isAnimationDataLoaded,
5 | preLoadAnimationData,
6 | createAvatar,
7 | registerSyncAvatarHeadAndCamera,
8 | setNonVRCameraMode,
9 | addMirrorHUD,
10 | Lipsync,
11 | SimpleBoundingBoxCollider,
12 | } from "three-avatar";
13 | import { setupScene, createTransformControls } from "./setup";
14 | import { PlayerController } from "./player-controller";
15 |
16 | let _avatar;
17 | let _avatarIK;
18 | let _collisionBoxes = [];
19 | let _collisionObjects = [];
20 | let _interactableObjects = [];
21 | let _teleportTargetObjects = [];
22 | let _isFPS = false;
23 | let _bc;
24 |
25 | export const main = (initialLoad /* :?()=>Promise*/) => {
26 | let lipSync;
27 | const ctx = setupScene((dt) => {
28 | _avatar?.tick(dt);
29 | _avatarIK?.tick(dt);
30 |
31 | if (ctx?.renderer.xr?.enabled) {
32 | _avatar?.headSync(ctx?.camera.rotation);
33 | }
34 | if (lipSync) {
35 | _avatar?.lipSync(...lipSync.update());
36 | }
37 | playerController?.tick(dt);
38 | }, true);
39 | _teleportTargetObjects.push(ctx.ground);
40 |
41 | const { scene, cameraContainer, renderer, camera } = ctx;
42 | const playerObj = new THREE.Group();
43 | playerObj.name = "playerObj";
44 | playerObj.add(cameraContainer);
45 | scene.add(playerObj);
46 |
47 | _bc = new SimpleBoundingBoxCollider(playerObj, () => _collisionBoxes);
48 |
49 | const playerController = new PlayerController(
50 | playerObj,
51 | cameraContainer,
52 | cameraContainer,
53 | scene,
54 | renderer,
55 | camera,
56 | {
57 | moveTo: _bc.moveTo.bind(_bc),
58 | getCollisionObjects: () => _collisionObjects,
59 | getInteractableObjects: () => _interactableObjects,
60 | getTeleportTargetObjects: () => _teleportTargetObjects,
61 | }
62 | );
63 |
64 | if (!initialLoad) {
65 | setupVR(ctx, playerController);
66 | } else {
67 | setTimeout(async () => {
68 | await initialLoad();
69 | setupVR(ctx, playerController);
70 | });
71 | }
72 | const setTestIKEnabled = setupTestIK(ctx, playerObj, playerController);
73 | let mirrorHUD;
74 | let lastUrl;
75 | let lastAnimationMap;
76 | let isLowSpecMode = false;
77 | return {
78 | scene,
79 | collisionObjects: _collisionObjects,
80 | interactableObjects: _interactableObjects,
81 | teleportTargetObjects: _teleportTargetObjects,
82 | getAvatar: () => _avatar,
83 | setLowSpecMode: (enabled) => {
84 | enabled = !!enabled;
85 | if (enabled === isLowSpecMode) {
86 | return;
87 | }
88 | isLowSpecMode = enabled;
89 | if (lastUrl) {
90 | return changeAvatar(
91 | camera,
92 | lastUrl,
93 | lastAnimationMap,
94 | playerObj,
95 | renderer,
96 | isLowSpecMode
97 | );
98 | }
99 | },
100 | changeAvatar: (url, animationMap) => {
101 | lastUrl = url;
102 | lastAnimationMap = animationMap;
103 | return changeAvatar(
104 | camera,
105 | url,
106 | animationMap,
107 | playerObj,
108 | renderer,
109 | isLowSpecMode
110 | );
111 | },
112 | setLipsyncEnabled: async (enabled) => {
113 | if (enabled) {
114 | const ms = await navigator.mediaDevices.getUserMedia({
115 | audio: true,
116 | });
117 | lipSync = new Lipsync(THREE.AudioContext.getContext(), ms);
118 | } else {
119 | lipSync = undefined;
120 | }
121 | },
122 | setTestIKEnabled,
123 | setFPSMode: (enabled) => {
124 | _isFPS = enabled;
125 | updateNonVrCameraMode(_isFPS, _avatar, ctx.camera);
126 | },
127 | showMirrorHUD: async (enabled) => {
128 | mirrorHUD?.removeFromParent();
129 | mirrorHUD?.dispose();
130 | if (enabled) {
131 | while (!_avatar) {
132 | await new Promise((resolve) => setTimeout(resolve));
133 | }
134 | mirrorHUD = addMirrorHUD(_avatar, playerObj, {
135 | xr: renderer.xr,
136 | });
137 | } else {
138 | mirrorHUD = undefined;
139 | }
140 | },
141 | };
142 | };
143 |
144 | const changeAvatar = async (
145 | camera,
146 | url,
147 | animationMap,
148 | playerObj,
149 | renderer,
150 | isLowSpecMode
151 | ) => {
152 | _avatar?.dispose();
153 |
154 | if (!isAnimationDataLoaded()) {
155 | await preLoadAnimationData(animationMap);
156 | }
157 |
158 | let resp = await fetch(url);
159 | const avatarData = new Uint8Array(await resp.arrayBuffer());
160 |
161 | _avatar = await createAvatar(avatarData, renderer, false, {
162 | isInvisibleFirstPerson: true,
163 | isLowSpecMode,
164 | });
165 | window._avatar = _avatar;
166 | _avatar.object3D.name = "myAvatar";
167 | playerObj.add(_avatar.object3D);
168 |
169 | _bc.setup(_avatar);
170 | _collisionBoxes.length = 0;
171 | [..._collisionObjects, ..._teleportTargetObjects].map((el) => {
172 | el.traverse((c) => {
173 | if (!c.isMesh) {
174 | return;
175 | }
176 | _collisionBoxes.push(new THREE.Box3().setFromObject(c));
177 | });
178 | });
179 |
180 | if (renderer.xr.isPresenting) {
181 | const getCameras = () =>
182 | [camera, renderer.xr?.getCamera()].filter((v) => !!v);
183 | _avatar.setFirstPersonMode(getCameras());
184 | } else {
185 | updateNonVrCameraMode(_isFPS, _avatar, camera);
186 | }
187 | };
188 |
189 | const setupVR = ({ camera, renderer }, playerController) => {
190 | const getCameras = () =>
191 | [camera, renderer.xr?.getCamera()].filter((v) => !!v);
192 |
193 | registerSyncAvatarHeadAndCamera(
194 | renderer.xr,
195 | camera,
196 | camera,
197 | camera.parent,
198 | () => _avatar,
199 | {
200 | onVR: async () => {
201 | playerController.isVR = true;
202 | await _avatar.setIKMode(true);
203 | _avatar.setFirstPersonMode(getCameras());
204 | _avatarIK?.dispose();
205 | _avatarIK = createAvatarIK(
206 | _avatar,
207 | {
208 | left: () => playerController.xrController.handHolder.leftHand,
209 | right: () => playerController.xrController.handHolder.rightHand,
210 | },
211 | { isDebug: true }
212 | );
213 | window._debugAik = _avatarIK;
214 | },
215 | onNonVR: async () => {
216 | await _avatar.setIKMode(false);
217 | updateNonVrCameraMode(_isFPS, _avatar, camera);
218 | _avatarIK?.dispose();
219 | _avatarIK = undefined;
220 |
221 | playerController.isVR = false;
222 | },
223 | }
224 | );
225 | };
226 | const updateNonVrCameraMode = (isFPS, avatar, camera) => {
227 | setNonVRCameraMode(camera, camera.parent, avatar, isFPS);
228 | };
229 | const setupTestIK = (
230 | { renderer, camera, scene },
231 | playerObj,
232 | playerController
233 | ) => {
234 | const createTarget = (pos) => {
235 | const tc = createTransformControls(
236 | camera,
237 | renderer.domElement,
238 | (enabled) => {
239 | playerController.enabled = !enabled;
240 | }
241 | );
242 | scene.add(tc);
243 | const mesh = new THREE.Mesh(
244 | new THREE.BoxGeometry(0.01, 0.01, 0.01),
245 | new THREE.MeshNormalMaterial()
246 | );
247 | mesh.position.set(...pos);
248 | playerObj.add(mesh);
249 | tc.attach(mesh);
250 | tc.visible = tc.enabled = mesh.visible = false;
251 | return { tc, mesh };
252 | };
253 | const targets = [
254 | createTarget([0.26, 0.7, -0.26]),
255 | createTarget([-0.26, 0.7, -0.26]),
256 | ];
257 |
258 | const f = async (enabled) => {
259 | targets.forEach((v) => {
260 | v.tc.visible = v.tc.enabled = v.mesh.visible = enabled;
261 | });
262 | await _avatar.setIKMode(enabled);
263 | _avatarIK?.dispose();
264 | if (enabled) {
265 | _avatarIK = createAvatarIK(
266 | _avatar,
267 | { right: () => targets[0].mesh, left: () => targets[1].mesh },
268 | { isDebug: true }
269 | );
270 | } else {
271 | _avatarIK = undefined;
272 | }
273 | window._debugAik = _avatarIK;
274 | };
275 | return f;
276 | };
277 |
--------------------------------------------------------------------------------
/example/player-controller.js:
--------------------------------------------------------------------------------
1 | import { TouchController } from "@verseengine/three-touch-controller";
2 | import { MoveController } from "@verseengine/three-move-controller";
3 | import { DefaultXrControllerSet } from "@verseengine/three-xr-controller";
4 |
5 | function isTouchDevice() {
6 | return "ontouchstart" in window || navigator.maxTouchPoints > 0;
7 | }
8 |
9 | export function isVRSupported() {
10 | return !!navigator.xr;
11 | }
12 |
13 | export class PlayerController {
14 | constructor(
15 | moveTarget /* :THREE.Object3D */,
16 | headRotationTarget /* :THREE.Object3D */,
17 | handContainer /* :THREE.Object3D */,
18 | scene /* :THREE.Scene */,
19 | renderer /* :THREE.WebGLRenderer */,
20 | camera /* :THREE.Camera */,
21 | controllerOptions
22 | ) {
23 | this._enabled = true;
24 | this._isVR = false;
25 | this.touchController = new TouchController(moveTarget, {
26 | moveTo: controllerOptions?.moveTo,
27 | });
28 | this.moveController = new MoveController(
29 | moveTarget,
30 | moveTarget,
31 | headRotationTarget,
32 | {
33 | moveTo: controllerOptions?.moveTo,
34 | minVerticalRotation: 1.2,
35 | maxVerticalRotation: 2.2,
36 | }
37 | );
38 | this.xrController = new DefaultXrControllerSet(
39 | renderer,
40 | camera,
41 | scene,
42 | handContainer,
43 | moveTarget,
44 | moveTarget,
45 | controllerOptions
46 | );
47 | this.isVR = renderer.xr.isPresenting;
48 | }
49 | set isVR(v) {
50 | this._isVR = v;
51 | if (v) {
52 | this.touchController.enabled = false;
53 | this.moveController.enabled = false;
54 | if (this.xrController) {
55 | this.xrController.enabled = true;
56 | }
57 | } else {
58 | this.touchController.enabled = isTouchDevice();
59 | this.moveController.enabled = !this.touchController.enabled;
60 | if (this.xrController) {
61 | this.xrController.enabled = false;
62 | }
63 | }
64 | }
65 | get isVR() {
66 | return this._isVR;
67 | }
68 | set enabled(v) {
69 | this._enabled = v;
70 | }
71 | get enabled() {
72 | return this._enabled;
73 | }
74 | tick(deltaTime /* : number // THREE.Clock.getDelta() */) {
75 | if (!this._enabled) {
76 | return;
77 | }
78 | this.touchController.tick(deltaTime);
79 | this.moveController.tick(deltaTime);
80 | this.xrController.tick(deltaTime);
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/example/setup.js:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import { Sky } from "three/examples/jsm/objects/Sky.js";
3 | import { TransformControls } from "three/examples/jsm/controls/TransformControls.js";
4 | import { VRButton } from "three/examples/jsm/webxr/VRButton.js";
5 | import Stats from "three/examples/jsm/libs/stats.module.js";
6 |
7 | export function setupScene(
8 | tickFunc /* ?: (deltaTime: number) => void */,
9 | withVR,
10 | withStats
11 | ) {
12 | console.log(getRenderInfo());
13 |
14 | const renderer = new THREE.WebGLRenderer({ antialias: true });
15 | renderer.outputEncoding = THREE.sRGBEncoding;
16 | /* renderer.toneMapping = THREE.ACESFilmicToneMapping;
17 | renderer.toneMappingExposure = 1; */
18 | renderer.setSize(window.innerWidth, window.innerHeight);
19 | renderer.setPixelRatio(window.devicePixelRatio);
20 | document.body.appendChild(renderer.domElement);
21 |
22 | window.addEventListener("resize", () => {
23 | camera.aspect = window.innerWidth / window.innerHeight;
24 | camera.updateProjectionMatrix();
25 |
26 | renderer.setSize(window.innerWidth, window.innerHeight);
27 | });
28 |
29 | const camera = new THREE.PerspectiveCamera(
30 | 60,
31 | window.innerWidth / window.innerHeight,
32 | 0.1,
33 | 100
34 | );
35 | camera.position.set(0.0, 1.5, 2.0);
36 | const cameraContainer = new THREE.Group();
37 | cameraContainer.add(camera);
38 |
39 | const scene = new THREE.Scene();
40 |
41 | {
42 | const light = new THREE.AmbientLight(0xffffff, 0.2);
43 | scene.add(light);
44 | staticize(light);
45 | }
46 | {
47 | const light = new THREE.DirectionalLight(0xffffff, 0.8);
48 | light.position.set(0, 10, -10).normalize();
49 | scene.add(light);
50 | staticize(light);
51 | }
52 | {
53 | const sky = new Sky();
54 | sky.name = "sky";
55 | sky.scale.setScalar(450000);
56 | scene.add(sky);
57 | staticize(sky);
58 |
59 | const uniforms = sky.material.uniforms;
60 | const phi = THREE.MathUtils.degToRad(90 - 30);
61 | const theta = THREE.MathUtils.degToRad(180);
62 |
63 | const sun = new THREE.Vector3();
64 | sun.setFromSphericalCoords(1, phi, theta);
65 |
66 | uniforms["sunPosition"].value.copy(sun);
67 | }
68 | let ground;
69 | {
70 | ground = new THREE.Mesh(
71 | new THREE.PlaneGeometry(50, 50, 1, 1),
72 | new THREE.MeshLambertMaterial({
73 | color: 0x5e5e5e,
74 | })
75 | );
76 | ground.name = "ground";
77 | ground.rotation.x = Math.PI / -2;
78 | scene.add(ground);
79 | staticize(ground);
80 | }
81 | {
82 | const gridHelper = new THREE.GridHelper(50, 50);
83 | scene.add(gridHelper);
84 | staticize(gridHelper);
85 | }
86 |
87 | let stats;
88 | if (withStats) {
89 | stats = createStats();
90 | document.body.appendChild(stats.stats.dom);
91 | stats.object3D.position.set(-0.5, -0.3, -1.5);
92 | stats.object3D.scale.set(0.003, 0.003, 0.003);
93 | camera.add(stats.object3D);
94 | stats.object3D.visible = false;
95 | renderer.xr.addEventListener("sessionstart", () => {
96 | stats.object3D.visible = true;
97 | });
98 | renderer.xr.addEventListener("sessionend", () => {
99 | stats.object3D.visible = false;
100 | });
101 | }
102 |
103 | const clock = new THREE.Clock();
104 | function animate() {
105 | if (withStats) {
106 | stats.stats.begin();
107 | }
108 |
109 | const dt = clock.getDelta();
110 | if (tickFunc) {
111 | tickFunc(dt);
112 | }
113 |
114 | renderer.render(scene, camera);
115 |
116 | if (withStats) {
117 | stats.stats.end();
118 | }
119 | }
120 | renderer.setAnimationLoop(animate);
121 |
122 | let vrButton = undefined;
123 | if (withVR) {
124 | if ("xr" in navigator) {
125 | navigator.xr
126 | .isSessionSupported("immersive-vr")
127 | .then(function (supported) {
128 | if (supported) {
129 | renderer.xr.enabled = true;
130 |
131 | document.addEventListener("keydown", function (e) {
132 | if (e.key === "Escape") {
133 | if (renderer.xr.isPresenting) {
134 | renderer.xr.getSession()?.end();
135 | }
136 | }
137 | });
138 | vrButton = VRButton.createButton(renderer);
139 | document.body.appendChild(vrButton);
140 | }
141 | });
142 | } else {
143 | if (window.isSecureContext === false) {
144 | console.warn("webxr needs https");
145 | } else {
146 | console.warn("webxr not available");
147 | }
148 | }
149 | }
150 |
151 | {
152 | // For Three.js Inspector (https://zz85.github.io/zz85-bookmarklets/threelabs.html)
153 | window.THREE = THREE;
154 | window._scene = scene;
155 | }
156 |
157 | const res = {
158 | camera,
159 | scene,
160 | renderer,
161 | cameraContainer,
162 | ground,
163 | stats,
164 | vrButton,
165 | };
166 | window._debugCtx = res; // debug
167 | return res;
168 | }
169 |
170 | export function createBridge() {
171 | const res = new THREE.Group();
172 | res.name = "bridge";
173 | const material = new THREE.MeshStandardMaterial({ color: 0xffd479 });
174 |
175 | let y = 0;
176 | let z = 0;
177 | for (let i = 0; i < 10; i++) {
178 | const m = new THREE.Mesh(new THREE.BoxGeometry(1, 0.2, 0.2), material);
179 | m.position.set(0, y, z);
180 | y += 0.2;
181 | z += 0.2;
182 | res.add(m);
183 | }
184 | z -= 0.1;
185 | {
186 | const m = new THREE.Mesh(new THREE.BoxGeometry(1, 0.2, 5), material);
187 | m.position.set(0, y, z + 2.5);
188 | res.add(m);
189 | z += 5;
190 | }
191 | y -= 0.2;
192 | z += 0.1;
193 | for (let i = 0; i < 10; i++) {
194 | const m = new THREE.Mesh(new THREE.BoxGeometry(1, 0.2, 0.2), material);
195 | m.position.set(0, y, z);
196 | y -= 0.2;
197 | z += 0.2;
198 | res.add(m);
199 | }
200 |
201 | return res;
202 | }
203 |
204 | export function createTransformControls(
205 | camera /* :THREE.Camera*/,
206 | domElement /* ?:HTMLElement*/,
207 | onToggleTransform /* ?:(enabled:boolean)=>void */
208 | ) {
209 | const tc = new TransformControls(camera, domElement);
210 | tc.addEventListener("dragging-changed", (e) => {
211 | if (onToggleTransform) {
212 | onToggleTransform(e.value);
213 | }
214 | });
215 | let downTm = 0;
216 | tc.addEventListener("mouseDown", (_e) => {
217 | downTm = new Date().getTime();
218 | //
219 | });
220 | tc.addEventListener("mouseUp", (_e) => {
221 | const now = new Date().getTime();
222 | if (now - downTm > 300) {
223 | return;
224 | }
225 | const prev = tc.getMode();
226 | if (prev === "translate") {
227 | tc.setMode("rotate");
228 | } else {
229 | tc.setMode("translate");
230 | }
231 | });
232 |
233 | return tc;
234 | }
235 |
236 | function createStats() {
237 | const stats = new Stats();
238 | const canvas = stats.dom.children[0];
239 | const panel = new THREE.Mesh(
240 | new THREE.PlaneGeometry(
241 | canvas.width * window.devicePixelRatio,
242 | canvas.height * window.devicePixelRatio
243 | ),
244 | new THREE.MeshBasicMaterial()
245 | );
246 | panel.name = "stats";
247 | const textureLoader = new THREE.TextureLoader();
248 |
249 | const updateMesh = () => {
250 | const img = canvas.toDataURL("image/png");
251 | textureLoader.load(img, (v) => {
252 | panel.material.map?.dispose();
253 | panel.material.map = v;
254 | panel.material.needsUpdate = true;
255 | });
256 | };
257 | setInterval(updateMesh, 1000);
258 |
259 | return {
260 | object3D: panel,
261 | stats,
262 | };
263 | }
264 |
265 | export function getRenderInfo() {
266 | try {
267 | const canvas = document.createElement("canvas");
268 | const gl = canvas.getContext("webgl2");
269 | const ext = gl.getExtension("WEBGL_debug_renderer_info");
270 | if (ext) {
271 | return {
272 | vendor: gl.getParameter(ext.UNMASKED_VENDOR_WEBGL),
273 | renderer: gl.getParameter(ext.UNMASKED_RENDERER_WEBGL),
274 | };
275 | }
276 | } catch (ex) {
277 | console.warn(ex);
278 | }
279 | }
280 |
281 | function staticize(o) {
282 | o.matrixAutoUpdate = false;
283 | o.matrixWorldAutoUpdate = false;
284 | o.updateMatrix();
285 | o.updateMatrixWorld();
286 | }
287 |
--------------------------------------------------------------------------------
/example/world.js:
--------------------------------------------------------------------------------
1 | import { createBridge } from "./setup";
2 | import * as THREE from "three";
3 |
4 | export function createWorldObjects({
5 | scene,
6 | collisionObjects,
7 | _interactableObjects,
8 | teleportTargetObjects,
9 | }) {
10 | {
11 | const o = createBridge();
12 | o.rotateY(180 * (Math.PI / 180));
13 | o.position.set(1, 0.1, -1.5);
14 | scene.add(o);
15 | teleportTargetObjects.push(o);
16 | }
17 | {
18 | const o = createBridge();
19 | o.rotateY(135 * (Math.PI / 180));
20 | o.position.set(2, 0.1, -1);
21 | scene.add(o);
22 | teleportTargetObjects.push(o);
23 | }
24 | const wallMaterial = new THREE.MeshLambertMaterial({
25 | color: 0x5e5e5e,
26 | side: THREE.DoubleSide,
27 | });
28 | {
29 | const wall = new THREE.Mesh(
30 | new THREE.PlaneGeometry(10, 3, 1, 1),
31 | wallMaterial
32 | );
33 | wall.position.set(-2, 0.5, 0);
34 | wall.rotation.y = Math.PI / 2;
35 | scene.add(wall);
36 | collisionObjects.push(wall);
37 | }
38 | {
39 | const wall = new THREE.Mesh(
40 | new THREE.PlaneGeometry(1, 3, 1, 1),
41 | wallMaterial
42 | );
43 | wall.position.set(-0.5, 0.5, -3);
44 | scene.add(wall);
45 | collisionObjects.push(wall);
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@verseengine/three-avatar",
3 | "version": "1.0.2",
4 | "description": "Avatar system for three.js",
5 | "author": "Appland, Inc",
6 | "license": "MIT",
7 | "main": "dist/index.js",
8 | "module": "dist/esm/index.js",
9 | "types": "dist/esm/index.d.ts",
10 | "repository": {
11 | "type": "git",
12 | "url": "https://github.com/VerseEngine/three-avatar"
13 | },
14 | "homepage": "https://verseengine.cloud/",
15 | "keywords": [
16 | "vr",
17 | "3d",
18 | "metaverse"
19 | ],
20 | "scripts": {
21 | "example": "npm run build && npx http-server -p 8080",
22 | "example-ssl": "npm run build && npx http-server -c-1 --ssl --key ./cert/localhost+2-key.pem --cert ./cert/localhost+2.pem -p 8080",
23 | "clean": "rimraf dist *.clean **/*.clean",
24 | "prepare": "npm run build",
25 | "prebuild": "rimraf dist",
26 | "build": "run-p build:*",
27 | "build:common": "esbuild --format=iife --sourcemap src/index.ts --tsconfig=tsconfig.json --bundle --packages=external --outfile=dist/index.js",
28 | "build:esm": "esbuild --format=esm --sourcemap src/index.ts --tsconfig=tsconfig.json --bundle --packages=external --outfile=dist/esm/index.js",
29 | "build:types": "tsc --emitDeclarationOnly --declaration --declarationDir dist/esm",
30 | "postbuild": "npx api-extractor run --local --verbose && npx api-documenter markdown -i dist/temp/ -o ./docs",
31 | "lint": "tsc --noEmit && npx eslint .",
32 | "check-update": "npx npm-check-updates"
33 | },
34 | "files": [
35 | "dist"
36 | ],
37 | "devDependencies": {
38 | "@microsoft/api-documenter": "^7.21.5",
39 | "@microsoft/api-extractor": "^7.34.4",
40 | "@types/three": ">=0.146.0",
41 | "@typescript-eslint/eslint-plugin": "^5.52.0",
42 | "@typescript-eslint/parser": "^5.52.0",
43 | "@verseengine/three-move-controller": "^1",
44 | "@verseengine/three-touch-controller": "^1",
45 | "@verseengine/three-xr-controller": "^1",
46 | "esbuild": "^0.17.8",
47 | "eslint": "^8.34.0",
48 | "eslint-plugin-tsdoc": "^0.2.17",
49 | "npm-run-all": "^4.1.5",
50 | "rimraf": "^4.1.2",
51 | "typescript": "^4.9.5"
52 | },
53 | "dependencies": {
54 | "@pixiv/three-vrm": "2",
55 | "three": "^0.153.0"
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/avatar-ik.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type { IKTargetBones } from "./types";
3 |
4 | import {
5 | CCDIKSolver,
6 | CCDIKHelper,
7 | } from "three/examples/jsm/animation/CCDIKSolver.js";
8 |
9 | const DEFAULT_INTERVAL_SEC = 1 / 60; // 60fps
10 |
11 | type IKTargetBonesStrict = [THREE.Bone, THREE.Bone, THREE.Bone];
12 | function isNotIncludeUndefined(
13 | bones: IKTargetBones
14 | ): bones is IKTargetBonesStrict {
15 | return !bones.includes(undefined);
16 | }
17 |
18 | class Tmps {
19 | vec: THREE.Vector3;
20 | vec1: THREE.Vector3;
21 | // vec2: THREE.Vector3;
22 | quat: THREE.Quaternion;
23 | quat1: THREE.Quaternion;
24 | mat: THREE.Matrix4;
25 |
26 | constructor() {
27 | this.vec = new THREE.Vector3();
28 | this.vec1 = new THREE.Vector3();
29 | // this.vec2 = new THREE.Vector3();
30 | this.quat = new THREE.Quaternion();
31 | this.quat1 = new THREE.Quaternion();
32 | this.mat = new THREE.Matrix4();
33 | }
34 | }
35 | let _tmps: Tmps;
36 |
37 | type SkeletonMesh = THREE.Object3D & { skeleton?: THREE.Skeleton };
38 | /**
39 | * Get XR controller objects.
40 | * Use to get the position of the IK hand.
41 | */
42 | export type VRHandGetter = {
43 | left?: () => THREE.Object3D | undefined;
44 | right?: () => THREE.Object3D | undefined;
45 | };
46 |
47 | /**
48 | * x,y,z
49 | */
50 | export type Vector3Tupple = [number, number, number];
51 | /**
52 | * Limit the range of rotation of joint.
53 | */
54 | export type RotationLimit = {
55 | rotationMin?: Vector3Tupple;
56 | rotationMax?: Vector3Tupple;
57 | };
58 | /**
59 | * Limit the range of rotation of joints.
60 | */
61 | export type RotationLimitSet = {
62 | leftArm: [RotationLimit, RotationLimit];
63 | rightArm: [RotationLimit, RotationLimit];
64 | };
65 | /**
66 | * Offset of rotation angle between the XR controller and the avatar's wrist.
67 | */
68 | export type WristRotationOffsetSet = {
69 | left: Vector3Tupple;
70 | right: Vector3Tupple;
71 | };
72 |
73 | /**
74 | * IK (Inverse Kinematics) to move the avatar's arms in sync with the XR controller's movements.
75 | * (Experimental Features)
76 | *
77 | * @example
78 | * {@link createAvatarIK}
79 | */
80 | export class AvatarIK {
81 | private _solver?: CCDIKSolver;
82 | private _vrMappings: VRMapping[];
83 | private _helper?: CCDIKHelper;
84 | private _intervalSec = 0;
85 | private _sec = 0;
86 |
87 | /**i
88 | *
89 | * @param target - Avatar object. Can be an object unrelated to the {@link Avatar} class.
90 | * @param maybeRightArmBones - Bones moved by IK. If it contains undefined, it is disabled.
91 | * @param maybeLeftArmBones - Bones moved by IK. If it contains undefined, it is disabled.
92 | * @param rotationLimitSet - Limit the range of rotation of joints.
93 | * @param vrHandGetter - Get XR controller objects.
94 | * @param wristRotationOffsetSet - Offset of rotation angle between the XR controller and the avatar's wrist.
95 | * @param options - Processing frequency of tick(). Default is 1 / 60 (30fps).
96 | */
97 | constructor(
98 | target: THREE.Object3D,
99 | maybeRightArmBones: IKTargetBones,
100 | maybeLeftArmBones: IKTargetBones,
101 | rotationLimitSet: RotationLimitSet,
102 | vrHandGetter: VRHandGetter,
103 | wristRotationOffsetSet: WristRotationOffsetSet,
104 | options?: {
105 | isDebug?: boolean;
106 | intervalSec?: number;
107 | }
108 | ) {
109 | if (!_tmps) {
110 | _tmps = new Tmps();
111 | }
112 | this._intervalSec =
113 | options?.intervalSec || options?.intervalSec === 0
114 | ? options.intervalSec
115 | : DEFAULT_INTERVAL_SEC;
116 |
117 | const person = target as SkeletonMesh;
118 |
119 | const leftArmBones = isNotIncludeUndefined(maybeLeftArmBones)
120 | ? maybeLeftArmBones
121 | : undefined;
122 | const rightArmBones = isNotIncludeUndefined(maybeRightArmBones)
123 | ? maybeRightArmBones
124 | : undefined;
125 |
126 | if (!person.skeleton) {
127 | const bones = [...(leftArmBones || []), ...(rightArmBones || [])];
128 | person.skeleton = new THREE.Skeleton(bones);
129 | }
130 |
131 | const skeleton = person.skeleton;
132 |
133 | const iks = [];
134 | const vrMappings: VRMapping[] = [];
135 | if (leftArmBones) {
136 | const { ik, vrMapping } = createIKSettings(
137 | skeleton,
138 | person,
139 | "leftArmIK",
140 | leftArmBones,
141 | rotationLimitSet.leftArm,
142 | vrHandGetter.left,
143 | wristRotationOffsetSet.left
144 | );
145 | iks.push(ik);
146 | vrMappings.push(vrMapping);
147 | }
148 | if (rightArmBones) {
149 | const { ik, vrMapping } = createIKSettings(
150 | skeleton,
151 | person,
152 | "rightArmIK",
153 | rightArmBones,
154 | rotationLimitSet.rightArm,
155 | vrHandGetter.right,
156 | wristRotationOffsetSet.right
157 | );
158 | iks.push(ik);
159 | vrMappings.push(vrMapping);
160 | }
161 |
162 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
163 | this._solver = new CCDIKSolver(person as any, iks as any);
164 | if (options?.isDebug) {
165 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
166 | this._helper = new CCDIKHelper(person as any, iks as any, 0.01);
167 | let el = person;
168 | while (el.parent) {
169 | el = el.parent;
170 | }
171 | el.add(this._helper);
172 | }
173 |
174 | this._vrMappings = vrMappings;
175 | }
176 | /**
177 | * Processes called periodically
178 | *
179 | * @example
180 | * ```ts
181 | * const clock = new THREE.Clock();
182 | * renderer.setAnimationLoop(() => {
183 | * const dt = clock.getDelta();
184 | * avatarIK.tick(dt);
185 | * });
186 | * ```
187 | * or
188 | * ```ts
189 | * const clock = new THREE.Clock();
190 | * setInterval(() => {
191 | * const dt = clock.getDelta();
192 | * avatarIK.tick(dt);
193 | * }, anything);
194 | * ```
195 | */
196 | tick(
197 | deltaTime: number // THREE.Clock.getDelta()
198 | ) {
199 | this._sec += deltaTime;
200 | if (this._sec < this._intervalSec) {
201 | return;
202 | }
203 | this._sec = 0;
204 |
205 | if (this._solver) {
206 | for (let i = 0; i < this._vrMappings.length; i++) {
207 | const m = this._vrMappings[i];
208 | const vrHandPos = m.getVrHand()?.position;
209 | if (
210 | vrHandPos &&
211 | !(vrHandPos.x === 0 && vrHandPos.y === 0 && vrHandPos.z === 0)
212 | ) {
213 | m.mapVRAvatar();
214 | } else {
215 | m.reset();
216 | }
217 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
218 | this._solver.updateOne((this._solver as any).iks[i]);
219 | }
220 | }
221 | }
222 | /**
223 | * Releases all resources allocated by this instance.
224 | */
225 | dispose() {
226 | delete this._solver;
227 | if (this._helper) {
228 | this._helper.removeFromParent();
229 | delete this._helper;
230 | }
231 | }
232 | }
233 |
234 | function findOrAddTargetBone(
235 | skeleton: THREE.Skeleton,
236 | person: THREE.Object3D,
237 | name: string,
238 | effectorBone: THREE.Bone
239 | ): THREE.Bone {
240 | const b = skeleton.bones.find((v) => v.name === name);
241 | if (b) {
242 | return b;
243 | }
244 | const targetBone = new THREE.Bone();
245 | targetBone.name = name;
246 | skeleton.bones.push(targetBone);
247 | skeleton.boneInverses.push(new THREE.Matrix4());
248 |
249 | skeleton.boneInverses[skeleton.boneInverses.length - 1]
250 | .copy(skeleton.bones[skeleton.bones.length - 1].matrixWorld)
251 | .invert();
252 | person.add(targetBone);
253 | person.updateMatrixWorld();
254 | const initPos = effectorBone.getWorldPosition(new THREE.Vector3());
255 | targetBone.position.copy(
256 | initPos.applyMatrix4(person.matrixWorld.clone().invert())
257 | );
258 | return targetBone;
259 | }
260 |
261 | const limitToVec = (ar: Vector3Tupple | undefined): THREE.Vector3 | undefined =>
262 | ar
263 | ? new THREE.Vector3().fromArray(ar.map(THREE.MathUtils.degToRad))
264 | : undefined;
265 |
266 | function createIKSettings(
267 | skeleton: THREE.Skeleton,
268 | person: THREE.Object3D,
269 | name: string,
270 | bones: IKTargetBonesStrict,
271 | rotationLimits: [RotationLimit, RotationLimit],
272 | vrHandGetter: (() => THREE.Object3D | undefined) | undefined,
273 | rotationOffset: Vector3Tupple
274 | ) {
275 | const targetBone = findOrAddTargetBone(skeleton, person, name, bones[2]);
276 | const ik = {
277 | target: skeleton.bones.findIndex((v) => v === targetBone),
278 | effector: skeleton.bones.findIndex((v) => v === bones[2]),
279 | iteration: 1,
280 | minAngle: -0.2,
281 | maxAngle: 0.2,
282 | links: [
283 | {
284 | index: skeleton.bones.findIndex((v) => v === bones[1]),
285 | rotationMin: limitToVec(rotationLimits[1].rotationMin),
286 | rotationMax: limitToVec(rotationLimits[1].rotationMax),
287 | },
288 | {
289 | index: skeleton.bones.findIndex((v) => v === bones[0]),
290 | rotationMin: limitToVec(rotationLimits[0].rotationMin),
291 | rotationMax: limitToVec(rotationLimits[0].rotationMax),
292 | },
293 | ],
294 | };
295 | const vrMapping = new VRMapping(
296 | person,
297 | () => vrHandGetter?.(),
298 | targetBone,
299 | bones[2],
300 | bones[0],
301 | new THREE.Quaternion().setFromEuler(
302 | new THREE.Euler(...rotationOffset.map((v) => THREE.MathUtils.degToRad(v)))
303 | )
304 | );
305 | return { ik, vrMapping };
306 | }
307 |
308 | class VRMapping {
309 | vrPerson: THREE.Object3D;
310 | getVrHand: () => THREE.Object3D | undefined;
311 | ikTarget: THREE.Object3D;
312 | ikTargetInitPos: THREE.Vector3;
313 | wristBone: THREE.Object3D;
314 | wristRotationOffset: THREE.Quaternion | undefined;
315 | wristBoneInitQuat: THREE.Quaternion;
316 | // maxDistanceSquared: number;
317 | maxShoulderToControllderDistanceSquared = 0;
318 | shoulderBone: THREE.Object3D;
319 |
320 | constructor(
321 | vrPerson: THREE.Object3D,
322 | getVrHand: () => THREE.Object3D | undefined,
323 | ikTarget: THREE.Object3D,
324 | wristBone: THREE.Object3D,
325 | shoulderBone: THREE.Object3D,
326 | wristRotationOffset?: THREE.Quaternion
327 | ) {
328 | this.vrPerson = vrPerson;
329 | this.getVrHand = getVrHand;
330 | this.ikTarget = ikTarget;
331 | this.ikTargetInitPos = new THREE.Vector3().copy(ikTarget.position);
332 | this.wristBone = wristBone;
333 | this.wristRotationOffset = wristRotationOffset;
334 | this.wristBoneInitQuat = new THREE.Quaternion().copy(wristBone.quaternion);
335 | this.shoulderBone = shoulderBone;
336 | /* this.maxDistanceSquared = shoulderBone
337 | .getWorldPosition(_tmps.vec)
338 | .distanceToSquared(wristBone.getWorldPosition(_tmps.vec1)); */
339 | }
340 |
341 | mapVRAvatar() {
342 | const vrHand = this.getVrHand();
343 | if (!vrHand) {
344 | return;
345 | }
346 |
347 | {
348 | /* {
349 | const v = this.shoulderBone
350 | .getWorldPosition(_tmps.vec)
351 | .distanceToSquared(this.wristBone.getWorldPosition(_tmps.vec1));
352 | if (this.maxDistanceSquared < v) {
353 | this.maxDistanceSquared = v;
354 | }
355 | }
356 | const controllerPos = vrHand.getWorldPosition(_tmps.vec);
357 | const shoulderPos = this.shoulderBone.getWorldPosition(_tmps.vec1);
358 | const shoulderToControllderDistanceSquared =
359 | shoulderPos.distanceToSquared(controllerPos);
360 | if (
361 | this.maxShoulderToControllderDistanceSquared <
362 | shoulderToControllderDistanceSquared
363 | ) {
364 | this.maxShoulderToControllderDistanceSquared =
365 | shoulderToControllderDistanceSquared;
366 | }
367 | if (shoulderToControllderDistanceSquared > this.maxDistanceSquared) {
368 | const alpha = Math.sqrt(
369 | this.maxDistanceSquared / this.maxShoulderToControllderDistanceSquared
370 | );
371 | this.ikTarget.position.copy(
372 | shoulderPos.lerp(controllerPos, alpha).applyMatrix4(
373 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
374 | _tmps.mat.copy(this.ikTarget.parent!.matrixWorld).invert()
375 | )
376 | );
377 | } else { */
378 | {
379 | const controllerPos = vrHand.getWorldPosition(_tmps.vec);
380 | this.ikTarget.position.copy(
381 | controllerPos.applyMatrix4(
382 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
383 | _tmps.mat.copy(this.ikTarget.parent!.matrixWorld).invert()
384 | )
385 | );
386 | }
387 | }
388 |
389 | {
390 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
391 | const parentWorldQuat = this.wristBone.parent!.getWorldQuaternion(
392 | _tmps.quat1
393 | );
394 | // https://github.com/mrdoob/three.js/issues/13704
395 | const vrHandQuat = vrHand
396 | .getWorldQuaternion(_tmps.quat)
397 | .premultiply(parentWorldQuat.invert());
398 |
399 | if (this.wristRotationOffset) {
400 | this.wristBone.quaternion.multiplyQuaternions(
401 | vrHandQuat,
402 | this.wristRotationOffset
403 | );
404 | } else {
405 | this.wristBone.quaternion.copy(vrHandQuat);
406 | }
407 | }
408 | }
409 | reset() {
410 | this.ikTarget.position.copy(this.ikTargetInitPos);
411 | this.wristBone.quaternion.copy(this.wristBoneInitQuat);
412 | }
413 | }
414 |
--------------------------------------------------------------------------------
/src/collider/simple-bounding-box-collider.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type { Avatar } from "../avatar";
3 |
4 | class Tmps {
5 | vec: THREE.Vector3;
6 | vec1: THREE.Vector3;
7 | vec2: THREE.Vector3;
8 | box: THREE.Box3;
9 | box1: THREE.Box3;
10 | mat: THREE.Matrix4;
11 | hits: [THREE.Box3, number][];
12 | constructor() {
13 | this.vec = new THREE.Vector3();
14 | this.vec1 = new THREE.Vector3();
15 | this.vec2 = new THREE.Vector3();
16 | this.box = new THREE.Box3();
17 | this.box1 = new THREE.Box3();
18 | this.mat = new THREE.Matrix4();
19 | this.hits = [];
20 | }
21 | }
22 | let _tmps: Tmps;
23 |
24 | /**
25 | * Avatar extension to determine collision using {@link https://threejs.org/docs/?q=Box3#api/en/math/Box3.setFromObject | BoundingBox}.
26 | */
27 | export class SimpleBoundingBoxCollider {
28 | private _getBoxes: () => THREE.Box3[] | undefined;
29 | private _moveTarget: THREE.Object3D;
30 | private _isSetup = false;
31 | private _height = 0;
32 | private _halfWidthX = 0;
33 | private _halfWidthZ = 0;
34 | private _lastX?: number;
35 | private _lastZ?: number;
36 |
37 | /**
38 | * @param moveTarget - Objects to move.
39 | * @param getBoxes - Get a list of bounding boxes of collisionable objects.
40 | * @param options - Processing frequency of tick(). Default is 1 / 30 (30fps).
41 | */
42 | constructor(
43 | moveTarget: THREE.Object3D,
44 | getBoxes: () => THREE.Box3[] | undefined
45 | ) {
46 | this._moveTarget = moveTarget;
47 | this._getBoxes = getBoxes;
48 | }
49 | setup(avatar: Avatar) {
50 | if (!_tmps) {
51 | _tmps = new Tmps();
52 | }
53 | this._isSetup = true;
54 | this._height = avatar.height;
55 | this._halfWidthX = avatar.widthX / 2;
56 | this._halfWidthZ = avatar.widthZ / 2;
57 | }
58 | moveTo(x: number, y: number, z: number) {
59 | if (!this._isSetup) {
60 | return;
61 | }
62 | const [targetBox, targetBoxUpDown] = this._getTargetBox(x, y, z);
63 | const targetPos = _tmps.vec2.set(x, y, z);
64 |
65 | {
66 | const hits = _tmps.hits;
67 | hits.length = 0;
68 | for (const box of this._getBoxes() || []) {
69 | if (targetBoxUpDown.intersectsBox(box)) {
70 | if (
71 | box.max.y >= targetBoxUpDown.min.y &&
72 | box.max.y <= targetBoxUpDown.max.y
73 | ) {
74 | hits.push([box, targetBoxUpDown.max.y - box.max.y]);
75 | } else if (
76 | targetBoxUpDown.max.y >= box.max.y &&
77 | targetBoxUpDown.min.y <= box.max.y
78 | ) {
79 | hits.push([box, box.max.y - targetBoxUpDown.min.y]);
80 | }
81 | }
82 | }
83 | if (hits.length !== 0) {
84 | hits.sort((a, b) => a[1] - b[1]);
85 | // console.log(hits.map((v) => `${v[0].max.y}(${v[1]})`).join(", "));
86 | if (hits[0][0].max.y !== targetBox.min.y) {
87 | const pos = this._moveTarget.getWorldPosition(_tmps.vec1);
88 | pos.y = hits[0][0].max.y;
89 | // console.log("move y", pos.y);
90 | targetPos.copy(
91 | pos.applyMatrix4(
92 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
93 | _tmps.mat.copy(this._moveTarget.parent!.matrixWorld).invert()
94 | )
95 | );
96 | targetBox.min.y = hits[0][0].max.y;
97 | targetBox.max.y = targetBox.min.y + this._height;
98 | }
99 | } else {
100 | return;
101 | }
102 | }
103 | for (const box of this._getBoxes() || []) {
104 | if (targetBox.intersectsBox(box)) {
105 | if (box.max.y === targetBox.min.y) {
106 | // ground
107 | } else {
108 | if (this._lastX !== undefined && this._lastZ !== undefined) {
109 | if (
110 | this._lastX > targetPos.x &&
111 | (box.max.x <= targetBox.min.x ||
112 | Math.abs(targetBox.min.x - box.max.x) <
113 | Math.abs(targetBox.max.x - box.max.x))
114 | ) {
115 | return;
116 | } else if (
117 | this._lastX < targetPos.x &&
118 | (box.min.x >= targetBox.max.x ||
119 | Math.abs(targetBox.max.x - box.min.x) <
120 | Math.abs(targetBox.min.x - box.min.x))
121 | ) {
122 | return;
123 | }
124 |
125 | if (
126 | this._lastZ > targetPos.z &&
127 | (box.max.z <= targetBox.min.z ||
128 | Math.abs(targetBox.min.z - box.max.z) <
129 | Math.abs(targetBox.max.z - box.max.z))
130 | ) {
131 | return;
132 | } else if (
133 | this._lastZ < targetPos.z &&
134 | (box.min.z >= targetBox.max.z ||
135 | Math.abs(targetBox.max.z - box.min.z) <
136 | Math.abs(targetBox.min.z - box.min.z))
137 | ) {
138 | return;
139 | }
140 | }
141 | }
142 | }
143 | }
144 | this._moveTarget.position.copy(targetPos);
145 | this._lastX = targetPos.x;
146 | this._lastZ = targetPos.z;
147 | }
148 | private _getTargetBox(
149 | x: number,
150 | y: number,
151 | z: number
152 | ): [THREE.Box3, THREE.Box3] {
153 | const targetBox = _tmps.box;
154 | targetBox.min.set(x - this._halfWidthX, y, z - this._halfWidthZ);
155 | targetBox.max.set(
156 | x + this._halfWidthX,
157 | y + this._height,
158 | z + this._halfWidthZ
159 | );
160 | const targetBoxUpDown = _tmps.box1.copy(targetBox);
161 | targetBoxUpDown.min.y = y - 0.5;
162 | targetBoxUpDown.max.y = y + 0.5;
163 |
164 | return [targetBox, targetBoxUpDown];
165 | }
166 | }
167 |
--------------------------------------------------------------------------------
/src/defaults.ts:
--------------------------------------------------------------------------------
1 | import {
2 | AvatarType,
3 | AvatarTypeVrmV0,
4 | AvatarTypeVrmV1,
5 | AvatarTypeReadyPlayerMe,
6 | } from "./avatar";
7 | import type { RotationLimitSet, WristRotationOffsetSet } from "./avatar-ik";
8 | const RotationLimitSetVrmV1: RotationLimitSet = {
9 | leftArm: [
10 | {
11 | rotationMin: [0, -120, -80],
12 | rotationMax: [0, 5, 80],
13 | },
14 | {
15 | rotationMin: [0, -150, 0],
16 | rotationMax: [0, 0, 0],
17 | },
18 | ],
19 | rightArm: [
20 | {
21 | rotationMin: [0, -5, -80],
22 | rotationMax: [0, 120, 80],
23 | },
24 | {
25 | rotationMin: [0, 0, 0],
26 | rotationMax: [0, 150, 0],
27 | },
28 | ],
29 | };
30 | const RotationLimitSetVrmV0: RotationLimitSet = {
31 | leftArm: [
32 | {
33 | rotationMin: [0, -120, -80],
34 | rotationMax: [0, 5, 80],
35 | },
36 | {
37 | rotationMin: [0, -150, 0],
38 | rotationMax: [0, 0, 0],
39 | },
40 | ],
41 | rightArm: [
42 | {
43 | rotationMin: [0, -5, -80],
44 | rotationMax: [0, 120, 80],
45 | },
46 | {
47 | rotationMin: [0, 0, 0],
48 | rotationMax: [0, 150, 0],
49 | },
50 | ],
51 | };
52 | const RotationLimitSetRpm: RotationLimitSet = {
53 | leftArm: [
54 | {
55 | rotationMin: [-80, 0, -5],
56 | rotationMax: [80, 0, 120],
57 | },
58 | {
59 | rotationMin: [0, 0, 0],
60 | rotationMax: [0, 0, 150],
61 | },
62 | ],
63 | rightArm: [
64 | {
65 | rotationMin: [-80, 0, -120],
66 | rotationMax: [80, 0, 5],
67 | },
68 | {
69 | rotationMin: [0, 0, -150],
70 | rotationMax: [0, 0, 0],
71 | },
72 | ],
73 | };
74 |
75 | export const RotationLimitHeadSync = {
76 | x: { min: -0.6, max: 0.6 },
77 | y: { min: -0.6, max: 0.6 },
78 | z: { min: 0, max: 0 },
79 | };
80 |
81 | /**
82 | * Default value of {@link RotationLimitSet}.
83 | */
84 | export function getDefaultRotationLimitSet(t: AvatarType): RotationLimitSet {
85 | switch (t) {
86 | case AvatarTypeVrmV0:
87 | return RotationLimitSetVrmV0;
88 | case AvatarTypeVrmV1:
89 | return RotationLimitSetVrmV1;
90 | case AvatarTypeReadyPlayerMe:
91 | return RotationLimitSetRpm;
92 | }
93 | throw new Error(`unknown avatar type: ${t}`);
94 | }
95 | const WristRotationOffsetSetVrmV0: WristRotationOffsetSet = {
96 | left: [110, 0, 70],
97 | right: [110, 0, -70],
98 | };
99 | const WristRotationOffsetSetVrmV1: WristRotationOffsetSet = {
100 | left: [-85, 28, 57],
101 | right: [-85, -28, -57],
102 | };
103 | const WristRotationOffsetSetRpm: WristRotationOffsetSet = {
104 | left: [-90, 90, 20],
105 | right: [-90, -90, 20],
106 | };
107 |
108 | /**
109 | * Default value of {@link WristRotationOffsetSet}.
110 | */
111 | export function getDefaultWristRotationOffsetSet(
112 | t: AvatarType
113 | ): WristRotationOffsetSet {
114 | switch (t) {
115 | case AvatarTypeVrmV0:
116 | return WristRotationOffsetSetVrmV0;
117 | case AvatarTypeVrmV1:
118 | return WristRotationOffsetSetVrmV1;
119 | case AvatarTypeReadyPlayerMe:
120 | return WristRotationOffsetSetRpm;
121 | }
122 | throw new Error(`unknown avatar type: ${t}`);
123 | }
124 |
--------------------------------------------------------------------------------
/src/ext/auto-walker.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type { Avatar } from "../avatar";
3 | import type { AvatarExtension } from "./avatar-extension";
4 |
5 | /**
6 | * Avatar extension for automatic walking animation when moving.
7 | */
8 | export class AutoWalker implements AvatarExtension {
9 | private _isWalking = false;
10 | private _walkCheckTimer = 0;
11 | private _avatar?: WeakRef;
12 | private _object3D?: THREE.Object3D;
13 | private _lastPosition?: THREE.Vector3;
14 | private _tmpVec?: THREE.Vector3;
15 |
16 | /**
17 | * {@inheritDoc AvatarExtension.setup}
18 | */
19 | setup(avatar: Avatar) {
20 | this._avatar = new WeakRef(avatar);
21 | this._object3D = avatar.object3D;
22 | this._tmpVec = new THREE.Vector3();
23 | }
24 | /**
25 | * {@inheritDoc AvatarExtension.tick}
26 | */
27 | tick(dt: number) {
28 | /* eslint-disable @typescript-eslint/no-non-null-assertion */
29 | const object3D = this._object3D;
30 | if (!object3D?.visible) {
31 | return;
32 | }
33 |
34 | const newPos = object3D.getWorldPosition(this._tmpVec!);
35 | if (!this._lastPosition) {
36 | this._lastPosition = new THREE.Vector3().copy(newPos);
37 | } else if (
38 | Math.abs(this._lastPosition.x - newPos.x) > 0.05 ||
39 | Math.abs(this._lastPosition.z - newPos.z) > 0.05
40 | ) {
41 | this._lastPosition.copy(newPos);
42 | if (!this._isWalking) {
43 | this._isWalking = true;
44 | this._avatar?.deref()?.playClip("walk");
45 | }
46 | this._walkCheckTimer = 0;
47 | } else {
48 | if (this._isWalking) {
49 | this._walkCheckTimer += dt;
50 | if (this._walkCheckTimer > 0.3) {
51 | this._isWalking = false;
52 | this._avatar?.deref()?.playClip("idle");
53 | }
54 | }
55 | }
56 | /* eslint-enable */
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/src/ext/avatar-extension.ts:
--------------------------------------------------------------------------------
1 | import type { Avatar } from "../avatar";
2 |
3 | /**
4 | * Interface to extend {@link Avatar}.
5 | */
6 | export interface AvatarExtension {
7 | /**
8 | * initialization
9 | */
10 | setup(avatar: Avatar): void;
11 | /**
12 | * Processes called periodically
13 | */
14 | tick(deltaTime: number): void;
15 | /**
16 | * Releases all resources allocated by this instance.
17 | */
18 | dispose?(): void;
19 | }
20 |
--------------------------------------------------------------------------------
/src/ext/blinker.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type * as THREE_VRM from "@pixiv/three-vrm";
3 | import type { Avatar } from "../avatar";
4 | import type { AvatarExtension } from "./avatar-extension";
5 |
6 | /**
7 | * Avatar extension to implement blink.
8 | */
9 | export class Blinker implements AvatarExtension {
10 | private _faceMesh?: THREE.Mesh;
11 | private _vrm?: THREE_VRM.VRM;
12 |
13 | private _blinking = false;
14 | private _blinkProgress = 0;
15 | private _blinkTiming = 0;
16 | private _blinkSpeed = 20;
17 | private _blinkPrev = 0;
18 |
19 | /**
20 | * {@inheritDoc AvatarExtension.setup}
21 | */
22 | setup(avatar: Avatar) {
23 | this._faceMesh = avatar.faceMesh;
24 | this._vrm = avatar.vrm;
25 | }
26 |
27 | /**
28 | * {@inheritDoc AvatarExtension.tick}
29 | */
30 | tick(dt: number) {
31 | if (!this._blinking) {
32 | this._blinkTiming += dt;
33 | if (this._blinkTiming > 1) {
34 | this._blinkTiming = 0;
35 | if (Math.random() < 0.2) {
36 | this._blinking = true;
37 | this._blinkProgress = 0;
38 | }
39 | }
40 | } else {
41 | this._blinkProgress += dt * this._blinkSpeed;
42 | let i = Math.sin(this._blinkProgress);
43 | if (i < 0.1 && i < this._blinkPrev) {
44 | this._blinking = false;
45 | this._blinkTiming = -2;
46 | i = 0;
47 | }
48 | this._blinkPrev = i;
49 | if (this._vrm) {
50 | this._vrm.expressionManager?.setValue("blink", i);
51 | } else if (this._faceMesh) {
52 | const faceMesh = this._faceMesh;
53 | const idx = faceMesh.morphTargetDictionary?.eyesClosed;
54 | if (idx && faceMesh.morphTargetInfluences) {
55 | faceMesh.morphTargetInfluences[idx] = i;
56 | }
57 | }
58 | }
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/external/threelipsync-mod/LICENSE.md:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2016 Gerard Llorach Tó
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/src/external/threelipsync-mod/README.md:
--------------------------------------------------------------------------------
1 | # ThreeLS - threelipsync
2 |
3 | Computes the weights of THREE blend shapes (kiss, lips closed and mouth open/jaw) from an audio stream in real-time. The algorithm calculates the energies of THREE frequency bands and maps them to the blend shapes with simple equations.
4 |
5 | To use the microphone call startMic(). To use an external audio file from an URL use startSample(URL). Remember that the webpage should be https in order to use the microphone.
6 |
7 | An explanation about the theory behind the algorithm can be found here:
8 | https://www.youtube.com/watch?v=89pBiGKXpZI
9 |
10 | Here is the package for Unity:
11 | https://doi.org/10.5281/zenodo.5765691
12 |
13 | Reference:
14 | Llorach, G., Evans, A., Blat, J., Grimm, G. and Hohmann, V., 2016, September. Web-based live speech-driven lip-sync. In 2016 8th International Conference on Games and Virtual Worlds for Serious Applications (VS-GAMES) (pp. 1-4). IEEE.
15 |
--------------------------------------------------------------------------------
/src/external/threelipsync-mod/threelipsync.ts:
--------------------------------------------------------------------------------
1 | // @ts-nocheck
2 |
3 | Lipsync.prototype.refFBins = [0, 500, 700, 3000, 6000];
4 | /**
5 | * Slightly customized {@link https://github.com/gerardllorach/threelipsync | threelipsync module}.
6 | *
7 | *
8 | * @remarks
9 | * ```
10 | --------------------- THREELIPSYNC MODULE --------------------
11 | Computes the values of THREE blend shapes (kiss, lips closed and mouth open/jaw)
12 | To do so, it computes the energy of THREE frequency bands in real time.
13 | he webpage needs to be https in order to get the microphone. If using external
14 | audio files from URL, they need to be from a https origin.
15 |
16 | Author: Gerard Llorach
17 | Paper: G. Llorach, A. Evans, J. Blat, G. Grimm, V. Hohmann. Web-based live speech-driven
18 | lip-sync, Proceedings of VS-Games 2016, September, Barcelona
19 | Date: Nov 2016
20 | https://repositori.upf.edu/bitstream/handle/10230/28139/llorach_VSG16_web.pdf
21 | License: MIT
22 | * ```
23 | */
24 | export function Lipsync(
25 | context: AudioContext,
26 | ms: MediaStream,
27 | threshold?: number,
28 | smoothness?: number,
29 | pitch?: number
30 | ) {
31 | this.context = context;
32 | // Freq analysis bins, energy and lipsync vectors
33 | this.energy = [0, 0, 0, 0, 0, 0, 0, 0];
34 | this.lipsyncBSW = [0, 0, 0];
35 |
36 | // Lipsync parameters
37 | this.threshold = threshold || 0.45;
38 | this.smoothness = smoothness || 0.6;
39 | this.pitch = pitch || 1;
40 | // Change freq bins according to pitch
41 | this.defineFBins(this.pitch);
42 |
43 | // Initialize buffers
44 | this.init();
45 |
46 | this.sample = this.context.createMediaStreamSource(ms);
47 | this.sample.connect(this.analyser);
48 | }
49 |
50 | // Define fBins
51 | Lipsync.prototype.defineFBins = function (pitch: number) {
52 | this.fBins = this.refFBins.map((v: number) => v * pitch);
53 | };
54 |
55 | // Audio buffers and analysers
56 | Lipsync.prototype.init = function () {
57 | // Sound source
58 | this.sample = this.context.createBufferSource();
59 | // Gain Node
60 | this.gainNode = this.context.createGain();
61 | // Analyser
62 | this.analyser = this.context.createAnalyser();
63 | // FFT size
64 | this.analyser.fftSize = 1024;
65 | // FFT smoothing
66 | this.analyser.smoothingTimeConstant = this.smoothness;
67 |
68 | // FFT buffer
69 | this.data = new Float32Array(this.analyser.frequencyBinCount);
70 | };
71 |
72 | // Update lipsync weights
73 | Lipsync.prototype.update = function (): [number, number, number] {
74 | // Short-term power spectrum
75 | this.analyser.getFloatFrequencyData(this.data);
76 | // Analyze energies
77 | this.binAnalysis();
78 | // Calculate lipsync blenshape weights
79 | this.lipAnalysis();
80 | // Return weights
81 | return this.lipsyncBSW;
82 | };
83 | // Analyze energies
84 | Lipsync.prototype.binAnalysis = function () {
85 | // Signal properties
86 | const nfft = this.analyser.frequencyBinCount;
87 | const fs = this.context.sampleRate;
88 |
89 | const fBins = this.fBins;
90 | const energy = this.energy;
91 |
92 | // Energy of bins
93 | for (let binInd = 0; binInd < fBins.length - 1; binInd++) {
94 | // Start and end of bin
95 | const indxIn = Math.round((fBins[binInd] * nfft) / (fs / 2));
96 | const indxEnd = Math.round((fBins[binInd + 1] * nfft) / (fs / 2));
97 |
98 | // Sum of freq values
99 | energy[binInd] = 0;
100 | for (let i = indxIn; i < indxEnd; i++) {
101 | // data goes from -25 to -160 approx
102 | // default threshold: 0.45
103 | let value = this.threshold + (this.data[i] + 20) / 140;
104 | // Zeroes negative values
105 | value = value > 0 ? value : 0;
106 |
107 | energy[binInd] += value;
108 | }
109 | // Divide by number of sumples
110 | energy[binInd] /= indxEnd - indxIn;
111 | }
112 | };
113 |
114 | // Calculate lipsyncBSW
115 | Lipsync.prototype.lipAnalysis = function () {
116 | const energy = this.energy;
117 |
118 | if (energy !== undefined) {
119 | let value = 0;
120 |
121 | // Kiss blend shape
122 | // When there is energy in the 1 and 2 bin, blend shape is 0
123 | value = (0.5 - energy[2]) * 2;
124 | if (energy[1] < 0.2) value = value * (energy[1] * 5);
125 | value = Math.max(0, Math.min(value, 1)); // Clip
126 | this.lipsyncBSW[0] = value;
127 |
128 | // Lips closed blend shape
129 | value = energy[3] * 3;
130 | value = Math.max(0, Math.min(value, 1)); // Clip
131 | this.lipsyncBSW[1] = value;
132 |
133 | // Jaw blend shape
134 | value = energy[1] * 0.8 - energy[3] * 0.8;
135 | value = Math.max(0, Math.min(value, 1)); // Clip
136 | this.lipsyncBSW[2] = value;
137 | }
138 | };
139 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Avatar system
3 | * @packageDocumentation
4 | */
5 | export {
6 | Avatar,
7 | AvatarOptions,
8 | AvatarType,
9 | AvatarTypeVrmV0,
10 | AvatarTypeVrmV1,
11 | AvatarTypeReadyPlayerMe,
12 | } from "./avatar";
13 | export {
14 | registerSyncAvatarHeadAndCamera,
15 | getRenderInfo,
16 | setNonVRCameraMode,
17 | addMirrorHUD,
18 | AddMirrorHUDOptions,
19 | isLowSpecDevice,
20 | isAndroid,
21 | isIOS,
22 | isTouchDevice,
23 | } from "./util";
24 | export type { AvatarModel, IKTargetBones } from "./types";
25 | export {
26 | preLoadAnimationData,
27 | isAnimationDataLoaded,
28 | loadAvatarModel,
29 | AvatarAnimationDataSource,
30 | DecordersOptions,
31 | } from "./loader";
32 | export {
33 | AvatarIK,
34 | Vector3Tupple,
35 | RotationLimit,
36 | RotationLimitSet,
37 | WristRotationOffsetSet,
38 | VRHandGetter,
39 | } from "./avatar-ik";
40 | export {
41 | getDefaultRotationLimitSet,
42 | getDefaultWristRotationOffsetSet,
43 | } from "./defaults";
44 |
45 | export type { AvatarExtension } from "./ext/avatar-extension";
46 | export { Blinker } from "./ext/blinker";
47 | export { AutoWalker } from "./ext/auto-walker";
48 | export { SimpleBoundingBoxCollider } from "./collider/simple-bounding-box-collider";
49 |
50 | // @ts-ignore
51 | export { Lipsync } from "./external/threelipsync-mod/threelipsync";
52 |
53 | import type * as THREE from "three";
54 | import { Avatar, AvatarOptions } from "./avatar";
55 | import { loadAvatarModel, DecordersOptions } from "./loader";
56 | import { Blinker } from "./ext/blinker";
57 | import { AutoWalker } from "./ext/auto-walker";
58 | import {
59 | AvatarIK,
60 | VRHandGetter,
61 | RotationLimitSet,
62 | WristRotationOffsetSet,
63 | } from "./avatar-ik";
64 | import {
65 | getDefaultRotationLimitSet,
66 | getDefaultWristRotationOffsetSet,
67 | } from "./defaults";
68 |
69 | export interface CreateAvatarOptions extends DecordersOptions, AvatarOptions {
70 | /**
71 | * Not displayed with a first-person camera. ( But it will be shown in mirrors, etc.).
72 | */
73 | isInvisibleFirstPerson?: boolean;
74 | /**
75 | * Reduces resource consumption by omitting some processes.
76 | */
77 | isLowSpecMode?: boolean;
78 | }
79 |
80 | /**
81 | * Create {@link Avatar}
82 | *
83 | * @param avatarData - Data from gltf or vrm files.
84 | * @param frustumCulled - {@link https://threejs.org/docs/?q=Mesh#api/en/core/Object3D.frustumCulled | Object3D.frustumCulled } applied recursively.
85 | *
86 | * @example
87 | ```ts
88 |
89 | let resp = await fetch(url);
90 | const avatarData = new Uint8Array(await resp.arrayBuffer());
91 | const avatar = createAvatar(avatarData, renderer, false, {
92 | isInvisibleFirstPerson: true,
93 | isLowSpecMode: maybeLowSpecDevice,
94 | });
95 | playerObj.add(avatar.object3D);
96 |
97 | ...
98 |
99 | const clock = new THREE.Clock();
100 | renderer.setAnimationLoop(() => {
101 | ...
102 | const dt = clock.getDelta();
103 | avatar.tick(dt);
104 | });
105 | ```
106 | */
107 | export async function createAvatar(
108 | avatarData: Uint8Array,
109 | renderer: THREE.WebGLRenderer,
110 | frustumCulled?: boolean,
111 | options?: CreateAvatarOptions
112 | ): Promise {
113 | const model = await loadAvatarModel(
114 | avatarData,
115 | renderer,
116 | frustumCulled,
117 | options
118 | );
119 | const res = new Avatar(model, options);
120 | if (options?.isInvisibleFirstPerson) {
121 | res.invisibleFirstPerson();
122 | }
123 | res.object3D.updateMatrixWorld();
124 | setDefaultExtensions(res);
125 | if (options?.isLowSpecMode) {
126 | if (res.vrm) {
127 | type unsafeVrm = {
128 | springBoneManager?: object;
129 | };
130 | delete (res.vrm as unsafeVrm).springBoneManager;
131 | }
132 | }
133 | return res;
134 | }
135 |
136 | /**
137 | * Set up default extensions for {@link Avatar}
138 | * @param moveTarget - Objects to move. Specify the object that contains {@link Avatar.object3D}.
139 | */
140 | export function setDefaultExtensions(avatar: Avatar): void {
141 | avatar.addExtension(new Blinker());
142 | avatar.addExtension(new AutoWalker());
143 | }
144 |
145 | export interface CreateAvatarIKOptions {
146 | /**
147 | * Limit the range of rotation of joints. Default is {@link getDefaultRotationLimitSet}.
148 | */
149 | rotationLimitSet: RotationLimitSet;
150 | /**
151 | * Offset of rotation angle between the XR controller and the avatar's wrist. Default is {@link getDefaultWristRotationOffsetSet}.
152 | */
153 | wristRotationOffsetSet: WristRotationOffsetSet;
154 | /**
155 | * Processing frequency of tick(). Default is 1 / 60 (30fps).
156 | */
157 | intervalSec?: number;
158 | isDebug?: boolean;
159 | }
160 |
161 | /**
162 | * Create {@link AvatarIK}.
163 | * (Experimental Features)
164 | *
165 | * @param handGetter - Get XR controller objects.
166 | *
167 | * @example
168 | * ```ts
169 |
170 | ...
171 |
172 | onVR: async () => {
173 | await avatar.setIKMode(true);
174 | avatarIK = createAvatarIK(
175 | avatar,
176 | {
177 | left: () => leftXRController,
178 | right: () => rightXRController,
179 | },
180 | );
181 | },
182 | onNonVR: async () => {
183 | await avatar.setIKMode(false);
184 | avatarIK?.dispose();
185 | }
186 |
187 | ...
188 |
189 |
190 | const clock = new THREE.Clock();
191 | renderer.setAnimationLoop(() => {
192 | ...
193 | const dt = clock.getDelta();
194 | avatar.tick(dt);
195 | avatarIK?.tick(dt);
196 | });
197 | * ```
198 | */
199 | export function createAvatarIK(
200 | avatar: Avatar,
201 | handGetter: VRHandGetter,
202 | option?: CreateAvatarIKOptions
203 | ) {
204 | return new AvatarIK(
205 | avatar.object3D,
206 | avatar.ikTargetRightArmBones,
207 | avatar.ikTargetLeftArmBones,
208 | option?.rotationLimitSet || getDefaultRotationLimitSet(avatar.type),
209 | handGetter,
210 | option?.wristRotationOffsetSet ||
211 | getDefaultWristRotationOffsetSet(avatar.type),
212 | {
213 | isDebug: option?.isDebug || false,
214 | intervalSec: option?.intervalSec,
215 | }
216 | );
217 | }
218 |
--------------------------------------------------------------------------------
/src/loader.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import * as THREE_VRM from "@pixiv/three-vrm";
3 | import {
4 | GLTFLoader,
5 | GLTFParser,
6 | } from "three/examples/jsm/loaders/GLTFLoader.js";
7 | import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader.js";
8 | import { DRACOLoader } from "three/examples/jsm/loaders/DRACOLoader.js";
9 | import { KTX2Loader } from "three/examples/jsm/loaders/KTX2Loader.js";
10 | import { convertForReadyPlayerMe, convertForVrm } from "./mixamo";
11 | import { AvatarModel } from "./types";
12 | // import { MeshoptDecoder } from "meshoptimizer";
13 |
14 | // https://github.com/google/draco
15 | const DRACO_DECODER_PATH =
16 | "https://www.gstatic.com/draco/versioned/decoders/1.5.5/";
17 | const BASIS_TRANSCODER_PATH =
18 | "https://cdn.jsdelivr.net/npm/three@0.149.0/examples/jsm/libs/basis/";
19 | const MESHOPT_DECODER_PATH =
20 | "https://cdn.jsdelivr.net/npm/meshoptimizer@0.18.1/meshopt_decoder.js";
21 |
22 | /**
23 | * URL of the library required if GLTF data is compressed. If omitted, decompress with default values.
24 | *
25 | * @remarks
26 | * {@link https://threejs.org/docs/#examples/en/loaders/DRACOLoader | DRACOLoader}, {@link https://threejs.org/docs/?q=KTX2Loader#examples/en/loaders/KTX2Loader | KTX2Loader}, {@link https://threejs.org/docs/?q=gltfl#examples/en/loaders/GLTFLoader | GLTFLoader}
27 | */
28 | export type DecordersOptions = {
29 | dracoDecoderPath?: string;
30 | basisTranscoderPath?: string;
31 | meshoptDecoderPath?: string;
32 | };
33 |
34 | type AvatarAnimationData = {
35 | walk: THREE.Group;
36 | idle: THREE.Group;
37 | [key: string]: THREE.Group;
38 | };
39 | let avatarAnimationData: AvatarAnimationData | null = null;
40 |
41 | // export type AvatarDataType = "mixamo" | "readyplayerme" | "vrm";
42 | // type FileType = "fbx" | "gltf";
43 |
44 | /**
45 | * {@link https://www.mixamo.com/ | mixamo} animation file URLs
46 | *
47 | * @example
48 | * ```ts
49 | const ANIMATION_MAP = {
50 | idle: "asset/animation/idle.fbx",
51 | walk: "asset/animation/walk.fbx",
52 | dance: "asset/animation/dance.fbx",
53 | };
54 | * ```
55 | */
56 | export type AvatarAnimationDataSource = {
57 | walk: string;
58 | idle: string;
59 | [key: string]: string;
60 | };
61 |
62 | /**
63 | * Whether animation data is pre-loaded or not
64 | */
65 | export function isAnimationDataLoaded() {
66 | return !!avatarAnimationData;
67 | }
68 |
69 | /**
70 | * Pre-load animation data.
71 | *
72 | * @example
73 | ```ts
74 | // Animation fbx files downloaded from mixamo.com
75 | const ANIMATION_MAP = {
76 | idle: "path/to/idle.fbx",
77 | walk: "path/to/walk.fbx",
78 | dance: "path/to/dance.fbx",
79 | ...
80 | };
81 | if (!isAnimationDataLoaded()) {
82 | await preLoadAnimationData(ANIMATION_MAP);
83 | }
84 | ```
85 | */
86 | export async function preLoadAnimationData(
87 | source: AvatarAnimationDataSource,
88 | fetchFunc: (url: string) => Promise = fetch
89 | ) {
90 | const ps = Object.entries(source).map(([k, v]) => {
91 | return (async (k) => {
92 | const res = await (await fetchFunc(v)).arrayBuffer();
93 | const asset = new FBXLoader().parse(
94 | res,
95 | "about:blank" // No additional data is needed. about:blank
96 | );
97 | return [k, asset];
98 | })(k);
99 | });
100 | avatarAnimationData = Object.fromEntries(await Promise.all(ps));
101 | }
102 |
103 | async function createGLTFLoader(
104 | renderer: THREE.WebGLRenderer,
105 | options?: DecordersOptions
106 | ) {
107 | const loader = new GLTFLoader();
108 |
109 | const dracoLoader = new DRACOLoader();
110 | dracoLoader.setDecoderPath(options?.dracoDecoderPath || DRACO_DECODER_PATH);
111 | loader.setDRACOLoader(dracoLoader);
112 |
113 | const ktx2Loader = new KTX2Loader();
114 | ktx2Loader
115 | .setTranscoderPath(options?.basisTranscoderPath || BASIS_TRANSCODER_PATH)
116 | .detectSupport(renderer);
117 | loader.setKTX2Loader(ktx2Loader);
118 |
119 | const meshoptDecoder = await fetchScript(
120 | options?.meshoptDecoderPath || MESHOPT_DECODER_PATH
121 | )
122 | .then(() => {
123 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
124 | return (window as any).MeshoptDecoder.ready;
125 | })
126 | .then(() => {
127 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
128 | return (window as any).MeshoptDecoder;
129 | });
130 | loader.setMeshoptDecoder(meshoptDecoder);
131 |
132 | loader.register((parser: GLTFParser) => {
133 | return new THREE_VRM.VRMLoaderPlugin(parser);
134 | });
135 | return loader;
136 | }
137 |
138 | /**
139 | * Load avatar file data.
140 | *
141 | * @param avatarData - Data from gltf or vrm files.
142 | * @param frustumCulled - {@link https://threejs.org/docs/?q=Mesh#api/en/core/Object3D.frustumCulled | Object3D.frustumCulled } applied recursively.
143 | *
144 | * @example
145 | ```ts
146 | let resp = await fetch(url);
147 | const avatarData = new Uint8Array(await resp.arrayBuffer());
148 | const model = await loadAvatarModel(avatarData, renderer, false);
149 | ```
150 | */
151 | export async function loadAvatarModel(
152 | avatarData: Uint8Array,
153 | renderer: THREE.WebGLRenderer,
154 | frustumCulled?: boolean,
155 | options?: DecordersOptions
156 | ): Promise {
157 | let model: THREE.Group | undefined;
158 | const gltfLoader = await createGLTFLoader(renderer, options);
159 | const gltf = await gltfLoader.parseAsync(
160 | avatarData.buffer,
161 | "about:blank" // Assume only avatars consisting of a single file.
162 | );
163 | const vrm: THREE_VRM.VRM = gltf.userData.vrm;
164 | if (vrm) {
165 | THREE_VRM.VRMUtils.removeUnnecessaryVertices(gltf.scene);
166 | THREE_VRM.VRMUtils.removeUnnecessaryJoints(gltf.scene);
167 | THREE_VRM.VRMUtils.rotateVRM0(vrm);
168 |
169 | model = vrm.scene;
170 | } else {
171 | model = gltf.scene || gltf.scenes[0];
172 | }
173 | model.animations = gltf.animations;
174 |
175 | if (!model) {
176 | throw new Error("invalid avatar data");
177 | }
178 |
179 | if (frustumCulled !== undefined) {
180 | // Keep body parts from disappearing when approaching the camera.
181 | model.traverse((obj) => {
182 | if (obj.frustumCulled !== undefined) {
183 | obj.frustumCulled = frustumCulled;
184 | }
185 | });
186 | }
187 |
188 | model.rotateY(180 * (Math.PI / 180));
189 |
190 | if (!avatarAnimationData) {
191 | throw new Error("preLoadAnimationData is required");
192 | }
193 | const animationData = avatarAnimationData;
194 | if (vrm) {
195 | model.animations = [
196 | ...model.animations,
197 | ...Object.entries(animationData).map(([k, v]) =>
198 | convertForVrm(v, vrm, k)
199 | ),
200 | ];
201 | } else {
202 | model.animations = [
203 | ...model.animations,
204 | ...Object.entries(animationData).map(([k, v]) =>
205 | convertForReadyPlayerMe(v, model as THREE.Object3D, k)
206 | ),
207 | ];
208 | }
209 |
210 | return {
211 | model,
212 | vrm,
213 | };
214 | }
215 |
216 | function fetchScript(src: string) {
217 | return new Promise(function (resolve, reject) {
218 | const script = document.createElement("script");
219 | document.body.appendChild(script);
220 | script.onload = resolve;
221 | script.onerror = reject;
222 | script.async = true;
223 | script.src = src;
224 | });
225 | }
226 |
227 | /* function getFileType(data: Uint8Array): FileType | undefined {
228 | const hasPrefix = (v: Uint8Array, prefix: Uint8Array) => {
229 | const n = prefix.length;
230 | for (let i = 0; i < n; i++) {
231 | if (v[i] !== prefix[i]) {
232 | return false;
233 | }
234 | }
235 | return true;
236 | };
237 | {
238 | const prefix = new TextEncoder().encode("glTF");
239 | if (hasPrefix(data, prefix)) {
240 | return "gltf";
241 | }
242 | }
243 | {
244 | const prefix = new TextEncoder().encode("Kaydara FBX Binary");
245 | if (hasPrefix(data, prefix)) {
246 | return "fbx";
247 | }
248 | }
249 | } */
250 |
--------------------------------------------------------------------------------
/src/mixamo.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type * as THREE_VRM from "@pixiv/three-vrm";
3 |
4 | export function convertForReadyPlayerMe(
5 | asset: THREE.Group,
6 | model: THREE.Object3D,
7 | name: string
8 | ) {
9 | const clip = THREE.AnimationClip.findByName(
10 | asset.animations,
11 | "mixamo.com"
12 | ).clone();
13 |
14 | const getHipsPositionScale = (asset: THREE.Object3D) => {
15 | // Adjust with reference to hips height.
16 | const vec3 = new THREE.Vector3();
17 | const motionHipsHeight =
18 | asset.getObjectByName("mixamorigHips")?.position?.y;
19 | if (motionHipsHeight === undefined) {
20 | throw new Error("mixamorigHips not found");
21 | }
22 | const hipsY = model.getObjectByName("Hips")?.getWorldPosition(vec3)?.y;
23 | if (hipsY === undefined) {
24 | throw new Error("hip not found");
25 | }
26 | const rootY = model.getWorldPosition(vec3).y;
27 | const hipsHeight = Math.abs(hipsY - rootY);
28 | return hipsHeight / motionHipsHeight;
29 | };
30 | const hipsPositionScale = getHipsPositionScale(asset);
31 |
32 | const tracks: THREE.KeyframeTrack[] = [];
33 | clip.tracks.forEach((track) => {
34 | const trackSplitted = track.name.split(".");
35 | const mixamoRigName = trackSplitted[0];
36 | const nodeName = mixamoRigName.replace("mixamorig", "");
37 |
38 | const propertyName = trackSplitted[1];
39 | if (track.ValueTypeName === "quaternion") {
40 | tracks.push(
41 | new THREE.QuaternionKeyframeTrack(
42 | `${nodeName}.${propertyName}`,
43 | track.times as any, // eslint-disable-line @typescript-eslint/no-explicit-any
44 | track.values as any // eslint-disable-line @typescript-eslint/no-explicit-any
45 | )
46 | );
47 | } else if (track.ValueTypeName === "vector") {
48 | const value = track.values.map((v, _i) => v * hipsPositionScale);
49 | tracks.push(
50 | new THREE.VectorKeyframeTrack(
51 | `${nodeName}.${propertyName}`,
52 | track.times as any, // eslint-disable-line @typescript-eslint/no-explicit-any
53 | value as any // eslint-disable-line @typescript-eslint/no-explicit-any
54 | )
55 | );
56 | }
57 | });
58 |
59 | return new THREE.AnimationClip(name, clip.duration, tracks);
60 | }
61 |
62 | export function convertForVrm(
63 | asset: THREE.Group,
64 | vrm: THREE_VRM.VRM,
65 | name: string
66 | ) {
67 | // reference: https://github.com/pixiv/three-vrm/blob/dev/packages/three-vrm/examples/humanoidAnimation/loadMixamoAnimation.js
68 | const clip = THREE.AnimationClip.findByName(
69 | asset.animations,
70 | "mixamo.com"
71 | ).clone();
72 |
73 | const tracks: THREE.KeyframeTrack[] = []; // KeyframeTracks compatible with VRM will be added here
74 |
75 | const restRotationInverse = new THREE.Quaternion();
76 | const parentRestWorldRotation = new THREE.Quaternion();
77 | const _quatA = new THREE.Quaternion();
78 | const _vec3 = new THREE.Vector3();
79 |
80 | // Adjust with reference to hips height.
81 | const motionHipsHeight = asset.getObjectByName("mixamorigHips")?.position?.y;
82 | if (motionHipsHeight === undefined) {
83 | throw new Error("mixamorigHips not found");
84 | }
85 | const hipsY = vrm.humanoid
86 | .getNormalizedBoneNode("hips")
87 | ?.getWorldPosition(_vec3)?.y;
88 | if (hipsY === undefined) {
89 | throw new Error("hip not found");
90 | }
91 | const rootY = vrm.scene.getWorldPosition(_vec3).y;
92 | const hipsHeight = Math.abs(hipsY - rootY);
93 | const hipsPositionScale = hipsHeight / motionHipsHeight;
94 |
95 | clip.tracks.forEach((track) => {
96 | // Convert each tracks for VRM use, and push to `tracks`
97 | const trackSplitted = track.name.split(".");
98 | const mixamoRigName = trackSplitted[0];
99 | const boneName = mixamoVRMRigMap[mixamoRigName];
100 | const nodeName = vrm.humanoid?.getNormalizedBoneNode(boneName)?.name;
101 | const mixamoRigNode = asset.getObjectByName(mixamoRigName);
102 |
103 | if (nodeName && mixamoRigNode) {
104 | const propertyName = trackSplitted[1];
105 |
106 | // Store rotations of rest-pose.
107 | mixamoRigNode.getWorldQuaternion(restRotationInverse).invert();
108 | if (!mixamoRigNode.parent) {
109 | throw new Error("mixamoRigNode.parent is null");
110 | }
111 | mixamoRigNode.parent.getWorldQuaternion(parentRestWorldRotation);
112 |
113 | if (track.ValueTypeName === "quaternion") {
114 | // Retarget rotation of mixamoRig to NormalizedBone.
115 | for (let i = 0; i < track.values.length; i += 4) {
116 | const flatQuaternion = track.values.slice(i, i + 4);
117 |
118 | _quatA.fromArray(flatQuaternion);
119 |
120 | // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆
121 | _quatA
122 | .premultiply(parentRestWorldRotation)
123 | .multiply(restRotationInverse);
124 |
125 | _quatA.toArray(flatQuaternion);
126 |
127 | flatQuaternion.forEach((v, index) => {
128 | track.values[index + i] = v;
129 | });
130 | }
131 |
132 | tracks.push(
133 | new THREE.QuaternionKeyframeTrack(
134 | `${nodeName}.${propertyName}`,
135 | track.times as any, // eslint-disable-line @typescript-eslint/no-explicit-any
136 | track.values.map((v, i) =>
137 | vrm.meta?.metaVersion === "0" && i % 2 === 0 ? -v : v
138 | ) as any // eslint-disable-line @typescript-eslint/no-explicit-any
139 | )
140 | );
141 | } else if (track.ValueTypeName === "vector") {
142 | const value = track.values.map(
143 | (v, i) =>
144 | (vrm.meta?.metaVersion === "0" && i % 3 !== 1 ? -v : v) *
145 | hipsPositionScale
146 | );
147 | tracks.push(
148 | new THREE.VectorKeyframeTrack(
149 | `${nodeName}.${propertyName}`,
150 | track.times as any, // eslint-disable-line @typescript-eslint/no-explicit-any
151 | value as any // eslint-disable-line @typescript-eslint/no-explicit-any
152 | )
153 | );
154 | }
155 | }
156 | });
157 |
158 | return new THREE.AnimationClip(name, clip.duration, tracks);
159 | }
160 |
161 | /**
162 | * A map from Mixamo rig name to VRM Humanoid bone name
163 | */
164 | const mixamoVRMRigMap: { [key: string]: THREE_VRM.VRMHumanBoneName } = {
165 | mixamorigHips: "hips",
166 | mixamorigSpine: "spine",
167 | mixamorigSpine1: "chest",
168 | mixamorigSpine2: "upperChest",
169 | mixamorigNeck: "neck",
170 | mixamorigHead: "head",
171 | mixamorigLeftShoulder: "leftShoulder",
172 | mixamorigLeftArm: "leftUpperArm",
173 | mixamorigLeftForeArm: "leftLowerArm",
174 | mixamorigLeftHand: "leftHand",
175 | mixamorigLeftHandThumb1: "leftThumbMetacarpal",
176 | mixamorigLeftHandThumb2: "leftThumbProximal",
177 | mixamorigLeftHandThumb3: "leftThumbDistal",
178 | mixamorigLeftHandIndex1: "leftIndexProximal",
179 | mixamorigLeftHandIndex2: "leftIndexIntermediate",
180 | mixamorigLeftHandIndex3: "leftIndexDistal",
181 | mixamorigLeftHandMiddle1: "leftMiddleProximal",
182 | mixamorigLeftHandMiddle2: "leftMiddleIntermediate",
183 | mixamorigLeftHandMiddle3: "leftMiddleDistal",
184 | mixamorigLeftHandRing1: "leftRingProximal",
185 | mixamorigLeftHandRing2: "leftRingIntermediate",
186 | mixamorigLeftHandRing3: "leftRingDistal",
187 | mixamorigLeftHandPinky1: "leftLittleProximal",
188 | mixamorigLeftHandPinky2: "leftLittleIntermediate",
189 | mixamorigLeftHandPinky3: "leftLittleDistal",
190 | mixamorigRightShoulder: "rightShoulder",
191 | mixamorigRightArm: "rightUpperArm",
192 | mixamorigRightForeArm: "rightLowerArm",
193 | mixamorigRightHand: "rightHand",
194 | mixamorigRightHandPinky1: "rightLittleProximal",
195 | mixamorigRightHandPinky2: "rightLittleIntermediate",
196 | mixamorigRightHandPinky3: "rightLittleDistal",
197 | mixamorigRightHandRing1: "rightRingProximal",
198 | mixamorigRightHandRing2: "rightRingIntermediate",
199 | mixamorigRightHandRing3: "rightRingDistal",
200 | mixamorigRightHandMiddle1: "rightMiddleProximal",
201 | mixamorigRightHandMiddle2: "rightMiddleIntermediate",
202 | mixamorigRightHandMiddle3: "rightMiddleDistal",
203 | mixamorigRightHandIndex1: "rightIndexProximal",
204 | mixamorigRightHandIndex2: "rightIndexIntermediate",
205 | mixamorigRightHandIndex3: "rightIndexDistal",
206 | mixamorigRightHandThumb1: "rightThumbMetacarpal",
207 | mixamorigRightHandThumb2: "rightThumbProximal",
208 | mixamorigRightHandThumb3: "rightThumbDistal",
209 | mixamorigLeftUpLeg: "leftUpperLeg",
210 | mixamorigLeftLeg: "leftLowerLeg",
211 | mixamorigLeftFoot: "leftFoot",
212 | mixamorigLeftToeBase: "leftToes",
213 | mixamorigRightUpLeg: "rightUpperLeg",
214 | mixamorigRightLeg: "rightLowerLeg",
215 | mixamorigRightFoot: "rightFoot",
216 | mixamorigRightToeBase: "rightToes",
217 | };
218 |
--------------------------------------------------------------------------------
/src/types.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import * as THREE_VRM from "@pixiv/three-vrm";
3 | /**
4 | * Avatar model.
5 | */
6 | export interface AvatarModel {
7 | model: THREE.Group;
8 | vrm?: THREE_VRM.VRM;
9 | }
10 |
11 | /**
12 | * Shoulder, Elbow, Wrist
13 | */
14 | export type IKTargetBones = [
15 | THREE.Bone | undefined,
16 | THREE.Bone | undefined,
17 | THREE.Bone | undefined
18 | ];
19 |
--------------------------------------------------------------------------------
/src/util.ts:
--------------------------------------------------------------------------------
1 | import * as THREE from "three";
2 | import type { Avatar } from "./avatar";
3 | import { Reflector } from "three/examples/jsm/objects/Reflector.js";
4 |
5 | /**
6 | * Automatically adjust camera when switching between VR and non-VR modes.
7 | *
8 | * @param getAvatar - If the avatar is changed after calling registerSyncAvatarHeadAndCamera, it will continue to work correctly.
9 | * @example
10 | * ```ts
11 | registerSyncAvatarHeadAndCamera(
12 | renderer.xr,
13 | camera,
14 | camera,
15 | camera.parent,
16 | () => _avatar,
17 | {
18 | onVR: async () => {
19 | playerController.isVR = true;
20 | },
21 | onNonVR: async () => {
22 | playerController.isVR = false;
23 | setNonVRCameraMode(camera, camera.parent, avatar, isFPS);
24 | },
25 | }
26 | );
27 | * ```
28 | */
29 | export async function registerSyncAvatarHeadAndCamera(
30 | xr: THREE.WebXRManager,
31 | nonVrCamera: THREE.PerspectiveCamera,
32 | head: THREE.Object3D,
33 | headOffset: THREE.Object3D,
34 | getAvatar: () => Avatar | undefined,
35 | options: {
36 | onVR: () => Promise | void;
37 | onNonVR: () => Promise | void;
38 | }
39 | ) {
40 | let positions: [THREE.Vector3, THREE.Vector3] | undefined;
41 | let rotation: THREE.Euler | undefined;
42 |
43 | const setupVR = async () => {
44 | positions = [head.position.clone(), headOffset.position.clone()];
45 | rotation = headOffset.rotation.clone(); // save Non VR camera rotation
46 | headOffset.rotation.set(0, 0, 0);
47 |
48 | let avatar: Avatar | undefined;
49 | while (!(avatar = getAvatar())) {
50 | await new Promise((resolve) => setTimeout(resolve, 0));
51 | }
52 |
53 | xr.updateCamera(nonVrCamera);
54 | // waitForVRCameraInitialized
55 | while (Math.round(xr.getCamera().position.y * 10) === 0) {
56 | await new Promise((resolve) => setTimeout(resolve, 0));
57 | }
58 |
59 | headOffset.position.setY(avatar.getHeadHeight() - head.position.y);
60 |
61 | const p = options?.onVR?.();
62 | if (p) {
63 | await p;
64 | }
65 | };
66 | const setupNonVR = async () => {
67 | head.rotation.set(0, 0, 0);
68 | if (rotation) {
69 | headOffset.rotation.copy(rotation); // restore Non VR camera rotation
70 | }
71 | if (positions) {
72 | head.position.copy(positions[0]);
73 | headOffset.position.copy(positions[1]);
74 | }
75 | const p = options?.onNonVR?.();
76 | if (p) {
77 | await p;
78 | }
79 | };
80 | xr.addEventListener("sessionstart", setupVR);
81 | xr.addEventListener("sessionend", setupNonVR);
82 |
83 | if (xr.isPresenting) {
84 | setupVR();
85 | } else {
86 | setupNonVR();
87 | }
88 | }
89 |
90 | /**
91 | * Whether the device does not have a GPU capable of processing enough.
92 | * (Experimental Features)
93 | *
94 | * @remarks
95 | * Intended to be used to adjust texture size, processing load, etc.
96 | * The current implementation is a very tentative decision and needs to be improved.
97 | */
98 | export function isLowSpecDevice(): boolean {
99 | const info = getRenderInfo();
100 | if (!info) {
101 | return false;
102 | }
103 | if (info.vendor === "Qualcomm") {
104 | return true;
105 | }
106 | if (info.vendor.includes("Intel")) {
107 | if (info.renderer.includes("Iris")) {
108 | return true;
109 | }
110 | }
111 | if (isAndroid() || isIOS()) {
112 | return true;
113 | }
114 | return false;
115 | }
116 | export function isAndroid() {
117 | return navigator.userAgent.includes("Android");
118 | }
119 | export function isIOS() {
120 | return navigator.userAgent.includes("Mac") && isTouchDevice();
121 | }
122 | export function isTouchDevice() {
123 | return "ontouchstart" in window || navigator.maxTouchPoints > 0;
124 | }
125 |
126 | /**
127 | * Get GPU information.
128 | * See also {@link https://developer.mozilla.org/en-US/docs/Web/API/WEBGL_debug_renderer_info | WEBGL_debug_renderer_info}
129 | */
130 | export function getRenderInfo():
131 | | { vendor: string; renderer: string }
132 | | undefined {
133 | try {
134 | const canvas = document.createElement("canvas");
135 | const gl = canvas.getContext("webgl2");
136 | if (!gl) {
137 | return;
138 | }
139 | const ext = gl.getExtension("WEBGL_debug_renderer_info");
140 | if (ext) {
141 | return {
142 | vendor: gl.getParameter(ext.UNMASKED_VENDOR_WEBGL) || "",
143 | renderer: gl.getParameter(ext.UNMASKED_RENDERER_WEBGL) || "",
144 | };
145 | }
146 | } catch (ex) {
147 | console.warn(ex);
148 | }
149 | }
150 |
151 | /**
152 | * Adjust the camera for non-VR mode.
153 | * @param isFirstPerson - First person view or 3rd person view.
154 | */
155 | export async function setNonVRCameraMode(
156 | camera: THREE.Camera,
157 | cameraOffset: THREE.Object3D,
158 | avatar: Avatar,
159 | isFirstPerson: boolean // first person view or third person view
160 | ) {
161 | if (isFirstPerson) {
162 | avatar.setFirstPersonMode([camera]);
163 | camera.position.set(0, avatar.getHeadHeight(), 0);
164 | cameraOffset.rotation.set(0, 0, 0);
165 | } else {
166 | avatar.setThirdPersonMode([camera]);
167 | camera.position.set(0.0, avatar.getHeadHeight() + 0.2, 2.0);
168 | cameraOffset.rotation.set(-0.2, 0, 0);
169 | }
170 | }
171 |
172 | const DEFAULT_MIRROR_DISTANCE_VR = 0.3;
173 | const DEFAULT_MIRROR_DISTANCE_NON_VR = 1.2;
174 |
175 | export interface AddMirrorHUDOptions {
176 | /**
177 | * `renderer.xr`
178 | */
179 | xr?: THREE.WebXRManager;
180 | /**
181 | * Distance between avatar and mirror (In VR mode). Default is 0.3.
182 | */
183 | distanceVR?: number;
184 | /**
185 | * Distance between avatar and mirror (In non-VR mode). Default is 1.2.
186 | */
187 | distanceNonVR?: number;
188 | }
189 |
190 | /**
191 | * Create a mirror in front of the avatar.
192 | *
193 | * @param container - Where to ADD mirrors.
194 | */
195 | export function addMirrorHUD(
196 | avatar: Avatar,
197 | container: THREE.Object3D,
198 | options?: AddMirrorHUDOptions
199 | ): Reflector {
200 | const distanceNonVR =
201 | options?.distanceNonVR || DEFAULT_MIRROR_DISTANCE_NON_VR;
202 | const distanceVR = options?.distanceVR || DEFAULT_MIRROR_DISTANCE_VR;
203 | const mirror = new Reflector(new THREE.PlaneGeometry(0.6, 1.2), {
204 | textureWidth: window.innerWidth * window.devicePixelRatio,
205 | textureHeight: window.innerHeight * window.devicePixelRatio,
206 | color: 0x777777,
207 | });
208 | mirror.camera.layers.enableAll();
209 | mirror.camera.layers.disable(avatar.firstPersonOnlyLayer);
210 | mirror.name = "mirrorHUD";
211 |
212 | if (options?.xr) {
213 | const xr = options.xr;
214 | const ref = new WeakRef(mirror);
215 | const update = () => {
216 | ref
217 | .deref()
218 | ?.position?.set(
219 | 0,
220 | avatar.getHeadHeight(),
221 | (options?.xr?.isPresenting ? distanceVR : distanceNonVR) * -1
222 | );
223 | };
224 | xr.addEventListener("sessionstart", update);
225 | xr.addEventListener("sessionend", update);
226 | const dispose = mirror.dispose.bind(mirror);
227 | mirror.dispose = () => {
228 | dispose();
229 | xr.removeEventListener("sessionstart", update);
230 | xr.removeEventListener("sessionend", update);
231 | };
232 | }
233 |
234 | mirror.position.set(
235 | 0,
236 | avatar.getHeadHeight(),
237 | (options?.xr?.isPresenting ? distanceVR : distanceNonVR) * -1
238 | );
239 | container.add(mirror);
240 | return mirror;
241 | }
242 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es2021",
4 | "moduleResolution": "node",
5 | "declaration": true,
6 | "declarationMap": true,
7 | "sourceMap": true,
8 | "strict": true,
9 | "skipLibCheck": true
10 | },
11 | "include": ["src"],
12 | "exclude": ["dist", "node_modules"]
13 | }
14 |
--------------------------------------------------------------------------------
/watch.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # set -euxo pipefail
3 | set -euo pipefail
4 | cd `/usr/bin/dirname $0`
5 |
6 | trap "final; exit 1" 2
7 |
8 | function final {
9 | echo "Ctrl+C pushed."
10 | }
11 |
12 | dir=$(greadlink -f .)
13 |
14 | while true; do
15 | set +e
16 | npm run build
17 | set -e
18 |
19 | out=$(fswatch \
20 | --one-event \
21 | --recursive \
22 | --exclude='.*' \
23 | --include=$dir'/src/.*\.ts$' \
24 | --include=$dir'/src/.*\.css$' \
25 | --include=$dir'/src/.*\.svg$' \
26 | .)
27 |
28 | if [ -z "$out" ]; then
29 | break
30 | fi
31 | done
32 |
--------------------------------------------------------------------------------