├── .eslintrc.json ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── next.config.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── public ├── character.png ├── characters │ ├── doux.png │ ├── doux_preview.png │ ├── mort.png │ ├── mort_preview.png │ ├── targ.png │ ├── targ_preview.png │ ├── vita.png │ └── vita_preview.png ├── disco.mp3 ├── favicon.ico ├── github.png ├── livekit │ ├── logo-reduced.svg │ └── logo.svg ├── next.svg ├── thirteen.svg ├── vercel.svg └── world │ ├── boombox.png │ └── map.png ├── src ├── app │ ├── globals.css │ ├── head.tsx │ ├── layout.tsx │ ├── page.tsx │ └── room │ │ └── [room_name] │ │ └── page.tsx ├── components │ ├── BottomBar.tsx │ ├── Camera.tsx │ ├── Character.tsx │ ├── CharacterSelector.tsx │ ├── DPad.tsx │ ├── EarshotRadius.tsx │ ├── GameView.tsx │ ├── GithubLink.tsx │ ├── JukeBox.tsx │ ├── JukeBoxModal.tsx │ ├── MicrophoneMuteButton.tsx │ ├── MicrophoneSelector.tsx │ ├── PoweredByLiveKit.tsx │ ├── RoomInfo.tsx │ ├── Shadows.tsx │ ├── Stage.tsx │ ├── UsernameInput.tsx │ ├── World.tsx │ └── icons │ │ ├── d-pad.svg │ │ ├── disco-ball.svg │ │ ├── mic-off.svg │ │ ├── mic.svg │ │ └── sine-wave.svg ├── controller │ ├── InputController.tsx │ ├── JukeBoxProvider.tsx │ ├── MyCharacterController.tsx │ ├── MyPlayerSpawnController.tsx │ ├── NetcodeController.tsx │ ├── RemotePlayersController.tsx │ ├── SpatialAudioController.tsx │ ├── WorldBoundaryController.tsx │ └── useTrackPositions.tsx ├── model │ ├── AnimationState.ts │ ├── GameState.ts │ ├── Inputs.ts │ ├── JukeBoxState.ts │ ├── Player.ts │ ├── Vector2.ts │ └── WorldBoundaries.ts ├── pages │ └── api │ │ ├── connection_details.ts │ │ └── room_info │ │ └── [room].ts ├── providers │ ├── animations.tsx │ └── audio │ │ └── webAudio.tsx └── util │ ├── useAudioTracksByName.tsx │ └── useMobile.tsx ├── tailwind.config.js ├── tsconfig.json └── yarn.lock /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | .env.development 4 | 5 | # dependencies 6 | /node_modules 7 | /.pnp 8 | .pnp.js 9 | 10 | # testing 11 | /coverage 12 | 13 | # next.js 14 | /.next/ 15 | /out/ 16 | 17 | # production 18 | /build 19 | 20 | # misc 21 | .DS_Store 22 | *.pem 23 | 24 | # debug 25 | npm-debug.log* 26 | yarn-debug.log* 27 | yarn-error.log* 28 | .pnpm-debug.log* 29 | 30 | # local env files 31 | .env*.local 32 | 33 | # vercel 34 | .vercel 35 | 36 | # typescript 37 | *.tsbuildinfo 38 | next-env.d.ts 39 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules/typescript/lib", 3 | "typescript.enablePromptUseWorkspaceTsdk": true 4 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spatial Audio with LiveKit 2 | 3 | [![Sample Gif](https://user-images.githubusercontent.com/8453967/221318613-861215da-1d71-492e-979f-dc7f18cb5c7f.gif)](https://spatial-audio-demo.livekit.io/) 4 | 5 | This is a demo of spatial audio using LiveKit. Users join a little 2D world, and hear other users' audio in stereo, based on their position and distance relative to you. 6 | 7 | ## Online demo 8 | 9 | You can try an online demo right now at . 10 | 11 | ## Running locally 12 | 13 | Clone the repo and install dependencies: 14 | 15 | ```bash 16 | git clone git@github.com:livekit-examples/spatial-audio.git 17 | cd spatial-audio 18 | npm install 19 | ``` 20 | 21 | Create a new LiveKit project at . Then create a new key in your [project settings](https://cloud.livekit.io/projects/p_/settings/keys). 22 | 23 | Create a new file at `spatial-audio/.env.development` and add your new API key and secret as well as your project's WebSocket URL (found at the top of ): 24 | 25 | ``` 26 | LIVEKIT_API_KEY= 27 | LIVEKIT_API_SECRET= 28 | LIVEKIT_WS_URL=wss://.livekit.cloud 29 | ``` 30 | 31 | (Note: this file is in `.gitignore`. Never commit your API secret to git.) 32 | 33 | Then run the development server: 34 | 35 | ```bash 36 | npm run dev 37 | ``` 38 | 39 | You can test it by opening in a browser. 40 | 41 | ## Deploying for production 42 | 43 | This demo is a Next.js app. You can deploy to your Vercel account with one click: 44 | 45 | [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flivekit-examples%2Fspatial-audio&env=LIVEKIT_API_KEY,LIVEKIT_API_SECRET,LIVEKIT_WS_URL&envDescription=Get%20these%20from%20your%20cloud%20livekit%20project.&envLink=https%3A%2F%2Fcloud.livekit.io&project-name=my-spatial-audio-app) 46 | 47 | Refer to the [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more about deploying to a production environment. 48 | 49 | ## Asset credits 50 | 51 | This demo uses the following assets: 52 | 53 | - [Field of Green](https://guttykreum.itch.io/field-of-green) and boombox sprite by [GuttyKreum](https://twitter.com/GuttyKreum) 54 | - [Dino Characters](https://arks.itch.io/dino-characters) by [Arks](https://arks.digital/) 55 | 56 | They're both wonderful artists, check out their work! 57 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: false, 4 | experimental: { 5 | appDir: true, 6 | }, 7 | webpack: (config) => { 8 | config.module.rules.push({ 9 | test: /\.svg$/, 10 | use: ["@svgr/webpack"] 11 | }); 12 | return config; 13 | } 14 | } 15 | 16 | module.exports = nextConfig 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "spatial-audio", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@livekit/components-core": "^0.6.8", 13 | "@livekit/components-react": "^1", 14 | "@next/font": "13.1.5", 15 | "@pixi/react": "^7.1.1", 16 | "@pixi/tilemap": "^4.0.0", 17 | "@svgr/webpack": "^6.5.1", 18 | "@types/node": "18.11.18", 19 | "@types/react": "18.0.27", 20 | "@types/react-dom": "18.0.10", 21 | "add": "^2.0.6", 22 | "daisyui": "^2.49.0", 23 | "eslint": "8.32.0", 24 | "eslint-config-next": "13.1.5", 25 | "livekit-client": "^1.6.3", 26 | "livekit-server-sdk": "^1.1.0", 27 | "next": "13.1.5", 28 | "pixi.js": "^7.1.1", 29 | "react": "18.2.0", 30 | "react-device-detect": "^2.2.3", 31 | "react-dom": "18.2.0", 32 | "react-hot-toast": "^2.4.0", 33 | "react-use": "^17.4.0", 34 | "set-interval-async": "^3.0.3", 35 | "typescript": "4.9.4", 36 | "use-resize-observer": "^9.1.0", 37 | "yarn": "^1.22.19" 38 | }, 39 | "devDependencies": { 40 | "autoprefixer": "^10.4.13", 41 | "postcss": "^8.4.21", 42 | "tailwindcss": "^3.2.4" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /public/character.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/character.png -------------------------------------------------------------------------------- /public/characters/doux.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/doux.png -------------------------------------------------------------------------------- /public/characters/doux_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/doux_preview.png -------------------------------------------------------------------------------- /public/characters/mort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/mort.png -------------------------------------------------------------------------------- /public/characters/mort_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/mort_preview.png -------------------------------------------------------------------------------- /public/characters/targ.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/targ.png -------------------------------------------------------------------------------- /public/characters/targ_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/targ_preview.png -------------------------------------------------------------------------------- /public/characters/vita.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/vita.png -------------------------------------------------------------------------------- /public/characters/vita_preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/characters/vita_preview.png -------------------------------------------------------------------------------- /public/disco.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/disco.mp3 -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/favicon.ico -------------------------------------------------------------------------------- /public/github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/github.png -------------------------------------------------------------------------------- /public/livekit/logo-reduced.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /public/livekit/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/thirteen.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/world/boombox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/world/boombox.png -------------------------------------------------------------------------------- /public/world/map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livekit-examples/spatial-audio/fe9c218afea4dd57de907e553e468ed4d8e76077/public/world/map.png -------------------------------------------------------------------------------- /src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; -------------------------------------------------------------------------------- /src/app/head.tsx: -------------------------------------------------------------------------------- 1 | export default function Head() { 2 | return ( 3 | <> 4 | LiveKit Spatial Audio Example 5 | 6 | 7 | 8 | 9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import "./globals.css"; 2 | 3 | export default function RootLayout({ 4 | children, 5 | }: { 6 | children: React.ReactNode; 7 | }) { 8 | return ( 9 | 10 | {/* 11 | will contain the components returned by the nearest parent 12 | head.tsx. Find out more at https://beta.nextjs.org/docs/api-reference/file-conventions/head 13 | */} 14 | 15 | 16 |
{children}
17 | 18 | 19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /src/app/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import Image from "next/image"; 4 | import { useRouter } from "next/navigation"; 5 | import { useCallback, useState } from "react"; 6 | import { toast, Toaster } from "react-hot-toast"; 7 | 8 | export default function Home() { 9 | const router = useRouter(); 10 | const [roomName, setRoomName] = useState(""); 11 | 12 | const joinRoom = useCallback(() => { 13 | if (roomName === "") { 14 | toast.error("Please enter a room name"); 15 | return; 16 | } 17 | router.push(`/room/${roomName}`); 18 | }, [roomName, router]); 19 | 20 | return ( 21 |
22 | 23 |
24 |

Spatial Audio LiveKit Example App

25 |
{ 27 | e.preventDefault(); 28 | joinRoom(); 29 | }} 30 | > 31 |
32 |
33 | setRoomName(e.currentTarget.value)} 36 | type="text" 37 | placeholder="Room Name" 38 | className="input input-bordered input-secondary" 39 | /> 40 | 41 |
42 |
43 |
44 | 50 |
51 |
52 | ); 53 | } 54 | -------------------------------------------------------------------------------- /src/app/room/[room_name]/page.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { WebAudioContext } from "@/providers/audio/webAudio"; 4 | import { BottomBar } from "@/components/BottomBar"; 5 | import { RoomInfo } from "@/components/RoomInfo"; 6 | import { UsernameInput } from "@/components/UsernameInput"; 7 | import { 8 | ConnectionDetails, 9 | ConnectionDetailsBody, 10 | } from "@/pages/api/connection_details"; 11 | import { LiveKitRoom } from "@livekit/components-react"; 12 | import { useCallback, useEffect, useMemo, useState } from "react"; 13 | import { toast, Toaster } from "react-hot-toast"; 14 | import { 15 | CharacterName, 16 | CharacterSelector, 17 | } from "@/components/CharacterSelector"; 18 | import { useMobile } from "@/util/useMobile"; 19 | import { GameView } from "@/components/GameView"; 20 | 21 | type Props = { 22 | params: { room_name: string }; 23 | }; 24 | 25 | export default function Page({ params: { room_name } }: Props) { 26 | const [connectionDetails, setConnectionDetails] = 27 | useState(null); 28 | const [selectedCharacter, setSelectedCharacter] = 29 | useState("doux"); 30 | const isMobile = useMobile(); 31 | const [audioContext, setAudioContext] = useState(null); 32 | 33 | useEffect(() => { 34 | setAudioContext(new AudioContext()); 35 | return () => { 36 | setAudioContext((prev) => { 37 | prev?.close(); 38 | return null; 39 | }); 40 | }; 41 | }, []); 42 | 43 | const humanRoomName = useMemo(() => { 44 | return decodeURI(room_name); 45 | }, [room_name]); 46 | 47 | const requestConnectionDetails = useCallback( 48 | async (username: string) => { 49 | const body: ConnectionDetailsBody = { 50 | room_name, 51 | username, 52 | character: selectedCharacter, 53 | }; 54 | const response = await fetch("/api/connection_details", { 55 | method: "POST", 56 | headers: { "Content-Type": "application/json" }, 57 | body: JSON.stringify(body), 58 | }); 59 | if (response.status === 200) { 60 | return response.json(); 61 | } 62 | 63 | const { error } = await response.json(); 64 | throw error; 65 | }, 66 | [room_name, selectedCharacter] 67 | ); 68 | 69 | if (!audioContext) { 70 | return null; 71 | } 72 | 73 | // If we don't have any connection details yet, show the username form 74 | if (connectionDetails === null) { 75 | return ( 76 |
77 | 78 |

{humanRoomName}

79 | 80 |
81 | 85 | { 88 | try { 89 | // TODO unify this kind of pattern across examples, either with the `useToken` hook or an equivalent 90 | const connectionDetails = await requestConnectionDetails( 91 | username 92 | ); 93 | setConnectionDetails(connectionDetails); 94 | } catch (e: any) { 95 | toast.error(e); 96 | } 97 | }} 98 | /> 99 |
100 | ); 101 | } 102 | 103 | // Show the room UI 104 | return ( 105 |
106 | 113 | 114 |
115 |
120 |
121 |
122 | 123 |
124 |
125 |
126 | 127 |
128 |
129 |
130 | 131 | 132 |
133 | ); 134 | } 135 | -------------------------------------------------------------------------------- /src/components/BottomBar.tsx: -------------------------------------------------------------------------------- 1 | import { useMobile } from "@/util/useMobile"; 2 | import { GithubLink } from "./GithubLink"; 3 | import { MicrophoneMuteButton } from "./MicrophoneMuteButton"; 4 | import { MicrophoneSelector } from "./MicrophoneSelector"; 5 | import { PoweredByLiveKit } from "./PoweredByLiveKit"; 6 | 7 | export function BottomBar() { 8 | const mobile = useMobile(); 9 | return ( 10 |
11 |
12 | 13 |
14 | 15 |
16 |
17 |
18 | {!mobile && ( 19 |
20 | 21 |
22 | )} 23 | 24 |
25 |
26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /src/components/Camera.tsx: -------------------------------------------------------------------------------- 1 | import { Vector2 } from "@/model/Vector2"; 2 | import { Container, useApp } from "@pixi/react"; 3 | 4 | type Props = { 5 | children?: React.ReactNode; 6 | targetPosition: Vector2; 7 | }; 8 | 9 | export const Camera = ({ children, targetPosition }: Props) => { 10 | const app = useApp(); 11 | 12 | return ( 13 | //@ts-ignore 14 | 20 | {children} 21 | 22 | ); 23 | }; 24 | -------------------------------------------------------------------------------- /src/components/Character.tsx: -------------------------------------------------------------------------------- 1 | import { AnimationState } from "@/model/AnimationState"; 2 | import { useAnimations } from "@/providers/animations"; 3 | import { AnimatedSprite, Container, Text } from "@pixi/react"; 4 | import { TextStyle } from "pixi.js"; 5 | import { useMemo } from "react"; 6 | import { CharacterName } from "./CharacterSelector"; 7 | 8 | type Props = { 9 | x: number; 10 | y: number; 11 | speaking: boolean; 12 | username: string; 13 | animation: AnimationState; 14 | character: CharacterName; 15 | }; 16 | 17 | export function Character({ 18 | x, 19 | y, 20 | username, 21 | animation, 22 | speaking, 23 | character, 24 | }: Props) { 25 | const animationSheet = useAnimations(character); 26 | 27 | const { color: usernameOutlineColor, thickness: usernameOutlineThickness } = 28 | useMemo(() => { 29 | if (speaking) { 30 | return { color: 0x00ff00, thickness: 6 }; 31 | } else { 32 | return { color: 0x000000, thickness: 4 }; 33 | } 34 | }, [speaking]); 35 | 36 | const animationName = useMemo( 37 | () => (animation.startsWith("idle_") ? "idle" : "walk"), 38 | [animation] 39 | ); 40 | 41 | const scale = useMemo( 42 | () => (animation.endsWith("_right") ? 1 : -1), 43 | [animation] 44 | ); 45 | 46 | return ( 47 | // @ts-ignore 48 | // pixi-react types don't support React 18 yet 49 | // See: https://github.com/pixijs/pixi-react/issues/350 50 | 51 | 64 | {["walk", "idle"].map( 65 | (a) => 66 | animationName === a && ( 67 | 75 | ) 76 | )} 77 | 78 | ); 79 | } 80 | -------------------------------------------------------------------------------- /src/components/CharacterSelector.tsx: -------------------------------------------------------------------------------- 1 | import Image from "next/image"; 2 | 3 | export type CharacterName = "doux" | "mort" | "targ" | "vita"; 4 | 5 | type Props = { 6 | selectedCharacter: CharacterName; 7 | onSelectedCharacterChange: (character: CharacterName) => void; 8 | }; 9 | 10 | export const CharacterSelector = ({ 11 | selectedCharacter, 12 | onSelectedCharacterChange, 13 | }: Props) => { 14 | return ( 15 |
16 |
17 |
onSelectedCharacterChange("doux")} 20 | > 21 | 26 |
27 |
onSelectedCharacterChange("mort")} 30 | > 31 | 36 |
37 |
onSelectedCharacterChange("targ")} 40 | > 41 | 46 |
47 |
onSelectedCharacterChange("vita")} 50 | > 51 | 56 |
57 |
58 |
59 | ); 60 | }; 61 | 62 | const Character = ({ 63 | name, 64 | image, 65 | selected, 66 | }: { 67 | name: string; 68 | image: string; 69 | selected: boolean; 70 | }) => { 71 | return ( 72 |
73 |
74 | {name} 82 |
83 |
{name}
84 |
85 | ); 86 | }; 87 | -------------------------------------------------------------------------------- /src/components/DPad.tsx: -------------------------------------------------------------------------------- 1 | import DPadSvg from "@/components/icons/d-pad.svg"; 2 | import { TouchEvent, useCallback, useRef } from "react"; 3 | 4 | type Props = { 5 | onInput: (x: number, y: number) => void; 6 | }; 7 | 8 | export const DPad = ({ onInput }: Props) => { 9 | const container = useRef(null); 10 | 11 | const calculateInput = useCallback((e: TouchEvent) => { 12 | if (!container.current) return { x: 0, y: 0 }; 13 | const boundingRect = container.current.getBoundingClientRect(); 14 | const paddedHeight = boundingRect.height * 0.8; 15 | 16 | // pad by 10px on each side to make it easier to acheive max input 17 | // clamp to -1 to 1 and invert y 18 | const x = Math.max( 19 | Math.min( 20 | (2 * (e.touches[0].clientX - boundingRect.left - 10)) / 21 | (boundingRect.width - 20) - 22 | 1, 23 | 1 24 | ), 25 | -1 26 | ); 27 | const y = 28 | Math.max( 29 | Math.min( 30 | (2 * (e.touches[0].clientY - boundingRect.top - 10)) / 31 | (boundingRect.height - 20) - 32 | 1, 33 | 1 34 | ), 35 | -1 36 | ) * -1; 37 | 38 | return { x, y }; 39 | }, []); 40 | 41 | const onTouchDown = useCallback( 42 | (e: TouchEvent) => { 43 | const input = calculateInput(e); 44 | onInput(input.x, input.y); 45 | }, 46 | [calculateInput, onInput] 47 | ); 48 | 49 | const onTouchUp = useCallback( 50 | (e: TouchEvent) => { 51 | onInput(0, 0); 52 | }, 53 | [onInput] 54 | ); 55 | 56 | const onTouchMove = useCallback( 57 | (e: TouchEvent) => { 58 | const input = calculateInput(e); 59 | onInput(input.x, input.y); 60 | }, 61 | [calculateInput, onInput] 62 | ); 63 | 64 | return ( 65 |
72 | 76 |
77 | ); 78 | }; 79 | -------------------------------------------------------------------------------- /src/components/EarshotRadius.tsx: -------------------------------------------------------------------------------- 1 | import { Vector2 } from "@/model/Vector2"; 2 | import { Graphics } from "pixi.js"; 3 | import { Graphics as GraphicsComponent } from "@pixi/react"; 4 | import { useCallback } from "react"; 5 | 6 | type Props = { 7 | earshotRadius: number; 8 | myPlayerPosition: Vector2; 9 | backgroundZIndex: number; 10 | render: boolean; 11 | }; 12 | 13 | export const EarshotRadius = ({ 14 | backgroundZIndex, 15 | myPlayerPosition, 16 | earshotRadius, 17 | }: Props) => { 18 | const draw = useCallback( 19 | (g: Graphics) => { 20 | g.clear(); 21 | g.beginFill(0xffffff, 0.1); 22 | g.drawCircle(myPlayerPosition.x, myPlayerPosition.y, earshotRadius); 23 | g.endFill(); 24 | }, 25 | [earshotRadius, myPlayerPosition.x, myPlayerPosition.y] 26 | ); 27 | 28 | return ; 29 | }; 30 | -------------------------------------------------------------------------------- /src/components/GameView.tsx: -------------------------------------------------------------------------------- 1 | import { NetcodeController } from "@/controller/NetcodeController"; 2 | import { 3 | useConnectionState, 4 | useIsSpeaking, 5 | useLocalParticipant, 6 | useParticipantInfo, 7 | useSpeakingParticipants, 8 | } from "@livekit/components-react"; 9 | import { Container } from "@pixi/react"; 10 | import { useCallback, useEffect, useMemo, useState } from "react"; 11 | import useResizeObserver from "use-resize-observer"; 12 | import { Character } from "./Character"; 13 | import { InputController } from "@/controller/InputController"; 14 | import { useGameState } from "@/model/GameState"; 15 | import { MyCharacterController } from "@/controller/MyCharacterController"; 16 | import { MyPlayerSpawnController } from "@/controller/MyPlayerSpawnController"; 17 | import { ConnectionState } from "livekit-client"; 18 | import { SpatialAudioController } from "@/controller/SpatialAudioController"; 19 | import { RemotePlayersController } from "@/controller/RemotePlayersController"; 20 | import { WorldBoundaryController } from "@/controller/WorldBoundaryController"; 21 | import { World } from "./World"; 22 | import { Camera } from "./Camera"; 23 | import { EarshotRadius } from "./EarshotRadius"; 24 | import { AnimationsProvider } from "@/providers/animations"; 25 | import { Shadows } from "./Shadows"; 26 | import { DPad } from "./DPad"; 27 | import { Inputs } from "@/model/Inputs"; 28 | import { useMobile } from "@/util/useMobile"; 29 | import { Stage } from "./Stage"; 30 | import { JukeBox } from "./JukeBox"; 31 | import { JukeBoxModal } from "./JukeBoxModal"; 32 | import { JukeBoxProvider } from "@/controller/JukeBoxProvider"; 33 | import { useTrackPositions } from "@/controller/useTrackPositions"; 34 | 35 | export function GameView() { 36 | const { ref, width = 1, height = 1 } = useResizeObserver(); 37 | const mobile = useMobile(); 38 | const connectionState = useConnectionState(); 39 | const { localParticipant } = useLocalParticipant(); 40 | const { metadata: localMetadata } = useParticipantInfo({ 41 | participant: localParticipant, 42 | }); 43 | const localSpeaking = useIsSpeaking(localParticipant); 44 | const speakingParticipants = useSpeakingParticipants(); 45 | const { 46 | inputs, 47 | remotePlayers, 48 | myPlayer, 49 | networkAnimations, 50 | networkPositions, 51 | worldBoundaries, 52 | earshotRadius, 53 | backgroundZIndex, 54 | playerSpeed, 55 | jukeBoxPosition, 56 | setMyPlayer, 57 | setInputs, 58 | setNetworkAnimations, 59 | setNetworkPositions, 60 | setRemotePlayers, 61 | } = useGameState(); 62 | 63 | const [mobileInputs, setMobileInputs] = useState({ 64 | direction: { x: 0, y: 0 }, 65 | }); 66 | 67 | const speakingLookup = useMemo(() => { 68 | const lookup = new Set(); 69 | for (const p of speakingParticipants) { 70 | lookup.add(p.identity); 71 | } 72 | return lookup; 73 | }, [speakingParticipants]); 74 | 75 | useEffect(() => { 76 | if (localParticipant) { 77 | setMyPlayer((prev) => prev && { ...prev, character: "targ" }); 78 | } 79 | }, [localParticipant, setMyPlayer]); 80 | 81 | const localCharacter = useMemo( 82 | () => JSON.parse(localMetadata || "{}").character || null, 83 | [localMetadata] 84 | ); 85 | 86 | const onMobileInput = useCallback((x: number, y: number) => { 87 | setMobileInputs({ direction: { x, y: -y } }); 88 | }, []); 89 | 90 | const distanceFromJukeBox = useMemo(() => { 91 | if (!myPlayer) return Infinity; 92 | return Math.sqrt( 93 | (myPlayer.position.x - jukeBoxPosition.x) ** 2 + 94 | (myPlayer.position.y - jukeBoxPosition.y) ** 2 95 | ); 96 | }, [jukeBoxPosition.x, jukeBoxPosition.y, myPlayer]); 97 | 98 | const trackPositions = useTrackPositions({ remotePlayers, jukeBoxPosition }); 99 | 100 | if (connectionState !== ConnectionState.Connected) { 101 | return null; 102 | } 103 | 104 | return ( 105 |
106 | 107 | {myPlayer && ( 108 | 113 | )} 114 | {myPlayer && ( 115 | 120 | )} 121 | 126 | 127 | 133 | {distanceFromJukeBox < 60 && ( 134 |
135 |
136 | 137 |
138 |
139 | )} 140 | {mobile && ( 141 |
142 | 143 |
144 | )} 145 | 153 | 154 | 155 | {/* @ts-ignore */} 156 | 157 | 162 | {myPlayer && ( 163 | 168 | )} 169 | {myPlayer && ( 170 | 178 | )} 179 | 183 | 187 | 192 | {remotePlayers.map((player) => ( 193 | 202 | ))} 203 | 209 | 210 | 211 | 212 | 213 | 214 |
215 | ); 216 | } 217 | -------------------------------------------------------------------------------- /src/components/GithubLink.tsx: -------------------------------------------------------------------------------- 1 | import Image from "next/image"; 2 | import React from "react"; 3 | 4 | export const GithubLink = () => { 5 | return ( 6 | 24 | ); 25 | }; 26 | -------------------------------------------------------------------------------- /src/components/JukeBox.tsx: -------------------------------------------------------------------------------- 1 | import { useJukeBox } from "@/controller/JukeBoxProvider"; 2 | import { Vector2 } from "@/model/Vector2"; 3 | import { Container, Graphics, Sprite, Text, useTick } from "@pixi/react"; 4 | import { SCALE_MODES, TextStyle, Texture } from "pixi.js"; 5 | import { useMemo, useState } from "react"; 6 | 7 | type Props = { 8 | position: Vector2; 9 | backgroundZIndex: number; 10 | }; 11 | 12 | const BASE_SCALE = 2; 13 | 14 | export const JukeBox = ({ position, backgroundZIndex }: Props) => { 15 | const { jukeBoxParticipant } = useJukeBox(); 16 | const [jukeboxY, setJukeboxY] = useState(0); 17 | const [scale, setScale] = useState(BASE_SCALE); 18 | 19 | const jukeboxTexture = useMemo(() => { 20 | return Texture.from("/world/boombox.png", { 21 | scaleMode: SCALE_MODES.NEAREST, 22 | }); 23 | }, []); 24 | 25 | useTick(() => { 26 | if (jukeBoxParticipant === null) { 27 | setScale(BASE_SCALE); 28 | setJukeboxY(0); 29 | return; 30 | } 31 | const newScale = 32 | BASE_SCALE + Math.abs(Math.sin((Date.now() * 8) / 1000) * 0.2); 33 | const newY = Math.abs(Math.sin((Date.now() * 8) / 1000) * 0.2) * -30; 34 | setScale(newScale); 35 | setJukeboxY(newY); 36 | }); 37 | 38 | return ( 39 | //@ts-ignore 40 | 46 | {jukeBoxParticipant && ( 47 | //@ts-ignore 48 | 61 | )} 62 | 68 | { 73 | g.clear(); 74 | g.beginFill(0x000000, 0.2); 75 | g.drawEllipse(0, 14, 35, 15); 76 | g.endFill(); 77 | }} 78 | /> 79 | 80 | ); 81 | }; 82 | -------------------------------------------------------------------------------- /src/components/JukeBoxModal.tsx: -------------------------------------------------------------------------------- 1 | import { useJukeBox } from "@/controller/JukeBoxProvider"; 2 | import { useMobile } from "@/util/useMobile"; 3 | import { useCallback, useEffect, useMemo } from "react"; 4 | 5 | export const JukeBoxModal = () => { 6 | const { 7 | playJukeBox, 8 | amIPlayingJukeBox, 9 | stopJukeBox, 10 | jukeBoxTrack, 11 | someoneElsePlayingJukeBox, 12 | jukeBoxParticipant, 13 | } = useJukeBox(); 14 | 15 | const isMobile = useMobile(); 16 | 17 | const inner = useMemo(() => { 18 | if (amIPlayingJukeBox) { 19 | // I'm playing 20 | return isMobile ? ( 21 |
22 |
23 | You are playing to the speaker 24 |
25 | 31 |
32 | ) : ( 33 |
34 | You are playing to the speaker. Press{" "} 35 | [x] to stop. 36 |
37 | ); 38 | } else if (jukeBoxParticipant !== null) { 39 | // Someone else is playing 40 | return ( 41 |
42 | {jukeBoxParticipant} is playing to 43 | the speaker 44 |
45 | ); 46 | } else { 47 | // No one is playing 48 | return isMobile ? ( 49 |
50 |
51 | Want to listen to spatially groovy tunes over WebRTC? Press play. 52 |
53 | 59 |
60 | ) : ( 61 |
62 | Press [x] to play spatially groovy 63 | tunes over WebRTC . 64 |
65 | ); 66 | } 67 | }, [ 68 | amIPlayingJukeBox, 69 | isMobile, 70 | jukeBoxParticipant, 71 | playJukeBox, 72 | stopJukeBox, 73 | ]); 74 | 75 | const keyDownListener = useCallback( 76 | (e: KeyboardEvent) => { 77 | if (e.key === "x") { 78 | if (amIPlayingJukeBox) { 79 | stopJukeBox(); 80 | } else { 81 | playJukeBox(); 82 | } 83 | } 84 | }, 85 | [amIPlayingJukeBox, playJukeBox, stopJukeBox] 86 | ); 87 | 88 | useEffect(() => { 89 | if (isMobile) return; 90 | document.addEventListener("keydown", keyDownListener); 91 | 92 | return () => { 93 | document.removeEventListener("keydown", keyDownListener); 94 | }; 95 | }, [isMobile, keyDownListener]); 96 | 97 | return ( 98 |
99 | {inner} 100 |
101 | ); 102 | }; 103 | -------------------------------------------------------------------------------- /src/components/MicrophoneMuteButton.tsx: -------------------------------------------------------------------------------- 1 | import { TrackToggle } from "@livekit/components-react"; 2 | import { Track } from "livekit-client"; 3 | 4 | export function MicrophoneMuteButton() { 5 | return ( 6 |
7 | 11 |
12 | ); 13 | } 14 | -------------------------------------------------------------------------------- /src/components/MicrophoneSelector.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | useMediaDeviceSelect, 3 | useRoomContext, 4 | } from "@livekit/components-react"; 5 | 6 | export function MicrophoneSelector() { 7 | // TODO remove roomContext, this is only needed because of a bug in `useMediaDeviceSelect` 8 | const roomContext = useRoomContext(); 9 | const { devices, activeDeviceId, setActiveMediaDevice } = 10 | useMediaDeviceSelect({ kind: "audioinput", room: roomContext }); 11 | 12 | return ( 13 |
14 |
15 | 31 |
32 |
33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /src/components/PoweredByLiveKit.tsx: -------------------------------------------------------------------------------- 1 | import Image from "next/image"; 2 | import React from "react"; 3 | 4 | export const PoweredByLiveKit = () => { 5 | return ( 6 | 24 | ); 25 | }; 26 | -------------------------------------------------------------------------------- /src/components/RoomInfo.tsx: -------------------------------------------------------------------------------- 1 | import { RoomInfo } from "@/pages/api/room_info/[room]"; 2 | import { useCallback, useEffect, useMemo, useState } from "react"; 3 | import { setIntervalAsync, clearIntervalAsync } from "set-interval-async"; 4 | 5 | type Props = { 6 | roomName: string; 7 | }; 8 | 9 | const DEFAULT_ROOM_INFO: RoomInfo = { num_participants: 0 }; 10 | 11 | export function RoomInfo({ roomName }: Props) { 12 | const [roomInfo, setRoomInfo] = useState(DEFAULT_ROOM_INFO); 13 | 14 | const fetchRoomInfo = useCallback(async () => { 15 | const res = await fetch(`/api/room_info/${roomName}`); 16 | const roomInfo = (await res.json()) as RoomInfo; 17 | setRoomInfo(roomInfo); 18 | }, [roomName]); 19 | 20 | useEffect(() => { 21 | fetchRoomInfo(); 22 | const interval = setIntervalAsync(fetchRoomInfo, 1000); 23 | return () => { 24 | clearIntervalAsync(interval); 25 | }; 26 | }, [fetchRoomInfo]); 27 | 28 | return ( 29 |
30 |
31 | 32 | 33 | 34 |
Participants currently in room
35 |
36 |
37 | ); 38 | } 39 | -------------------------------------------------------------------------------- /src/components/Shadows.tsx: -------------------------------------------------------------------------------- 1 | import { Player } from "@/model/Player"; 2 | import { Graphics } from "@pixi/react"; 3 | 4 | type Props = { 5 | myPlayer: Player | null; 6 | remotePlayers: Player[]; 7 | backgroundZIndex: number; 8 | }; 9 | export const Shadows = ({ 10 | myPlayer, 11 | remotePlayers, 12 | backgroundZIndex, 13 | }: Props) => { 14 | const players = [...remotePlayers, myPlayer]; 15 | return ( 16 | <> 17 | {players.map((player) => { 18 | if (!player) return null; 19 | const { x, y } = player.position; 20 | return ( 21 | { 27 | g.clear(); 28 | g.beginFill(0x000000, 0.3); 29 | g.drawEllipse(0, 17, 15, 8); 30 | g.endFill(); 31 | }} 32 | /> 33 | ); 34 | })} 35 | 36 | ); 37 | }; 38 | -------------------------------------------------------------------------------- /src/components/Stage.tsx: -------------------------------------------------------------------------------- 1 | // adapted context bridge from https://pixijs.io/pixi-react/context-bridge/ for providing LiveKit room context to pixi components 2 | import { Stage as PixiStage } from "@pixi/react"; 3 | import { RoomContext } from "@livekit/components-react"; 4 | import { JukeBoxContext } from "@/controller/JukeBoxProvider"; 5 | 6 | interface ContextBridgeProps extends React.PropsWithChildren { 7 | render: (tree: React.ReactElement) => React.ReactNode; 8 | } 9 | 10 | // the context bridge: 11 | const ContextBridge = ({ children, render }: ContextBridgeProps) => { 12 | return ( 13 | 14 | {(roomValue) => ( 15 | 16 | {(jukeBoxValue) => 17 | render( 18 | 19 | 20 | {children} 21 | 22 | 23 | ) 24 | } 25 | 26 | )} 27 | 28 | ); 29 | }; 30 | 31 | // custom pixi stage with livekit room context 32 | export const Stage = ({ children, ...props }: PixiStage["props"]) => { 33 | return ( 34 | {children}} 36 | > 37 | {children} 38 | 39 | ); 40 | }; 41 | -------------------------------------------------------------------------------- /src/components/UsernameInput.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from "react"; 2 | 3 | type Props = { 4 | submitText: string; 5 | onSubmit: (username: string) => void; 6 | }; 7 | 8 | export function UsernameInput({ submitText, onSubmit }: Props) { 9 | const [username, setUsername] = useState(""); 10 | return ( 11 |
{ 13 | e.preventDefault(); 14 | onSubmit(username); 15 | }} 16 | > 17 |
18 |
19 | setUsername(e.currentTarget.value)} 22 | type="text" 23 | placeholder="Username" 24 | className="input input-bordered input-secondary" 25 | /> 26 | 27 |
28 |
29 |
30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /src/components/World.tsx: -------------------------------------------------------------------------------- 1 | import { WorldBoundaries } from "@/model/WorldBoundaries"; 2 | import { Sprite } from "@pixi/react"; 3 | 4 | type Props = { 5 | worldBoundaries: WorldBoundaries; 6 | backgroundZIndex: number; 7 | }; 8 | 9 | export const World = ({ worldBoundaries, backgroundZIndex }: Props) => { 10 | return ( 11 | 16 | ); 17 | }; 18 | -------------------------------------------------------------------------------- /src/components/icons/d-pad.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/components/icons/disco-ball.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/components/icons/mic-off.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/components/icons/mic.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/components/icons/sine-wave.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/controller/InputController.tsx: -------------------------------------------------------------------------------- 1 | import { Inputs } from "@/model/Inputs"; 2 | import { useMobile } from "@/util/useMobile"; 3 | import { 4 | Dispatch, 5 | SetStateAction, 6 | useCallback, 7 | useEffect, 8 | useRef, 9 | } from "react"; 10 | 11 | type Props = { 12 | mobileInputs?: Inputs; 13 | setInputs: Dispatch>; 14 | }; 15 | 16 | export const InputController = ({ setInputs, mobileInputs }: Props) => { 17 | const isMobile = useMobile(); 18 | const keyDownListener = useCallback( 19 | (e: KeyboardEvent) => { 20 | setInputs((prev) => { 21 | const newInputs = { ...prev }; 22 | if (e.key === "ArrowUp" || e.key === "w") { 23 | newInputs.direction.y = -1; 24 | } else if (e.key === "ArrowDown" || e.key === "s") { 25 | newInputs.direction.y = 1; 26 | } 27 | 28 | if (e.key === "ArrowLeft" || e.key === "a") { 29 | newInputs.direction.x = -1; 30 | } else if (e.key === "ArrowRight" || e.key === "d") { 31 | newInputs.direction.x = 1; 32 | } 33 | return newInputs; 34 | }); 35 | }, 36 | [setInputs] 37 | ); 38 | 39 | const keyUpListener = useCallback( 40 | (e: KeyboardEvent) => { 41 | setInputs((prev) => { 42 | const newInputs = { ...prev }; 43 | if ((e.key === "ArrowUp" || e.key === "w") && prev.direction.y === -1) { 44 | newInputs.direction.y = 0; 45 | } else if ( 46 | (e.key === "ArrowDown" || e.key === "s") && 47 | prev.direction.y === 1 48 | ) { 49 | newInputs.direction.y = 0; 50 | } 51 | 52 | if ( 53 | (e.key === "ArrowLeft" || e.key === "a") && 54 | prev.direction.x === -1 55 | ) { 56 | newInputs.direction.x = 0; 57 | } else if ( 58 | (e.key === "ArrowRight" || e.key === "d") && 59 | prev.direction.x === 1 60 | ) { 61 | newInputs.direction.x = 0; 62 | } 63 | 64 | return newInputs; 65 | }); 66 | }, 67 | [setInputs] 68 | ); 69 | 70 | useEffect(() => { 71 | if (isMobile) return; 72 | document.addEventListener("keydown", keyDownListener); 73 | document.addEventListener("keyup", keyUpListener); 74 | 75 | return () => { 76 | document.removeEventListener("keydown", keyDownListener); 77 | document.removeEventListener("keyup", keyUpListener); 78 | }; 79 | }, [isMobile, keyDownListener, keyUpListener, mobileInputs]); 80 | 81 | useEffect(() => { 82 | if (!isMobile || !mobileInputs) return; 83 | setInputs(mobileInputs); 84 | }, [isMobile, mobileInputs, setInputs]); 85 | 86 | return null; 87 | }; 88 | -------------------------------------------------------------------------------- /src/controller/JukeBoxProvider.tsx: -------------------------------------------------------------------------------- 1 | import { useCallback, useMemo, useRef } from "react"; 2 | 3 | import React, { useContext } from "react"; 4 | import { 5 | useLocalParticipant, 6 | useRemoteParticipants, 7 | } from "@livekit/components-react"; 8 | import { useUnmount } from "react-use"; 9 | import { 10 | TrackWithIdentity, 11 | useTracksByName, 12 | } from "@/util/useAudioTracksByName"; 13 | import { useWebAudioContext } from "@/providers/audio/webAudio"; 14 | import { LocalTrack, LocalTrackPublication, Track } from "livekit-client"; 15 | 16 | type Data = { 17 | jukeBoxTrack: TrackWithIdentity | null; 18 | amIPlayingJukeBox: boolean; 19 | jukeBoxParticipant: string | null; 20 | someoneElsePlayingJukeBox: boolean; 21 | playJukeBox: () => Promise; 22 | stopJukeBox: () => Promise; 23 | }; 24 | 25 | const defaultData: Data = { 26 | jukeBoxTrack: null, 27 | jukeBoxParticipant: null, 28 | amIPlayingJukeBox: false, 29 | someoneElsePlayingJukeBox: false, 30 | playJukeBox: () => Promise.resolve(), 31 | stopJukeBox: () => Promise.resolve(), 32 | }; 33 | 34 | type Props = { 35 | children: React.ReactNode; 36 | }; 37 | 38 | export const JukeBoxContext = React.createContext(defaultData); 39 | 40 | export const JukeBoxProvider = ({ children }: Props) => { 41 | const { localParticipant } = useLocalParticipant(); 42 | const existingJukeBoxTracks = useTracksByName("jukebox"); 43 | const audioContext = useWebAudioContext(); 44 | 45 | const audioElContainer = useRef(null); 46 | const audioEl = useRef(null); 47 | const source = useRef(null); 48 | const sink = useRef(null); 49 | 50 | const jukeBoxTrack = useMemo(() => { 51 | return existingJukeBoxTracks[0] || null; 52 | }, [existingJukeBoxTracks]); 53 | 54 | const jukeBoxParticipant = useMemo( 55 | () => jukeBoxTrack?.identity || null, 56 | [jukeBoxTrack?.identity] 57 | ); 58 | 59 | const amIPlayingJukeBox = useMemo( 60 | () => 61 | existingJukeBoxTracks.findIndex( 62 | (t) => t.identity === localParticipant.identity 63 | ) > -1, 64 | [existingJukeBoxTracks, localParticipant.identity] 65 | ); 66 | 67 | const cleanup = useRef(() => { 68 | if (sink.current) sink.current.disconnect(); 69 | if (source.current) source.current.disconnect(); 70 | if (audioEl.current) { 71 | audioEl.current.pause(); 72 | audioEl.current.remove(); 73 | } 74 | }); 75 | 76 | const stopJukeBox = useCallback(async () => { 77 | cleanup.current(); 78 | const myJukeBoxTracks = existingJukeBoxTracks 79 | .filter((t) => t.track instanceof LocalTrackPublication && t.track.track) 80 | .map((t) => t.track.track as LocalTrack); 81 | myJukeBoxTracks.forEach((t) => localParticipant.unpublishTrack(t)); 82 | }, [existingJukeBoxTracks, localParticipant]); 83 | 84 | const playJukeBox = useCallback(async () => { 85 | if (!audioElContainer.current) return; 86 | 87 | // Stop any existing jukebox 88 | await stopJukeBox(); 89 | 90 | audioEl.current = new Audio("/disco.mp3"); 91 | audioEl.current.setAttribute("muted", "false"); 92 | audioEl.current.setAttribute("loop", "true"); 93 | audioEl.current.setAttribute("autoplay", "true"); 94 | audioElContainer.current.appendChild(audioEl.current); 95 | source.current = audioContext.createMediaElementSource(audioEl.current); 96 | sink.current = audioContext.createMediaStreamDestination(); 97 | source.current.connect(sink.current); 98 | localParticipant.publishTrack(sink.current.stream.getAudioTracks()[0], { 99 | name: "jukebox", 100 | source: Track.Source.Unknown, 101 | }); 102 | }, [audioContext, localParticipant, stopJukeBox]); 103 | 104 | useUnmount(cleanup.current); 105 | 106 | return ( 107 | 0 && !amIPlayingJukeBox, 114 | playJukeBox, 115 | stopJukeBox, 116 | }} 117 | > 118 | {children} 119 |
120 | 121 | ); 122 | }; 123 | 124 | export function useJukeBox() { 125 | const ctx = useContext(JukeBoxContext); 126 | if (!ctx) { 127 | throw "useJukeBox must be used within a WebAudioProvider"; 128 | } 129 | return ctx; 130 | } 131 | -------------------------------------------------------------------------------- /src/controller/MyCharacterController.tsx: -------------------------------------------------------------------------------- 1 | import { Inputs } from "@/model/Inputs"; 2 | import { Player } from "@/model/Player"; 3 | import { useTick } from "@pixi/react"; 4 | import { Dispatch, SetStateAction } from "react"; 5 | 6 | type Props = { 7 | playerSpeed: number; 8 | inputs: Inputs; 9 | setMyPlayer: Dispatch>; 10 | }; 11 | 12 | export function MyCharacterController({ 13 | playerSpeed, 14 | inputs, 15 | setMyPlayer, 16 | }: Props) { 17 | useTick((delta) => { 18 | setMyPlayer((prev) => { 19 | if (!prev) { 20 | return prev; 21 | } 22 | 23 | const magnitude = Math.sqrt( 24 | inputs.direction.x ** 2 + inputs.direction.y ** 2 25 | ); 26 | let newAnimation = prev.animation; 27 | let newPosition = { ...prev.position }; 28 | let walking = magnitude > 0.01; 29 | 30 | const velocity = { 31 | x: magnitude > 0 ? (inputs.direction.x * playerSpeed) / magnitude : 0, 32 | y: magnitude > 0 ? (inputs.direction.y * playerSpeed) / magnitude : 0, 33 | }; 34 | 35 | if (velocity.x > 0 && walking) { 36 | newAnimation = "walk_right"; 37 | } else if (velocity.x < 0 && walking) { 38 | newAnimation = "walk_left"; 39 | } else { 40 | if (walking) { 41 | if (prev.animation.endsWith("_right")) { 42 | newAnimation = "walk_right"; 43 | } else { 44 | newAnimation = "walk_left"; 45 | } 46 | } else { 47 | if (prev.animation.endsWith("_right")) { 48 | newAnimation = "idle_right"; 49 | } else { 50 | newAnimation = "idle_left"; 51 | } 52 | } 53 | } 54 | 55 | newPosition = { 56 | x: prev.position.x + velocity.x * delta, 57 | y: prev.position.y + velocity.y * delta, 58 | }; 59 | 60 | return { ...prev, position: newPosition, animation: newAnimation }; 61 | }); 62 | }); 63 | return null; 64 | } 65 | -------------------------------------------------------------------------------- /src/controller/MyPlayerSpawnController.tsx: -------------------------------------------------------------------------------- 1 | import { CharacterName } from "@/components/CharacterSelector"; 2 | import { Player } from "@/model/Player"; 3 | import { Participant } from "livekit-client"; 4 | import { Dispatch, SetStateAction, useEffect } from "react"; 5 | 6 | type Props = { 7 | localParticipant: Participant | null; 8 | localCharacter: CharacterName | null; 9 | myPlayer: Player | null; 10 | setMyPlayer: Dispatch>; 11 | }; 12 | 13 | export function MyPlayerSpawnController({ 14 | setMyPlayer, 15 | myPlayer, 16 | localParticipant, 17 | localCharacter, 18 | }: Props) { 19 | useEffect(() => { 20 | if (myPlayer === null && localParticipant?.identity && localCharacter) { 21 | setMyPlayer({ 22 | username: localParticipant.identity, 23 | position: { x: 10, y: 0 }, 24 | animation: "idle_left", 25 | character: localCharacter, 26 | }); 27 | } 28 | }, [localCharacter, localParticipant, myPlayer, setMyPlayer]); 29 | return null; 30 | } 31 | -------------------------------------------------------------------------------- /src/controller/NetcodeController.tsx: -------------------------------------------------------------------------------- 1 | "use-client"; 2 | 3 | import { AnimationState } from "@/model/AnimationState"; 4 | import { Player } from "@/model/Player"; 5 | import { Vector2 } from "@/model/Vector2"; 6 | import { 7 | useConnectionState, 8 | useLocalParticipant, 9 | useRemoteParticipants, 10 | useRoomContext, 11 | } from "@livekit/components-react"; 12 | import { 13 | ConnectionState, 14 | DataPacket_Kind, 15 | RemoteParticipant, 16 | RoomEvent, 17 | } from "livekit-client"; 18 | import React, { 19 | Dispatch, 20 | SetStateAction, 21 | useCallback, 22 | useEffect, 23 | useMemo, 24 | useRef, 25 | } from "react"; 26 | import { useInterval } from "react-use"; 27 | 28 | type Props = { 29 | myPlayer: Player; 30 | setNetworkPositions: Dispatch>>; 31 | setNetworkAnimations: Dispatch>>; 32 | }; 33 | 34 | export function NetcodeController({ 35 | myPlayer, 36 | setNetworkAnimations, 37 | setNetworkPositions, 38 | }: Props) { 39 | // LiveKit state 40 | const roomCtx = useRoomContext(); 41 | const connectionState = useConnectionState(); 42 | const remoteParticipants = useRemoteParticipants({}); 43 | const { localParticipant } = useLocalParticipant(); 44 | 45 | // Position and animation state 46 | const _playerPositions = useRef>( 47 | new Map() 48 | ); 49 | const _animations = useRef>(new Map()); 50 | const _myPosition = useRef(myPlayer.position); 51 | const _myAnimation = useRef(myPlayer.animation); 52 | 53 | const positionSendLock = useRef(false); 54 | const animationSendLock = useRef(false); 55 | const textEncoder = useRef(new TextEncoder()); 56 | const textDecoder = useRef(new TextDecoder()); 57 | 58 | const onDataChannel = useCallback( 59 | (payload: Uint8Array, participant: RemoteParticipant | undefined) => { 60 | if (!participant) return; 61 | const data = JSON.parse(textDecoder.current.decode(payload)); 62 | if (data.channelId === "position") { 63 | const { x, y } = data.payload; 64 | _playerPositions.current.set(participant.identity, { x, y }); 65 | } else if (data.channelId === "animation") { 66 | _animations.current.set(participant.identity, data.payload); 67 | } 68 | }, 69 | [] 70 | ); 71 | 72 | // Setup datachannel listener 73 | useEffect(() => { 74 | roomCtx.on(RoomEvent.DataReceived, onDataChannel); 75 | 76 | return () => { 77 | roomCtx.off(RoomEvent.DataReceived, onDataChannel); 78 | }; 79 | }, [onDataChannel, roomCtx]); 80 | 81 | // Take myPosition and myAnimation out of react-land so we can send them reliable on a fixed interval 82 | useEffect(() => { 83 | _myPosition.current = myPlayer.position; 84 | _myAnimation.current = myPlayer.animation; 85 | }, [myPlayer]); 86 | 87 | const sendMyPosition = useCallback(async () => { 88 | if (positionSendLock.current) return; 89 | positionSendLock.current = true; 90 | try { 91 | const payload: Uint8Array = textEncoder.current.encode( 92 | JSON.stringify({ payload: _myPosition.current, channelId: "position" }) 93 | ); 94 | await localParticipant.publishData(payload, DataPacket_Kind.LOSSY); 95 | } finally { 96 | positionSendLock.current = false; 97 | } 98 | }, [localParticipant]); 99 | 100 | const sendMyAnimation = useCallback(async () => { 101 | if (animationSendLock.current) return; 102 | animationSendLock.current = true; 103 | try { 104 | const payload: Uint8Array = textEncoder.current.encode( 105 | JSON.stringify({ 106 | payload: _myAnimation.current, 107 | channelId: "animation", 108 | }) 109 | ); 110 | await localParticipant.publishData(payload, DataPacket_Kind.LOSSY); 111 | } finally { 112 | animationSendLock.current = false; 113 | } 114 | }, [localParticipant]); 115 | 116 | const remoteParticipantLookup = useMemo(() => { 117 | return new Set(remoteParticipants.map((p) => p.identity)); 118 | }, [remoteParticipants]); 119 | 120 | // Cleanup positions and animations for participants that have left 121 | useEffect(() => { 122 | const positionIdentities = Array.from(_playerPositions.current.keys()); 123 | const animationIdentities = Array.from(_animations.current.keys()); 124 | for (const identity of positionIdentities) { 125 | if (!remoteParticipantLookup.has(identity)) { 126 | _playerPositions.current.delete(identity); 127 | } 128 | } 129 | for (const identity of animationIdentities) { 130 | if (!remoteParticipantLookup.has(identity)) { 131 | _animations.current.delete(identity); 132 | } 133 | } 134 | }, [remoteParticipantLookup]); 135 | 136 | const setNetworkValues = useCallback(() => { 137 | setNetworkAnimations(new Map(_animations.current)); 138 | setNetworkPositions(new Map(_playerPositions.current)); 139 | }, [setNetworkAnimations, setNetworkPositions]); 140 | 141 | const sendUpdate = useCallback(() => { 142 | if (connectionState !== ConnectionState.Connected) return; 143 | sendMyPosition(); 144 | sendMyAnimation(); 145 | }, [connectionState, sendMyAnimation, sendMyPosition]); 146 | 147 | useInterval(setNetworkValues, 100); 148 | useInterval(sendUpdate, 100); 149 | 150 | return null; 151 | } 152 | -------------------------------------------------------------------------------- /src/controller/RemotePlayersController.tsx: -------------------------------------------------------------------------------- 1 | "use-client"; 2 | 3 | import { CharacterName } from "@/components/CharacterSelector"; 4 | import { AnimationState } from "@/model/AnimationState"; 5 | import { Player } from "@/model/Player"; 6 | import { Vector2 } from "@/model/Vector2"; 7 | import { useRemoteParticipants } from "@livekit/components-react"; 8 | import { Dispatch, SetStateAction, useCallback, useMemo, useRef } from "react"; 9 | import { useInterval } from "react-use"; 10 | 11 | type Props = { 12 | networkPositions: Map; 13 | networkAnimations: Map; 14 | setRemotePlayers: Dispatch>; 15 | }; 16 | 17 | export function RemotePlayersController({ 18 | networkPositions, 19 | networkAnimations, 20 | setRemotePlayers, 21 | }: Props) { 22 | const remoteParticipants = useRemoteParticipants({}); 23 | 24 | const remoteCharacterLookup = useMemo(() => { 25 | const lookup = new Map(); 26 | for (const rp of remoteParticipants) { 27 | const metadata = JSON.parse(rp.metadata || "{}"); 28 | lookup.set(rp.identity, metadata.character || ("doux" as CharacterName)); 29 | } 30 | return lookup; 31 | }, [remoteParticipants]); 32 | 33 | const applyNetworkValues = useCallback(() => { 34 | setRemotePlayers((previousRemotePlayers) => { 35 | const participantIdentities = remoteParticipants.map((rp) => rp.identity); 36 | const previousPlayersLookup = new Map(); 37 | for (const rp of previousRemotePlayers) { 38 | previousPlayersLookup.set(rp.username, rp); 39 | } 40 | 41 | // cleanup players that no longer have a remote participant or 42 | // we haven't received network data yet and make copies to keep 43 | // react updates working 44 | const newRemotePlayers: Player[] = participantIdentities 45 | .filter( 46 | (identity) => 47 | networkAnimations.has(identity) && networkPositions.has(identity) 48 | ) 49 | .map((identity) => ({ 50 | username: identity, 51 | position: 52 | previousPlayersLookup.get(identity)?.position || 53 | networkPositions.get(identity)!, // use the network position if we don't have a previous one 54 | animation: networkAnimations.get(identity)!, 55 | character: remoteCharacterLookup.get(identity)! || "doux", 56 | })); 57 | 58 | // Crude interpolation that tries to match the 0.1 second send interval 59 | for (const p of newRemotePlayers) { 60 | p.position = { 61 | x: 62 | p.position.x + 63 | (networkPositions.get(p.username)!.x - p.position.x) * (10 / 30), 64 | y: 65 | p.position.y + 66 | (networkPositions.get(p.username)!.y - p.position.y) * (10 / 30), 67 | }; 68 | } 69 | 70 | return newRemotePlayers; 71 | }); 72 | }, [ 73 | networkAnimations, 74 | networkPositions, 75 | remoteCharacterLookup, 76 | remoteParticipants, 77 | setRemotePlayers, 78 | ]); 79 | 80 | useInterval(applyNetworkValues, 1000 / 30); 81 | 82 | return null; 83 | } 84 | -------------------------------------------------------------------------------- /src/controller/SpatialAudioController.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Vector2 } from "@/model/Vector2"; 4 | import { useMobile } from "@/util/useMobile"; 5 | import { 6 | LocalTrackPublication, 7 | RemoteTrackPublication, 8 | TrackPublication, 9 | } from "livekit-client"; 10 | import React, { 11 | useCallback, 12 | useEffect, 13 | useMemo, 14 | useRef, 15 | useState, 16 | } from "react"; 17 | import { useWebAudioContext } from "../providers/audio/webAudio"; 18 | 19 | export type TrackPosition = { 20 | trackPublication: TrackPublication; 21 | position: Vector2; 22 | }; 23 | 24 | type SpatialAudioControllerProps = { 25 | trackPositions: TrackPosition[]; 26 | myPosition: Vector2; 27 | maxHearableDistance: number; 28 | }; 29 | 30 | export function SpatialAudioController({ 31 | trackPositions, 32 | myPosition, 33 | maxHearableDistance, 34 | }: SpatialAudioControllerProps) { 35 | const audioContext = useWebAudioContext(); 36 | if (!audioContext) return null; 37 | return ( 38 | <> 39 | {trackPositions.map((tp) => { 40 | return ( 41 | 48 | ); 49 | })} 50 | 51 | ); 52 | } 53 | 54 | type SpatialParticipantPlaybackProps = { 55 | maxHearableDistance: number; 56 | trackPublication: TrackPublication; 57 | myPosition: { x: number; y: number }; 58 | position: { x: number; y: number }; 59 | }; 60 | 61 | function SpatialPublicationPlayback({ 62 | maxHearableDistance, 63 | trackPublication, 64 | myPosition, 65 | position, 66 | }: SpatialParticipantPlaybackProps) { 67 | const distance = useMemo(() => { 68 | const dx = myPosition.x - position.x; 69 | const dy = myPosition.y - position.y; 70 | return Math.sqrt(dx * dx + dy * dy); 71 | }, [myPosition.x, myPosition.y, position.x, position.y]); 72 | 73 | const hearable = useMemo( 74 | () => distance <= maxHearableDistance, 75 | [distance, maxHearableDistance] 76 | ); 77 | 78 | // Selective subscription 79 | useEffect(() => { 80 | if (!(trackPublication instanceof RemoteTrackPublication)) { 81 | return; 82 | } 83 | trackPublication?.setSubscribed(hearable); 84 | }, [hearable, trackPublication]); 85 | 86 | return ( 87 |
88 | {hearable && ( 89 | 94 | )} 95 |
96 | ); 97 | } 98 | 99 | type PublicationRendererProps = { 100 | trackPublication: TrackPublication; 101 | position: { x: number; y: number }; 102 | myPosition: { x: number; y: number }; 103 | }; 104 | 105 | function PublicationRenderer({ 106 | trackPublication, 107 | position, 108 | myPosition, 109 | }: PublicationRendererProps) { 110 | const mobile = useMobile(); 111 | const audioEl = useRef(null); 112 | const audioContext = useWebAudioContext(); 113 | const sourceNode = useRef(null); 114 | const panner = useRef(null); 115 | const gain = useRef(null); 116 | const [relativePosition, setRelativePosition] = useState<{ 117 | x: number; 118 | y: number; 119 | }>({ 120 | x: 1000, 121 | y: 1000, 122 | }); // set as far away initially 123 | const mediaStream = usePublicationAudioMediaStream(trackPublication); 124 | 125 | const cleanupWebAudio = useCallback(() => { 126 | if (panner.current) panner.current.disconnect(); 127 | if (sourceNode.current) sourceNode.current.disconnect(); 128 | if (gain.current) gain.current.disconnect(); 129 | 130 | gain.current = null; 131 | panner.current = null; 132 | sourceNode.current = null; 133 | }, []); 134 | 135 | // calculate relative position when position changes 136 | useEffect(() => { 137 | setRelativePosition((prev) => { 138 | return { 139 | x: position.x - myPosition.x, 140 | y: position.y - myPosition.y, 141 | }; 142 | }); 143 | }, [myPosition.x, myPosition.y, position.x, position.y]); 144 | 145 | // setup panner node for desktop 146 | useEffect(() => { 147 | cleanupWebAudio(); 148 | 149 | if ( 150 | !audioEl.current || 151 | !mediaStream || 152 | mediaStream.getAudioTracks().length === 0 153 | ) { 154 | return; 155 | } 156 | 157 | sourceNode.current = audioContext.createMediaStreamSource(mediaStream); 158 | 159 | // if on mobile, the panner node has no effect 160 | if (mobile) { 161 | gain.current = audioContext.createGain(); 162 | gain.current.gain.setValueAtTime(0, 0); 163 | sourceNode.current 164 | .connect(gain.current) 165 | .connect(audioContext.destination); 166 | } else { 167 | panner.current = audioContext.createPanner(); 168 | panner.current.coneOuterAngle = 360; 169 | panner.current.coneInnerAngle = 360; 170 | panner.current.positionX.setValueAtTime(relativePosition.x, 0); // set far away initially so we don't hear it at full volume 171 | panner.current.positionY.setValueAtTime(relativePosition.y, 0); 172 | panner.current.positionZ.setValueAtTime(0, 0); 173 | panner.current.distanceModel = "exponential"; 174 | panner.current.coneOuterGain = 1; 175 | panner.current.refDistance = 100; 176 | panner.current.maxDistance = 500; 177 | panner.current.rolloffFactor = 2; 178 | sourceNode.current 179 | .connect(panner.current) 180 | .connect(audioContext.destination); 181 | audioEl.current.srcObject = mediaStream; 182 | audioEl.current.play(); 183 | } 184 | 185 | return cleanupWebAudio; 186 | }, [ 187 | panner, 188 | mobile, 189 | trackPublication.track, 190 | cleanupWebAudio, 191 | audioContext, 192 | trackPublication, 193 | mediaStream, 194 | ]); 195 | 196 | // On mobile we use volume because panner nodes have no effect 197 | // https://developer.apple.com/forums/thread/696034 198 | useEffect(() => { 199 | if (!audioEl.current) return; 200 | 201 | // for mobile we use the gain node 202 | if (mobile) { 203 | if (!gain.current) return; 204 | const distance = Math.sqrt( 205 | relativePosition.x ** 2 + relativePosition.y ** 2 206 | ); 207 | if (distance < 50) { 208 | gain.current.gain.setTargetAtTime(1, 0, 0.2); 209 | } else { 210 | if (distance > 250) { 211 | gain.current.gain.setTargetAtTime(0, 0, 0.2); 212 | return; 213 | } 214 | gain.current.gain.setTargetAtTime(1 - (distance - 50) / 200, 0, 0.2); 215 | } 216 | } else { 217 | if (!panner.current) return; 218 | panner.current.positionX.setTargetAtTime(relativePosition.x, 0, 0.02); 219 | panner.current.positionZ.setTargetAtTime(relativePosition.y, 0, 0.02); 220 | } 221 | }, [mobile, relativePosition.x, relativePosition.y, panner]); 222 | 223 | return ( 224 | <> 225 |