├── LICENSE ├── README.md ├── agent ├── .env.sample ├── .gitignore ├── package-lock.json ├── package.json ├── playground_agent.py ├── playground_agent.ts ├── pnpm-lock.yaml ├── requirements.txt ├── ruff.toml └── tsconfig.json └── web ├── .env.sample ├── .eslintrc.json ├── .gitignore ├── components.json ├── next.config.mjs ├── package-lock.json ├── package.json ├── pnpm-lock.yaml ├── postcss.config.mjs ├── public ├── favicon.ico ├── fonts │ ├── CommitMono-400-Regular.otf │ └── CommitMono-700-Regular.otf └── og-image.png ├── src ├── app │ ├── api │ │ └── token │ │ │ └── route.ts │ ├── globals.css │ ├── layout.tsx │ └── page.tsx ├── assets │ ├── heart.svg │ └── lk.svg ├── components │ ├── agent │ │ ├── agent-control-bar.tsx │ │ ├── animation-sequences │ │ │ ├── connecting-sequence.ts │ │ │ ├── listening-sequence.ts │ │ │ └── thinking-sequence.ts │ │ ├── animators │ │ │ ├── use-bar-animator.ts │ │ │ ├── use-grid-animator.ts │ │ │ └── use-radial-animator.ts │ │ ├── data │ │ │ └── visualizer-variations.ts │ │ └── visualizers │ │ │ ├── bar-visualizer.tsx │ │ │ ├── grid-visualizer.tsx │ │ │ ├── multiband-bar-visualizer.tsx │ │ │ └── radial-visualizer.tsx │ ├── auth.tsx │ ├── authBanner.tsx │ ├── chat-controls.tsx │ ├── chat.tsx │ ├── code-viewer.tsx │ ├── configuration-form-drawer.tsx │ ├── configuration-form.tsx │ ├── connect-button.tsx │ ├── header.tsx │ ├── instructions-editor.tsx │ ├── instructions.tsx │ ├── lk.tsx │ ├── max-output-tokens-selector.tsx │ ├── modalities-selector.tsx │ ├── model-selector.tsx │ ├── preset-save.tsx │ ├── preset-selector.tsx │ ├── preset-share.tsx │ ├── room-component.tsx │ ├── session-config.tsx │ ├── session-controls.tsx │ ├── temperature-selector.tsx │ ├── top-p-selector.tsx │ ├── transcript-drawer.tsx │ ├── transcript.tsx │ ├── transcription-selector.tsx │ ├── turn-detection-selector.tsx │ ├── ui │ │ ├── alert-dialog.tsx │ │ ├── badge.tsx │ │ ├── button.tsx │ │ ├── checkbox.tsx │ │ ├── command.tsx │ │ ├── dialog.tsx │ │ ├── drawer.tsx │ │ ├── dropdown-menu.tsx │ │ ├── form.tsx │ │ ├── hover-card.tsx │ │ ├── input.tsx │ │ ├── label.tsx │ │ ├── popover.tsx │ │ ├── select.tsx │ │ ├── separator.tsx │ │ ├── slider.tsx │ │ ├── switch.tsx │ │ ├── tabs.tsx │ │ ├── textarea.tsx │ │ ├── toast.tsx │ │ ├── toaster.tsx │ │ └── tooltip.tsx │ ├── vad-prefix-padding-selector.tsx │ ├── vad-silence-duration-selector.tsx │ ├── vad-threshold-selector.tsx │ └── voice-selector.tsx ├── data │ ├── agent.ts │ ├── modalities.ts │ ├── models.ts │ ├── playground-state.ts │ ├── presets.ts │ ├── transcription-models.ts │ ├── turn-end-types.ts │ └── voices.ts ├── hooks │ ├── use-agent.tsx │ ├── use-connection.tsx │ ├── use-multiband-track-volume.tsx │ ├── use-mutation-observer.ts │ ├── use-playground-state.tsx │ └── use-toast.ts ├── lib │ ├── agent │ │ └── audio-visualizer.ts │ ├── playground-state-helpers.ts │ └── utils.ts └── types │ └── svg.d.ts ├── tailwind.config.ts └── tsconfig.json /README.md: -------------------------------------------------------------------------------- 1 | # LiveKit + OpenAI Realtime Playground 2 | 3 | This project is an interactive playground that demonstrates the capabilities of OpenAI's Realtime API, allowing users to experiment with the API directly in their browser. It's built on top of LiveKit Agents. 4 | 5 | See it in action at [realtime-playground.livekit.io](https://realtime-playground.livekit.io) 6 | 7 | ## Repository Structure 8 | 9 | ### /agent 10 | 11 | This directory contains the agent implementation. There are two versions, one in Python and one in TypeScript. 12 | 13 | 1. `playground_agent.py` - Built on the LiveKit [Python Agents framework](https://github.com/livekit/agents) 14 | 2. `playground_agent.ts` - Built on the LiveKit [Node.js Agents framework](https://github.com/livekit/agents-js) 15 | 16 | ### /web 17 | 18 | This directory houses the web frontend, built with Next.js. 19 | 20 | ## Prerequisites 21 | 22 | - Node.js and pnpm (for web frontend and Node.js agent) 23 | - Python 3.9 or higher (for Python agent) 24 | - pip (Python package installer) 25 | - LiveKit Cloud or self-hosted LiveKit server 26 | 27 | ## Getting Started 28 | 29 | ### Agent Setup 30 | 31 | 1. Navigate to the `/agent` directory 32 | 2. Copy the sample environment file: `cp .env.sample .env.local` 33 | 3. Open `.env.local` in a text editor and enter your LiveKit credentials 34 | 35 | #### Python Version 36 | 37 | 1. Create a virtual environment: `python -m venv .venv` 38 | 2. Activate the virtual environment: 39 | - On macOS and Linux: `source .venv/bin/activate` 40 | - On Windows: `.venv\Scripts\activate` 41 | 3. Load the environment variables: 42 | - On macOS and Linux: `source .env.local` 43 | - On Windows: `set -a; . .env.local; set +a` 44 | 4. Install dependencies: `pip install -r requirements.txt` 45 | 5. Run the agent in development mode: `python playground_agent.py dev` 46 | 47 | #### Node.js Version 48 | 49 | 1. Install dependencies: `pnpm install` 50 | 2. Run the agent in development mode: `pnpm dev` 51 | 52 | ### Web Frontend Setup 53 | 54 | 1. Navigate to the `/web` directory 55 | 2. Copy the sample environment file: `cp .env.sample .env.local` 56 | 3. Open `.env.local` in a text editor and enter your LiveKit credentials: 57 | 4. Install dependencies: `pnpm install` 58 | 5. Run the development server: `pnpm dev` 59 | 6. Open [http://localhost:3000](http://localhost:3000) in your browser 60 | 61 | ## Deployment 62 | 63 | The agent can be deployed in a variety of ways: [Deployment & Scaling Guide](https://docs.livekit.io/agents/deployment/) 64 | 65 | The web frontend can be deployed using your preferred Next.js hosting solution, such as [Vercel](https://vercel.com/). 66 | 67 | ## Troubleshooting 68 | 69 | Ensure the following: 70 | 71 | - Both web and agent are running 72 | - Environment variables are set up correctly 73 | - Correct versions of Node.js, pnpm, and Python are installed 74 | 75 | ## Additional Resources 76 | 77 | For more information or support, please refer to [LiveKit docs](https://docs.livekit.io/). 78 | 79 | ## License 80 | 81 | Apache 2.0 82 | -------------------------------------------------------------------------------- /agent/.env.sample: -------------------------------------------------------------------------------- 1 | # Sign Up for LiveKit Cloud --> https://cloud.livekit.io 2 | # Copy this file to .env.local and fill in the values. it should match ../web/.env.local 3 | export LIVEKIT_URL= 4 | export LIVEKIT_API_KEY= 5 | export LIVEKIT_API_SECRET= 6 | -------------------------------------------------------------------------------- /agent/.gitignore: -------------------------------------------------------------------------------- 1 | .env.local 2 | node_modules/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # poetry 101 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 102 | # This is especially recommended for binary packages to ensure reproducibility, and is more 103 | # commonly ignored for libraries. 104 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 105 | #poetry.lock 106 | 107 | # pdm 108 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 109 | #pdm.lock 110 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 111 | # in version control. 112 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 113 | .pdm.toml 114 | .pdm-python 115 | .pdm-build/ 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # mypy 147 | .mypy_cache/ 148 | .dmypy.json 149 | dmypy.json 150 | 151 | # Pyre type checker 152 | .pyre/ 153 | 154 | # pytype static type analyzer 155 | .pytype/ 156 | 157 | # Cython debug symbols 158 | cython_debug/ 159 | 160 | # PyCharm 161 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 162 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 163 | # and can be added to the global gitignore or merged into this file. For a more nuclear 164 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 165 | #.idea/ 166 | 167 | -------------------------------------------------------------------------------- /agent/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "name": "realtime-playground-agent", 4 | "type": "module", 5 | "scripts": { 6 | "build": "tsc", 7 | "clean": "rm -rf dist", 8 | "clean:build": "pnpm clean && pnpm build", 9 | "lint": "eslint -f unix \"src/**/*.ts\"", 10 | "dev": "pnpm exec tsx playground_agent.ts dev", 11 | "start": "pnpm exec tsx playground_agent.ts start", 12 | "format:check": "prettier --check \"**/*.{ts,tsx,md,json}\"", 13 | "format:write": "prettier --write \"**/*.{ts,tsx,md,json}\"" 14 | }, 15 | "devDependencies": { 16 | "@types/node": "^22.7.5", 17 | "@types/uuid": "^10.0.0", 18 | "dotenv": "^16.4.5", 19 | "typescript": "^5.0.0" 20 | }, 21 | "dependencies": { 22 | "@livekit/agents": "^0.3.2", 23 | "@livekit/agents-plugin-openai": "^0.3.2", 24 | "@livekit/rtc-node": "^0.9.1", 25 | "eslint": "^8", 26 | "eslint-config-prettier": "^8.10.0", 27 | "eslint-plugin-prettier": "^5.1.3", 28 | "eslint-plugin-unused-imports": "^3.0.0", 29 | "uuid": "^10.0.0", 30 | "zod": "^3.23.8" 31 | }, 32 | "version": "0.0.1", 33 | "packageManager": "pnpm@9.6.0" 34 | } 35 | -------------------------------------------------------------------------------- /agent/playground_agent.ts: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2024 LiveKit, Inc. 2 | // 3 | // SPDX-License-Identifier: Apache-2.0 4 | import { type JobContext, WorkerOptions, cli, defineAgent, multimodal } from "@livekit/agents"; 5 | import * as openai from "@livekit/agents-plugin-openai"; 6 | import type { LocalParticipant, Participant, TrackPublication } from "@livekit/rtc-node"; 7 | import { RemoteParticipant, TrackSource } from "@livekit/rtc-node"; 8 | import dotenv from "dotenv"; 9 | import { fileURLToPath } from "node:url"; 10 | import { v4 as uuidv4 } from "uuid"; 11 | 12 | dotenv.config({ path: ".env.local" }); 13 | 14 | function safeLogConfig(config: SessionConfig): string { 15 | const safeConfig = { ...config, openaiApiKey: "[REDACTED]" }; 16 | return JSON.stringify(safeConfig); 17 | } 18 | 19 | export default defineAgent({ 20 | entry: async (ctx: JobContext) => { 21 | await ctx.connect(); 22 | 23 | const participant = await ctx.waitForParticipant(); 24 | 25 | await runMultimodalAgent(ctx, participant); 26 | } 27 | }); 28 | 29 | type TurnDetectionType = { 30 | type: "server_vad"; 31 | threshold?: number; 32 | prefix_padding_ms?: number; 33 | silence_duration_ms?: number; 34 | }; 35 | 36 | interface SessionConfig { 37 | openaiApiKey: string; 38 | instructions: string; 39 | voice: string; 40 | temperature: number; 41 | maxOutputTokens?: number; 42 | modalities: string[]; 43 | turnDetection: TurnDetectionType; 44 | } 45 | 46 | function parseSessionConfig(data: any): SessionConfig { 47 | return { 48 | openaiApiKey: data.openai_api_key || "", 49 | instructions: data.instructions || "", 50 | voice: data.voice || "", 51 | temperature: parseFloat(data.temperature || "0.8"), 52 | maxOutputTokens: data.max_output_tokens === "inf" ? Infinity : parseInt(data.max_output_tokens) || undefined, 53 | modalities: modalitiesFromString(data.modalities || "text_and_audio"), 54 | turnDetection: data.turn_detection ? JSON.parse(data.turn_detection) : null 55 | }; 56 | } 57 | 58 | function modalitiesFromString(modalities: string): ["text", "audio"] | ["text"] { 59 | const modalitiesMap: { [key: string]: ["text", "audio"] | ["text"] } = { 60 | text_and_audio: ["text", "audio"], 61 | text_only: ["text"] 62 | }; 63 | return modalitiesMap[modalities] || ["text", "audio"]; 64 | } 65 | 66 | function getMicrophoneTrackSid(participant: Participant): string | undefined { 67 | return Array.from(participant.trackPublications.values()).find((track: TrackPublication) => track.source === TrackSource.SOURCE_MICROPHONE)?.sid; 68 | } 69 | 70 | async function runMultimodalAgent(ctx: JobContext, participant: RemoteParticipant) { 71 | const metadata = JSON.parse(participant.metadata); 72 | const config = parseSessionConfig(metadata); 73 | console.log(`starting multimodal agent with config: ${safeLogConfig(config)}`); 74 | 75 | const model = new openai.realtime.RealtimeModel({ 76 | apiKey: config.openaiApiKey, 77 | instructions: config.instructions, 78 | voice: config.voice, 79 | temperature: config.temperature, 80 | maxResponseOutputTokens: config.maxOutputTokens, 81 | modalities: config.modalities as ["text", "audio"] | ["text"], 82 | turnDetection: config.turnDetection 83 | }); 84 | 85 | const agent = new multimodal.MultimodalAgent({ model }); 86 | const session = (await agent.start(ctx.room)) as openai.realtime.RealtimeSession; 87 | 88 | session.conversation.item.create({ 89 | type: "message", 90 | role: "user", 91 | content: [ 92 | { 93 | type: "input_text", 94 | text: "Please begin the interaction with the user in a manner consistent with your instructions." 95 | } 96 | ] 97 | }); 98 | session.response.create(); 99 | 100 | ctx.room.on("participantAttributesChanged", (changedAttributes: Record, changedParticipant: Participant) => { 101 | if (changedParticipant !== participant) { 102 | return; 103 | } 104 | const newConfig = parseSessionConfig({ 105 | ...changedParticipant.attributes, 106 | ...changedAttributes 107 | }); 108 | 109 | session.sessionUpdate({ 110 | instructions: newConfig.instructions, 111 | temperature: newConfig.temperature, 112 | maxResponseOutputTokens: newConfig.maxOutputTokens, 113 | modalities: newConfig.modalities as ["text", "audio"] | ["text"], 114 | turnDetection: newConfig.turnDetection 115 | }); 116 | }); 117 | 118 | async function sendTranscription(ctx: JobContext, participant: Participant, trackSid: string, segmentId: string, text: string, isFinal: boolean = true) { 119 | const transcription = { 120 | participantIdentity: participant.identity, 121 | trackSid: trackSid, 122 | segments: [ 123 | { 124 | id: segmentId, 125 | text: text, 126 | startTime: BigInt(0), 127 | endTime: BigInt(0), 128 | language: "", 129 | final: isFinal 130 | } 131 | ] 132 | }; 133 | await (ctx.room.localParticipant as LocalParticipant).publishTranscription(transcription); 134 | } 135 | 136 | session.on("response_done", (response: openai.realtime.RealtimeResponse) => { 137 | let message: string | undefined; 138 | if (response.status === "incomplete") { 139 | if (response.statusDetails?.reason) { 140 | const reason = response.statusDetails.reason; 141 | switch (reason) { 142 | case "max_output_tokens": 143 | message = "🚫 Max output tokens reached"; 144 | break; 145 | case "content_filter": 146 | message = "🚫 Content filter applied"; 147 | break; 148 | default: 149 | message = `🚫 Response incomplete: ${reason}`; 150 | break; 151 | } 152 | } else { 153 | message = "🚫 Response incomplete"; 154 | } 155 | } else if (response.status === "failed") { 156 | if (response.statusDetails?.error) { 157 | switch (response.statusDetails.error.code) { 158 | case "server_error": 159 | message = `⚠️ Server error`; 160 | break; 161 | case "rate_limit_exceeded": 162 | message = `⚠️ Rate limit exceeded`; 163 | break; 164 | default: 165 | message = `⚠️ Response failed`; 166 | break; 167 | } 168 | } else { 169 | message = "⚠️ Response failed"; 170 | } 171 | } else { 172 | return; 173 | } 174 | 175 | const localParticipant = ctx.room.localParticipant as LocalParticipant; 176 | const trackSid = getMicrophoneTrackSid(localParticipant); 177 | 178 | if (trackSid) { 179 | sendTranscription(ctx, localParticipant, trackSid, "status-" + uuidv4(), message); 180 | } 181 | }); 182 | } 183 | 184 | cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) })); 185 | -------------------------------------------------------------------------------- /agent/requirements.txt: -------------------------------------------------------------------------------- 1 | livekit >= 0.15.0 2 | livekit-protocol 3 | livekit-agents>=0.10.0 4 | livekit-plugins-openai>=0.10.0 5 | -------------------------------------------------------------------------------- /agent/ruff.toml: -------------------------------------------------------------------------------- 1 | line-length = 88 2 | indent-width = 4 3 | 4 | target-version = "py39" 5 | 6 | [lint] 7 | extend-select = ["I"] 8 | 9 | [lint.pydocstyle] 10 | convention = "numpy" 11 | 12 | [format] 13 | docstring-code-format = true 14 | -------------------------------------------------------------------------------- /agent/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["./playground_agent.ts"], 3 | "compilerOptions": { 4 | "rootDir": "./", 5 | "outDir": "./dist", 6 | "target": "ES2017", 7 | "module": "ES2022", 8 | "moduleResolution": "node", 9 | "declaration": true, 10 | "declarationMap": true, 11 | "allowJs": false, 12 | "strict": true, 13 | "skipLibCheck": true, 14 | "esModuleInterop": true, 15 | "sourceMap": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /web/.env.sample: -------------------------------------------------------------------------------- 1 | # Sign Up for LiveKit Cloud and create a project --> https://cloud.livekit.io 2 | # Copy this file to .env.local and fill in the values. it should match ../agent/.env.local 3 | export LIVEKIT_URL= 4 | export LIVEKIT_API_KEY= 5 | export LIVEKIT_API_SECRET= 6 | -------------------------------------------------------------------------------- /web/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals", 3 | "plugins": ["unused-imports"], 4 | "rules": { 5 | "unused-imports/no-unused-imports": "error" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | .pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | web/coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | .env.local 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | -------------------------------------------------------------------------------- /web/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.ts", 8 | "css": "src/app/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": false, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /web/next.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | webpack(config) { 4 | config.module.rules.push({ 5 | test: /\.svg$/, // Look for .svg files 6 | use: ["@svgr/webpack"], // Use @svgr/webpack to handle them 7 | }); 8 | 9 | return config; // Always return the modified config 10 | }, 11 | }; 12 | 13 | export default nextConfig; 14 | -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "playground", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint", 10 | "lint:fix": "next lint --fix", 11 | "format:check": "prettier --check \"**/*.{ts,tsx,md,json}\"", 12 | "format:write": "prettier --write \"**/*.{ts,tsx,md,json}\"" 13 | }, 14 | "dependencies": { 15 | "@hookform/resolvers": "^3.9.0", 16 | "@livekit/components-react": "^2.6.4", 17 | "@livekit/components-styles": "^1.1.2", 18 | "@livekit/krisp-noise-filter": "^0.2.12", 19 | "@radix-ui/react-alert-dialog": "^1.1.1", 20 | "@radix-ui/react-checkbox": "^1.1.1", 21 | "@radix-ui/react-dialog": "^1.1.1", 22 | "@radix-ui/react-dropdown-menu": "^2.1.1", 23 | "@radix-ui/react-hover-card": "^1.1.1", 24 | "@radix-ui/react-icons": "^1.3.0", 25 | "@radix-ui/react-label": "^2.1.0", 26 | "@radix-ui/react-popover": "^1.1.1", 27 | "@radix-ui/react-scroll-area": "^1.1.0", 28 | "@radix-ui/react-select": "^2.1.1", 29 | "@radix-ui/react-separator": "^1.1.0", 30 | "@radix-ui/react-slider": "^1.2.0", 31 | "@radix-ui/react-slot": "^1.1.0", 32 | "@radix-ui/react-switch": "^1.1.0", 33 | "@radix-ui/react-tabs": "^1.1.0", 34 | "@radix-ui/react-toast": "^1.2.1", 35 | "@radix-ui/react-tooltip": "^1.1.2", 36 | "@svgr/webpack": "^8.1.0", 37 | "@tiptap/extension-placeholder": "^2.6.6", 38 | "@tiptap/pm": "^2.6.6", 39 | "@tiptap/react": "^2.6.6", 40 | "@tiptap/starter-kit": "^2.6.6", 41 | "class-variance-authority": "^0.7.0", 42 | "clsx": "^2.1.1", 43 | "cmdk": "1.0.0", 44 | "framer-motion": "^11.5.4", 45 | "js-cookie": "^3.0.5", 46 | "livekit-client": "^2.5.4", 47 | "livekit-server-sdk": "^2.6.1", 48 | "lucide-react": "^0.437.0", 49 | "next": "14.2.7", 50 | "react": "^18", 51 | "react-dom": "^18", 52 | "react-hook-form": "^7.53.0", 53 | "react-syntax-highlighter": "^15.5.0", 54 | "tailwind-merge": "^2.5.2", 55 | "tailwindcss-animate": "^1.0.7", 56 | "vaul": "^0.9.1", 57 | "zod": "^3.23.8" 58 | }, 59 | "devDependencies": { 60 | "@types/node": "^20", 61 | "@types/react": "^18", 62 | "@types/react-dom": "^18", 63 | "@types/react-syntax-highlighter": "^15.5.13", 64 | "@typescript-eslint/eslint-plugin": "^7.2.0", 65 | "@typescript-eslint/parser": "^7.2.0", 66 | "eslint": "^8", 67 | "eslint-config-next": "14.2.7", 68 | "eslint-config-prettier": "^8.10.0", 69 | "eslint-plugin-prettier": "^5.1.3", 70 | "eslint-plugin-unused-imports": "^3.0.0", 71 | "postcss": "^8", 72 | "react-syntax-highlighter": "^15.5.0", 73 | "tailwindcss": "^3.4.1", 74 | "typescript": "^5" 75 | }, 76 | "packageManager": "pnpm@9.6.0" 77 | } 78 | -------------------------------------------------------------------------------- /web/postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/6400b53e0b0f6cec213e96a6cd284e38ecd6c722/web/public/favicon.ico -------------------------------------------------------------------------------- /web/public/fonts/CommitMono-400-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/6400b53e0b0f6cec213e96a6cd284e38ecd6c722/web/public/fonts/CommitMono-400-Regular.otf -------------------------------------------------------------------------------- /web/public/fonts/CommitMono-700-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/6400b53e0b0f6cec213e96a6cd284e38ecd6c722/web/public/fonts/CommitMono-700-Regular.otf -------------------------------------------------------------------------------- /web/public/og-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/6400b53e0b0f6cec213e96a6cd284e38ecd6c722/web/public/og-image.png -------------------------------------------------------------------------------- /web/src/app/api/token/route.ts: -------------------------------------------------------------------------------- 1 | import { AccessToken } from "livekit-server-sdk"; 2 | import { PlaygroundState } from "@/data/playground-state"; 3 | 4 | export async function POST(request: Request) { 5 | const { 6 | instructions, 7 | openaiAPIKey, 8 | sessionConfig: { 9 | turnDetection, 10 | modalities, 11 | voice, 12 | temperature, 13 | maxOutputTokens, 14 | vadThreshold, 15 | vadSilenceDurationMs, 16 | vadPrefixPaddingMs, 17 | }, 18 | }: PlaygroundState = await request.json(); 19 | 20 | const roomName = Math.random().toString(36).substring(7); 21 | const apiKey = process.env.LIVEKIT_API_KEY; 22 | const apiSecret = process.env.LIVEKIT_API_SECRET; 23 | if (!apiKey || !apiSecret) { 24 | throw new Error("LIVEKIT_API_KEY and LIVEKIT_API_SECRET must be set"); 25 | } 26 | 27 | const at = new AccessToken(apiKey, apiSecret, { 28 | identity: "human", 29 | metadata: JSON.stringify({ 30 | instructions: instructions, 31 | modalities: modalities, 32 | voice: voice, 33 | temperature: temperature, 34 | max_output_tokens: maxOutputTokens, 35 | openai_api_key: openaiAPIKey, 36 | turn_detection: JSON.stringify({ 37 | type: turnDetection, 38 | threshold: vadThreshold, 39 | silence_duration_ms: vadSilenceDurationMs, 40 | prefix_padding_ms: vadPrefixPaddingMs, 41 | }), 42 | }), 43 | }); 44 | at.addGrant({ 45 | room: roomName, 46 | roomJoin: true, 47 | canPublish: true, 48 | canPublishData: true, 49 | canSubscribe: true, 50 | canUpdateOwnMetadata: true, 51 | }); 52 | return Response.json({ 53 | accessToken: await at.toJwt(), 54 | url: process.env.LIVEKIT_URL, 55 | }); 56 | } 57 | -------------------------------------------------------------------------------- /web/src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 0, 0, 0; 7 | --background-start-rgb: 214, 219, 220; 8 | --background-end-rgb: 255, 255, 255; 9 | } 10 | 11 | @media (prefers-color-scheme: dark) { 12 | :root { 13 | --foreground-rgb: 255, 255, 255; 14 | --background-start-rgb: 0, 0, 0; 15 | --background-end-rgb: 0, 0, 0; 16 | } 17 | } 18 | 19 | @layer utilities { 20 | .text-balance { 21 | text-wrap: balance; 22 | } 23 | } 24 | 25 | @layer base { 26 | :root { 27 | --radius: 0.5rem; 28 | } 29 | } 30 | 31 | * { 32 | box-sizing: border-box; 33 | } 34 | 35 | body, 36 | html { 37 | margin: 0; 38 | padding: 0; 39 | height: 100vh; 40 | width: 100vw; 41 | display: flex; 42 | flex-direction: column; 43 | } 44 | 45 | .tiptap p.is-editor-empty:first-child::before { 46 | color: #adb5bd; 47 | font-weight: 300; 48 | content: attr(data-placeholder); 49 | float: left; 50 | height: 0; 51 | pointer-events: none; 52 | } 53 | 54 | .lk-audio-bar-visualizer { 55 | gap: 12px !important; 56 | 57 | & > .lk-audio-bar { 58 | /* aspect-ratio: 1/1; */ 59 | /* width: auto !important; */ 60 | width: 64px !important; 61 | min-height: 64px !important; 62 | background-color: rgba(0, 0, 0, 0.05) !important; 63 | } 64 | 65 | &[data-lk-va-state='speaking'] > .lk-audio-bar, 66 | & > .lk-audio-bar.lk-highlighted, 67 | & > [data-lk-highlighted='true'] { 68 | @apply bg-black !important; 69 | } 70 | 71 | & > [data-lk-highlighted='false'] { 72 | @apply bg-black/10 !important; 73 | } 74 | } 75 | 76 | @font-face { 77 | font-family: 'Commit Mono'; 78 | src: url('/fonts/CommitMono-400-Regular.otf') format('opentype'); 79 | font-weight: 400; 80 | font-style: normal; 81 | } 82 | 83 | @font-face { 84 | font-family: 'Commit Mono'; 85 | src: url('/fonts/CommitMono-700-Regular.otf') format('opentype'); 86 | font-weight: 700; 87 | font-style: normal; 88 | } 89 | -------------------------------------------------------------------------------- /web/src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from "next"; 2 | import "./globals.css"; 3 | import { PlaygroundStateProvider } from "@/hooks/use-playground-state"; 4 | import { ConnectionProvider } from "@/hooks/use-connection"; 5 | import { TooltipProvider } from "@/components/ui/tooltip"; 6 | import { Toaster } from "@/components/ui/toaster"; 7 | import { Public_Sans } from "next/font/google"; 8 | 9 | // Configure the Public Sans font 10 | const publicSans = Public_Sans({ 11 | subsets: ["latin"], 12 | weight: ["100", "200", "300", "400", "500", "600", "700", "800", "900"], 13 | style: ["normal", "italic"], 14 | display: "swap", 15 | }); 16 | 17 | import "@livekit/components-styles"; 18 | 19 | export const metadata: Metadata = { 20 | title: "Realtime Playground", 21 | description: 22 | "Try OpenAI's new Realtime API right from your browser. Built on LiveKit Agents.", 23 | openGraph: { 24 | title: "Realtime Playground", 25 | description: 26 | "Try OpenAI's new Realtime API right from your browser. Built on LiveKit Agents.", 27 | type: "website", 28 | url: "https://playground.livekit.io/", 29 | images: [ 30 | { 31 | url: "https://playground.livekit.io/og-image.png", 32 | width: 1200, 33 | height: 675, 34 | type: "image/png", 35 | alt: "Realtime Playground", 36 | }, 37 | ], 38 | }, 39 | }; 40 | 41 | export default function RootLayout({ 42 | children, 43 | }: Readonly<{ 44 | children: React.ReactNode; 45 | }>) { 46 | return ( 47 | 48 | 49 | 50 | 51 | 52 | {children} 53 | 54 | 55 | 56 | 57 | 58 | 59 | ); 60 | } 61 | -------------------------------------------------------------------------------- /web/src/app/page.tsx: -------------------------------------------------------------------------------- 1 | import { Header } from "@/components/header"; 2 | import { RoomComponent } from "@/components/room-component"; 3 | import { Auth } from "@/components/auth"; 4 | import LK from "@/components/lk"; 5 | import Heart from "@/assets/heart.svg"; 6 | import { GitHubLogoIcon } from "@radix-ui/react-icons"; 7 | export default function Dashboard() { 8 | return ( 9 |
10 |
11 | 12 | 13 |
14 |
15 |
16 | 17 |
18 | 42 |
43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /web/src/assets/heart.svg: -------------------------------------------------------------------------------- 1 | 8 | 15 | 22 | 29 | 36 | 43 | 50 | 57 | 64 | 71 | 78 | 85 | 92 | 99 | 106 | 113 | 120 | 127 | 134 | 141 | 148 | 149 | 150 | 151 | 158 | 165 | 172 | 179 | 183 | 190 | 197 | 204 | -------------------------------------------------------------------------------- /web/src/assets/lk.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /web/src/components/agent/agent-control-bar.tsx: -------------------------------------------------------------------------------- 1 | import { AgentState } from "@/data/agent"; 2 | import { ComponentType, CSSProperties } from "react"; 3 | import { AgentBarVisualizer } from "./visualizers/bar-visualizer"; 4 | import { AgentGridVisualizer } from "./visualizers/grid-visualizer"; 5 | import { AgentRadialBarVisualizer } from "@/components/agent/visualizers/radial-visualizer"; 6 | 7 | export type GridAnimationOptions = { 8 | interval?: number; 9 | connectingRing?: number; 10 | onTransition?: string; 11 | offTransition?: string; 12 | }; 13 | 14 | export type AgentVisualizerOptions = { 15 | baseStyle: CSSProperties; 16 | gridComponent?: ComponentType<{ style: CSSProperties }>; 17 | gridSpacing?: string; 18 | onStyle?: CSSProperties; 19 | offStyle?: CSSProperties; 20 | transformer?: (distanceFromCenter: number) => CSSProperties; 21 | rowCount?: number; 22 | animationOptions?: GridAnimationOptions; 23 | maxHeight?: number; 24 | minHeight?: number; 25 | radiusFactor?: number; 26 | radial?: boolean; 27 | stateOptions?: { 28 | [key in AgentState]: AgentVisualizerOptions; 29 | }; 30 | }; 31 | 32 | export type AgentVisualizerProps = { 33 | style?: "grid" | "bar" | "radial" | "waveform"; 34 | state: AgentState; 35 | volumeBands: number[]; 36 | options?: AgentVisualizerOptions; 37 | }; 38 | 39 | export type GridAnimatorState = "paused" | "active"; 40 | 41 | // Separate style and semantic properties 42 | // Semantic, react component 43 | // style: CSS 44 | // Animator is a hooks 45 | // Animation properties in css vars 46 | export const AgentVisualizer = ({ 47 | style = "grid", 48 | state, 49 | volumeBands, 50 | options, 51 | }: AgentVisualizerProps) => { 52 | if (style === "grid") { 53 | return ( 54 | 59 | ); 60 | } else if (style === "bar") { 61 | return ( 62 | 67 | ); 68 | } else if (style === "radial") { 69 | return ( 70 | 75 | ); 76 | } 77 | return ( 78 | 83 | ); 84 | }; 85 | 86 | export const AgentWaveformVisualizer = ({ 87 | state, 88 | volumeBands, 89 | options, 90 | }: AgentVisualizerProps) => { 91 | return
; 92 | }; 93 | -------------------------------------------------------------------------------- /web/src/components/agent/animation-sequences/connecting-sequence.ts: -------------------------------------------------------------------------------- 1 | export const generateConnectingSequence = ( 2 | rows: number, 3 | columns: number, 4 | ringDistance: number, 5 | ) => { 6 | let seq = []; 7 | const centerX = Math.floor(columns / 2); 8 | const centerY = Math.floor(rows / 2); 9 | 10 | // Calculate the boundaries of the ring based on the ring distance 11 | const topLeft = { 12 | x: Math.max(0, centerY - ringDistance), 13 | y: Math.max(0, centerY - ringDistance), 14 | }; 15 | const bottomRight = { 16 | x: columns - 1 - topLeft.x, 17 | y: Math.min(rows - 1, centerY + ringDistance), 18 | }; 19 | 20 | // Top edge 21 | for (let x = topLeft.x; x <= bottomRight.x; x++) { 22 | seq.push({ x, y: topLeft.y }); 23 | } 24 | 25 | // Right edge 26 | for (let y = topLeft.y + 1; y <= bottomRight.y; y++) { 27 | seq.push({ x: bottomRight.x, y }); 28 | } 29 | 30 | // Bottom edge 31 | for (let x = bottomRight.x - 1; x >= topLeft.x; x--) { 32 | seq.push({ x, y: bottomRight.y }); 33 | } 34 | 35 | // Left edge 36 | for (let y = bottomRight.y - 1; y > topLeft.y; y--) { 37 | seq.push({ x: topLeft.x, y }); 38 | } 39 | 40 | return seq; 41 | }; 42 | 43 | export const generateConnectingSequenceBar = ( 44 | columns: number, 45 | ): number[] | number[][] => { 46 | let seq = []; 47 | 48 | for (let x = 0; x <= columns; x++) { 49 | seq.push([x, columns - 1 - x]); 50 | } 51 | 52 | return seq; 53 | }; 54 | 55 | export const generateConnectingSequenceRadialBar = ( 56 | columns: number, 57 | ): number[] | number[][] => { 58 | let seq = []; 59 | 60 | for (let x = 0; x <= columns; x++) { 61 | seq.push([x, columns - 1 - x]); 62 | } 63 | 64 | return seq; 65 | }; 66 | -------------------------------------------------------------------------------- /web/src/components/agent/animation-sequences/listening-sequence.ts: -------------------------------------------------------------------------------- 1 | export const generateListeningSequence = (rows: number, columns: number) => { 2 | const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }; 3 | const noIndex = { x: -1, y: -1 }; 4 | 5 | return [ 6 | center, 7 | noIndex, 8 | noIndex, 9 | noIndex, 10 | noIndex, 11 | noIndex, 12 | noIndex, 13 | noIndex, 14 | noIndex, 15 | ]; 16 | }; 17 | 18 | export const generateListeningSequenceBar = (columns: number) => { 19 | const center = Math.floor(columns / 2); 20 | const noIndex = -1; 21 | 22 | return [center, noIndex]; 23 | }; 24 | 25 | export const generateListeningSequenceRadialBar = (columns: number) => { 26 | const evenNumbers = Array.from({ length: columns }, (_, i) => i).filter( 27 | (num) => num % 2 === 0, 28 | ); 29 | const oddNumbers = Array.from({ length: columns }, (_, i) => i).filter( 30 | (num) => num % 2 !== 0, 31 | ); 32 | const noIndex = -1; 33 | 34 | return [evenNumbers, noIndex, oddNumbers, noIndex]; 35 | }; 36 | -------------------------------------------------------------------------------- /web/src/components/agent/animation-sequences/thinking-sequence.ts: -------------------------------------------------------------------------------- 1 | export const generateThinkingSequence = (rows: number, columns: number) => { 2 | let seq = []; 3 | let y = Math.floor(rows / 2); 4 | for (let x = 0; x < columns; x++) { 5 | seq.push({ x, y }); 6 | } 7 | for (let x = columns - 1; x >= 0; x--) { 8 | seq.push({ x, y }); 9 | } 10 | 11 | return seq; 12 | }; 13 | 14 | export const generateThinkingSequenceBar = (columns: number) => { 15 | let seq = []; 16 | for (let x = 0; x < columns; x++) { 17 | seq.push(x); 18 | } 19 | 20 | for (let x = columns - 1; x >= 0; x--) { 21 | seq.push(x); 22 | } 23 | 24 | return seq; 25 | }; 26 | 27 | export const generateThinkingSequenceRadialBar = (columns: number) => { 28 | let seq = []; 29 | for (let x = 0; x < columns; x++) { 30 | seq.push(x); 31 | } 32 | 33 | return seq; 34 | }; 35 | -------------------------------------------------------------------------------- /web/src/components/agent/animators/use-bar-animator.ts: -------------------------------------------------------------------------------- 1 | import { AgentState } from "@/data/agent"; 2 | import { useEffect, useRef, useState } from "react"; 3 | import { 4 | GridAnimationOptions, 5 | GridAnimatorState, 6 | } from "@/components/agent/agent-control-bar"; 7 | import { generateConnectingSequenceBar } from "@/components/agent/animation-sequences/connecting-sequence"; 8 | import { generateListeningSequenceBar } from "@/components/agent/animation-sequences/listening-sequence"; 9 | import { generateThinkingSequenceBar } from "@/components/agent/animation-sequences/thinking-sequence"; 10 | 11 | export const useBarAnimator = ( 12 | type: AgentState, 13 | columns: number, 14 | interval: number, 15 | state: GridAnimatorState, 16 | animationOptions?: GridAnimationOptions, 17 | ): number | number[] => { 18 | const [index, setIndex] = useState(0); 19 | const [sequence, setSequence] = useState<(number | number[])[]>([]); 20 | 21 | useEffect(() => { 22 | if (type === "thinking") { 23 | setSequence(generateThinkingSequenceBar(columns)); 24 | } else if (type === "connecting") { 25 | const sequence = [...generateConnectingSequenceBar(columns)]; 26 | setSequence(sequence); 27 | } else if (type === "listening") { 28 | setSequence(generateListeningSequenceBar(columns)); 29 | } else { 30 | setSequence([]); 31 | } 32 | setIndex(0); 33 | }, [type, columns, state, animationOptions?.connectingRing]); 34 | 35 | const animationFrameId = useRef(null); 36 | useEffect(() => { 37 | if (state === "paused") { 38 | return; 39 | } 40 | 41 | let startTime = performance.now(); 42 | 43 | const animate = (time: DOMHighResTimeStamp) => { 44 | const timeElapsed = time - startTime; 45 | 46 | if (timeElapsed >= interval) { 47 | setIndex((prev) => prev + 1); 48 | startTime = time; 49 | } 50 | 51 | animationFrameId.current = requestAnimationFrame(animate); 52 | }; 53 | 54 | animationFrameId.current = requestAnimationFrame(animate); 55 | 56 | return () => { 57 | if (animationFrameId.current !== null) { 58 | cancelAnimationFrame(animationFrameId.current); 59 | } 60 | }; 61 | }, [interval, columns, state, type, sequence.length]); 62 | 63 | return sequence[index % sequence.length]; 64 | }; 65 | -------------------------------------------------------------------------------- /web/src/components/agent/animators/use-grid-animator.ts: -------------------------------------------------------------------------------- 1 | import { AgentState } from "@/data/agent"; 2 | import { useEffect, useState } from "react"; 3 | import { 4 | GridAnimationOptions, 5 | GridAnimatorState, 6 | } from "@/components/agent/agent-control-bar"; 7 | import { generateConnectingSequence } from "@/components/agent/animation-sequences/connecting-sequence"; 8 | import { generateListeningSequence } from "@/components/agent/animation-sequences/listening-sequence"; 9 | import { generateThinkingSequence } from "@/components/agent/animation-sequences/thinking-sequence"; 10 | 11 | export const useGridAnimator = ( 12 | type: AgentState, 13 | rows: number, 14 | columns: number, 15 | interval: number, 16 | state: GridAnimatorState, 17 | animationOptions?: GridAnimationOptions, 18 | ): { x: number; y: number } => { 19 | const [index, setIndex] = useState(0); 20 | const [sequence, setSequence] = useState<{ x: number; y: number }[]>([]); 21 | 22 | useEffect(() => { 23 | if (type === "thinking") { 24 | setSequence(generateThinkingSequence(rows, columns)); 25 | } else if (type === "connecting") { 26 | const sequence = [ 27 | ...generateConnectingSequence( 28 | rows, 29 | columns, 30 | animationOptions?.connectingRing ?? 1, 31 | ), 32 | ]; 33 | setSequence(sequence); 34 | } else if (type === "listening") { 35 | setSequence(generateListeningSequence(rows, columns)); 36 | } else { 37 | setSequence([]); 38 | } 39 | setIndex(0); 40 | }, [type, rows, columns, state, animationOptions?.connectingRing]); 41 | 42 | useEffect(() => { 43 | if (state === "paused") { 44 | return; 45 | } 46 | const indexInterval = setInterval(() => { 47 | setIndex((prev) => { 48 | return prev + 1; 49 | }); 50 | }, interval); 51 | return () => clearInterval(indexInterval); 52 | }, [interval, columns, rows, state, type, sequence.length]); 53 | 54 | return sequence[index % sequence.length]; 55 | }; 56 | -------------------------------------------------------------------------------- /web/src/components/agent/animators/use-radial-animator.ts: -------------------------------------------------------------------------------- 1 | import { AgentState } from "@/data/agent"; 2 | import { useEffect, useState } from "react"; 3 | import { 4 | GridAnimationOptions, 5 | GridAnimatorState, 6 | } from "@/components/agent/agent-control-bar"; 7 | import { generateConnectingSequenceRadialBar } from "@/components/agent/animation-sequences/connecting-sequence"; 8 | import { generateListeningSequenceRadialBar } from "@/components/agent/animation-sequences/listening-sequence"; 9 | import { generateThinkingSequenceRadialBar } from "@/components/agent/animation-sequences/thinking-sequence"; 10 | 11 | export const useRadialBarAnimator = ( 12 | type: AgentState, 13 | columns: number, 14 | interval: number, 15 | state: GridAnimatorState, 16 | animationOptions?: GridAnimationOptions, 17 | ): number | number[] => { 18 | const [index, setIndex] = useState(0); 19 | const [sequence, setSequence] = useState<(number | number[])[]>([]); 20 | 21 | useEffect(() => { 22 | if (type === "thinking") { 23 | setSequence(generateThinkingSequenceRadialBar(columns)); 24 | } else if (type === "connecting") { 25 | const sequence = [...generateConnectingSequenceRadialBar(columns)]; 26 | setSequence(sequence); 27 | } else if (type === "listening") { 28 | setSequence(generateListeningSequenceRadialBar(columns)); 29 | } else { 30 | setSequence([]); 31 | } 32 | setIndex(0); 33 | }, [type, columns, state, animationOptions?.connectingRing]); 34 | 35 | useEffect(() => { 36 | if (state === "paused") { 37 | return; 38 | } 39 | const indexInterval = setInterval(() => { 40 | setIndex((prev) => { 41 | return prev + 1; 42 | }); 43 | }, interval); 44 | return () => clearInterval(indexInterval); 45 | }, [interval, columns, state, type, sequence.length]); 46 | 47 | return sequence[index % sequence.length]; 48 | }; 49 | -------------------------------------------------------------------------------- /web/src/components/agent/data/visualizer-variations.ts: -------------------------------------------------------------------------------- 1 | export const visualizerVariations: { 2 | name: string; 3 | style: "grid" | "bar" | "radial" | "waveform"; 4 | count: number; 5 | options: any; 6 | }[] = [ 7 | { 8 | name: "Grid 1", 9 | style: "grid", 10 | count: 5, 11 | options: { 12 | baseStyle: { 13 | width: "32px", 14 | height: "32px", 15 | borderRadius: "4px", 16 | }, 17 | offStyle: { 18 | backgroundColor: "rgba(255, 255, 255, 0.05)", 19 | }, 20 | onStyle: { 21 | backgroundColor: "#FFF", 22 | boxShadow: "0px 0px 32px 8px rgba(255, 255, 255, 0.20)", 23 | }, 24 | gridSpacing: "48px", 25 | animationOptions: { 26 | interval: 70, 27 | connectingRing: 2, 28 | }, 29 | minHeight: 32, 30 | maxHeight: 128, 31 | }, 32 | }, 33 | { 34 | name: "Grid 2", 35 | style: "grid", 36 | count: 20, 37 | options: { 38 | baseStyle: { 39 | width: "8px", 40 | height: "8px", 41 | borderRadius: "2px", 42 | }, 43 | offStyle: { 44 | backgroundColor: "rgba(255, 255, 255, 0.05)", 45 | }, 46 | onStyle: { 47 | backgroundColor: "#FFF", 48 | boxShadow: "0px 0px 4px 2px rgba(255, 255, 255, 0.20)", 49 | }, 50 | gridSpacing: "24px", 51 | animationOptions: { 52 | interval: 75, 53 | onTransition: "all 0.02s ease-out", 54 | offTransition: "all 0.5s ease-out", 55 | connectingRing: 2, 56 | }, 57 | stateOptions: { 58 | listening: { 59 | animationOptions: { 60 | interval: 100, 61 | onTransition: "all 0.2s ease-out", 62 | offTransition: "all 0.5s ease-out", 63 | }, 64 | }, 65 | }, 66 | }, 67 | }, 68 | { 69 | name: "Bar 1", 70 | style: "bar", 71 | count: 5, 72 | options: { 73 | baseStyle: { 74 | width: "64px", 75 | borderRadius: "32px", 76 | }, 77 | offStyle: { 78 | backgroundColor: "rgba(255, 255, 255, 0.05)", 79 | }, 80 | onStyle: { 81 | backgroundColor: "#FFF", 82 | boxShadow: "0px 0px 32px 8px rgba(255, 255, 255, 0.20)", 83 | }, 84 | gridSpacing: "24px", 85 | animationOptions: { 86 | interval: 70, 87 | onTransition: "all 0.2s ease-out", 88 | offTransition: "all 0.5s ease-out", 89 | }, 90 | minHeight: 64, 91 | maxHeight: 256, 92 | stateOptions: { 93 | thinking: { 94 | animationOptions: { 95 | onTransition: "all 0.075s ease-out", 96 | interval: 40, 97 | }, 98 | }, 99 | connecting: { 100 | animationOptions: { 101 | interval: 350, 102 | }, 103 | }, 104 | speaking: { 105 | animationOptions: { 106 | onTransition: "none", 107 | offTransition: "none", 108 | }, 109 | }, 110 | listening: { 111 | animationOptions: { 112 | interval: 500, 113 | }, 114 | offStyle: { 115 | // backgroundColor: 'rgba(255, 255, 255, 0.1)', 116 | }, 117 | onStyle: { 118 | transform: "scale(1.05)", 119 | }, 120 | }, 121 | }, 122 | }, 123 | }, 124 | { 125 | name: "Bar 2", 126 | style: "bar", 127 | count: 50, 128 | options: { 129 | baseStyle: { 130 | width: "3px", 131 | borderRadius: "4px", 132 | }, 133 | offStyle: { 134 | backgroundColor: "rgba(0, 0, 0, 0.25)", 135 | }, 136 | onStyle: { 137 | backgroundColor: "#5ABE82", 138 | boxShadow: "0px 0px 32px 8px rgba(255, 255, 255, 0.20)", 139 | }, 140 | gridSpacing: "4px", 141 | animationOptions: { 142 | interval: 25, 143 | onTransition: "all 0.0s ease-out", 144 | offTransition: "all 0.5s ease-out", 145 | }, 146 | minHeight: 2, 147 | maxHeight: 256, 148 | stateOptions: { 149 | thinking: { 150 | animationOptions: { 151 | onTransition: "all 0.04s ease-out", 152 | interval: 40, 153 | }, 154 | }, 155 | connecting: { 156 | animationOptions: { 157 | interval: 200, 158 | onTransition: "all 0.2s ease-out", 159 | offTransition: "all 1.5s ease-out", 160 | }, 161 | }, 162 | speaking: { 163 | animationOptions: { 164 | onTransition: "none", 165 | offTransition: "none", 166 | }, 167 | }, 168 | listening: { 169 | animationOptions: { 170 | interval: 500, 171 | }, 172 | offStyle: { 173 | // backgroundColor: 'rgba(255, 255, 255, 0.1)', 174 | }, 175 | onStyle: { 176 | transform: "scale(1.05)", 177 | }, 178 | }, 179 | }, 180 | }, 181 | }, 182 | { 183 | name: "Radial", 184 | style: "radial", 185 | count: 14, 186 | options: { 187 | baseStyle: { 188 | borderRadius: "12px", 189 | width: "32px", 190 | background: "#FFF", 191 | }, 192 | offStyle: { 193 | background: "rgba(255, 255, 255, 0.05)", 194 | }, 195 | onStyle: { 196 | background: "#FFF", 197 | boxShadow: "0px 0px 8px 2px rgba(255, 255, 255, 0.4)", 198 | }, 199 | animationOptions: { 200 | interval: 500, 201 | onTransition: "all 0.2s ease-out", 202 | offTransition: "all 0.5s ease-out", 203 | }, 204 | stateOptions: { 205 | speaking: { 206 | animationOptions: { 207 | interval: 500, 208 | onTransition: "none", 209 | offTransition: "none", 210 | }, 211 | }, 212 | thinking: { 213 | animationOptions: { 214 | interval: 50, 215 | onTransition: "all 0.05s ease-out", 216 | offTransition: "all 0.5s ease-out", 217 | }, 218 | }, 219 | }, 220 | gridSpacing: "8px", 221 | minHeight: 24, 222 | maxHeight: 84, 223 | radiusFactor: 3, 224 | radial: true, 225 | }, 226 | }, 227 | ]; 228 | -------------------------------------------------------------------------------- /web/src/components/agent/visualizers/bar-visualizer.tsx: -------------------------------------------------------------------------------- 1 | import { useMemo } from "react"; 2 | import { AgentVisualizerProps } from "@/components/agent/agent-control-bar"; 3 | import { useBarAnimator } from "@/components/agent/animators/use-bar-animator"; 4 | 5 | export const AgentBarVisualizer = ({ 6 | state, 7 | volumeBands, 8 | options, 9 | }: AgentVisualizerProps) => { 10 | const gridColumns = volumeBands.length; 11 | const gridArray = Array.from({ length: gridColumns }).map((_, i) => i); 12 | const minHeight = options?.minHeight ?? 8; 13 | const maxHeight = options?.maxHeight ?? 64; 14 | const midpoint = Math.floor(gridColumns / 2.0); 15 | 16 | let animationOptions = options?.animationOptions; 17 | 18 | if (options?.stateOptions) { 19 | animationOptions = { 20 | ...animationOptions, 21 | ...options.stateOptions[state]?.animationOptions, 22 | }; 23 | } 24 | const highlightedIndex = useBarAnimator( 25 | state, 26 | gridColumns, 27 | animationOptions?.interval ?? 100, 28 | state !== "speaking" ? "active" : "paused", 29 | animationOptions, 30 | ); 31 | 32 | // TODO: Remove useMemo 33 | const bars = useMemo(() => { 34 | let baseStyle = options?.baseStyle ?? {}; 35 | let onStyle = { 36 | ...baseStyle, 37 | ...(options?.onStyle ?? {}), 38 | ...(options?.stateOptions ? options.stateOptions[state]?.onStyle : {}), 39 | }; 40 | let offStyle = { 41 | ...baseStyle, 42 | ...(options?.offStyle ?? {}), 43 | ...(options?.stateOptions ? options.stateOptions[state]?.offStyle : {}), 44 | }; 45 | return gridArray.map((x) => { 46 | const height = volumeBands[x] * (maxHeight - minHeight) + minHeight; 47 | //console.log("height", height, volumeBands[x], minHeight, maxHeight); 48 | const distanceFromCenter = Math.abs(midpoint - x); 49 | const isIndexHighlighted = 50 | typeof highlightedIndex === "object" 51 | ? highlightedIndex.includes(x) 52 | : highlightedIndex === x; 53 | return ( 54 |
67 | ); 68 | }); 69 | }, [ 70 | options, 71 | gridArray, 72 | volumeBands, 73 | maxHeight, 74 | minHeight, 75 | midpoint, 76 | highlightedIndex, 77 | state, 78 | animationOptions, 79 | ]); 80 | 81 | return ( 82 |
83 |
90 | {bars} 91 |
92 |
93 | ); 94 | }; 95 | -------------------------------------------------------------------------------- /web/src/components/agent/visualizers/grid-visualizer.tsx: -------------------------------------------------------------------------------- 1 | import { AgentVisualizerProps } from "@/components/agent/agent-control-bar"; 2 | import { useGridAnimator } from "@/components/agent/animators/use-grid-animator"; 3 | 4 | export const AgentGridVisualizer = ({ 5 | state, 6 | volumeBands, 7 | options, 8 | }: AgentVisualizerProps) => { 9 | const gridColumns = volumeBands.length; 10 | const gridRows = options?.rowCount ?? gridColumns; 11 | const gridArray = Array.from({ length: gridColumns }).map((_, i) => i); 12 | const gridRowsArray = Array.from({ length: gridRows }).map((_, i) => i); 13 | const highlightedIndex = useGridAnimator( 14 | state, 15 | gridRows, 16 | gridColumns, 17 | options?.animationOptions?.interval ?? 100, 18 | state !== "speaking" ? "active" : "paused", 19 | options?.animationOptions, 20 | ); 21 | 22 | const rowMidPoint = Math.floor(gridRows / 2.0); 23 | const volumeChunks = 1 / (rowMidPoint + 1); 24 | 25 | let baseStyle = options?.baseStyle ?? {}; 26 | let onStyle = { ...baseStyle, ...(options?.onStyle ?? {}) }; 27 | let offStyle = { ...baseStyle, ...(options?.offStyle ?? {}) }; 28 | const GridComponent = options?.gridComponent || "div"; 29 | 30 | const grid = gridArray.map((x) => { 31 | return ( 32 |
39 | {gridRowsArray.map((y) => { 40 | const distanceToMid = Math.abs(rowMidPoint - y); 41 | const threshold = distanceToMid * volumeChunks; 42 | let targetStyle; 43 | if (state !== "speaking") { 44 | if (highlightedIndex?.x === x && highlightedIndex?.y === y) { 45 | targetStyle = { 46 | transition: `all ${ 47 | (options?.animationOptions?.interval ?? 100) / 1000 48 | }s ease-out`, 49 | ...onStyle, 50 | }; 51 | } else { 52 | targetStyle = { 53 | transition: `all ${ 54 | (options?.animationOptions?.interval ?? 100) / 100 55 | }s ease-out`, 56 | ...offStyle, 57 | }; 58 | } 59 | } else { 60 | if (volumeBands[x] >= threshold) { 61 | targetStyle = onStyle; 62 | } else { 63 | targetStyle = offStyle; 64 | } 65 | } 66 | 67 | let distanceFromCenter = Math.sqrt( 68 | Math.pow(rowMidPoint - x, 2) + Math.pow(rowMidPoint - y, 2), 69 | ); 70 | return ( 71 | 78 | ); 79 | })} 80 |
81 | ); 82 | }); 83 | return ( 84 |
90 | {grid} 91 |
92 | ); 93 | }; 94 | -------------------------------------------------------------------------------- /web/src/components/agent/visualizers/multiband-bar-visualizer.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react"; 2 | 3 | type VisualizerState = "listening" | "idle" | "speaking" | "thinking"; 4 | type MultibandAudioVisualizerProps = { 5 | state: VisualizerState; 6 | barWidth: number; 7 | minBarHeight: number; 8 | maxBarHeight: number; 9 | frequencies: Float32Array[] | number[][]; 10 | borderRadius: number; 11 | gap: number; 12 | }; 13 | 14 | export const MultibandAudioVisualizer = ({ 15 | state, 16 | barWidth, 17 | minBarHeight, 18 | maxBarHeight, 19 | frequencies, 20 | borderRadius, 21 | gap, 22 | }: MultibandAudioVisualizerProps) => { 23 | const summedFrequencies = frequencies.map((bandFrequencies) => { 24 | const sum = (bandFrequencies as number[]).reduce((a, b) => a + b, 0); 25 | return Math.sqrt(sum / bandFrequencies.length); 26 | }); 27 | 28 | const [thinkingIndex, setThinkingIndex] = useState( 29 | Math.floor(summedFrequencies.length / 2), 30 | ); 31 | const [thinkingDirection, setThinkingDirection] = useState<"left" | "right">( 32 | "right", 33 | ); 34 | 35 | useEffect(() => { 36 | if (state !== "thinking") { 37 | setThinkingIndex(Math.floor(summedFrequencies.length / 2)); 38 | return; 39 | } 40 | const timeout = setTimeout(() => { 41 | if (thinkingDirection === "right") { 42 | if (thinkingIndex === summedFrequencies.length - 1) { 43 | setThinkingDirection("left"); 44 | setThinkingIndex((prev) => prev - 1); 45 | } else { 46 | setThinkingIndex((prev) => prev + 1); 47 | } 48 | } else { 49 | if (thinkingIndex === 0) { 50 | setThinkingDirection("right"); 51 | setThinkingIndex((prev) => prev + 1); 52 | } else { 53 | setThinkingIndex((prev) => prev - 1); 54 | } 55 | } 56 | }, 200); 57 | 58 | return () => clearTimeout(timeout); 59 | }, [state, summedFrequencies.length, thinkingDirection, thinkingIndex]); 60 | 61 | return ( 62 |
68 | {summedFrequencies.map((frequency, index) => { 69 | const isCenter = index === Math.floor(summedFrequencies.length / 2); 70 | let transform; 71 | 72 | return ( 73 |
94 | ); 95 | })} 96 |
97 | ); 98 | }; 99 | -------------------------------------------------------------------------------- /web/src/components/agent/visualizers/radial-visualizer.tsx: -------------------------------------------------------------------------------- 1 | import { useMemo } from "react"; 2 | import { AgentVisualizerProps } from "@/components/agent/agent-control-bar"; 3 | import { useRadialBarAnimator } from "@/components/agent/animators/use-radial-animator"; 4 | 5 | export const AgentRadialBarVisualizer = ({ 6 | state, 7 | volumeBands, 8 | options, 9 | }: AgentVisualizerProps) => { 10 | const gridColumns = volumeBands.length; 11 | const gridArray = Array.from({ length: gridColumns }).map((_, i) => i); 12 | 13 | const minHeight = options?.minHeight ?? 64; 14 | const maxHeight = options?.maxHeight ?? 8; 15 | const midpoint = Math.floor(gridColumns / 2.0); 16 | 17 | let deg = 360 / gridColumns; 18 | 19 | let animationOptions = options?.animationOptions; 20 | if (options?.stateOptions) { 21 | animationOptions = { 22 | ...animationOptions, 23 | ...options.stateOptions[state]?.animationOptions, 24 | }; 25 | } 26 | const highlightedIndex = useRadialBarAnimator( 27 | state, 28 | gridColumns, 29 | animationOptions?.interval ?? 100, 30 | state !== "speaking" ? "active" : "paused", 31 | animationOptions, 32 | ); 33 | 34 | const bars = useMemo(() => { 35 | let baseStyle = options?.baseStyle ?? {}; 36 | let onStyle = { 37 | ...baseStyle, 38 | ...(options?.onStyle ?? {}), 39 | ...(options?.stateOptions ? options.stateOptions[state]?.onStyle : {}), 40 | }; 41 | let offStyle = { 42 | ...baseStyle, 43 | ...(options?.offStyle ?? {}), 44 | ...(options?.stateOptions ? options.stateOptions[state]?.offStyle : {}), 45 | }; 46 | return gridArray.map((x) => { 47 | const isIndexHighlighted = 48 | typeof highlightedIndex === "object" 49 | ? highlightedIndex.includes(x) 50 | : highlightedIndex === x; 51 | const height = volumeBands[x] * (maxHeight - minHeight) + minHeight; 52 | const distanceFromCenter = Math.abs(midpoint - x); 53 | return ( 54 |
64 |
68 |
80 |
81 |
82 | ); 83 | }); 84 | }, [ 85 | options, 86 | state, 87 | gridArray, 88 | highlightedIndex, 89 | volumeBands, 90 | maxHeight, 91 | minHeight, 92 | midpoint, 93 | deg, 94 | animationOptions?.onTransition, 95 | animationOptions?.offTransition, 96 | ]); 97 | 98 | return ( 99 |
105 | {bars} 106 |
107 | ); 108 | }; 109 | -------------------------------------------------------------------------------- /web/src/components/chat-controls.tsx: -------------------------------------------------------------------------------- 1 | import { Button } from "@/components/ui/button"; 2 | import { Edit, Settings, MessageSquareQuote, AudioLines } from "lucide-react"; 3 | import { ConfigurationFormDrawer } from "@/components/configuration-form-drawer"; 4 | import { TranscriptDrawer } from "@/components/transcript-drawer"; 5 | 6 | interface ChatControlsProps { 7 | showEditButton: boolean; 8 | isEditingInstructions: boolean; 9 | onToggleEdit: () => void; 10 | } 11 | 12 | export function ChatControls({ 13 | showEditButton, 14 | isEditingInstructions, 15 | onToggleEdit, 16 | }: ChatControlsProps) { 17 | return ( 18 |
19 |
20 | 21 | 24 | 25 |
26 |
27 | {showEditButton && ( 28 | 35 | )} 36 | 37 | 40 | 41 |
42 |
43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /web/src/components/chat.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useState, useEffect } from "react"; 4 | import { Instructions } from "@/components/instructions"; 5 | import { SessionControls } from "@/components/session-controls"; 6 | import { ConnectButton } from "./connect-button"; 7 | import { ConnectionState } from "livekit-client"; 8 | import { motion, AnimatePresence } from "framer-motion"; 9 | import { 10 | useConnectionState, 11 | useVoiceAssistant, 12 | BarVisualizer, 13 | } from "@livekit/components-react"; 14 | import { ChatControls } from "@/components/chat-controls"; 15 | import { useAgent } from "@/hooks/use-agent"; 16 | import { useConnection } from "@/hooks/use-connection"; 17 | import { toast } from "@/hooks/use-toast"; 18 | 19 | export function Chat() { 20 | const connectionState = useConnectionState(); 21 | const { audioTrack, state } = useVoiceAssistant(); 22 | const [isChatRunning, setIsChatRunning] = useState(false); 23 | const { agent } = useAgent(); 24 | const { disconnect } = useConnection(); 25 | const [isEditingInstructions, setIsEditingInstructions] = useState(false); 26 | 27 | const [hasSeenAgent, setHasSeenAgent] = useState(false); 28 | 29 | useEffect(() => { 30 | let disconnectTimer: NodeJS.Timeout | undefined; 31 | let appearanceTimer: NodeJS.Timeout | undefined; 32 | 33 | if (connectionState === ConnectionState.Connected && !agent) { 34 | appearanceTimer = setTimeout(() => { 35 | disconnect(); 36 | setHasSeenAgent(false); 37 | 38 | toast({ 39 | title: "Agent Unavailable", 40 | description: 41 | "Unable to connect to an agent right now. Please try again later.", 42 | variant: "destructive", 43 | }); 44 | }, 5000); 45 | } 46 | 47 | if (agent) { 48 | setHasSeenAgent(true); 49 | } 50 | 51 | if ( 52 | connectionState === ConnectionState.Connected && 53 | !agent && 54 | hasSeenAgent 55 | ) { 56 | // Agent disappeared while connected, wait 5s before disconnecting 57 | disconnectTimer = setTimeout(() => { 58 | if (!agent) { 59 | disconnect(); 60 | setHasSeenAgent(false); 61 | } 62 | 63 | toast({ 64 | title: "Agent Disconnected", 65 | description: 66 | "The AI agent has unexpectedly left the conversation. Please try again.", 67 | variant: "destructive", 68 | }); 69 | }, 5000); 70 | } 71 | 72 | setIsChatRunning( 73 | connectionState === ConnectionState.Connected && hasSeenAgent, 74 | ); 75 | 76 | return () => { 77 | if (disconnectTimer) clearTimeout(disconnectTimer); 78 | if (appearanceTimer) clearTimeout(appearanceTimer); 79 | }; 80 | }, [connectionState, agent, disconnect, hasSeenAgent]); 81 | 82 | const toggleInstructionsEdit = () => 83 | setIsEditingInstructions(!isEditingInstructions); 84 | 85 | const renderVisualizer = () => ( 86 |
87 |
88 | 94 |
95 |
96 | ); 97 | 98 | const renderConnectionControl = () => ( 99 | 100 | 107 | {isChatRunning ? : } 108 | 109 | 110 | ); 111 | 112 | return ( 113 |
114 | 119 |
120 |
121 |
122 |
123 | {isChatRunning && !isEditingInstructions ? ( 124 | renderVisualizer() 125 | ) : ( 126 | 127 | )} 128 |
129 |
130 | 131 |
132 |
133 |
134 |
135 | {isChatRunning && !isEditingInstructions && renderVisualizer()} 136 |
137 |
138 |
139 | 140 |
141 | {renderConnectionControl()} 142 |
143 |
144 |
145 | ); 146 | } 147 | -------------------------------------------------------------------------------- /web/src/components/configuration-form-drawer.tsx: -------------------------------------------------------------------------------- 1 | import { Drawer, DrawerContent, DrawerTrigger } from "@/components/ui/drawer"; 2 | import { ConfigurationForm } from "@/components/configuration-form"; 3 | 4 | interface ConfigurationFormDrawerProps { 5 | children: React.ReactNode; 6 | } 7 | 8 | export function ConfigurationFormDrawer({ 9 | children, 10 | }: ConfigurationFormDrawerProps) { 11 | return ( 12 | 13 | {children} 14 | 15 |
16 |
17 | 18 |
19 |
20 | 21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /web/src/components/connect-button.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useState, useEffect, useCallback } from "react"; 4 | import { Button } from "@/components/ui/button"; 5 | import { useConnection } from "@/hooks/use-connection"; 6 | import { Loader2, Mic } from "lucide-react"; 7 | import { usePlaygroundState } from "@/hooks/use-playground-state"; 8 | import { AuthDialog } from "./auth"; 9 | 10 | export function ConnectButton() { 11 | const { connect, disconnect, shouldConnect } = useConnection(); 12 | const [connecting, setConnecting] = useState(false); 13 | const { pgState } = usePlaygroundState(); 14 | const [showAuthDialog, setShowAuthDialog] = useState(false); 15 | const [initiateConnectionFlag, setInitiateConnectionFlag] = useState(false); 16 | 17 | const handleConnectionToggle = async () => { 18 | if (shouldConnect) { 19 | await disconnect(); 20 | } else { 21 | if (!pgState.openaiAPIKey) { 22 | setShowAuthDialog(true); 23 | } else { 24 | await initiateConnection(); 25 | } 26 | } 27 | }; 28 | 29 | const initiateConnection = useCallback(async () => { 30 | setConnecting(true); 31 | try { 32 | await connect(); 33 | } catch (error) { 34 | console.error("Connection failed:", error); 35 | } finally { 36 | setConnecting(false); 37 | } 38 | }, [connect]); 39 | 40 | const handleAuthComplete = () => { 41 | setShowAuthDialog(false); 42 | setInitiateConnectionFlag(true); 43 | }; 44 | 45 | useEffect(() => { 46 | if (initiateConnectionFlag && pgState.openaiAPIKey) { 47 | initiateConnection(); 48 | setInitiateConnectionFlag(false); 49 | } 50 | }, [initiateConnectionFlag, initiateConnection, pgState.openaiAPIKey]); 51 | 52 | return ( 53 | <> 54 | 71 | 76 | 77 | ); 78 | } 79 | -------------------------------------------------------------------------------- /web/src/components/header.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { CodeViewer } from "@/components/code-viewer"; 4 | import { PresetSave } from "@/components/preset-save"; 5 | import { PresetSelector } from "@/components/preset-selector"; 6 | import { PresetShare } from "@/components/preset-share"; 7 | 8 | export function Header() { 9 | return ( 10 |
11 |
12 |
13 |
14 |
15 |

Realtime Playground

16 |

17 | Try OpenAI's new Realtime API right from your browser. 18 |

19 |
20 |
21 |
22 |
23 |
24 | 25 | 26 | 27 | 28 |
29 |
30 |
31 |
32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /web/src/components/instructions-editor.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useEffect, useState } from "react"; 4 | import { usePlaygroundState } from "@/hooks/use-playground-state"; 5 | import { useConnectionState } from "@livekit/components-react"; 6 | import { ConnectionState } from "livekit-client"; 7 | 8 | export interface InstructionsEditorProps { 9 | instructions?: string; 10 | onFocus?: () => void; 11 | onBlur?: () => void; 12 | onDirty?: () => void; 13 | } 14 | 15 | export function InstructionsEditor({ 16 | instructions, 17 | onFocus, 18 | onBlur, 19 | onDirty, 20 | }: InstructionsEditorProps) { 21 | const connectionState = useConnectionState(); 22 | const { pgState, dispatch } = usePlaygroundState(); 23 | const [dirty, setDirty] = useState(false); 24 | const [inputValue, setInputValue] = useState(instructions || ""); 25 | 26 | const handleInputChange = (event: React.ChangeEvent) => { 27 | const newValue = event.target.value; 28 | setInputValue(newValue); 29 | 30 | if ( 31 | connectionState === ConnectionState.Connected && 32 | newValue !== pgState.instructions 33 | ) { 34 | setDirty(true); 35 | if (onDirty) { 36 | onDirty(); 37 | } 38 | } 39 | }; 40 | 41 | const handleBlur = () => { 42 | dispatch({ type: "SET_INSTRUCTIONS", payload: inputValue }); 43 | setDirty(false); 44 | if (onBlur) { 45 | onBlur(); 46 | } 47 | }; 48 | 49 | useEffect(() => { 50 | if (instructions !== undefined && instructions !== inputValue) { 51 | setInputValue(instructions); 52 | setDirty(false); 53 | } 54 | // eslint-disable-next-line react-hooks/exhaustive-deps 55 | }, [instructions]); 56 | 57 | return ( 58 |