├── web ├── public │ ├── favicon.ico │ ├── og-image.png │ └── fonts │ │ ├── CommitMono-400-Regular.otf │ │ └── CommitMono-700-Regular.otf ├── .eslintrc.json ├── postcss.config.mjs ├── src │ ├── types │ │ └── svg.d.ts │ ├── data │ │ ├── agent.ts │ │ ├── models.ts │ │ ├── transcription-models.ts │ │ ├── voices.ts │ │ ├── modalities.ts │ │ ├── turn-end-types.ts │ │ └── playground-state.ts │ ├── components │ │ ├── lk.tsx │ │ ├── configuration-form-drawer.tsx │ │ ├── ui │ │ │ ├── label.tsx │ │ │ ├── separator.tsx │ │ │ ├── textarea.tsx │ │ │ ├── toaster.tsx │ │ │ ├── input.tsx │ │ │ ├── slider.tsx │ │ │ ├── tooltip.tsx │ │ │ ├── hover-card.tsx │ │ │ ├── checkbox.tsx │ │ │ ├── switch.tsx │ │ │ ├── popover.tsx │ │ │ ├── badge.tsx │ │ │ ├── tabs.tsx │ │ │ ├── button.tsx │ │ │ ├── drawer.tsx │ │ │ ├── dialog.tsx │ │ │ ├── form.tsx │ │ │ ├── alert-dialog.tsx │ │ │ ├── toast.tsx │ │ │ ├── command.tsx │ │ │ └── select.tsx │ │ ├── agent │ │ │ ├── animation-sequences │ │ │ │ ├── thinking-sequence.ts │ │ │ │ ├── listening-sequence.ts │ │ │ │ └── connecting-sequence.ts │ │ │ ├── animators │ │ │ │ ├── use-radial-animator.ts │ │ │ │ ├── use-grid-animator.ts │ │ │ │ └── use-bar-animator.ts │ │ │ ├── agent-control-bar.tsx │ │ │ ├── visualizers │ │ │ │ ├── bar-visualizer.tsx │ │ │ │ ├── grid-visualizer.tsx │ │ │ │ ├── multiband-bar-visualizer.tsx │ │ │ │ └── radial-visualizer.tsx │ │ │ └── data │ │ │ │ └── visualizer-variations.ts │ │ ├── session-config.tsx │ │ ├── header.tsx │ │ ├── transcript-drawer.tsx │ │ ├── chat-controls.tsx │ │ ├── top-p-selector.tsx │ │ ├── instructions-editor.tsx │ │ ├── instructions.tsx │ │ ├── max-output-tokens-selector.tsx │ │ ├── connect-button.tsx │ │ ├── room-component.tsx │ │ ├── voice-selector.tsx │ │ ├── model-selector.tsx │ │ ├── modalities-selector.tsx │ │ ├── transcription-selector.tsx │ │ ├── preset-share.tsx │ │ ├── turn-detection-selector.tsx │ │ ├── vad-threshold-selector.tsx │ │ ├── temperature-selector.tsx │ │ ├── vad-prefix-padding-selector.tsx │ │ ├── vad-silence-duration-selector.tsx │ │ ├── preset-save.tsx │ │ ├── transcript.tsx │ │ ├── session-controls.tsx │ │ └── chat.tsx │ ├── lib │ │ ├── utils.ts │ │ ├── agent │ │ │ └── audio-visualizer.ts │ │ └── playground-state-helpers.ts │ ├── hooks │ │ ├── use-mutation-observer.ts │ │ ├── use-multiband-track-volume.tsx │ │ ├── use-connection.tsx │ │ ├── use-agent.tsx │ │ └── use-toast.ts │ ├── assets │ │ ├── lk.svg │ │ └── heart.svg │ └── app │ │ ├── page.tsx │ │ ├── api │ │ └── token │ │ │ └── route.ts │ │ ├── layout.tsx │ │ └── globals.css ├── .env.sample ├── next.config.mjs ├── components.json ├── .gitignore ├── tsconfig.json ├── tailwind.config.ts └── package.json ├── agent ├── requirements.txt ├── ruff.toml ├── .env.sample ├── tsconfig.json ├── package.json ├── .gitignore └── playground_agent.ts └── README.md /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/HEAD/web/public/favicon.ico -------------------------------------------------------------------------------- /agent/requirements.txt: -------------------------------------------------------------------------------- 1 | livekit >= 0.15.0 2 | livekit-protocol 3 | livekit-agents>=0.10.0 4 | livekit-plugins-openai>=0.10.0 5 | -------------------------------------------------------------------------------- /web/public/og-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/HEAD/web/public/og-image.png -------------------------------------------------------------------------------- /web/public/fonts/CommitMono-400-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/HEAD/web/public/fonts/CommitMono-400-Regular.otf -------------------------------------------------------------------------------- /web/public/fonts/CommitMono-700-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mckaywrigley/realtime-ai-livekit-playground/HEAD/web/public/fonts/CommitMono-700-Regular.otf -------------------------------------------------------------------------------- /web/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals", 3 | "plugins": ["unused-imports"], 4 | "rules": { 5 | "unused-imports/no-unused-imports": "error" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /web/postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /agent/ruff.toml: -------------------------------------------------------------------------------- 1 | line-length = 88 2 | indent-width = 4 3 | 4 | target-version = "py39" 5 | 6 | [lint] 7 | extend-select = ["I"] 8 | 9 | [lint.pydocstyle] 10 | convention = "numpy" 11 | 12 | [format] 13 | docstring-code-format = true 14 | -------------------------------------------------------------------------------- /web/src/types/svg.d.ts: -------------------------------------------------------------------------------- 1 | declare module "*.svg" { 2 | import * as React from "react"; 3 | const ReactComponent: React.FunctionComponent< 4 | React.SVGProps & { title?: string } 5 | >; 6 | export default ReactComponent; 7 | } 8 | -------------------------------------------------------------------------------- /agent/.env.sample: -------------------------------------------------------------------------------- 1 | # Sign Up for LiveKit Cloud --> https://cloud.livekit.io 2 | # Copy this file to .env.local and fill in the values. it should match ../web/.env.local 3 | export LIVEKIT_URL= 4 | export LIVEKIT_API_KEY= 5 | export LIVEKIT_API_SECRET= 6 | -------------------------------------------------------------------------------- /web/src/data/agent.ts: -------------------------------------------------------------------------------- 1 | export type AgentState = 2 | | "offline" 3 | | "connecting" 4 | | "listening" 5 | | "thinking" 6 | | "speaking"; 7 | export const AgentStates: AgentState[] = [ 8 | "offline", 9 | "connecting", 10 | "listening", 11 | "thinking", 12 | "speaking", 13 | ]; 14 | -------------------------------------------------------------------------------- /web/.env.sample: -------------------------------------------------------------------------------- 1 | # Sign Up for LiveKit Cloud and create a project --> https://cloud.livekit.io 2 | # Copy this file to .env.local and fill in the values. it should match ../agent/.env.local 3 | export LIVEKIT_URL= 4 | export LIVEKIT_API_KEY= 5 | export LIVEKIT_API_SECRET= 6 | -------------------------------------------------------------------------------- /web/src/data/models.ts: -------------------------------------------------------------------------------- 1 | export enum ModelId { 2 | gpt_4o_realtime = "gpt-4o-realtime", 3 | } 4 | 5 | export interface Model { 6 | id: ModelId; 7 | name: string; 8 | } 9 | 10 | export const models: Model[] = [ 11 | { 12 | id: ModelId.gpt_4o_realtime, 13 | name: "gpt-4o-realtime", 14 | }, 15 | ]; 16 | -------------------------------------------------------------------------------- /web/src/data/transcription-models.ts: -------------------------------------------------------------------------------- 1 | export enum TranscriptionModelId { 2 | whisper1 = "whisper-1", 3 | } 4 | 5 | export interface TranscriptionModel { 6 | id: TranscriptionModelId; 7 | name: string; 8 | } 9 | 10 | export const transcriptionModels: TranscriptionModel[] = [ 11 | { 12 | id: TranscriptionModelId.whisper1, 13 | name: "whisper-1", 14 | }, 15 | ]; 16 | -------------------------------------------------------------------------------- /web/src/components/lk.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import Logo from "@/assets/lk.svg"; 4 | 5 | export default function LK() { 6 | return ( 7 | 13 | 14 | 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /web/next.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | webpack(config) { 4 | config.module.rules.push({ 5 | test: /\.svg$/, // Look for .svg files 6 | use: ["@svgr/webpack"], // Use @svgr/webpack to handle them 7 | }); 8 | 9 | return config; // Always return the modified config 10 | }, 11 | }; 12 | 13 | export default nextConfig; 14 | -------------------------------------------------------------------------------- /agent/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["./playground_agent.ts"], 3 | "compilerOptions": { 4 | "rootDir": "./", 5 | "outDir": "./dist", 6 | "target": "ES2017", 7 | "module": "ES2022", 8 | "moduleResolution": "node", 9 | "declaration": true, 10 | "declarationMap": true, 11 | "allowJs": false, 12 | "strict": true, 13 | "skipLibCheck": true, 14 | "esModuleInterop": true, 15 | "sourceMap": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /web/src/data/voices.ts: -------------------------------------------------------------------------------- 1 | export enum VoiceId { 2 | alloy = "alloy", 3 | shimmer = "shimmer", 4 | echo = "echo", 5 | } 6 | 7 | export interface Voice { 8 | id: VoiceId; 9 | name: string; 10 | } 11 | 12 | export const voices: Voice[] = [ 13 | { 14 | id: VoiceId.alloy, 15 | name: "Alloy", 16 | }, 17 | { 18 | id: VoiceId.shimmer, 19 | name: "Shimmer", 20 | }, 21 | { 22 | id: VoiceId.echo, 23 | name: "Echo", 24 | }, 25 | ]; 26 | -------------------------------------------------------------------------------- /web/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.ts", 8 | "css": "src/app/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": false, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /web/src/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { clsx, type ClassValue } from "clsx"; 2 | import { twMerge } from "tailwind-merge"; 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)); 6 | } 7 | 8 | export function ellipsisMiddle( 9 | text: string, 10 | startLength: number, 11 | endLength: number, 12 | ): string { 13 | if (text.length <= startLength + endLength) { 14 | return text; 15 | } 16 | const start = text.slice(0, startLength); 17 | const end = text.slice(-endLength); 18 | return `${start}...${end}`; 19 | } 20 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | node_modules 5 | .pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | web/coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | .env.local 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | -------------------------------------------------------------------------------- /web/src/data/modalities.ts: -------------------------------------------------------------------------------- 1 | export enum ModalitiesId { 2 | text_and_audio = "text_and_audio", 3 | text_only = "text_only", 4 | } 5 | 6 | export interface Modalities { 7 | id: ModalitiesId; 8 | name: string; 9 | description: string; 10 | } 11 | 12 | export const modalities: Modalities[] = [ 13 | { 14 | id: ModalitiesId.text_and_audio, 15 | name: "Audio + Text", 16 | description: "The model will produce both audio and text.", 17 | }, 18 | { 19 | id: ModalitiesId.text_only, 20 | name: "Text Only", 21 | description: "The model will produce text only.", 22 | }, 23 | ]; 24 | -------------------------------------------------------------------------------- /web/src/hooks/use-mutation-observer.ts: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | export const useMutationObserver = ( 4 | ref: React.MutableRefObject, 5 | callback: MutationCallback, 6 | options = { 7 | attributes: true, 8 | characterData: true, 9 | childList: true, 10 | subtree: true, 11 | }, 12 | ) => { 13 | React.useEffect(() => { 14 | if (ref.current) { 15 | const observer = new MutationObserver(callback); 16 | observer.observe(ref.current, options); 17 | return () => observer.disconnect(); 18 | } 19 | }, [ref, callback, options]); 20 | }; 21 | -------------------------------------------------------------------------------- /web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "plugins": [ 16 | { 17 | "name": "next" 18 | } 19 | ], 20 | "paths": { 21 | "@/*": ["./src/*"] 22 | } 23 | }, 24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 25 | "exclude": ["node_modules"] 26 | } 27 | -------------------------------------------------------------------------------- /web/src/data/turn-end-types.ts: -------------------------------------------------------------------------------- 1 | export enum TurnDetectionTypeId { 2 | server_vad = "server_vad", 3 | none = "none", 4 | } 5 | 6 | export interface TurnDetectionType { 7 | id: TurnDetectionTypeId; 8 | name: string; 9 | description: string; 10 | } 11 | 12 | export const turnDetectionTypes: TurnDetectionType[] = [ 13 | { 14 | id: TurnDetectionTypeId.server_vad, 15 | name: "Server VAD", 16 | description: 17 | "The model will automatically detect when the user has finished speaking and end the turn.", 18 | }, 19 | { 20 | id: TurnDetectionTypeId.none, 21 | name: "None", 22 | description: 23 | "The client must perform its own turn logic and inform the model.", 24 | }, 25 | ]; 26 | -------------------------------------------------------------------------------- /web/src/components/configuration-form-drawer.tsx: -------------------------------------------------------------------------------- 1 | import { Drawer, DrawerContent, DrawerTrigger } from "@/components/ui/drawer"; 2 | import { ConfigurationForm } from "@/components/configuration-form"; 3 | 4 | interface ConfigurationFormDrawerProps { 5 | children: React.ReactNode; 6 | } 7 | 8 | export function ConfigurationFormDrawer({ 9 | children, 10 | }: ConfigurationFormDrawerProps) { 11 | return ( 12 | 13 | {children} 14 | 15 |
16 |
17 | 18 |
19 |
20 | 21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /web/src/components/ui/label.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as LabelPrimitive from "@radix-ui/react-label"; 5 | import { cva, type VariantProps } from "class-variance-authority"; 6 | 7 | import { cn } from "@/lib/utils"; 8 | 9 | const labelVariants = cva( 10 | "text-sm font-light leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70", 11 | ); 12 | 13 | const Label = React.forwardRef< 14 | React.ElementRef, 15 | React.ComponentPropsWithoutRef & 16 | VariantProps 17 | >(({ className, ...props }, ref) => ( 18 | 23 | )); 24 | Label.displayName = LabelPrimitive.Root.displayName; 25 | 26 | export { Label }; 27 | -------------------------------------------------------------------------------- /web/src/components/agent/animation-sequences/thinking-sequence.ts: -------------------------------------------------------------------------------- 1 | export const generateThinkingSequence = (rows: number, columns: number) => { 2 | let seq = []; 3 | let y = Math.floor(rows / 2); 4 | for (let x = 0; x < columns; x++) { 5 | seq.push({ x, y }); 6 | } 7 | for (let x = columns - 1; x >= 0; x--) { 8 | seq.push({ x, y }); 9 | } 10 | 11 | return seq; 12 | }; 13 | 14 | export const generateThinkingSequenceBar = (columns: number) => { 15 | let seq = []; 16 | for (let x = 0; x < columns; x++) { 17 | seq.push(x); 18 | } 19 | 20 | for (let x = columns - 1; x >= 0; x--) { 21 | seq.push(x); 22 | } 23 | 24 | return seq; 25 | }; 26 | 27 | export const generateThinkingSequenceRadialBar = (columns: number) => { 28 | let seq = []; 29 | for (let x = 0; x < columns; x++) { 30 | seq.push(x); 31 | } 32 | 33 | return seq; 34 | }; 35 | -------------------------------------------------------------------------------- /web/src/components/ui/separator.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as SeparatorPrimitive from "@radix-ui/react-separator"; 5 | 6 | import { cn } from "@/lib/utils"; 7 | 8 | const Separator = React.forwardRef< 9 | React.ElementRef, 10 | React.ComponentPropsWithoutRef 11 | >( 12 | ( 13 | { className, orientation = "horizontal", decorative = true, ...props }, 14 | ref, 15 | ) => ( 16 | 27 | ), 28 | ); 29 | Separator.displayName = SeparatorPrimitive.Root.displayName; 30 | 31 | export { Separator }; 32 | -------------------------------------------------------------------------------- /web/src/components/ui/textarea.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { cn } from "@/lib/utils"; 4 | 5 | export interface TextareaProps 6 | extends React.TextareaHTMLAttributes {} 7 | 8 | const Textarea = React.forwardRef( 9 | ({ className, ...props }, ref) => { 10 | return ( 11 |