├── LICENSE ├── README.md ├── webapp ├── .env.example ├── .gitignore ├── app │ ├── api │ │ └── twilio │ │ │ ├── numbers │ │ │ └── route.ts │ │ │ ├── route.ts │ │ │ └── webhook-local │ │ │ └── route.ts │ ├── favicon.ico │ ├── globals.css │ ├── layout.tsx │ └── page.tsx ├── components.json ├── components │ ├── backend-tag.tsx │ ├── call-interface.tsx │ ├── checklist-and-config.tsx │ ├── function-calls-panel.tsx │ ├── phone-number-checklist.tsx │ ├── session-configuration-panel.tsx │ ├── tool-configuration-dialog.tsx │ ├── top-bar.tsx │ ├── transcript.tsx │ ├── types.ts │ └── ui │ │ ├── alert.tsx │ │ ├── badge.tsx │ │ ├── button.tsx │ │ ├── card.tsx │ │ ├── checkbox.tsx │ │ ├── dialog.tsx │ │ ├── input.tsx │ │ ├── label.tsx │ │ ├── scroll-area.tsx │ │ ├── select.tsx │ │ └── textarea.tsx ├── lib │ ├── handle-realtime-event.ts │ ├── tool-templates.ts │ ├── twilio.ts │ ├── use-backend-tools.ts │ └── utils.ts ├── next.config.mjs ├── package-lock.json ├── package.json ├── postcss.config.mjs ├── public │ ├── next.svg │ └── vercel.svg ├── tailwind.config.ts └── tsconfig.json └── websocket-server ├── .env.example ├── .gitignore ├── package-lock.json ├── package.json ├── src ├── functionHandlers.ts ├── server.ts ├── sessionManager.ts ├── twiml.xml └── types.ts └── tsconfig.json /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 OpenAI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Realtime API with Twilio Quickstart 2 | 3 | Combine OpenAI's Realtime API and Twilio's phone calling capability to build an AI calling assistant. 4 | 5 | Screenshot 2024-12-18 at 4 59 30 PM 6 | 7 | ## Quick Setup 8 | 9 | Open three terminal windows: 10 | 11 | | Terminal | Purpose | Quick Reference (see below for more) | 12 | | -------- | ----------------------------- | ------------------------------------ | 13 | | 1 | To run the `webapp` | `npm run dev` | 14 | | 2 | To run the `websocket-server` | `npm run dev` | 15 | | 3 | To run `ngrok` | `ngrok http 8081` | 16 | 17 | Make sure all vars in `webapp/.env` and `websocket-server/.env` are set correctly. See [full setup](#full-setup) section for more. 18 | 19 | ## Overview 20 | 21 | This repo implements a phone calling assistant with the Realtime API and Twilio, and had two main parts: the `webapp`, and the `websocket-server`. 22 | 23 | 1. `webapp`: NextJS app to serve as a frontend for call configuration and transcripts 24 | 2. `websocket-server`: Express backend that handles connection from Twilio, connects it to the Realtime API, and forwards messages to the frontend 25 | Screenshot 2024-12-20 at 10 32 40 AM 26 | 27 | Twilio uses TwiML (a form of XML) to specify how to handle a phone call. When a call comes in we tell Twilio to start a bi-directional stream to our backend, where we forward messages between the call and the Realtime API. (`{{WS_URL}}` is replaced with our websocket endpoint.) 28 | 29 | ```xml 30 | 31 | 32 | 33 | 34 | Connected 35 | 36 | 37 | 38 | Disconnected 39 | 40 | ``` 41 | 42 | We use `ngrok` to make our server reachable by Twilio. 43 | 44 | ### Life of a phone call 45 | 46 | Setup 47 | 48 | 1. We run ngrok to make our server reachable by Twilio 49 | 1. We set the Twilio webhook to our ngrok address 50 | 1. Frontend connects to the backend (`wss://[your_backend]/logs`), ready for a call 51 | 52 | Call 53 | 54 | 1. Call is placed to Twilio-managed number 55 | 1. Twilio queries the webhook (`http://[your_backend]/twiml`) for TwiML instructions 56 | 1. Twilio opens a bi-directional stream to the backend (`wss://[your_backend]/call`) 57 | 1. The backend connects to the Realtime API, and starts forwarding messages: 58 | - between Twilio and the Realtime API 59 | - between the frontend and the Realtime API 60 | 61 | ### Function Calling 62 | 63 | This demo mocks out function calls so you can provide sample responses. In reality you could handle the function call, execute some code, and then supply the response back to the model. 64 | 65 | ## Full Setup 66 | 67 | 1. Make sure your [auth & env](#detailed-auth--env) is configured correctly. 68 | 69 | 2. Run webapp. 70 | 71 | ```shell 72 | cd webapp 73 | npm install 74 | npm run dev 75 | ``` 76 | 77 | 3. Run websocket server. 78 | 79 | ```shell 80 | cd websocket-server 81 | npm install 82 | npm run dev 83 | ``` 84 | 85 | ## Detailed Auth & Env 86 | 87 | ### OpenAI & Twilio 88 | 89 | Set your credentials in `webapp/.env` and `websocket-server` - see `webapp/.env.example` and `websocket-server.env.example` for reference. 90 | 91 | ### Ngrok 92 | 93 | Twilio needs to be able to reach your websocket server. If you're running it locally, your ports are inaccessible by default. [ngrok](https://ngrok.com/) can make them temporarily accessible. 94 | 95 | We have set the `websocket-server` to run on port `8081` by default, so that is the port we will be forwarding. 96 | 97 | ```shell 98 | ngrok http 8081 99 | ``` 100 | 101 | Make note of the `Forwarding` URL. (e.g. `https://54c5-35-170-32-42.ngrok-free.app`) 102 | 103 | ### Websocket URL 104 | 105 | Your server should now be accessible at the `Forwarding` URL when run, so set the `PUBLIC_URL` in `websocket-server/.env`. See `websocket-server/.env.example` for reference. 106 | 107 | # Additional Notes 108 | 109 | This repo isn't polished, and the security practices leave some to be desired. Please only use this as reference, and make sure to audit your app with security and engineering before deploying! 110 | -------------------------------------------------------------------------------- /webapp/.env.example: -------------------------------------------------------------------------------- 1 | # rename this to .env 2 | 3 | TWILIO_ACCOUNT_SID="" 4 | TWILIO_AUTH_TOKEN="" -------------------------------------------------------------------------------- /webapp/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | .env 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | -------------------------------------------------------------------------------- /webapp/app/api/twilio/numbers/route.ts: -------------------------------------------------------------------------------- 1 | import twilioClient from "@/lib/twilio"; 2 | 3 | export async function GET() { 4 | if (!twilioClient) { 5 | return Response.json( 6 | { error: "Twilio client not initialized" }, 7 | { status: 500 } 8 | ); 9 | } 10 | 11 | const incomingPhoneNumbers = await twilioClient.incomingPhoneNumbers.list({ 12 | limit: 20, 13 | }); 14 | return Response.json(incomingPhoneNumbers); 15 | } 16 | 17 | export async function POST(req: Request) { 18 | if (!twilioClient) { 19 | return Response.json( 20 | { error: "Twilio client not initialized" }, 21 | { status: 500 } 22 | ); 23 | } 24 | 25 | const { phoneNumberSid, voiceUrl } = await req.json(); 26 | const incomingPhoneNumber = await twilioClient 27 | .incomingPhoneNumbers(phoneNumberSid) 28 | .update({ voiceUrl }); 29 | 30 | return Response.json(incomingPhoneNumber); 31 | } 32 | -------------------------------------------------------------------------------- /webapp/app/api/twilio/route.ts: -------------------------------------------------------------------------------- 1 | export async function GET() { 2 | const credentialsSet = Boolean( 3 | process.env.TWILIO_ACCOUNT_SID && process.env.TWILIO_AUTH_TOKEN 4 | ); 5 | return Response.json({ credentialsSet }); 6 | } 7 | -------------------------------------------------------------------------------- /webapp/app/api/twilio/webhook-local/route.ts: -------------------------------------------------------------------------------- 1 | export async function GET() { 2 | return Response.json({ webhookUrl: process.env.TWILIO_WEBHOOK_URL }); 3 | } 4 | -------------------------------------------------------------------------------- /webapp/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openai/openai-realtime-twilio-demo/4e9f754f6349fd6ecb82c2f35e66691268cdf11d/webapp/app/favicon.ico -------------------------------------------------------------------------------- /webapp/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer base { 6 | :root { 7 | --background: 0 0% 100%; 8 | --foreground: 222.2 84% 4.9%; 9 | --card: 0 0% 100%; 10 | --card-foreground: 222.2 84% 4.9%; 11 | --popover: 0 0% 100%; 12 | --popover-foreground: 222.2 84% 4.9%; 13 | --primary: 222.2 47.4% 11.2%; 14 | --primary-foreground: 210 40% 98%; 15 | --secondary: 210 40% 96.1%; 16 | --secondary-foreground: 222.2 47.4% 11.2%; 17 | --muted: 210 40% 96.1%; 18 | --muted-foreground: 215.4 16.3% 46.9%; 19 | --accent: 210 40% 96.1%; 20 | --accent-foreground: 222.2 47.4% 11.2%; 21 | --destructive: 0 84.2% 60.2%; 22 | --destructive-foreground: 210 40% 98%; 23 | --border: 214.3 31.8% 91.4%; 24 | --input: 214.3 31.8% 91.4%; 25 | --ring: 222.2 84% 4.9%; 26 | --radius: 0.5rem; 27 | --chart-1: 12 76% 61%; 28 | --chart-2: 173 58% 39%; 29 | --chart-3: 197 37% 24%; 30 | --chart-4: 43 74% 66%; 31 | --chart-5: 27 87% 67%; 32 | } 33 | 34 | .dark { 35 | --background: 222.2 84% 4.9%; 36 | --foreground: 210 40% 98%; 37 | --card: 222.2 84% 4.9%; 38 | --card-foreground: 210 40% 98%; 39 | --popover: 222.2 84% 4.9%; 40 | --popover-foreground: 210 40% 98%; 41 | --primary: 210 40% 98%; 42 | --primary-foreground: 222.2 47.4% 11.2%; 43 | --secondary: 217.2 32.6% 17.5%; 44 | --secondary-foreground: 210 40% 98%; 45 | --muted: 217.2 32.6% 17.5%; 46 | --muted-foreground: 215 20.2% 65.1%; 47 | --accent: 217.2 32.6% 17.5%; 48 | --accent-foreground: 210 40% 98%; 49 | --destructive: 0 62.8% 30.6%; 50 | --destructive-foreground: 210 40% 98%; 51 | --border: 217.2 32.6% 17.5%; 52 | --input: 217.2 32.6% 17.5%; 53 | --ring: 212.7 26.8% 83.9%; 54 | --chart-1: 220 70% 50%; 55 | --chart-2: 160 60% 45%; 56 | --chart-3: 30 80% 55%; 57 | --chart-4: 280 65% 60%; 58 | --chart-5: 340 75% 55%; 59 | } 60 | } 61 | 62 | @layer base { 63 | * { 64 | @apply border-border; 65 | } 66 | body { 67 | @apply bg-background text-foreground; 68 | } 69 | } -------------------------------------------------------------------------------- /webapp/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from "next"; 2 | import { Inter } from "next/font/google"; 3 | import "./globals.css"; 4 | 5 | const inter = Inter({ subsets: ["latin"] }); 6 | 7 | export const metadata: Metadata = { 8 | title: "OpenAI Realtime + Twilio", 9 | description: 10 | "Sample phone call assistant app for OpenAI Realtime API and Twilio", 11 | }; 12 | 13 | export default function RootLayout({ 14 | children, 15 | }: Readonly<{ 16 | children: React.ReactNode; 17 | }>) { 18 | return ( 19 | 20 | {children} 21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /webapp/app/page.tsx: -------------------------------------------------------------------------------- 1 | import CallInterface from "@/components/call-interface"; 2 | 3 | export default function Page() { 4 | return ; 5 | } 6 | -------------------------------------------------------------------------------- /webapp/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "default", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.ts", 8 | "css": "app/globals.css", 9 | "baseColor": "slate", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils" 16 | } 17 | } -------------------------------------------------------------------------------- /webapp/components/backend-tag.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export const BackendTag = () => ( 4 | 5 | backend 6 | 7 | ); 8 | -------------------------------------------------------------------------------- /webapp/components/call-interface.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import React, { useState, useEffect } from "react"; 4 | import TopBar from "@/components/top-bar"; 5 | import ChecklistAndConfig from "@/components/checklist-and-config"; 6 | import SessionConfigurationPanel from "@/components/session-configuration-panel"; 7 | import Transcript from "@/components/transcript"; 8 | import FunctionCallsPanel from "@/components/function-calls-panel"; 9 | import { Item } from "@/components/types"; 10 | import handleRealtimeEvent from "@/lib/handle-realtime-event"; 11 | import PhoneNumberChecklist from "@/components/phone-number-checklist"; 12 | 13 | const CallInterface = () => { 14 | const [selectedPhoneNumber, setSelectedPhoneNumber] = useState(""); 15 | const [allConfigsReady, setAllConfigsReady] = useState(false); 16 | const [items, setItems] = useState([]); 17 | const [callStatus, setCallStatus] = useState("disconnected"); 18 | const [ws, setWs] = useState(null); 19 | 20 | useEffect(() => { 21 | if (allConfigsReady && !ws) { 22 | const newWs = new WebSocket("ws://localhost:8081/logs"); 23 | 24 | newWs.onopen = () => { 25 | console.log("Connected to logs websocket"); 26 | setCallStatus("connected"); 27 | }; 28 | 29 | newWs.onmessage = (event) => { 30 | const data = JSON.parse(event.data); 31 | console.log("Received logs event:", data); 32 | handleRealtimeEvent(data, setItems); 33 | }; 34 | 35 | newWs.onclose = () => { 36 | console.log("Logs websocket disconnected"); 37 | setWs(null); 38 | setCallStatus("disconnected"); 39 | }; 40 | 41 | setWs(newWs); 42 | } 43 | }, [allConfigsReady, ws]); 44 | 45 | return ( 46 |
47 | 53 | 54 |
55 |
56 | {/* Left Column */} 57 |
58 | { 61 | if (ws && ws.readyState === WebSocket.OPEN) { 62 | const updateEvent = { 63 | type: "session.update", 64 | session: { 65 | ...config, 66 | }, 67 | }; 68 | console.log("Sending update event:", updateEvent); 69 | ws.send(JSON.stringify(updateEvent)); 70 | } 71 | }} 72 | /> 73 |
74 | 75 | {/* Middle Column: Transcript */} 76 |
77 | 82 | 83 |
84 | 85 | {/* Right Column: Function Calls */} 86 |
87 | 88 |
89 |
90 |
91 |
92 | ); 93 | }; 94 | 95 | export default CallInterface; 96 | -------------------------------------------------------------------------------- /webapp/components/checklist-and-config.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import React, { useEffect, useState, useMemo } from "react"; 4 | import { 5 | Dialog, 6 | DialogContent, 7 | DialogHeader, 8 | DialogTitle, 9 | DialogDescription, 10 | } from "@/components/ui/dialog"; 11 | import { Input } from "@/components/ui/input"; 12 | import { Button } from "@/components/ui/button"; 13 | import { Circle, CheckCircle, Loader2 } from "lucide-react"; 14 | import { PhoneNumber } from "@/components/types"; 15 | import { 16 | Select, 17 | SelectContent, 18 | SelectItem, 19 | SelectTrigger, 20 | SelectValue, 21 | } from "@/components/ui/select"; 22 | 23 | export default function ChecklistAndConfig({ 24 | ready, 25 | setReady, 26 | selectedPhoneNumber, 27 | setSelectedPhoneNumber, 28 | }: { 29 | ready: boolean; 30 | setReady: (val: boolean) => void; 31 | selectedPhoneNumber: string; 32 | setSelectedPhoneNumber: (val: string) => void; 33 | }) { 34 | const [hasCredentials, setHasCredentials] = useState(false); 35 | const [phoneNumbers, setPhoneNumbers] = useState([]); 36 | const [currentNumberSid, setCurrentNumberSid] = useState(""); 37 | const [currentVoiceUrl, setCurrentVoiceUrl] = useState(""); 38 | 39 | const [publicUrl, setPublicUrl] = useState(""); 40 | const [localServerUp, setLocalServerUp] = useState(false); 41 | const [publicUrlAccessible, setPublicUrlAccessible] = useState(false); 42 | 43 | const [allChecksPassed, setAllChecksPassed] = useState(false); 44 | const [webhookLoading, setWebhookLoading] = useState(false); 45 | const [ngrokLoading, setNgrokLoading] = useState(false); 46 | 47 | const appendedTwimlUrl = publicUrl ? `${publicUrl}/twiml` : ""; 48 | const isWebhookMismatch = 49 | appendedTwimlUrl && currentVoiceUrl && appendedTwimlUrl !== currentVoiceUrl; 50 | 51 | useEffect(() => { 52 | let polling = true; 53 | 54 | const pollChecks = async () => { 55 | try { 56 | // 1. Check credentials 57 | let res = await fetch("/api/twilio"); 58 | if (!res.ok) throw new Error("Failed credentials check"); 59 | const credData = await res.json(); 60 | setHasCredentials(!!credData?.credentialsSet); 61 | 62 | // 2. Fetch numbers 63 | res = await fetch("/api/twilio/numbers"); 64 | if (!res.ok) throw new Error("Failed to fetch phone numbers"); 65 | const numbersData = await res.json(); 66 | if (Array.isArray(numbersData) && numbersData.length > 0) { 67 | setPhoneNumbers(numbersData); 68 | // If currentNumberSid not set or not in the list, use first 69 | const selected = 70 | numbersData.find((p: PhoneNumber) => p.sid === currentNumberSid) || 71 | numbersData[0]; 72 | setCurrentNumberSid(selected.sid); 73 | setCurrentVoiceUrl(selected.voiceUrl || ""); 74 | setSelectedPhoneNumber(selected.friendlyName || ""); 75 | } 76 | 77 | // 3. Check local server & public URL 78 | let foundPublicUrl = ""; 79 | try { 80 | const resLocal = await fetch("http://localhost:8081/public-url"); 81 | if (resLocal.ok) { 82 | const pubData = await resLocal.json(); 83 | foundPublicUrl = pubData?.publicUrl || ""; 84 | setLocalServerUp(true); 85 | setPublicUrl(foundPublicUrl); 86 | } else { 87 | throw new Error("Local server not responding"); 88 | } 89 | } catch { 90 | setLocalServerUp(false); 91 | setPublicUrl(""); 92 | } 93 | } catch (err) { 94 | console.error(err); 95 | } 96 | }; 97 | 98 | pollChecks(); 99 | const intervalId = setInterval(() => polling && pollChecks(), 1000); 100 | return () => { 101 | polling = false; 102 | clearInterval(intervalId); 103 | }; 104 | }, [currentNumberSid, setSelectedPhoneNumber]); 105 | 106 | const updateWebhook = async () => { 107 | if (!currentNumberSid || !appendedTwimlUrl) return; 108 | try { 109 | setWebhookLoading(true); 110 | const res = await fetch("/api/twilio/numbers", { 111 | method: "POST", 112 | headers: { "Content-Type": "application/json" }, 113 | body: JSON.stringify({ 114 | phoneNumberSid: currentNumberSid, 115 | voiceUrl: appendedTwimlUrl, 116 | }), 117 | }); 118 | if (!res.ok) throw new Error("Failed to update webhook"); 119 | setCurrentVoiceUrl(appendedTwimlUrl); 120 | } catch (err) { 121 | console.error(err); 122 | } finally { 123 | setWebhookLoading(false); 124 | } 125 | }; 126 | 127 | const checkNgrok = async () => { 128 | if (!localServerUp || !publicUrl) return; 129 | setNgrokLoading(true); 130 | let success = false; 131 | for (let i = 0; i < 5; i++) { 132 | try { 133 | const resTest = await fetch(publicUrl + "/public-url"); 134 | if (resTest.ok) { 135 | setPublicUrlAccessible(true); 136 | success = true; 137 | break; 138 | } 139 | } catch { 140 | // retry 141 | } 142 | if (i < 4) { 143 | await new Promise((r) => setTimeout(r, 3000)); 144 | } 145 | } 146 | if (!success) { 147 | setPublicUrlAccessible(false); 148 | } 149 | setNgrokLoading(false); 150 | }; 151 | 152 | const checklist = useMemo(() => { 153 | return [ 154 | { 155 | label: "Set up Twilio account", 156 | done: hasCredentials, 157 | description: "Then update account details in webapp/.env", 158 | field: ( 159 | 165 | ), 166 | }, 167 | { 168 | label: "Set up Twilio phone number", 169 | done: phoneNumbers.length > 0, 170 | description: "Costs around $1.15/month", 171 | field: 172 | phoneNumbers.length > 0 ? ( 173 | phoneNumbers.length === 1 ? ( 174 | 175 | ) : ( 176 | 198 | ) 199 | ) : ( 200 | 211 | ), 212 | }, 213 | { 214 | label: "Start local WebSocket server", 215 | done: localServerUp, 216 | description: "cd websocket-server && npm run dev", 217 | field: null, 218 | }, 219 | { 220 | label: "Start ngrok", 221 | done: publicUrlAccessible, 222 | description: "Then set ngrok URL in websocket-server/.env", 223 | field: ( 224 |
225 |
226 | 227 |
228 |
229 | 241 |
242 |
243 | ), 244 | }, 245 | { 246 | label: "Update Twilio webhook URL", 247 | done: !!publicUrl && !isWebhookMismatch, 248 | description: "Can also be done manually in Twilio console", 249 | field: ( 250 |
251 |
252 | 253 |
254 |
255 | 266 |
267 |
268 | ), 269 | }, 270 | ]; 271 | }, [ 272 | hasCredentials, 273 | phoneNumbers, 274 | currentNumberSid, 275 | localServerUp, 276 | publicUrl, 277 | publicUrlAccessible, 278 | currentVoiceUrl, 279 | isWebhookMismatch, 280 | appendedTwimlUrl, 281 | webhookLoading, 282 | ngrokLoading, 283 | setSelectedPhoneNumber, 284 | ]); 285 | 286 | useEffect(() => { 287 | setAllChecksPassed(checklist.every((item) => item.done)); 288 | }, [checklist]); 289 | 290 | useEffect(() => { 291 | if (!ready) { 292 | checkNgrok(); 293 | } 294 | }, [localServerUp, ready]); 295 | 296 | useEffect(() => { 297 | if (!allChecksPassed) { 298 | setReady(false); 299 | } 300 | }, [allChecksPassed, setReady]); 301 | 302 | const handleDone = () => setReady(true); 303 | 304 | return ( 305 | 306 | 307 | 308 | Setup Checklist 309 | 310 | This sample app requires a few steps before you get started 311 | 312 | 313 | 314 |
315 | {checklist.map((item, i) => ( 316 |
320 |
321 |
322 | {item.done ? ( 323 | 324 | ) : ( 325 | 326 | )} 327 | {item.label} 328 |
329 | {item.description && ( 330 |

331 | {item.description} 332 |

333 | )} 334 |
335 |
{item.field}
336 |
337 | ))} 338 |
339 | 340 |
341 | 348 |
349 | 350 | 351 | ); 352 | } 353 | -------------------------------------------------------------------------------- /webapp/components/function-calls-panel.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; 3 | import { ScrollArea } from "@/components/ui/scroll-area"; 4 | import { Input } from "@/components/ui/input"; 5 | import { Badge } from "@/components/ui/badge"; 6 | import { Button } from "@/components/ui/button"; 7 | import { Item } from "@/components/types"; 8 | 9 | type FunctionCallsPanelProps = { 10 | items: Item[]; 11 | ws?: WebSocket | null; // pass down ws from parent 12 | }; 13 | 14 | const FunctionCallsPanel: React.FC = ({ 15 | items, 16 | ws, 17 | }) => { 18 | const [responses, setResponses] = useState>({}); 19 | 20 | // Filter function_call items 21 | const functionCalls = items.filter((it) => it.type === "function_call"); 22 | 23 | // For each function_call, check for a corresponding function_call_output 24 | const functionCallsWithStatus = functionCalls.map((call) => { 25 | const outputs = items.filter( 26 | (it) => it.type === "function_call_output" && it.call_id === call.call_id 27 | ); 28 | const outputItem = outputs[0]; 29 | const completed = call.status === "completed" || !!outputItem; 30 | const response = outputItem ? outputItem.output : undefined; 31 | return { 32 | ...call, 33 | completed, 34 | response, 35 | }; 36 | }); 37 | 38 | const handleChange = (call_id: string, value: string) => { 39 | setResponses((prev) => ({ ...prev, [call_id]: value })); 40 | }; 41 | 42 | const handleSubmit = (call: Item) => { 43 | if (!ws || ws.readyState !== WebSocket.OPEN) return; 44 | const call_id = call.call_id || ""; 45 | ws.send( 46 | JSON.stringify({ 47 | type: "conversation.item.create", 48 | item: { 49 | type: "function_call_output", 50 | call_id: call_id, 51 | output: JSON.stringify(responses[call_id] || ""), 52 | }, 53 | }) 54 | ); 55 | // Ask the model to continue after providing the tool response 56 | ws.send(JSON.stringify({ type: "response.create" })); 57 | }; 58 | 59 | return ( 60 | 61 | 62 | 63 | Function Calls 64 | 65 | 66 | 67 | 68 |
69 | {functionCallsWithStatus.map((call) => ( 70 |
74 |
75 |

{call.name}

76 | 77 | {call.completed ? "Completed" : "Pending"} 78 | 79 |
80 | 81 |
82 | {JSON.stringify(call.params)} 83 |
84 | 85 | {!call.completed ? ( 86 |
87 | 91 | handleChange(call.call_id || "", e.target.value) 92 | } 93 | /> 94 | 103 |
104 | ) : ( 105 |
106 | {JSON.stringify(JSON.parse(call.response || ""))} 107 |
108 | )} 109 |
110 | ))} 111 |
112 | 113 | 114 | 115 | ); 116 | }; 117 | 118 | export default FunctionCallsPanel; 119 | -------------------------------------------------------------------------------- /webapp/components/phone-number-checklist.tsx: -------------------------------------------------------------------------------- 1 | // PhoneNumberChecklist.tsx 2 | "use client"; 3 | 4 | import React, { useState } from "react"; 5 | import { Card } from "@/components/ui/card"; 6 | import { CheckCircle, Circle, Eye, EyeOff } from "lucide-react"; 7 | import { Button } from "@/components/ui/button"; 8 | 9 | type PhoneNumberChecklistProps = { 10 | selectedPhoneNumber: string; 11 | allConfigsReady: boolean; 12 | setAllConfigsReady: (ready: boolean) => void; 13 | }; 14 | 15 | const PhoneNumberChecklist: React.FC = ({ 16 | selectedPhoneNumber, 17 | allConfigsReady, 18 | setAllConfigsReady, 19 | }) => { 20 | const [isVisible, setIsVisible] = useState(true); 21 | 22 | return ( 23 | 24 |
25 | Number 26 |
27 | 28 | {isVisible ? selectedPhoneNumber || "None" : "••••••••••"} 29 | 30 | 42 |
43 |
44 |
45 |
46 | {allConfigsReady ? ( 47 | 48 | ) : ( 49 | 50 | )} 51 | 52 | {allConfigsReady ? "Setup Ready" : "Setup Not Ready"} 53 | 54 |
55 | 62 |
63 | 64 | ); 65 | }; 66 | 67 | export default PhoneNumberChecklist; 68 | -------------------------------------------------------------------------------- /webapp/components/session-configuration-panel.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; 3 | import { ScrollArea } from "@/components/ui/scroll-area"; 4 | import { 5 | Select, 6 | SelectContent, 7 | SelectItem, 8 | SelectTrigger, 9 | SelectValue, 10 | } from "@/components/ui/select"; 11 | import { Textarea } from "@/components/ui/textarea"; 12 | import { Button } from "@/components/ui/button"; 13 | import { Plus, Edit, Trash, Check, AlertCircle } from "lucide-react"; 14 | import { toolTemplates } from "@/lib/tool-templates"; 15 | import { ToolConfigurationDialog } from "./tool-configuration-dialog"; 16 | import { BackendTag } from "./backend-tag"; 17 | import { useBackendTools } from "@/lib/use-backend-tools"; 18 | 19 | interface SessionConfigurationPanelProps { 20 | callStatus: string; 21 | onSave: (config: any) => void; 22 | } 23 | 24 | const SessionConfigurationPanel: React.FC = ({ 25 | callStatus, 26 | onSave, 27 | }) => { 28 | const [instructions, setInstructions] = useState( 29 | "You are a helpful assistant in a phone call." 30 | ); 31 | const [voice, setVoice] = useState("ash"); 32 | const [tools, setTools] = useState([]); 33 | const [editingIndex, setEditingIndex] = useState(null); 34 | const [editingSchemaStr, setEditingSchemaStr] = useState(""); 35 | const [isJsonValid, setIsJsonValid] = useState(true); 36 | const [openDialog, setOpenDialog] = useState(false); 37 | const [selectedTemplate, setSelectedTemplate] = useState(""); 38 | const [saveStatus, setSaveStatus] = useState< 39 | "idle" | "saving" | "saved" | "error" 40 | >("idle"); 41 | const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false); 42 | 43 | // Custom hook to fetch backend tools every 3 seconds 44 | const backendTools = useBackendTools("http://localhost:8081/tools", 3000); 45 | 46 | // Track changes to determine if there are unsaved modifications 47 | useEffect(() => { 48 | setHasUnsavedChanges(true); 49 | }, [instructions, voice, tools]); 50 | 51 | // Reset save status after a delay when saved 52 | useEffect(() => { 53 | if (saveStatus === "saved") { 54 | const timer = setTimeout(() => { 55 | setSaveStatus("idle"); 56 | }, 3000); 57 | return () => clearTimeout(timer); 58 | } 59 | }, [saveStatus]); 60 | 61 | const handleSave = async () => { 62 | setSaveStatus("saving"); 63 | try { 64 | await onSave({ 65 | instructions, 66 | voice, 67 | tools: tools.map((tool) => JSON.parse(tool)), 68 | }); 69 | setSaveStatus("saved"); 70 | setHasUnsavedChanges(false); 71 | } catch (error) { 72 | setSaveStatus("error"); 73 | } 74 | }; 75 | 76 | const handleAddTool = () => { 77 | setEditingIndex(null); 78 | setEditingSchemaStr(""); 79 | setSelectedTemplate(""); 80 | setIsJsonValid(true); 81 | setOpenDialog(true); 82 | }; 83 | 84 | const handleEditTool = (index: number) => { 85 | setEditingIndex(index); 86 | setEditingSchemaStr(tools[index] || ""); 87 | setSelectedTemplate(""); 88 | setIsJsonValid(true); 89 | setOpenDialog(true); 90 | }; 91 | 92 | const handleDeleteTool = (index: number) => { 93 | const newTools = [...tools]; 94 | newTools.splice(index, 1); 95 | setTools(newTools); 96 | }; 97 | 98 | const handleDialogSave = () => { 99 | try { 100 | JSON.parse(editingSchemaStr); 101 | } catch { 102 | return; 103 | } 104 | const newTools = [...tools]; 105 | if (editingIndex === null) { 106 | newTools.push(editingSchemaStr); 107 | } else { 108 | newTools[editingIndex] = editingSchemaStr; 109 | } 110 | setTools(newTools); 111 | setOpenDialog(false); 112 | }; 113 | 114 | const handleTemplateChange = (val: string) => { 115 | setSelectedTemplate(val); 116 | 117 | // Determine if the selected template is from local or backend 118 | let templateObj = 119 | toolTemplates.find((t) => t.name === val) || 120 | backendTools.find((t: any) => t.name === val); 121 | 122 | if (templateObj) { 123 | setEditingSchemaStr(JSON.stringify(templateObj, null, 2)); 124 | setIsJsonValid(true); 125 | } 126 | }; 127 | 128 | const onSchemaChange = (value: string) => { 129 | setEditingSchemaStr(value); 130 | try { 131 | JSON.parse(value); 132 | setIsJsonValid(true); 133 | } catch { 134 | setIsJsonValid(false); 135 | } 136 | }; 137 | 138 | const getToolNameFromSchema = (schema: string): string => { 139 | try { 140 | const parsed = JSON.parse(schema); 141 | return parsed?.name || "Untitled Tool"; 142 | } catch { 143 | return "Invalid JSON"; 144 | } 145 | }; 146 | 147 | const isBackendTool = (name: string): boolean => { 148 | return backendTools.some((t: any) => t.name === name); 149 | }; 150 | 151 | return ( 152 | 153 | 154 |
155 | 156 | Session Configuration 157 | 158 |
159 | {saveStatus === "error" ? ( 160 | 161 | 162 | Save failed 163 | 164 | ) : hasUnsavedChanges ? ( 165 | Not saved 166 | ) : ( 167 | 168 | 169 | Saved 170 | 171 | )} 172 |
173 |
174 | 175 | 176 | 177 |
178 |
179 | 182 |