├── .env.local.example ├── .gitignore ├── .vscode ├── extensions.json └── settings.json ├── App.tsx ├── README.md ├── android ├── .gitignore ├── app │ ├── build.gradle │ ├── debug.keystore │ ├── proguard-rules.pro │ └── src │ │ ├── debug │ │ └── AndroidManifest.xml │ │ └── main │ │ ├── AndroidManifest.xml │ │ ├── java │ │ └── com │ │ │ └── thorwebdev │ │ │ └── expowebrtcopenairealtime │ │ │ ├── MainActivity.kt │ │ │ └── MainApplication.kt │ │ └── res │ │ ├── drawable-hdpi │ │ └── splashscreen_logo.png │ │ ├── drawable-mdpi │ │ └── splashscreen_logo.png │ │ ├── drawable-xhdpi │ │ └── splashscreen_logo.png │ │ ├── drawable-xxhdpi │ │ └── splashscreen_logo.png │ │ ├── drawable-xxxhdpi │ │ └── splashscreen_logo.png │ │ ├── drawable │ │ ├── ic_launcher_background.xml │ │ └── rn_edit_text_material.xml │ │ ├── mipmap-anydpi-v26 │ │ ├── ic_launcher.xml │ │ └── ic_launcher_round.xml │ │ ├── mipmap-hdpi │ │ ├── ic_launcher.webp │ │ ├── ic_launcher_foreground.webp │ │ └── ic_launcher_round.webp │ │ ├── mipmap-mdpi │ │ ├── ic_launcher.webp │ │ ├── ic_launcher_foreground.webp │ │ └── ic_launcher_round.webp │ │ ├── mipmap-xhdpi │ │ ├── ic_launcher.webp │ │ ├── ic_launcher_foreground.webp │ │ └── ic_launcher_round.webp │ │ ├── mipmap-xxhdpi │ │ ├── ic_launcher.webp │ │ ├── ic_launcher_foreground.webp │ │ └── ic_launcher_round.webp │ │ ├── mipmap-xxxhdpi │ │ ├── ic_launcher.webp │ │ ├── ic_launcher_foreground.webp │ │ └── ic_launcher_round.webp │ │ ├── values-night │ │ └── colors.xml │ │ └── values │ │ ├── colors.xml │ │ ├── strings.xml │ │ └── styles.xml ├── build.gradle ├── gradle.properties ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat └── settings.gradle ├── app.json ├── assets ├── adaptive-icon.png ├── favicon.png ├── icon.png └── splash-icon.png ├── index.ts ├── ios ├── .gitignore ├── .xcode.env ├── Podfile ├── Podfile.lock ├── Podfile.properties.json ├── expowebrtcopenairealtime.xcodeproj │ ├── project.pbxproj │ └── xcshareddata │ │ └── xcschemes │ │ └── expowebrtcopenairealtime.xcscheme ├── expowebrtcopenairealtime.xcworkspace │ └── contents.xcworkspacedata └── expowebrtcopenairealtime │ ├── AppDelegate.h │ ├── AppDelegate.mm │ ├── Images.xcassets │ ├── AppIcon.appiconset │ │ ├── App-Icon-1024x1024@1x.png │ │ └── Contents.json │ ├── Contents.json │ ├── SplashScreenBackground.colorset │ │ └── Contents.json │ └── SplashScreenLogo.imageset │ │ ├── Contents.json │ │ ├── image.png │ │ ├── image@2x.png │ │ └── image@3x.png │ ├── Info.plist │ ├── PrivacyInfo.xcprivacy │ ├── SplashScreen.storyboard │ ├── Supporting │ └── Expo.plist │ ├── expowebrtcopenairealtime-Bridging-Header.h │ ├── expowebrtcopenairealtime.entitlements │ ├── main.m │ └── noop-file.swift ├── package-lock.json ├── package.json ├── supabase ├── .gitignore ├── config.toml └── functions │ ├── .env.example │ ├── _shared │ └── cors.ts │ └── token │ └── index.ts ├── tsconfig.json └── utils ├── supabase.ts └── tools.ts /.env.local.example: -------------------------------------------------------------------------------- 1 | # Local Supabase vars prefilled 2 | EXPO_PUBLIC_SUPABASE_URL=http://127.0.0.1:54321 3 | EXPO_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files 2 | 3 | # dependencies 4 | node_modules/ 5 | 6 | # Expo 7 | .expo/ 8 | dist/ 9 | web-build/ 10 | expo-env.d.ts 11 | 12 | # Native 13 | *.orig.* 14 | *.jks 15 | *.p8 16 | *.p12 17 | *.key 18 | *.mobileprovision 19 | 20 | # Metro 21 | .metro-health-check* 22 | 23 | # debug 24 | npm-debug.* 25 | yarn-debug.* 26 | yarn-error.* 27 | 28 | # macOS 29 | .DS_Store 30 | *.pem 31 | 32 | # local env files 33 | .env*.local 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["denoland.vscode-deno"] 3 | } 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enablePaths": [ 3 | "supabase/functions" 4 | ], 5 | "deno.lint": true, 6 | "deno.unstable": [ 7 | "bare-node-builtins", 8 | "byonm", 9 | "sloppy-imports", 10 | "unsafe-proto", 11 | "webgpu", 12 | "broadcast-channel", 13 | "worker-options", 14 | "cron", 15 | "kv", 16 | "ffi", 17 | "fs", 18 | "http", 19 | "net" 20 | ], 21 | "[typescript]": { 22 | "editor.defaultFormatter": "denoland.vscode-deno" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import { Button, SafeAreaView, StyleSheet, View, Text } from 'react-native'; 3 | import { StatusBar } from 'expo-status-bar'; 4 | import { supabase } from './utils/supabase'; 5 | import { Audio } from 'expo-av'; 6 | import InCallManager from 'react-native-incall-manager'; 7 | import { 8 | mediaDevices, 9 | RTCPeerConnection, 10 | MediaStream, 11 | RTCView, 12 | } from 'react-native-webrtc-web-shim'; 13 | import { clientTools, clientToolsSchema } from './utils/tools'; 14 | 15 | import * as Brightness from 'expo-brightness'; 16 | 17 | const App = () => { 18 | useEffect(() => { 19 | (async () => { 20 | const { status } = await Brightness.requestPermissionsAsync(); 21 | console.log('brightness status', status); 22 | // if (status === 'granted') { 23 | // Brightness.setSystemBrightnessAsync(0); 24 | // } 25 | })(); 26 | }, []); 27 | 28 | const [isSessionActive, setIsSessionActive] = useState(false); 29 | const [events, setEvents] = useState([]); 30 | const [transcript, setTranscript] = useState(''); 31 | const [dataChannel, setDataChannel] = useState>(null); 34 | const peerConnection = useRef(null); 35 | const [localMediaStream, setLocalMediaStream] = useState( 36 | null 37 | ); 38 | const remoteMediaStream = useRef(new MediaStream()); 39 | const isVoiceOnly = true; 40 | 41 | async function startSession() { 42 | // Get an ephemeral key from the Supabase Edge Function: 43 | const { data, error } = await supabase.functions.invoke('token'); 44 | if (error) throw error; 45 | const EPHEMERAL_KEY = data.client_secret.value; 46 | console.log('token response', EPHEMERAL_KEY); 47 | 48 | // Enable audio 49 | await Audio.setAudioModeAsync({ playsInSilentModeIOS: true }); 50 | // Start InCallManager and force speaker 51 | InCallManager.start({ media: 'audio' }); 52 | InCallManager.setForceSpeakerphoneOn(true); 53 | 54 | // Create a peer connection 55 | const pc = new RTCPeerConnection(); 56 | // Set up some event listeners 57 | pc.addEventListener('connectionstatechange', (e) => { 58 | console.log('connectionstatechange', e); 59 | }); 60 | pc.addEventListener('track', (event) => { 61 | if (event.track) remoteMediaStream.current.addTrack(event.track); 62 | }); 63 | 64 | // Add local audio track for microphone input in the browser 65 | const ms = await mediaDevices.getUserMedia({ 66 | audio: true, 67 | }); 68 | if (isVoiceOnly) { 69 | let videoTrack = await ms.getVideoTracks()[0]; 70 | if (videoTrack) videoTrack.enabled = false; 71 | } 72 | 73 | setLocalMediaStream(ms); 74 | pc.addTrack(ms.getTracks()[0]); 75 | 76 | // Set up data channel for sending and receiving events 77 | const dc = pc.createDataChannel('oai-events'); 78 | setDataChannel(dc); 79 | 80 | // Start the session using the Session Description Protocol (SDP) 81 | const offer = await pc.createOffer({}); 82 | await pc.setLocalDescription(offer); 83 | 84 | const baseUrl = 'https://api.openai.com/v1/realtime'; 85 | const model = 'gpt-4o-realtime-preview-2024-12-17'; 86 | const sdpResponse = await fetch(`${baseUrl}?model=${model}`, { 87 | method: 'POST', 88 | body: offer.sdp, 89 | headers: { 90 | Authorization: `Bearer ${EPHEMERAL_KEY}`, 91 | 'Content-Type': 'application/sdp', 92 | }, 93 | }); 94 | 95 | const answer = { 96 | type: 'answer', 97 | sdp: await sdpResponse.text(), 98 | }; 99 | await pc.setRemoteDescription(answer); 100 | 101 | peerConnection.current = pc; 102 | } 103 | 104 | // Stop current session, clean up peer connection and data channel 105 | function stopSession() { 106 | // Stop InCallManager 107 | InCallManager.stop(); 108 | 109 | if (dataChannel) { 110 | dataChannel.close(); 111 | } 112 | if (peerConnection.current) { 113 | peerConnection.current.close(); 114 | } 115 | 116 | setIsSessionActive(false); 117 | setDataChannel(null); 118 | peerConnection.current = null; 119 | } 120 | 121 | // Attach event listeners to the data channel when a new one is created 122 | useEffect(() => { 123 | function configureTools() { 124 | console.log('Configuring the client side tools'); 125 | const event = { 126 | type: 'session.update', 127 | session: { 128 | modalities: ['text', 'audio'], 129 | instructions: 130 | 'You are a helpful assistant. You have access to certain tools that allow you to check the user device battery level and change the display brightness. Use these tolls if the user asks about them. Otherwise, just answer the question.', 131 | // Provide the tools. Note they match the keys in the `clientTools` object above. 132 | tools: clientToolsSchema, 133 | }, 134 | }; 135 | dataChannel.send(JSON.stringify(event)); 136 | } 137 | 138 | if (dataChannel) { 139 | // Append new server events to the list 140 | // TODO: load types from OpenAI SDK. 141 | dataChannel.addEventListener('message', async (e: any) => { 142 | const data = JSON.parse(e.data); 143 | console.log('dataChannel message', data); 144 | setEvents((prev) => [data, ...prev]); 145 | // Get transcript. 146 | if (data.type === 'response.audio_transcript.done') { 147 | setTranscript(data.transcript); 148 | } 149 | // Handle function calls 150 | if (data.type === 'response.function_call_arguments.done') { 151 | // TODO: improve types. 152 | const functionName: keyof typeof clientTools = data.name; 153 | const tool: any = clientTools[functionName]; 154 | if (tool !== undefined) { 155 | console.log( 156 | `Calling local function ${data.name} with ${data.arguments}` 157 | ); 158 | const args = JSON.parse(data.arguments); 159 | const result = await tool(args); 160 | console.log('result', result); 161 | // Let OpenAI know that the function has been called and share it's output 162 | const event = { 163 | type: 'conversation.item.create', 164 | item: { 165 | type: 'function_call_output', 166 | call_id: data.call_id, // call_id from the function_call message 167 | output: JSON.stringify(result), // result of the function 168 | }, 169 | }; 170 | dataChannel.send(JSON.stringify(event)); 171 | // Force a response to the user 172 | dataChannel.send( 173 | JSON.stringify({ 174 | type: 'response.create', 175 | }) 176 | ); 177 | } 178 | } 179 | }); 180 | 181 | // Set session active when the data channel is opened 182 | dataChannel.addEventListener('open', () => { 183 | setIsSessionActive(true); 184 | setEvents([]); 185 | // Configure the client side tools 186 | configureTools(); 187 | }); 188 | } 189 | }, [dataChannel]); 190 | 191 | return ( 192 | <> 193 | 194 | 195 | 196 | {!isSessionActive ? ( 197 |