├── .gitignore ├── .swiftpm └── xcode │ └── package.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ └── IDEWorkspaceChecks.plist ├── CHANGELOG.md ├── LICENSE ├── Package.swift ├── PipecatClientIOS.podspec ├── README.md ├── Sources └── PipecatClientIOS │ ├── Legacy.swift │ ├── RTVIClient.swift │ ├── RTVIClientDelegate.swift │ ├── RTVIClientOptions.swift │ ├── RTVIClientParams.swift │ ├── RTVIClientVersion.swift │ ├── RTVIError.swift │ ├── helper │ ├── LLMHelper.swift │ └── RTVIClientHelper.swift │ ├── transport │ ├── AuthBundle.swift │ ├── RTVIMessageInbound.swift │ ├── RTVIMessageOutbound.swift │ └── Transport.swift │ ├── types │ ├── ActionDescription.swift │ ├── MediaDeviceId.swift │ ├── MediaDeviceInfo.swift │ ├── MediaTrackId.swift │ ├── Option.swift │ ├── OptionDescription.swift │ ├── OptionType.swift │ ├── Participant.swift │ ├── ParticipantId.swift │ ├── ParticipantTracks.swift │ ├── PipecatMetrics.swift │ ├── PipecatMetricsData.swift │ ├── RTVIURLEndpoints.swift │ ├── RegisteredHelper.swift │ ├── ServiceConfig.swift │ ├── ServiceConfigDescription.swift │ ├── Tracks.swift │ ├── Transcript.swift │ ├── TransportState.swift │ ├── Value.swift │ ├── httpMessages │ │ └── TextRequest.swift │ └── voiceMessages │ │ ├── ActionRequest.swift │ │ ├── ActionResponse.swift │ │ ├── BotError.swift │ │ ├── BotLLMText.swift │ │ ├── BotReadyResponse.swift │ │ ├── BotTTSText.swift │ │ ├── ConfigResponse.swift │ │ ├── DescribeActionsResponse.swift │ │ ├── DescribeConfigResponse.swift │ │ ├── ErrorResponse.swift │ │ └── StorageItemStoredData.swift │ └── utils │ ├── ConnectionBundle.swift │ ├── HTTPMessageDispatcher.swift │ ├── Logger.swift │ ├── MessageDispatcher.swift │ └── Promise.swift ├── Tests └── PipecatClientIOSTests │ └── RTVIClientIOSTests.swift ├── pipecat-ios.png └── scripts └── generateDocs.sh /.gitignore: -------------------------------------------------------------------------------- 1 | /.swiftpm/xcode/xcuserdata 2 | tmpDocs/ 3 | -------------------------------------------------------------------------------- /.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.3.5 - 2025-04-02 2 | 3 | ### Added 4 | 5 | - Triggering `onBotStartedSpeaking` and `onBotStoppedSpeaking` based on the RTVI events that we receive from Pipecat. 6 | 7 | # 0.3.4 - 2025-03-20 8 | 9 | ### Added 10 | 11 | - Made changes to enable transports to handle function calling. 12 | 13 | # 0.3.3 - 2025-03-19 14 | 15 | ### Added 16 | 17 | - Added improvements to support new transports to send transcription events. 18 | 19 | # 0.3.2 - 2025-01-15 20 | 21 | ### Changed 22 | 23 | - Refactored the `PipecatClientIOS` target to enable publishing to CocoaPods. 24 | - **Breaking Change:** Replace imports of `RTVIClientIOS` with `PipecatClientIOS`. 25 | 26 | # 0.3.1 - 2025-01-03 27 | 28 | ### Added 29 | 30 | - Added improvements to support the `GeminiLiveWebSocket` transport. 31 | 32 | ## 0.3.0 - 2024-12-10 33 | 34 | ### Changed 35 | 36 | - Renamed the package from `RTVIClientIOS` to `PipecatClientIOS`. 37 | 38 | ## 0.2.0 - 2024-10-10 39 | 40 | - Adding support for the HTTP action. 41 | - Renamed: 42 | - `VoiceClient` to `RTVIClient` 43 | - `VoiceClientOptions` to `RTVIClientOptions` 44 | - `VoiceClientDelegate` to `RTVIClientDelete` 45 | - `VoiceError` to `RTVIError` 46 | - `VoiceClientHelper` to `RTVIClientHelper` 47 | - `FailedToFetchAuthBundle` to `HttpError` 48 | - `RTVIClient()` constructor parameter changes 49 | - `options` is now mandatory 50 | - `baseUrl` has been moved to `options.params.baseUrl` 51 | - `baseUrl` and `endpoints` are now separate, and the endpoint names are appended to the `baseUrl` 52 | - Moved `RTVIClientOptions.config` to `RTVIClientOptions.params.config` 53 | - Moved `RTVIClientOptions.customHeaders` to `RTVIClientOptions.params.headers` 54 | - Moved `RTVIClientOptions.customBodyParams` to `RTVIClientOptions.params.requestData` 55 | - `TransportState` changes 56 | - Removed `Idle` state, replaced with `Disconnected` 57 | - Added `Disconnecting` state 58 | - Added callbacks 59 | - `onBotLLMText()` 60 | - `onBotTTSText()` 61 | - `onStorageItemStored()` 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2025, Daily 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.5 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "PipecatClientIOS", 8 | platforms: [ 9 | .iOS(.v13) 10 | ], 11 | products: [ 12 | // Products define the executables and libraries a package produces, making them visible to other packages. 13 | .library( 14 | name: "PipecatClientIOS", 15 | targets: ["PipecatClientIOS"]), 16 | ], 17 | targets: [ 18 | // Targets are the basic building blocks of a package, defining a module or a test suite. 19 | // Targets can depend on other targets in this package and products from dependencies. 20 | .target( 21 | name: "PipecatClientIOS"), 22 | .testTarget( 23 | name: "PipecatClientIOSTests", 24 | dependencies: ["PipecatClientIOS"]), 25 | ] 26 | ) 27 | -------------------------------------------------------------------------------- /PipecatClientIOS.podspec: -------------------------------------------------------------------------------- 1 | Pod::Spec.new do |s| 2 | s.name = 'PipecatClientIOS' 3 | s.version = '0.3.4' 4 | s.summary = 'Pipecat iOS client library.' 5 | s.description = <<-DESC 6 | Pipecat iOS client library that implements the RTVI-AI standard. 7 | DESC 8 | s.homepage = 'https://github.com/pipecat-ai/pipecat-client-ios' 9 | s.documentation_url = "https://docs.pipecat.ai/client/ios/introduction" 10 | s.license = { :type => 'BSD-2', :file => 'LICENSE' } 11 | s.author = { "Daily.co" => "help@daily.co" } 12 | s.source = { :git => 'https://github.com/pipecat-ai/pipecat-client-ios.git', :tag => "0.3.2" } 13 | s.ios.deployment_target = '13.0' 14 | s.source_files = 'Sources/**/*.{swift,h,m}' 15 | s.exclude_files = 'Sources/Exclude' 16 | s.swift_version = '5.5' 17 | end 18 | 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |  pipecat 3 |

4 | 5 | The [Pipecat](https://github.com/pipecat-ai/) project uses [RTVI-AI](https://docs.pipecat.ai/client/introduction), an open standard for Real-Time Voice [and Video] Inference. 6 | 7 | This iOS core library exports a VoiceClient that has no associated transport. 8 | 9 | When building an RTVI application, you should use your transport-specific export (see [here](https://docs.pipecat.ai/client/ios/transports/daily) for available first-party packages.) 10 | The base class has no out-of-the-box bindings included. 11 | 12 | ## Install 13 | 14 | To depend on the client package, you can add this package via Xcode's package manager using the URL of this git repository directly, or you can declare your dependency in your `Package.swift`: 15 | 16 | ```swift 17 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios.git", from: "0.3.3"), 18 | ``` 19 | 20 | and add `"PipecatClientIOS"` to your application/library target, `dependencies`, e.g. like this: 21 | 22 | ```swift 23 | .target(name: "YourApp", dependencies: [ 24 | .product(name: "PipecatClientIOS", package: "pipecat-client-ios") 25 | ], 26 | ``` 27 | 28 | ## References 29 | - [pipecat-client-ios reference docs](https://docs-ios.pipecat.ai/PipecatClientIOS/documentation/pipecatclientios). 30 | - [pipecat-client-ios SDK docs](https://docs.pipecat.ai/client/ios/introduction). 31 | 32 | ## Contributing 33 | 34 | We are welcoming contributions to this project in form of issues and pull request. For questions about RTVI head over to the [Pipecat discord server](https://discord.gg/pipecat) and check the [#rtvi](https://discord.com/channels/1239284677165056021/1265086477964935218) channel. 35 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/Legacy.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | @available(*, deprecated, message: "VoiceClient is renamed to RTVIClient") 4 | public typealias VoiceClient = RTVIClient 5 | 6 | @available(*, deprecated, message: "VoiceClientOptions is renamed to RTVIClientOptions") 7 | public typealias VoiceClientOptions = RTVIClientOptions 8 | 9 | @available(*, deprecated, message: "VoiceClientDelegate is renamed to RTVIClientDelegate") 10 | public typealias VoiceClientDelegate = RTVIClientDelegate 11 | 12 | @available(*, deprecated, message: "VoiceError renamed to RTVIError") 13 | public typealias VoiceError = RTVIError 14 | 15 | @available(*, deprecated, message: "VoiceClientHelper is renamed to RTVIClientHelper") 16 | public typealias VoiceClientHelper = RTVIClientHelper 17 | 18 | @available(*, deprecated, message: "VoiceMessageInbound is renamed to RTVIMessageInbound") 19 | public typealias VoiceMessageInbound = RTVIMessageInbound 20 | 21 | @available(*, deprecated, message: "VoiceMessageOutbound is renamed to RTVIMessageOutbound") 22 | public typealias VoiceMessageOutbound = RTVIMessageOutbound 23 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIClient.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI client. Connects to an RTVI backend and handles bidirectional audio and video. 4 | @MainActor 5 | open class RTVIClient { 6 | 7 | private let options: RTVIClientOptions 8 | private var transport: Transport 9 | private var baseUrl: String? // see RTVIClientParams for explanation of optionality 10 | 11 | private let messageDispatcher: MessageDispatcher 12 | private var helpers: [String: RegisteredHelper] = [:] 13 | private var devicesInitialized: Bool = false 14 | 15 | private var disconnectRequested: Bool = false 16 | 17 | private lazy var onMessage: (RTVIMessageInbound) -> Void = {(voiceMessage: RTVIMessageInbound) in 18 | guard let type = voiceMessage.type else { 19 | // Ignoring the message, it doesn't have a type 20 | return 21 | } 22 | Logger.shared.debug("Received voice message \(voiceMessage)") 23 | switch type { 24 | case RTVIMessageInbound.MessageType.BOT_READY: 25 | self.transport.setState(state: .ready) 26 | if let botReadyData = try? JSONDecoder().decode(BotReadyData.self, from: Data(voiceMessage.data!.utf8)) { 27 | self.delegate?.onBotReady(botReadyData: botReadyData) 28 | } 29 | case RTVIMessageInbound.MessageType.USER_TRANSCRIPTION: 30 | if let transcript = try? JSONDecoder().decode(Transcript.self, from: Data(voiceMessage.data!.utf8)) { 31 | self.delegate?.onUserTranscript(data: transcript) 32 | } 33 | case RTVIMessageInbound.MessageType.BOT_TRANSCRIPTION: 34 | if let transcript = try? JSONDecoder().decode(Transcript.self, from: Data(voiceMessage.data!.utf8)) { 35 | self.delegate?.onBotTranscript(data: transcript.text) 36 | } 37 | case RTVIMessageInbound.MessageType.BOT_LLM_STARTED: 38 | self.delegate?.onBotLLMStarted() 39 | case RTVIMessageInbound.MessageType.BOT_LLM_TEXT: 40 | if let botLLMText = try? JSONDecoder().decode(BotLLMText.self, from: Data(voiceMessage.data!.utf8)) { 41 | self.delegate?.onBotLLMText(data: botLLMText) 42 | } 43 | case RTVIMessageInbound.MessageType.BOT_LLM_STOPPED: 44 | self.delegate?.onBotLLMStopped() 45 | case RTVIMessageInbound.MessageType.BOT_TTS_STARTED: 46 | self.delegate?.onBotTTSStarted() 47 | case RTVIMessageInbound.MessageType.BOT_TTS_TEXT: 48 | if let botTTSText = try? JSONDecoder().decode(BotTTSText.self, from: Data(voiceMessage.data!.utf8)) { 49 | self.delegate?.onBotTTSText(data: botTTSText) 50 | } 51 | case RTVIMessageInbound.MessageType.BOT_TTS_STTOPED: 52 | self.delegate?.onBotTTSStopped() 53 | case RTVIMessageInbound.MessageType.STORAGE_ITEM_STORED: 54 | if let storedData = try? JSONDecoder().decode(StorageItemStoredData.self, from: Data(voiceMessage.data!.utf8)) { 55 | self.delegate?.onStorageItemStored(data: storedData) 56 | } 57 | case RTVIMessageInbound.MessageType.PIPECAT_METRICS: 58 | guard let metrics = voiceMessage.metrics else { 59 | return 60 | } 61 | self.delegate?.onMetrics(data: metrics) 62 | case RTVIMessageInbound.MessageType.USER_STARTED_SPEAKING: 63 | self.delegate?.onUserStartedSpeaking() 64 | case RTVIMessageInbound.MessageType.USER_STOPPED_SPEAKING: 65 | self.delegate?.onUserStoppedSpeaking() 66 | case RTVIMessageInbound.MessageType.BOT_STARTED_SPEAKING: 67 | self.delegate?.onBotStartedSpeaking() 68 | case RTVIMessageInbound.MessageType.BOT_STOPPED_SPEAKING: 69 | self.delegate?.onBotStoppedSpeaking() 70 | case RTVIMessageInbound.MessageType.ACTION_RESPONSE: 71 | _ = self.messageDispatcher.resolve(message: voiceMessage) 72 | case RTVIMessageInbound.MessageType.DESCRIBE_ACTION_RESPONSE: 73 | _ = self.messageDispatcher.resolve(message: voiceMessage) 74 | case RTVIMessageInbound.MessageType.DESCRIBE_CONFIG_RESPONSE: 75 | _ = self.messageDispatcher.resolve(message: voiceMessage) 76 | case RTVIMessageInbound.MessageType.CONFIG_RESPONSE: 77 | _ = self.messageDispatcher.resolve(message: voiceMessage) 78 | case RTVIMessageInbound.MessageType.ERROR_RESPONSE: 79 | Logger.shared.warn("RECEIVED ON ERROR_RESPONSE \(voiceMessage)") 80 | _ = self.messageDispatcher.reject(message: voiceMessage) 81 | if let errorResponse = try? JSONDecoder().decode(ErrorResponse.self, from: Data(voiceMessage.data!.utf8)) { 82 | self.delegate?.onError(message: errorResponse.error) 83 | } 84 | case RTVIMessageInbound.MessageType.ERROR: 85 | Logger.shared.warn("RECEIVED ON ERROR \(voiceMessage)") 86 | _ = self.messageDispatcher.reject(message: voiceMessage) 87 | if let botError = try? JSONDecoder().decode(BotError.self, from: Data(voiceMessage.data!.utf8)) { 88 | let errorMessage = "Received an error from the Bot: \(botError.error)" 89 | self.delegate?.onError(message: errorMessage) 90 | if(botError.fatal ?? false) { 91 | self.disconnect(completion: nil) 92 | } 93 | } 94 | default: 95 | // Check if we have handlers to deal with the message 96 | var match = false 97 | for entry in self.helpers.values where entry.supportedMessages.contains(type) { 98 | match = true 99 | entry.helper.handleMessage(msg: voiceMessage) 100 | } 101 | if !match { 102 | Logger.shared.debug("Unexpected message type: \(type), message: \(voiceMessage)") 103 | self.delegate?.onGenericMessage(message: voiceMessage) 104 | } 105 | } 106 | } 107 | 108 | /// The object that acts as the delegate of the voice client. 109 | private weak var _delegate: RTVIClientDelegate? = nil 110 | public weak var delegate: RTVIClientDelegate? { 111 | get { 112 | return _delegate 113 | } 114 | set { 115 | _delegate = newValue 116 | self.transport.delegate = _delegate 117 | } 118 | } 119 | 120 | public init(baseUrl:String? = nil, transport:Transport, options: RTVIClientOptions) { 121 | Logger.shared.info("Initializing RTVI Client iOS version \(RTVIClient.libraryVersion)") 122 | 123 | self.baseUrl = baseUrl ?? options.params.baseUrl 124 | self.options = options 125 | self.transport = transport 126 | 127 | let headers = options.customHeaders ?? options.params.headers 128 | let requestData = RTVIClient.appendRtviClientVersion(options.customBodyParams ?? options.params.requestData) 129 | 130 | var httpMessageDispatcher: HTTPMessageDispatcher? = nil 131 | if let baseUrl { 132 | httpMessageDispatcher = HTTPMessageDispatcher.init(baseUrl: baseUrl, endpoints: self.options.params.endpoints, headers: headers, requestData: requestData) 133 | } 134 | self.messageDispatcher = MessageDispatcher.init(transport: transport, httpMessageDispatcher: httpMessageDispatcher) 135 | 136 | httpMessageDispatcher?.onMessage = self.onMessage 137 | self.transport.onMessage = self.onMessage 138 | } 139 | 140 | /// Initialize local media devices such as camera and microphone. 141 | public func initDevices(completion: ((Result) -> Void)?) { 142 | Task { 143 | do { 144 | try await self.initDevices() 145 | completion?(.success(())) 146 | } catch { 147 | completion?(.failure(AsyncExecutionError(functionName: "initDevices", underlyingError: error))) 148 | } 149 | } 150 | } 151 | 152 | /// Initialize local media devices such as camera and microphone. 153 | public func initDevices() async throws { 154 | if (self.devicesInitialized) { 155 | // There is nothing to do in this case 156 | return 157 | } 158 | try await self.transport.initDevices() 159 | self.devicesInitialized = true 160 | } 161 | 162 | private func connectUrl() -> String? { 163 | if let baseUrl { 164 | return baseUrl + self.options.params.endpoints.connect 165 | } 166 | return nil 167 | } 168 | 169 | private func fetchAuthBundle() async throws -> AuthBundle? { 170 | guard let connectUrl = self.connectUrl() else { 171 | // Assume we're using a transport that doesn't communicate with an RTVI server. 172 | return nil 173 | } 174 | 175 | guard let url = URL(string: connectUrl) else { 176 | throw InvalidAuthBundleError() 177 | } 178 | 179 | var request = URLRequest(url: url) 180 | request.httpMethod = "POST" 181 | request.setValue("application/json", forHTTPHeaderField: "Content-Type") 182 | 183 | // Adding the custom headers if they have been provided 184 | let headers = options.customHeaders ?? options.params.headers 185 | for header in headers { 186 | for (key, value) in header { 187 | request.setValue(value, forHTTPHeaderField: key) 188 | } 189 | } 190 | 191 | do { 192 | let requestData = RTVIClient.appendRtviClientVersion(options.customBodyParams ?? options.params.requestData) 193 | let config = options.config ?? options.params.config 194 | if let customBodyParams = requestData { 195 | var customBundle:Value = try await customBodyParams.convertToRtviValue() 196 | try customBundle.addProperty(key: "services", value: try await self.options.services.convertToRtviValue()) 197 | try customBundle.addProperty(key: "config", value: try await config.convertToRtviValue()) 198 | request.httpBody = try JSONEncoder().encode( customBundle ) 199 | } else { 200 | request.httpBody = try JSONEncoder().encode( 201 | ConnectionBundle(services: self.options.services, config: config) 202 | ) 203 | } 204 | 205 | Logger.shared.debug("Will request bundle \(String(data: request.httpBody!, encoding: .utf8) ?? "")") 206 | 207 | let (data, response) = try await URLSession.shared.data(for: request) 208 | 209 | guard let httpResponse = response as? HTTPURLResponse, ( httpResponse.statusCode >= 200 && httpResponse.statusCode <= 299 ) else { 210 | let errorMessage = String(data: data, encoding: .utf8) ?? "Unknown error" 211 | let message = "Failed while authenticating: \(errorMessage)" 212 | Logger.shared.error(message) 213 | throw HttpError(message: message) 214 | } 215 | 216 | return AuthBundle(data: String(data: data, encoding: .utf8)!) 217 | } catch { 218 | Logger.shared.error(error.localizedDescription) 219 | throw HttpError(message: "Failed while authenticating.", underlyingError: error) 220 | } 221 | } 222 | 223 | /// Initiate an RTVI session, connecting to the backend. 224 | public func start() async throws { 225 | do { 226 | self.disconnectRequested = false 227 | if(!self.devicesInitialized) { 228 | try await self.initDevices() 229 | } 230 | 231 | if(self.bailIfDisconnected()) { 232 | return 233 | } 234 | 235 | self.transport.setState(state: .authenticating) 236 | // Send POST request to the provided baseUrl to connect and start the bot 237 | let authBundle = try await fetchAuthBundle() 238 | 239 | if(self.bailIfDisconnected()) { 240 | return 241 | } 242 | 243 | try await self.transport.connect(authBundle: authBundle) 244 | 245 | if(self.bailIfDisconnected()) { 246 | return 247 | } 248 | } catch { 249 | self.disconnect(completion: nil) 250 | self.transport.setState(state: .disconnected) 251 | throw StartBotError(underlyingError: error) 252 | } 253 | } 254 | 255 | private func bailIfDisconnected() -> Bool { 256 | if (self.disconnectRequested) { 257 | if (self.transport.state() != .disconnecting && self.transport.state() != .disconnected) { 258 | self.disconnect(completion: nil) 259 | } 260 | return true 261 | } 262 | return false 263 | } 264 | 265 | /// Initiate an RTVI session, connecting to the backend. 266 | public func start(completion: ((Result) -> Void)?) { 267 | Task { 268 | do { 269 | try await self.start() 270 | completion?(.success(())) 271 | } catch { 272 | completion?(.failure(AsyncExecutionError(functionName: "start", underlyingError: error))) 273 | } 274 | } 275 | } 276 | 277 | /// Directly send a message to the bot via the transport. 278 | func sendMessage(msg: RTVIMessageOutbound) async throws { 279 | try await self.transport.sendMessage(message: msg) 280 | } 281 | 282 | /// Directly send a message to the bot via the transport. 283 | func sendMessage(msg: RTVIMessageOutbound, completion: ((Result) -> Void)?) { 284 | Task { 285 | do { 286 | try await self.sendMessage(msg: msg) 287 | completion?(.success(())) 288 | } catch { 289 | completion?(.failure(AsyncExecutionError(functionName: "sendMessage", underlyingError: error))) 290 | } 291 | } 292 | } 293 | 294 | /// Disconnect an active RTVI session. 295 | public func disconnect() async throws { 296 | self.transport.setState(state: .disconnecting) 297 | self.disconnectRequested = true 298 | try await self.transport.disconnect() 299 | } 300 | 301 | /// Disconnect an active RTVI session. 302 | public func disconnect(completion: ((Result) -> Void)?) { 303 | Task { 304 | do { 305 | try await self.disconnect() 306 | completion?(.success(())) 307 | } catch { 308 | completion?(.failure(AsyncExecutionError(functionName: "disconnect", underlyingError: error))) 309 | } 310 | } 311 | } 312 | 313 | /// The current state of the session. 314 | public var state: TransportState { 315 | self.transport.state() 316 | } 317 | 318 | /// Check if the transport is connected 319 | public func isConnected() -> Bool { 320 | return self.transport.isConnected() 321 | } 322 | 323 | /// Returns a list of available audio input devices. 324 | public func getAllMics() -> [MediaDeviceInfo] { 325 | return self.transport.getAllMics() 326 | } 327 | 328 | /// Returns a list of available video input devices. 329 | public func getAllCams() -> [MediaDeviceInfo] { 330 | self.transport.getAllCams() 331 | } 332 | 333 | /// Returns the selected audio input device. 334 | public var selectedMic: MediaDeviceInfo? { 335 | return self.transport.selectedMic() 336 | } 337 | 338 | /// Returns the selected video input device. 339 | public var selectedCam: MediaDeviceInfo? { 340 | return self.transport.selectedCam() 341 | } 342 | 343 | /// Use the specified audio input device. 344 | public func updateMic(micId: MediaDeviceId) async throws { 345 | try await self.transport.updateMic(micId: micId) 346 | } 347 | 348 | /// Use the specified audio input device. 349 | public func updateMic(micId: MediaDeviceId, completion: ((Result) -> Void)?) { 350 | Task { 351 | do { 352 | try await self.updateMic(micId: micId) 353 | completion?(.success(())) 354 | } catch { 355 | completion?(.failure(AsyncExecutionError(functionName: "updateMic", underlyingError: error))) 356 | } 357 | } 358 | } 359 | 360 | /// Use the specified video input device. 361 | public func updateCam(camId: MediaDeviceId) async throws { 362 | try await self.transport.updateCam(camId: camId) 363 | } 364 | 365 | /// Use the specified video input device. 366 | public func updateCam(camId: MediaDeviceId, completion: ((Result) -> Void)?) { 367 | Task { 368 | do { 369 | try await self.updateCam(camId: camId) 370 | completion?(.success(())) 371 | } catch { 372 | completion?(.failure(AsyncExecutionError(functionName: "updateCam", underlyingError: error))) 373 | } 374 | } 375 | } 376 | 377 | /// Enables or disables the audio input device. 378 | public func enableMic(enable: Bool) async throws { 379 | try await self.transport.enableMic(enable: enable) 380 | } 381 | 382 | /// Enables or disables the audio input device. 383 | public func enableMic(enable: Bool, completion: ((Result) -> Void)?) { 384 | Task { 385 | do { 386 | try await self.enableMic(enable: enable) 387 | completion?(.success(())) 388 | } catch { 389 | completion?(.failure(AsyncExecutionError(functionName: "enableMic", underlyingError: error))) 390 | } 391 | } 392 | } 393 | 394 | /// Enables or disables the video input device. 395 | public func enableCam(enable: Bool) async throws { 396 | try await self.transport.enableCam(enable: enable) 397 | } 398 | 399 | /// Enables or disables the video input device. 400 | public func enableCam(enable: Bool, completion: ((Result) -> Void)?) { 401 | Task { 402 | do { 403 | try await self.enableCam(enable: enable) 404 | completion?(.success(())) 405 | } catch { 406 | completion?(.failure(AsyncExecutionError(functionName: "enableCam", underlyingError: error))) 407 | } 408 | } 409 | } 410 | 411 | /// Returns true if the microphone is enabled, false otherwise. 412 | public var isMicEnabled: Bool { 413 | self.transport.isMicEnabled() 414 | } 415 | 416 | /// Returns true if the camera is enabled, false otherwise. 417 | public var isCamEnabled: Bool { 418 | self.transport.isCamEnabled() 419 | } 420 | 421 | /// Returns a list of participant media tracks. 422 | var tracks: Tracks? { 423 | return self.transport.tracks() 424 | } 425 | 426 | /// Request the bot to send its current configuration 427 | public func getConfig() async throws -> ConfigResponse { 428 | try self.assertReady() 429 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: RTVIMessageOutbound.getConfig()) 430 | let configResponse = try JSONDecoder().decode(ConfigResponse.self, from: Data(voiceMessageResponse.data!.utf8)) 431 | self.delegate?.onConfigUpdated(config: configResponse.config) 432 | return configResponse 433 | } 434 | 435 | /// Request the bot to send its current configuration 436 | public func getConfig(completion: ((Result) -> Void)?) { 437 | Task { 438 | do { 439 | let config = try await self.getConfig() 440 | completion?(.success((config))) 441 | } catch { 442 | completion?(.failure(AsyncExecutionError(functionName: "getConfig", underlyingError: error))) 443 | } 444 | } 445 | } 446 | 447 | /// Request the bot to describe its current configuration 448 | public func describeConfig() async throws -> DescribeConfigResponse { 449 | try self.assertReady() 450 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: RTVIMessageOutbound.describeConfig()) 451 | let describedConfig = try JSONDecoder().decode(DescribeConfigResponse.self, from: Data(voiceMessageResponse.data!.utf8)) 452 | self.delegate?.onConfigDescribed(config: describedConfig.config) 453 | return describedConfig 454 | } 455 | 456 | /// Request the bot to describe its current configuration 457 | public func describeConfig(completion: ((Result) -> Void)?) { 458 | Task { 459 | do { 460 | let configDescription = try await self.describeConfig() 461 | completion?(.success((configDescription))) 462 | } catch { 463 | completion?(.failure(AsyncExecutionError(functionName: "describeConfig", underlyingError: error))) 464 | } 465 | } 466 | } 467 | 468 | /// Updates the config on the server. 469 | public func updateConfig(config: [ServiceConfig], interrupt: Bool = false) async throws -> ConfigResponse { 470 | try self.assertReady() 471 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: RTVIMessageOutbound.updateConfig(config: config, interrupt: interrupt)) 472 | let configResponse = try JSONDecoder().decode(ConfigResponse.self, from: Data(voiceMessageResponse.data!.utf8)) 473 | self.delegate?.onConfigUpdated(config: configResponse.config) 474 | return configResponse 475 | } 476 | 477 | /// Updates the config on the server. 478 | public func updateConfig(config: [ServiceConfig], interrupt: Bool = false, completion: ((Result) -> Void)?) { 479 | Task { 480 | do { 481 | let configDescription = try await self.updateConfig(config:config, interrupt: interrupt) 482 | completion?(.success((configDescription))) 483 | } catch { 484 | completion?(.failure(AsyncExecutionError(functionName: "updateConfig", underlyingError: error))) 485 | } 486 | } 487 | } 488 | 489 | /// Dispatch an action message to the bot 490 | public func action(action: ActionRequest, resultType: T.Type, unwrapResult: Bool = true) async throws -> T { 491 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: RTVIMessageOutbound.action(actionData: action)) 492 | if unwrapResult { 493 | return (try JSONDecoder().decode(ActionResponseWrapper.self, from: Data(voiceMessageResponse.data!.utf8))).result 494 | } else { 495 | return try JSONDecoder().decode(resultType, from: Data(voiceMessageResponse.data!.utf8)) 496 | } 497 | } 498 | 499 | /// Dispatch an action message to the bot 500 | public func action(action: ActionRequest) async throws -> ActionResponse { 501 | try await self.action(action:action, resultType: ActionResponse.self, unwrapResult: false) 502 | } 503 | 504 | /// Dispatch an action message to the bot 505 | public func action(action: ActionRequest, completion: ((Result) -> Void)?) { 506 | Task { 507 | do { 508 | let configDescription = try await self.action(action:action) 509 | completion?(.success((configDescription))) 510 | } catch { 511 | completion?(.failure(AsyncExecutionError(functionName: "action", underlyingError: error))) 512 | } 513 | } 514 | } 515 | 516 | /// Describe available / registered actions the bot has 517 | public func describeActions() async throws -> DescribeActionResponse { 518 | try self.assertReady() 519 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: RTVIMessageOutbound.describeActions()) 520 | let describedActions = try JSONDecoder().decode(DescribeActionResponse.self, from: Data(voiceMessageResponse.data!.utf8)) 521 | self.delegate?.onActionsAvailable(actions: describedActions.actions) 522 | return describedActions 523 | } 524 | 525 | /// Describe available / registered actions the bot has 526 | public func describeActions(completion: ((Result) -> Void)?) { 527 | Task { 528 | do { 529 | let actionDescription = try await self.describeActions() 530 | completion?(.success((actionDescription))) 531 | } catch { 532 | completion?(.failure(AsyncExecutionError(functionName: "describeActions", underlyingError: error))) 533 | } 534 | } 535 | } 536 | 537 | /// Registers a new helper with the client. 538 | public func registerHelper(service: String, helper: T.Type) throws -> T { 539 | if helpers.keys.contains(service) { 540 | throw OtherError(message: "Helper with name '\(service)' already registered") 541 | } 542 | 543 | let clientHelper = helper.init(service: service, voiceClient: self) 544 | let entry = RegisteredHelper( 545 | helper: clientHelper, 546 | supportedMessages: Set(clientHelper.getMessageTypes()) 547 | ) 548 | 549 | helpers[service] = entry 550 | 551 | return clientHelper 552 | } 553 | 554 | /// Unregisters a helper from the client. 555 | public func unregisterHelper(service: String) throws { 556 | if !helpers.keys.contains(service) { 557 | throw OtherError(message: "Helper with name '\(service)' not registered") 558 | } 559 | _ = helpers.removeValue(forKey: service) 560 | } 561 | 562 | /// Retrieves a helper from the client. 563 | public func getHelper(service: String) throws -> T { 564 | guard let entry = helpers[service] else { 565 | throw OtherError(message: "Helper with name '\(service)' not registered") 566 | } 567 | 568 | guard let helper = entry.helper as? T else { 569 | throw OtherError(message: "Helper registered for service '\(service)' is not of expected type") 570 | } 571 | 572 | return helper 573 | } 574 | 575 | /// Destroys this VoiceClient and cleans up any allocated resources. 576 | public func release() { 577 | self.transport.release() 578 | } 579 | 580 | /// The expiry time for the transport session, if applicable. Measured in seconds since the UNIX epoch (UTC). 581 | public func expiry() -> Int? { 582 | self.transport.expiry() 583 | } 584 | 585 | func assertReady() throws -> Void{ 586 | if self.state != .ready { 587 | throw BotNotReadyError() 588 | } 589 | } 590 | 591 | } 592 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIClientDelegate.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Callbacks invoked when changes occur in the voice session. 4 | public protocol RTVIClientDelegate: AnyObject { 5 | /// Invoked when the underlying transport has connected. 6 | func onConnected() 7 | 8 | /// Invoked when the underlying transport has disconnected. 9 | func onDisconnected() 10 | 11 | /// Invoked when the session state has changed. 12 | func onTransportStateChanged( state: TransportState) 13 | 14 | /// Invoked when the session configuration has returned or changed. 15 | func onConfigUpdated( config: [ServiceConfig]) 16 | 17 | /// Invoked when the configs have been described. 18 | func onConfigDescribed(config: [ServiceConfigDescription]) 19 | 20 | /// Invoked when the actions are available. 21 | func onActionsAvailable(actions: [ActionDescription]) 22 | 23 | /// Invoked when the bot has connected to the session. 24 | func onBotConnected( participant: Participant) 25 | 26 | /// Invoked when the bot has indicated it is ready for commands. 27 | func onBotReady(botReadyData: BotReadyData) 28 | 29 | /// Invoked when the bot has disconnected from the session. 30 | func onBotDisconnected( participant: Participant) 31 | 32 | /// Invoked when a participant has joined the session. 33 | func onParticipantJoined( participant: Participant) 34 | 35 | /// Invoked when a participant has left the session. 36 | func onParticipantLeft( participant: Participant) 37 | 38 | /// Invoked when the list of available cameras has changed. 39 | func onAvailableCamsUpdated( cams: [MediaDeviceInfo]) 40 | 41 | /// Invoked when the list of available microphones has updated. 42 | func onAvailableMicsUpdated( mics: [MediaDeviceInfo]) 43 | 44 | /// Invoked when the selected cam has changed. 45 | func onCamUpdated( cam: MediaDeviceInfo?) 46 | 47 | /// Invoked when the selected microphone has changed. 48 | func onMicUpdated( mic: MediaDeviceInfo?) 49 | 50 | /// Invoked regularly with the volume of the locally captured audio. 51 | func onUserAudioLevel( level: Float) 52 | 53 | /// Invoked regularly with the audio volume of each remote participant. 54 | func onRemoteAudioLevel( level: Float, participant: Participant) 55 | 56 | /// Invoked when the bot starts talking. 57 | @available(*, deprecated, message: "Use onBotStartedSpeaking() instead") 58 | func onBotStartedSpeaking( participant: Participant) 59 | 60 | /// Invoked when the bot starts talking. 61 | func onBotStartedSpeaking() 62 | 63 | /// Invoked when the bot stops talking. 64 | @available(*, deprecated, message: "Use onBotStoppedSpeaking() instead") 65 | func onBotStoppedSpeaking( participant: Participant) 66 | 67 | /// Invoked when the bot stops talking. 68 | func onBotStoppedSpeaking() 69 | 70 | /// Invoked when the local user starts talking. 71 | func onUserStartedSpeaking() 72 | 73 | /// Invoked when the local user stops talking. 74 | func onUserStoppedSpeaking() 75 | 76 | /// Invoked when session metrics are received. 77 | func onMetrics( data: PipecatMetrics) 78 | 79 | /// Invoked when user transcript data is avaiable. 80 | func onUserTranscript( data: Transcript) 81 | 82 | /// Invoked when bot transcript data is avaiable. 83 | func onBotTranscript( data: String) 84 | 85 | /// Invoked when the bot LLM text has started. 86 | func onBotLLMStarted() 87 | 88 | /// Invoked when received the bot transcription from the LLM. 89 | func onBotLLMText(data: BotLLMText) 90 | 91 | /// Invoked when the bot LLM text has stopped. 92 | func onBotLLMStopped() 93 | 94 | /// Invoked when the bot TTS text has started. 95 | func onBotTTSStarted() 96 | 97 | /// Invoked when text is spoken by the bot. 98 | func onBotTTSText(data: BotTTSText) 99 | 100 | /// Invoked when the bot TTS text has stopped. 101 | func onBotTTSStopped() 102 | 103 | /// Invoked when data is stored by the bot. 104 | func onStorageItemStored(data: StorageItemStoredData) 105 | 106 | /// Invoked when we receive an error message from the backend 107 | func onError( message: String) 108 | 109 | /// Invoked when a message from the backend is received which was not handled by the VoiceClient or a registered helper. 110 | func onGenericMessage (message:RTVIMessageInbound) 111 | 112 | /// Invoked when the set of available cam/mic tracks changes. 113 | func onTracksUpdated(tracks: Tracks) 114 | } 115 | 116 | public extension RTVIClientDelegate { 117 | func onConnected() {} 118 | func onDisconnected() {} 119 | func onTransportStateChanged( state: TransportState) {} 120 | func onConfigUpdated( config: [ServiceConfig]) {} 121 | func onConfigDescribed(config: [ServiceConfigDescription]) {} 122 | func onActionsAvailable(actions: [ActionDescription]) {} 123 | func onBotConnected( participant: Participant) {} 124 | func onBotReady(botReadyData: BotReadyData) {} 125 | func onBotDisconnected( participant: Participant) {} 126 | func onParticipantJoined( participant: Participant) {} 127 | func onParticipantLeft( participant: Participant) {} 128 | func onAvailableCamsUpdated( cams: [MediaDeviceInfo]) {} 129 | func onAvailableMicsUpdated( mics: [MediaDeviceInfo]) {} 130 | func onCamUpdated( cam: MediaDeviceInfo?) {} 131 | func onMicUpdated( mic: MediaDeviceInfo?) {} 132 | func onUserAudioLevel( level: Float) {} 133 | func onRemoteAudioLevel( level: Float, participant: Participant) {} 134 | @available(*, deprecated, message: "Use onBotStartedSpeaking() instead") 135 | func onBotStartedSpeaking( participant: Participant) {} 136 | func onBotStartedSpeaking() {} 137 | @available(*, deprecated, message: "Use onBotStoppedSpeaking() instead") 138 | func onBotStoppedSpeaking( participant: Participant) {} 139 | func onBotStoppedSpeaking() {} 140 | func onUserStartedSpeaking() {} 141 | func onUserStoppedSpeaking() {} 142 | func onMetrics( data: PipecatMetrics) {} 143 | func onUserTranscript( data: Transcript) {} 144 | func onBotTranscript( data: String) {} 145 | func onError( message: String) {} 146 | func onGenericMessage (message:RTVIMessageInbound) {} 147 | func onTracksUpdated(tracks: Tracks) {} 148 | func onBotLLMStarted() {} 149 | func onBotLLMText(data: BotLLMText) {} 150 | func onBotLLMStopped() {} 151 | func onBotTTSStarted() {} 152 | func onBotTTSText(data: BotTTSText) {} 153 | func onBotTTSStopped() {} 154 | func onStorageItemStored(data: StorageItemStoredData) {} 155 | } 156 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIClientOptions.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Configuration options when instantiating a VoiceClient. 4 | public struct RTVIClientOptions: Codable { 5 | 6 | /// Connection parameters. 7 | public let params: RTVIClientParams 8 | 9 | /// Enable the user mic input. Defaults to true. 10 | public let enableMic: Bool 11 | 12 | /// Enable user cam input. Defaults to false. 13 | public let enableCam: Bool 14 | 15 | /// A list of services to use on the backend. 16 | public let services: [String: String] 17 | 18 | /// Further configuration options for the backend. 19 | @available(*, deprecated, message: "Use params.config.") 20 | public let config: [ServiceConfig]? 21 | 22 | /// Custom HTTP headers to be sent with the POST request to baseUrl. 23 | @available(*, deprecated, message: "Use params.headers.") 24 | public let customHeaders: [[String: String]]? 25 | 26 | /// Custom HTTP body params to be sent with the POST request to baseUrl. 27 | @available(*, deprecated, message: "Use params.requestData.") 28 | public let customBodyParams: Value? 29 | 30 | public init( 31 | enableMic: Bool = true, 32 | enableCam: Bool = false, 33 | params: RTVIClientParams, 34 | services: [String: String] = [:], 35 | config: [ServiceConfig]? = nil, 36 | customHeaders: [[String: String]]? = nil, 37 | customBodyParams: Value? = nil 38 | ) { 39 | self.enableMic = enableMic 40 | self.enableCam = enableCam 41 | self.params = params 42 | self.services = services 43 | self.config = config 44 | self.customHeaders = customHeaders 45 | self.customBodyParams = customBodyParams 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIClientParams.swift: -------------------------------------------------------------------------------- 1 | /// Configuration params when instantiating a RTVIClient. 2 | public struct RTVIClientParams: Codable { 3 | 4 | /// The base URL for the RTVI POST request. 5 | /// Not needed when using certain transports that don't communicate with an RTVI server. 6 | public let baseUrl: String? 7 | 8 | /// Custom HTTP headers to be sent with the POST request to baseUrl. 9 | public let headers: [[String: String]] 10 | 11 | /// API endpoint names for the RTVI POST requests. 12 | public let endpoints: RTVIURLEndpoints 13 | 14 | /// Custom parameters to add to the auth request body. 15 | public let requestData: Value? 16 | 17 | /// Further configuration options for the backend. 18 | public let config: [ServiceConfig] 19 | 20 | public init( 21 | baseUrl: String? = nil, 22 | headers: [[String: String]] = [], 23 | endpoints: RTVIURLEndpoints = RTVIURLEndpoints(), 24 | requestData: Value? = nil, 25 | config: [ServiceConfig] = [] 26 | ) { 27 | self.baseUrl = baseUrl 28 | self.headers = headers 29 | self.endpoints = endpoints 30 | self.requestData = requestData 31 | self.config = config 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIClientVersion.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | extension RTVIClient { 4 | 5 | public static let libraryVersion = "0.2.0" 6 | 7 | static func appendRtviClientVersion(_ requestData: Value?) -> Value? { 8 | var requestDataWithVersion = requestData 9 | do { 10 | if (requestData == nil) { 11 | requestDataWithVersion = Value.object([ 12 | "rtvi_client_version": .string(RTVIClient.libraryVersion) 13 | ]) 14 | } else { 15 | try requestDataWithVersion?.addProperty(key: "rtvi_client_version", value: .string(RTVIClient.libraryVersion)) 16 | } 17 | } catch { 18 | Logger.shared.error("Failed to add rtvi_client_version \(error.localizedDescription)") 19 | } 20 | return requestDataWithVersion 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIError.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A protocol representing a base error occurring during an operation. 4 | public protocol RTVIError: Error { 5 | /// A human-readable description of the error. 6 | var message: String { get } 7 | 8 | /// If the error was caused by another error, this value is set. 9 | var underlyingError: Error? { get } 10 | } 11 | 12 | public extension RTVIError { 13 | var underlyingError: Error? { return nil } 14 | 15 | /// Provides a detailed description of the error, including any underlying error. 16 | var localizedDescription: String { 17 | if let underlyingError = self.underlyingError as? RTVIError { 18 | // Finding the root cause 19 | var rootCauseError: RTVIError = underlyingError 20 | while let underlyingError = rootCauseError.underlyingError as? RTVIError { 21 | rootCauseError = underlyingError 22 | } 23 | return "\(message) \(rootCauseError.localizedDescription)" 24 | } else { 25 | return message 26 | } 27 | } 28 | } 29 | 30 | /// Invalid or malformed auth bundle provided to Transport. 31 | public struct InvalidAuthBundleError: RTVIError { 32 | public let message: String = "Invalid or malformed auth bundle provided to Transport." 33 | public let underlyingError: Error? 34 | 35 | public init(underlyingError: Error? = nil) { 36 | self.underlyingError = underlyingError 37 | } 38 | } 39 | 40 | /// Failed to fetch the authentication bundle from the RTVI backend. 41 | public struct HttpError: RTVIError { 42 | public let message: String 43 | public let underlyingError: Error? 44 | 45 | public init(message: String, underlyingError: Error? = nil) { 46 | self.underlyingError = underlyingError 47 | self.message = message 48 | } 49 | } 50 | 51 | /// Failed to fetch the auth bundle. 52 | public struct StartBotError: RTVIError { 53 | public let message: String = "Failed to connect / invalid auth bundle from base url" 54 | public let underlyingError: Error? 55 | 56 | public init(underlyingError: Error? = nil) { 57 | self.underlyingError = underlyingError 58 | } 59 | } 60 | 61 | /// Unable to update configuration. 62 | public struct ConfigUpdateError: RTVIError { 63 | public let message: String = "Unable to update configuration." 64 | public let underlyingError: Error? 65 | 66 | public init(underlyingError: Error? = nil) { 67 | self.underlyingError = underlyingError 68 | } 69 | } 70 | 71 | /// Bot is not ready yet. 72 | public struct BotNotReadyError: RTVIError { 73 | public let message: String = "Bot is not ready yet." 74 | public let underlyingError: Error? 75 | 76 | public init(underlyingError: Error? = nil) { 77 | self.underlyingError = underlyingError 78 | } 79 | } 80 | 81 | /// Received error response from backend. 82 | public struct BotResponseError: RTVIError { 83 | public let message: String 84 | public let underlyingError: Error? 85 | 86 | public init(message: String = "Received error response from backend.", underlyingError: Error? = nil) { 87 | self.message = message 88 | self.underlyingError = underlyingError 89 | } 90 | } 91 | 92 | /// The operation timed out before it could complete. 93 | public struct ResponseTimeoutError: RTVIError { 94 | public let message: String = "The operation timed out before it could complete." 95 | public let underlyingError: Error? 96 | 97 | public init(underlyingError: Error? = nil) { 98 | self.underlyingError = underlyingError 99 | } 100 | } 101 | 102 | /// Received an error response when trying to execute the function. 103 | public struct AsyncExecutionError: RTVIError { 104 | public let message: String 105 | public let underlyingError: Error? 106 | 107 | public init(functionName: String, underlyingError: Error? = nil) { 108 | self.message = "Received an error response when trying to execute the function \(functionName)." 109 | self.underlyingError = underlyingError 110 | } 111 | } 112 | 113 | /// An unknown error occurred.. 114 | public struct OtherError: RTVIError { 115 | public let message: String 116 | public let underlyingError: Error? 117 | 118 | public init(message: String, underlyingError: Error? = nil) { 119 | self.message = message 120 | self.underlyingError = underlyingError 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/helper/LLMHelper.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public protocol LLMHelperDelegate { 4 | func onLLMJsonCompletion(jsonString: String) 5 | /// Invoked when the LLM attempts to invoke a function. The provided callback must be provided with a return value. 6 | func onLLMFunctionCall(functionCallData: LLMFunctionCallData, onResult: ((Value) async -> Void)) async 7 | func onLLMFunctionCallStart(functionName: String) 8 | } 9 | 10 | public extension LLMHelperDelegate { 11 | func onLLMJsonCompletion(jsonString: String) {} 12 | func onLLMFunctionCall(functionCallData: LLMFunctionCallData, onResult: ((Value) async -> Void)) async {} 13 | func onLLMFunctionCallStart(functionName: String) {} 14 | } 15 | 16 | public struct LLMMessageType { 17 | public struct Incoming { 18 | public static let LLMFunctionCall = "llm-function-call" 19 | public static let LLMFunctionCallStart = "llm-function-call-start" 20 | public static let LLMJsonCompletion = "llm-json-completion" 21 | } 22 | 23 | public struct Outgoing { 24 | public static let LLMFunctionCallResult = "llm-function-call-result" 25 | } 26 | } 27 | 28 | public struct LLMFunctionCallData: Codable { 29 | public let functionName: String 30 | public let toolCallID: String 31 | public let args: Value 32 | 33 | enum CodingKeys: String, CodingKey { 34 | case functionName = "function_name" 35 | case toolCallID = "tool_call_id" 36 | case args 37 | } 38 | 39 | public init(functionName: String, toolCallID: String, args: Value) { 40 | self.functionName = functionName 41 | self.toolCallID = toolCallID 42 | self.args = args 43 | } 44 | } 45 | 46 | public struct LLMFunctionCallResult: Codable { 47 | let functionName: String 48 | let toolCallID: String 49 | let arguments: Value 50 | let result: Value 51 | 52 | enum CodingKeys: String, CodingKey { 53 | case functionName = "function_name" 54 | case toolCallID = "tool_call_id" 55 | case arguments 56 | case result 57 | } 58 | } 59 | 60 | public struct LLMContextMessage: Codable { 61 | public let role: String 62 | public let content: String 63 | 64 | public init(role: String, content: String) { 65 | self.role = role 66 | self.content = content 67 | } 68 | } 69 | 70 | public struct LLMContext: Codable { 71 | public let messages: [LLMContextMessage]? 72 | 73 | public init(messages: [LLMContextMessage]?) { 74 | self.messages = messages 75 | } 76 | } 77 | 78 | struct FunctionCallParams { 79 | let functionName: String 80 | let arguments: Value 81 | } 82 | 83 | typealias FunctionCallCallback = (FunctionCallParams) -> Promise 84 | 85 | /// Helper for interacting with an LLM service. 86 | public class LLMHelper: RTVIClientHelper { 87 | 88 | var voiceClient: RTVIClient 89 | var service: String 90 | public var delegate: LLMHelperDelegate? 91 | 92 | required public init(service: String, voiceClient: RTVIClient) { 93 | self.voiceClient = voiceClient 94 | self.service = service 95 | } 96 | 97 | public func getMessageTypes() -> Set { 98 | return [ 99 | LLMMessageType.Incoming.LLMFunctionCall, 100 | LLMMessageType.Incoming.LLMFunctionCallStart, 101 | LLMMessageType.Incoming.LLMJsonCompletion 102 | ] 103 | } 104 | 105 | private func isVoiceClientReady() -> Bool { 106 | return self.voiceClient.state == .ready 107 | } 108 | 109 | private func _getMessagesKey() -> String { 110 | return self.isVoiceClientReady() ? "messages" : "initial_messages" 111 | } 112 | 113 | // --- Actions 114 | 115 | /// Returns the bot's current LLM context. Bot must be in the ready state. 116 | public func getContext() async throws -> LLMContext? { 117 | try await self.voiceClient.action(action: ActionRequest.init( 118 | service: self.service, 119 | action: "get_context" 120 | ), resultType: LLMContext.self) 121 | } 122 | 123 | /// Returns the bot's current LLM context. Bot must be in the ready state. 124 | public func getContext(completion: ((Result) -> Void)?) { 125 | Task { 126 | do { 127 | let llmContext = try await self.getContext() 128 | completion?(.success((llmContext))) 129 | } catch { 130 | completion?(.failure(AsyncExecutionError(functionName: "getContext", underlyingError: error))) 131 | } 132 | } 133 | } 134 | 135 | /// Update the bot's LLM context. 136 | public func setContext(context: LLMContext, interrupt: Bool = false) async throws { 137 | try await self.voiceClient.action(action: ActionRequest.init( 138 | service: self.service, 139 | action: "set_context", 140 | arguments: [ 141 | Argument( 142 | name: self._getMessagesKey(), 143 | value: (context.messages ?? []).convertToRtviValue() 144 | ), 145 | Argument( 146 | name: "interrupt", 147 | value: interrupt.convertToRtviValue() 148 | ) 149 | ] 150 | )) 151 | } 152 | 153 | /// Update the bot's LLM context. 154 | public func setContext(context: LLMContext, interrupt: Bool = false, completion: ((Result) -> Void)?) { 155 | Task { 156 | do { 157 | try await self.setContext(context: context, interrupt:interrupt) 158 | completion?(.success(())) 159 | } catch { 160 | completion?(.failure(AsyncExecutionError(functionName: "setContext", underlyingError: error))) 161 | } 162 | } 163 | } 164 | 165 | /// Append a new message to the LLM context. 166 | public func appendToMessages(message: LLMContextMessage, runImmediately: Bool = false) async throws { 167 | try await self.voiceClient.action(action: ActionRequest.init( 168 | service: self.service, 169 | action: "append_to_messages", 170 | arguments: [ 171 | Argument( 172 | name: "messages", 173 | value: [message].convertToRtviValue() 174 | ), 175 | Argument( 176 | name: "run_immediately", 177 | value: runImmediately.convertToRtviValue() 178 | ) 179 | ] 180 | )) 181 | } 182 | 183 | /// Append a new message to the LLM context. 184 | public func appendToMessages(message: LLMContextMessage, runImmediately: Bool = false, completion: ((Result) -> Void)?) { 185 | Task { 186 | do { 187 | try await self.appendToMessages(message: message, runImmediately:runImmediately) 188 | completion?(.success(())) 189 | } catch { 190 | completion?(.failure(AsyncExecutionError(functionName: "appendToMessages", underlyingError: error))) 191 | } 192 | } 193 | } 194 | 195 | /// Run the bot's current LLM context. 196 | /// Useful when appending messages to the context without runImmediately set to true. 197 | public func run(interrupt: Bool = false) async throws { 198 | try await self.voiceClient.action(action: ActionRequest.init( 199 | service: self.service, 200 | action: "run", 201 | arguments: [ 202 | Argument( 203 | name: "interrupt", 204 | value: interrupt.convertToRtviValue() 205 | ) 206 | ] 207 | )) 208 | } 209 | 210 | /// Run the bot's current LLM context. 211 | /// Useful when appending messages to the context without runImmediately set to true. 212 | public func run(interrupt: Bool = false, completion: ((Result) -> Void)?) { 213 | Task { 214 | do { 215 | try await self.run(interrupt: interrupt) 216 | completion?(.success(())) 217 | } catch { 218 | completion?(.failure(AsyncExecutionError(functionName: "run", underlyingError: error))) 219 | } 220 | } 221 | } 222 | 223 | public func handleMessage(msg: RTVIMessageInbound) { 224 | guard let type = msg.type else { 225 | // Ignoring the message, it doesn't have a type 226 | return 227 | } 228 | Logger.shared.debug("LLMHelper, received voice message \(msg)") 229 | switch type { 230 | case LLMMessageType.Incoming.LLMJsonCompletion: 231 | if let jsonData = msg.data { 232 | self.delegate?.onLLMJsonCompletion(jsonString: jsonData) 233 | } 234 | case LLMMessageType.Incoming.LLMFunctionCallStart: 235 | if let functionCallData = try? JSONDecoder().decode(LLMFunctionCallData.self, from: Data(msg.data!.utf8)) { 236 | self.delegate?.onLLMFunctionCallStart(functionName: functionCallData.functionName) 237 | } 238 | case LLMMessageType.Incoming.LLMFunctionCall: 239 | if let functionCallData = try? JSONDecoder().decode(LLMFunctionCallData.self, from: Data(msg.data!.utf8)) { 240 | Task { 241 | await self.delegate?.onLLMFunctionCall(functionCallData: functionCallData) { result in 242 | let resultData = try? await LLMFunctionCallResult( 243 | functionName: functionCallData.functionName, 244 | toolCallID: functionCallData.toolCallID, 245 | arguments: functionCallData.args, 246 | result: result 247 | ).convertToRtviValue() 248 | let resultMessage = RTVIMessageOutbound( 249 | type: LLMMessageType.Outgoing.LLMFunctionCallResult, 250 | data: resultData 251 | ) 252 | voiceClient.sendMessage(msg: resultMessage){ result in 253 | if case .failure(let error) = result { 254 | Logger.shared.error("Failing to send app result message \(error)") 255 | } 256 | } 257 | } 258 | } 259 | } 260 | default: 261 | // Ignoring any other message 262 | return 263 | } 264 | } 265 | 266 | } 267 | 268 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/helper/RTVIClientHelper.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | @MainActor 4 | public protocol RTVIClientHelper: AnyObject { 5 | init(service: String, voiceClient: RTVIClient) 6 | 7 | /// Handle a message received from the backend. 8 | func handleMessage(msg: RTVIMessageInbound) 9 | 10 | /// Returns a list of message types supported by this helper. Messages received from the backend which have these types will be passed to [handleMessage]. 11 | func getMessageTypes() -> Set 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/AuthBundle.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A bundle of initialization data received from the RTVI backend. 4 | public struct AuthBundle: Decodable { 5 | public let data: String 6 | } 7 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/RTVIMessageInbound.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI control message received the by the Transport. 4 | public struct RTVIMessageInbound: Codable { 5 | 6 | let id: String? 7 | let label: String? 8 | let type: String? 9 | let data: String? 10 | let metrics: PipecatMetrics? 11 | 12 | /// Messages from the server to the client. 13 | public enum MessageType { 14 | /// Bot is connected and ready to receive messages 15 | public static let BOT_READY = "bot-ready" 16 | 17 | /// Received an error response from the server 18 | public static let ERROR_RESPONSE = "error-response" 19 | 20 | /// Received an error from the server 21 | public static let ERROR = "error" 22 | 23 | /// STT transcript (both local and remote) flagged with partial final or sentence 24 | public static let TRANSCRIPT = "transcript" 25 | 26 | /// Get or update config response 27 | public static let CONFIG_RESPONSE = "config" 28 | 29 | /// Configuration options available on the bot 30 | public static let DESCRIBE_CONFIG_RESPONSE = "config-available" 31 | 32 | /// Actions available on the bot 33 | public static let DESCRIBE_ACTION_RESPONSE = "actions-available" 34 | 35 | public static let ACTION_RESPONSE = "action-response" 36 | 37 | /// STT transcript from the user 38 | public static let USER_TRANSCRIPTION = "user-transcription" 39 | 40 | /// STT transcript from the bot 41 | public static let BOT_TRANSCRIPTION = "bot-transcription" 42 | 43 | /// User started speaking 44 | public static let USER_STARTED_SPEAKING = "user-started-speaking" 45 | 46 | // User stopped speaking 47 | public static let USER_STOPPED_SPEAKING = "user-stopped-speaking" 48 | 49 | // Bot started speaking 50 | public static let BOT_STARTED_SPEAKING = "bot-started-speaking" 51 | 52 | // Bot stopped speaking 53 | public static let BOT_STOPPED_SPEAKING = "bot-stopped-speaking" 54 | 55 | /// Pipecat metrics 56 | public static let PIPECAT_METRICS = "pipecat-metrics" 57 | 58 | /// LLM transcript from the bot 59 | public static let BOT_LLM_TEXT = "bot-llm-text" 60 | /// LLM transcript from the bot has started 61 | public static let BOT_LLM_STARTED = "bot-llm-started" 62 | /// LLM transcript from the bot has stopped 63 | public static let BOT_LLM_STOPPED = "bot-llm-stopped" 64 | 65 | /// TTS transcript from the bot 66 | public static let BOT_TTS_TEXT = "bot-tts-text" 67 | /// LLM transcript from the bot has started 68 | public static let BOT_TTS_STARTED = "bot-tts-started" 69 | /// LLM transcript from the bot has stopped 70 | public static let BOT_TTS_STTOPED = "bot-tts-stopped" 71 | 72 | /// Text has been stored 73 | public static let STORAGE_ITEM_STORED = "storage-item-stored" 74 | } 75 | 76 | public init(type: String?, data: String?) { 77 | self.init(type: type, data: data, id: String(UUID().uuidString.prefix(8)), label: "rtvi-ai", metrics: nil) 78 | } 79 | 80 | public init(type: String?, data: String?, id: String?, label: String? = "rtvi-ai", metrics: PipecatMetrics? = nil) { 81 | self.id = id 82 | self.label = label 83 | self.type = type 84 | self.data = data 85 | self.metrics = metrics 86 | } 87 | 88 | private enum CodingKeys: String, CodingKey { 89 | case id 90 | case label 91 | case type 92 | case data 93 | case metrics 94 | } 95 | 96 | public init(from decoder: Decoder) throws { 97 | let container = try decoder.container(keyedBy: CodingKeys.self) 98 | 99 | let type = try container.decode(String.self, forKey: .type) 100 | 101 | let datavalue = try? container.decodeIfPresent(Value.self, forKey: .data) 102 | let data: String? 103 | if(datavalue != nil) { 104 | data = try? String(data: JSONEncoder().encode(datavalue), encoding: .utf8) 105 | }else { 106 | data = nil 107 | } 108 | 109 | let metrics = try? container.decodeIfPresent(PipecatMetrics.self, forKey: .metrics) 110 | 111 | let label = try? container.decodeIfPresent(String.self, forKey: .label) 112 | let id = try? container.decodeIfPresent(String.self, forKey: .id) 113 | 114 | self.init(type: type, data: data, id: id, label: label, metrics: metrics) 115 | } 116 | } 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/RTVIMessageOutbound.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI control message sent to the Transport. 4 | public struct RTVIMessageOutbound: Encodable { 5 | 6 | public let id: String 7 | public let label: String 8 | public let type: String 9 | public let data: Value? 10 | 11 | /// Messages from the client to the server. 12 | public enum MessageType { 13 | public static let UPDATE_CONFIG = "update-config" 14 | public static let GET_CONFIG = "get-config" 15 | public static let DESCRIBE_CONFIG = "describe-config" 16 | public static let ACTION = "action" 17 | public static let DESCRIBE_ACTIONS = "describe-actions" 18 | public static let CLIENT_READY = "client-ready" 19 | } 20 | 21 | public init(id: String = String (UUID().uuidString.prefix(8)), label: String = "rtvi-ai", type: String, data: Value? = nil) { 22 | self.id = id 23 | self.label = label 24 | self.type = type 25 | self.data = data 26 | } 27 | 28 | static func action(actionData: ActionRequest) async throws -> RTVIMessageOutbound { 29 | return RTVIMessageOutbound( 30 | type: RTVIMessageOutbound.MessageType.ACTION, 31 | data: try await actionData.convertToRtviValue() 32 | ) 33 | } 34 | 35 | static func updateConfig(config: [ServiceConfig], interrupt: Bool) async throws -> RTVIMessageOutbound { 36 | let configAsValue = try await config.convertToRtviValue() 37 | let data = Value.object(["config": configAsValue, "interrupt": Value.boolean(interrupt)]) 38 | return RTVIMessageOutbound( 39 | type: RTVIMessageOutbound.MessageType.UPDATE_CONFIG, 40 | data: data 41 | ) 42 | } 43 | 44 | static func describeConfig() -> RTVIMessageOutbound { 45 | return RTVIMessageOutbound( 46 | type: RTVIMessageOutbound.MessageType.DESCRIBE_CONFIG 47 | ) 48 | } 49 | 50 | static func getConfig() -> RTVIMessageOutbound { 51 | return RTVIMessageOutbound( 52 | type: RTVIMessageOutbound.MessageType.GET_CONFIG 53 | ) 54 | } 55 | 56 | static func describeActions() -> RTVIMessageOutbound { 57 | return RTVIMessageOutbound( 58 | type: RTVIMessageOutbound.MessageType.DESCRIBE_ACTIONS 59 | ) 60 | } 61 | 62 | // Decode action data, if this outbound message represents an action request. 63 | // This is useful for implementing transports that can intercept and handle action requests in their own way. 64 | public func decodeActionData() -> ActionRequest? { 65 | if type == RTVIMessageOutbound.MessageType.ACTION { 66 | do { 67 | let encodedData = try JSONEncoder().encode(data) 68 | return try JSONDecoder().decode(ActionRequest.self, from: encodedData) 69 | } catch {} 70 | } 71 | return nil 72 | } 73 | } 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/Transport.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI transport. 4 | @MainActor 5 | public protocol Transport { 6 | init(options: RTVIClientOptions) 7 | var delegate: RTVIClientDelegate? { get set } 8 | var onMessage: ((RTVIMessageInbound) -> Void)? { get set } 9 | 10 | func initDevices() async throws 11 | func release() 12 | func connect(authBundle: AuthBundle?) async throws 13 | func disconnect() async throws 14 | func getAllMics() -> [MediaDeviceInfo] 15 | func getAllCams() -> [MediaDeviceInfo] 16 | func updateMic(micId: MediaDeviceId) async throws 17 | func updateCam(camId: MediaDeviceId) async throws 18 | func selectedMic() -> MediaDeviceInfo? 19 | func selectedCam() -> MediaDeviceInfo? 20 | func enableMic(enable: Bool) async throws 21 | func enableCam(enable: Bool) async throws 22 | func isCamEnabled() -> Bool 23 | func isMicEnabled() -> Bool 24 | func sendMessage(message: RTVIMessageOutbound) throws 25 | func state() -> TransportState 26 | func setState(state: TransportState) 27 | func isConnected() -> Bool 28 | func tracks() -> Tracks? 29 | func expiry() -> Int? 30 | } 31 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ActionDescription.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ActionDescription: Codable { 4 | let service: String 5 | let action: String 6 | let arguments: [OptionDescription] 7 | let result: OptionType 8 | 9 | init(service: String, action: String, arguments: [OptionDescription], result: OptionType) { 10 | self.service = service 11 | self.action = action 12 | self.arguments = arguments 13 | self.result = result 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaDeviceId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A unique identifier for a media device. 4 | public struct MediaDeviceId: Equatable { 5 | public let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaDeviceInfo.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Information about a media device. 4 | public struct MediaDeviceInfo: Equatable { 5 | 6 | public let id: MediaDeviceId 7 | public let name: String 8 | 9 | public init(id: MediaDeviceId, name: String) { 10 | self.id = id 11 | self.name = name 12 | } 13 | 14 | public static func == (lhs: MediaDeviceInfo, rhs: MediaDeviceInfo) -> Bool { 15 | lhs.id == rhs.id 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaTrackId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An identifier for a media track. 4 | public struct MediaTrackId: Hashable, Equatable { 5 | let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Option.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct Option: Codable { 4 | public let name: String 5 | public let value: Value 6 | 7 | public init(name: String, value: Value) { 8 | self.name = name 9 | self.value = value 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/OptionDescription.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct OptionDescription: Codable { 4 | let name: String 5 | let type: OptionType 6 | 7 | public init(name: String, type: OptionType) { 8 | self.name = name 9 | self.type = type 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/OptionType.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public enum OptionType: String, Codable { 4 | case str = "string" 5 | case bool = "bool" 6 | case number = "number" 7 | case array = "array" 8 | case object = "object" 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Participant.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Information about a session participant. 4 | public struct Participant { 5 | public let id: ParticipantId 6 | public let name: String? 7 | /// True if this participant represents the local user, false otherwise. 8 | public let local: Bool 9 | 10 | public init(id: ParticipantId, name: String?, local: Bool) { 11 | self.id = id 12 | self.name = name 13 | self.local = local 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ParticipantId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A unique identifier for a session participant. 4 | public struct ParticipantId: Equatable { 5 | let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ParticipantTracks.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Media tracks associated with a participant. 4 | public struct ParticipantTracks: Equatable { 5 | public let audio: MediaTrackId? 6 | public let video: MediaTrackId? 7 | 8 | public init(audio: MediaTrackId?, video: MediaTrackId?) { 9 | self.audio = audio 10 | self.video = video 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/PipecatMetrics.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Metrics received from a Pipecat instance. 4 | public struct PipecatMetrics: Codable { 5 | let processing: [PipecatMetricsData]? 6 | let ttfb: [PipecatMetricsData]? 7 | } 8 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/PipecatMetricsData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Metrics data received from a Pipecat instance. 4 | public struct PipecatMetricsData: Codable { 5 | let processor: String 6 | let value: Double 7 | } 8 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/RTVIURLEndpoints.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct RTVIURLEndpoints: Codable { 4 | public let connect: String 5 | public let action: String 6 | 7 | public init() { 8 | self.init(connect: "/connect", action: "/action") 9 | } 10 | 11 | public init(connect: String = "/connect", action: String = "/action") { 12 | self.connect = connect 13 | self.action = action 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/RegisteredHelper.swift: -------------------------------------------------------------------------------- 1 | internal class RegisteredHelper { 2 | let helper: RTVIClientHelper 3 | let supportedMessages: Set 4 | 5 | init(helper: RTVIClientHelper, supportedMessages: Set) { 6 | self.helper = helper 7 | self.supportedMessages = supportedMessages 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ServiceConfig.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ServiceConfig: Codable { 4 | public let service: String 5 | public let options: [Option] 6 | 7 | public init(service: String, options: [Option]) { 8 | self.service = service 9 | self.options = options 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ServiceConfigDescription.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ServiceConfigDescription: Codable { 4 | // TODO check if we should receive service or name here 5 | // we are currently receiving name 6 | let service: String? 7 | let name: String? 8 | let options: [OptionDescription] 9 | 10 | init(service: String?, name: String?, options: [OptionDescription]) { 11 | self.service = service 12 | self.name = name 13 | self.options = options 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Tracks.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Media tracks for the local user and remote bot. 4 | public struct Tracks: Equatable { 5 | public let local: ParticipantTracks 6 | public let bot: ParticipantTracks? 7 | 8 | public init(local: ParticipantTracks, bot: ParticipantTracks?) { 9 | self.local = local 10 | self.bot = bot 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Transcript.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A written transcript of some spoken words. 4 | public struct Transcript: Codable { 5 | public let text: String 6 | public let final: Bool? 7 | public let timestamp: String? 8 | public let userId: String? 9 | 10 | enum CodingKeys: String, CodingKey { 11 | case text 12 | case final 13 | case timestamp 14 | case userId = "user_id" 15 | } 16 | 17 | public init(text: String, final: Bool? = false, timestamp: String? = nil, userId: String? = nil) { 18 | self.text = text 19 | self.final = final 20 | self.timestamp = timestamp 21 | self.userId = userId 22 | } 23 | 24 | public init(from decoder: any Decoder) throws { 25 | let container = try decoder.container(keyedBy: CodingKeys.self) 26 | self.text = try container.decode(String.self, forKey: .text) 27 | self.final = try container.decodeIfPresent(Bool.self, forKey: .final) 28 | self.timestamp = try container.decodeIfPresent(String.self, forKey: .timestamp) 29 | self.userId = try container.decodeIfPresent(String.self, forKey: .userId) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/TransportState.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// The current state of the session transport. 4 | public enum TransportState { 5 | case disconnected 6 | case initializing 7 | case initialized 8 | case authenticating 9 | case connecting 10 | case connected 11 | case ready 12 | case disconnecting 13 | case error 14 | } 15 | 16 | extension TransportState { 17 | public var description: String { 18 | switch self { 19 | case .initializing: 20 | return "Initializing" 21 | case .initialized: 22 | return "Initialized" 23 | case .authenticating: 24 | return "Handshaking" 25 | case .connecting: 26 | return "Connecting" 27 | case .connected: 28 | return "Connected" 29 | case .ready: 30 | return "Ready" 31 | case .disconnecting: 32 | return "Disconnecting" 33 | case .disconnected: 34 | return "Disconnected" 35 | case .error: 36 | return "Error" 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Value.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Generic json representation used for serialization. 4 | public enum Value: Codable, Equatable { 5 | struct ValueDecodingError: Error { 6 | let message: String 7 | } 8 | 9 | case boolean(Bool) 10 | case number(Double) 11 | case string(String) 12 | case array([Value?]) 13 | case object([String: Value?]) 14 | 15 | public init(from decoder: Decoder) throws { 16 | let container = try decoder.singleValueContainer() 17 | if let boolean = try? container.decode(Bool.self) { 18 | self = .boolean(boolean) 19 | } else if let number = try? container.decode(Double.self) { 20 | self = .number(number) 21 | } else if let string = try? container.decode(String.self) { 22 | self = .string(string) 23 | } else if let array = try? container.decode([Value].self) { 24 | self = .array(array) 25 | } else if let dictionary = try? container.decode([String: Value].self) { 26 | self = .object(dictionary) 27 | } else { 28 | throw ValueDecodingError(message: "could not decode a valid Value") 29 | } 30 | } 31 | 32 | public func encode(to encoder: Encoder) throws { 33 | var container = encoder.singleValueContainer() 34 | switch self { 35 | case .boolean(let value): 36 | try container.encode(value) 37 | case .number(let value): 38 | try container.encode(value) 39 | case .string(let value): 40 | try container.encode(value) 41 | case .array(let values): 42 | try container.encode(values) 43 | case .object(let valueDictionary): 44 | try container.encode(valueDictionary) 45 | } 46 | } 47 | 48 | public mutating func addProperty(key: String, value: Value?) throws { 49 | guard case .object(var dictionary) = self else { 50 | throw ValueDecodingError(message: "Cannot add properties to non-object Value") 51 | } 52 | dictionary[key] = value 53 | self = .object(dictionary) 54 | } 55 | } 56 | 57 | extension Value: ExpressibleByBooleanLiteral { 58 | public init(booleanLiteral value: Bool) { 59 | self = .boolean(value) 60 | } 61 | } 62 | 63 | extension Value: ExpressibleByIntegerLiteral { 64 | public init(integerLiteral value: Int) { 65 | self = .number(Double(value)) 66 | } 67 | } 68 | 69 | extension Value: ExpressibleByFloatLiteral { 70 | public init(floatLiteral value: Double) { 71 | self = .number(value) 72 | } 73 | } 74 | 75 | extension Value: ExpressibleByStringLiteral { 76 | public init(stringLiteral value: String) { 77 | self = .string(value) 78 | } 79 | } 80 | 81 | extension Value: ExpressibleByArrayLiteral { 82 | public init(arrayLiteral elements: Value?...) { 83 | self = .array(elements) 84 | } 85 | } 86 | 87 | extension Value: ExpressibleByDictionaryLiteral { 88 | public init(dictionaryLiteral elements: (String, Value?)...) { 89 | self = .object(Dictionary.init(uniqueKeysWithValues: elements)) 90 | } 91 | } 92 | 93 | extension Encodable { 94 | public func convertToRtviValue() throws -> Value { 95 | // Encode the current object to JSON data 96 | let jsonData = try JSONEncoder().encode(self) 97 | // Decode the JSON data into a Value object 98 | let value = try JSONDecoder().decode(Value.self, from: jsonData) 99 | return value 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/httpMessages/TextRequest.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct TextRequest: Encodable { 4 | let actions: [RTVIMessageOutbound] 5 | 6 | init(action: RTVIMessageOutbound) { 7 | self.actions = [action] 8 | } 9 | 10 | enum CodingKeys: String, CodingKey { 11 | case actions 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/ActionRequest.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public typealias Argument = Option 4 | 5 | public struct ActionRequest: Codable { 6 | 7 | public let service: String 8 | public let action: String 9 | public let arguments: [Argument]? 10 | 11 | public init(service: String, action: String, arguments: [Option]?=nil) { 12 | self.service = service 13 | self.action = action 14 | self.arguments = arguments 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/ActionResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ActionResponse: Codable { 4 | let result: Value 5 | 6 | public init(result: Value) { 7 | self.result = result 8 | } 9 | } 10 | 11 | struct ActionResponseWrapper: Decodable { 12 | let result: T 13 | } 14 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotError.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotError: Codable { 4 | public let error: String 5 | public let fatal: Bool? 6 | 7 | init(error: String, fatal: Bool) { 8 | self.error = error 9 | self.fatal = fatal 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotLLMText.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotLLMText: Codable { 4 | public let text: String 5 | 6 | public init(text: String) { 7 | self.text = text 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotReadyResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotReadyData: Codable { 4 | 5 | public let version: String 6 | public let config: [ServiceConfig] 7 | 8 | public init(version: String, config: [ServiceConfig]) { 9 | self.version = version 10 | self.config = config 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotTTSText.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotTTSText: Codable { 4 | public let text: String 5 | 6 | public init(text: String) { 7 | self.text = text 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/ConfigResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ConfigResponse: Codable { 4 | public let config: [ServiceConfig] 5 | 6 | init(config: [ServiceConfig]) { 7 | self.config = config 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/DescribeActionsResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct DescribeActionResponse: Codable { 4 | public let actions: [ActionDescription] 5 | 6 | init(actions: [ActionDescription]) { 7 | self.actions = actions 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/DescribeConfigResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct DescribeConfigResponse: Codable { 4 | public let config: [ServiceConfigDescription] 5 | 6 | init(config: [ServiceConfigDescription]) { 7 | self.config = config 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/ErrorResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ErrorResponse: Codable { 4 | public let error: String 5 | 6 | init(error: String) { 7 | self.error = error 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/StorageItemStoredData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct StorageItemStoredData: Codable { 4 | public let action: String 5 | public let items: Value 6 | } 7 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/ConnectionBundle.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ConnectionBundle: Codable { 4 | let services: [String: String] 5 | let config: [ServiceConfig] 6 | 7 | init(services: [String : String], config: [ServiceConfig]) { 8 | self.services = services 9 | self.config = config 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/HTTPMessageDispatcher.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Helper class for sending messages to the server through HTTP and awaiting the response. 4 | class HTTPMessageDispatcher { 5 | 6 | private let baseUrl: String 7 | 8 | private let endpoints: RTVIURLEndpoints 9 | 10 | /// Custom HTTP headers to be sent with the POST request to baseUrl. 11 | public let headers: [[String: String]] 12 | 13 | /// Custom HTTP body params to be sent with the POST request to baseUrl. 14 | public let requestData: Value? 15 | 16 | // callback for the messages that we are going to receive from the HTTP 17 | public var onMessage: ((RTVIMessageInbound) -> Void)? = nil 18 | 19 | init(baseUrl: String, endpoints: RTVIURLEndpoints, headers:[[String: String]], requestData: Value?) { 20 | self.baseUrl = baseUrl 21 | self.headers = headers 22 | self.requestData = requestData 23 | self.endpoints = endpoints 24 | } 25 | 26 | private func actionUrl() -> String { 27 | return self.baseUrl + self.endpoints.action 28 | } 29 | 30 | func sendMessage(message: RTVIMessageOutbound) throws { 31 | Task { 32 | let actionUrl = self.actionUrl() 33 | guard let url = URL(string: actionUrl) else { 34 | throw InvalidAuthBundleError() 35 | } 36 | 37 | Logger.shared.info("ActionUrl: \(actionUrl)") 38 | 39 | var request = URLRequest(url: url) 40 | request.httpMethod = "POST" 41 | request.setValue("application/json", forHTTPHeaderField: "Content-Type") 42 | request.setValue("no-cache", forHTTPHeaderField: "Cache-Control") 43 | request.setValue("keep-alive", forHTTPHeaderField: "Connection") 44 | 45 | // Adding the custom headers if they have been provided 46 | for header in self.headers { 47 | for (key, value) in header { 48 | request.setValue(value, forHTTPHeaderField: key) 49 | } 50 | } 51 | 52 | do { 53 | if let customBodyParams = self.requestData { 54 | var customBundle:Value = try await customBodyParams.convertToRtviValue() 55 | try customBundle.addProperty(key: "actions", value: Value.array([try await message.convertToRtviValue()])) 56 | request.httpBody = try JSONEncoder().encode( customBundle ) 57 | } else { 58 | let textRequest = TextRequest.init(action: message) 59 | let messageToSend = try JSONEncoder().encode(textRequest); 60 | request.httpBody = messageToSend; 61 | } 62 | 63 | Logger.shared.info("Will request action: \(String(data: request.httpBody!, encoding: .utf8) ?? "")") 64 | 65 | let streamDelegate = StreamDelegate(messageId: message.id) 66 | streamDelegate.onMessage = self.onMessage 67 | let session = URLSession(configuration: .default, delegate: streamDelegate, delegateQueue: nil) 68 | let task = session.dataTask(with: request) 69 | task.resume() 70 | } catch { 71 | Logger.shared.error(error.localizedDescription) 72 | throw HttpError(message: "Failed while requesting a single time action.", underlyingError: error) 73 | } 74 | } 75 | } 76 | 77 | // Stream handling using URLSessionDelegate 78 | class StreamDelegate: NSObject, URLSessionDataDelegate { 79 | 80 | // callback for the messages that we are going to receive from the HTTP 81 | public var onMessage: ((RTVIMessageInbound) -> Void)? = nil 82 | 83 | private var buffer: String = "" 84 | private var isEventStream: Bool = false 85 | 86 | private var messageId: String? 87 | 88 | init(messageId: String) { 89 | super.init() 90 | self.messageId = messageId 91 | } 92 | 93 | private func base64Decode(_ encodedData: String) -> String? { 94 | guard let data = Data(base64Encoded: encodedData) else { return nil } 95 | return String(data: data, encoding: .utf8) 96 | } 97 | 98 | private func parseStreamData() throws { 99 | while let boundaryRange = self.buffer.range(of: "\n\n") { 100 | let message = String(self.buffer[.. Void) { 126 | if let httpResponse = response as? HTTPURLResponse { 127 | if httpResponse.statusCode != 200 { 128 | Logger.shared.error("Failed with status code: \(httpResponse.statusCode)") 129 | do { 130 | let errorData = try? String(data: JSONEncoder() 131 | .encode( 132 | ErrorResponse(error: "Request failed with status code \(httpResponse.statusCode)") 133 | ), encoding: .utf8) 134 | let errorMessage = RTVIMessageInbound( 135 | type: RTVIMessageInbound.MessageType.ERROR_RESPONSE, 136 | data: errorData, 137 | id: self.messageId 138 | ) 139 | self.onMessage?(errorMessage) 140 | } catch { 141 | } 142 | completionHandler(.cancel) 143 | return 144 | } 145 | 146 | // Check Content-Type for event-stream 147 | if let contentType = httpResponse.allHeaderFields["Content-Type"] as? String, contentType.contains("text/event-stream") { 148 | self.isEventStream = true 149 | } else { 150 | self.isEventStream = false 151 | } 152 | Logger.shared.debug("isEventStream \(isEventStream)") 153 | 154 | // Continue processing the data 155 | completionHandler(.allow) 156 | } 157 | } 158 | 159 | // This function is called whenever new data arrives 160 | func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { 161 | if isEventStream { 162 | // Handle event stream 163 | if let chunk = String(data: data, encoding: .utf8) { 164 | self.buffer += chunk 165 | do { 166 | try parseStreamData() 167 | } catch { 168 | Logger.shared.error("Error parsing stream data: \(error)") 169 | } 170 | } 171 | } else { 172 | // Handle non-streamed data (regular JSON response) 173 | do { 174 | let appMessage = try JSONDecoder().decode(RTVIMessageInbound.self, from: data) 175 | self.onMessage?(appMessage) 176 | } catch { 177 | Logger.shared.error("Failed to parse regular JSON: \(error)") 178 | } 179 | } 180 | } 181 | 182 | // Handle task completion 183 | func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { 184 | if let error = error { 185 | Logger.shared.error("Stream task completed with error: \(error)") 186 | } else { 187 | Logger.shared.debug("Stream task completed successfully.") 188 | } 189 | } 190 | } 191 | 192 | } 193 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/Logger.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import os.log 3 | 4 | /// Log level used for logging by the framework. 5 | public enum LogLevel: UInt8 { 6 | /// A level lower than all log levels. 7 | case off = 0 8 | /// Corresponds to the `.error` log level. 9 | case error = 1 10 | /// Corresponds to the `.warn` log level. 11 | case warn = 2 12 | /// Corresponds to the `.debug` log level. 13 | case debug = 4 14 | /// Corresponds to the `.info` log level. 15 | case info = 3 16 | /// Corresponds to the `.trace` log level. 17 | case trace = 5 18 | } 19 | 20 | extension LogLevel { 21 | fileprivate var logType: OSLogType? { 22 | switch self { 23 | case .off: 24 | return nil 25 | case .error: 26 | return .error 27 | case .warn: 28 | return .default 29 | case .info: 30 | return .info 31 | case .debug: 32 | return .debug 33 | case .trace: 34 | return .debug 35 | } 36 | } 37 | } 38 | 39 | /// Sets the log level for logs produce by the framework. 40 | public func setLogLevel(_ logLevel: LogLevel) { 41 | Logger.shared.level = logLevel 42 | } 43 | 44 | internal final class Logger { 45 | fileprivate var level: LogLevel = .warn 46 | 47 | fileprivate let osLog: OSLog = .init(subsystem: "co.daily.rtvi", category: "main") 48 | 49 | internal static let shared: Logger = .init() 50 | 51 | @inlinable 52 | internal func error(_ message: @autoclosure () -> String) { 53 | self.log(.error, message()) 54 | } 55 | 56 | @inlinable 57 | internal func warn(_ message: @autoclosure () -> String) { 58 | self.log(.warn, message()) 59 | } 60 | 61 | @inlinable 62 | internal func info(_ message: @autoclosure () -> String) { 63 | self.log(.info, message()) 64 | } 65 | 66 | @inlinable 67 | internal func debug(_ message: @autoclosure () -> String) { 68 | self.log(.debug, message()) 69 | } 70 | 71 | @inlinable 72 | internal func trace(_ message: @autoclosure () -> String) { 73 | self.log(.trace, message()) 74 | } 75 | 76 | @inlinable 77 | internal func log(_ level: LogLevel, _ message: @autoclosure () -> String) { 78 | guard self.level.rawValue >= level.rawValue else { 79 | return 80 | } 81 | 82 | guard self.level != .off else { 83 | return 84 | } 85 | 86 | let log = self.osLog 87 | 88 | // The following force-unwrap is okay since we check for `.off` above: 89 | // swiftlint:disable:next force_unwrapping 90 | let type = level.logType! 91 | 92 | os_log("%@", log: log, type: type, message()) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/MessageDispatcher.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | enum MessageDispatcherError: Error { 4 | /// HTTP messages not supported when using certain transports that don't communicate with an RTVI server. 5 | case httpMessagesNotSupported 6 | } 7 | 8 | /// Helper class for sending messages to the server and awaiting the response. 9 | class MessageDispatcher { 10 | 11 | private let transport: Transport 12 | private let httpMessageDispatcher: HTTPMessageDispatcher? 13 | 14 | /// How long to wait before resolving the message/ 15 | private var gcTime: TimeInterval 16 | @MainActor 17 | private var queue: [QueuedVoiceMessage] = [] 18 | private var gcTimer: Timer? 19 | 20 | init(transport: Transport, httpMessageDispatcher: HTTPMessageDispatcher?) { 21 | self.gcTime = 10.0 // 10 seconds 22 | self.transport = transport 23 | self.httpMessageDispatcher = httpMessageDispatcher 24 | startGCTimer() 25 | } 26 | 27 | deinit { 28 | stopGCTimer() 29 | } 30 | 31 | @MainActor 32 | func dispatch(message: RTVIMessageOutbound) throws-> Promise { 33 | let promise = Promise() 34 | self.queue.append(QueuedVoiceMessage( 35 | message: message, 36 | timestamp: Date(), 37 | promise: promise 38 | )) 39 | do { 40 | if self.transport.isConnected() { 41 | try self.transport.sendMessage(message: message) 42 | } else { 43 | if let httpMessageDispatcher { 44 | try httpMessageDispatcher.sendMessage(message: message) 45 | } else { 46 | throw MessageDispatcherError.httpMessagesNotSupported 47 | } 48 | } 49 | } catch { 50 | Logger.shared.error("Failed to send app message \(error)") 51 | if let index = queue.firstIndex(where: { $0.message.id == message.id }) { 52 | // Removing the item that we have failed to send 53 | self.queue.remove(at: index) 54 | } 55 | throw error 56 | } 57 | return promise 58 | } 59 | 60 | @MainActor 61 | func dispatchAsync(message: RTVIMessageOutbound) async throws -> RTVIMessageInbound { 62 | try await withCheckedThrowingContinuation { continuation in 63 | do { 64 | let promise = try self.dispatch(message: message) 65 | promise.onResolve = { (inboundMessage: RTVIMessageInbound) in 66 | continuation.resume(returning: inboundMessage) 67 | } 68 | promise.onReject = { (error: Error) in 69 | continuation.resume(throwing: error) 70 | } 71 | } catch { 72 | continuation.resume(throwing: error) 73 | } 74 | } 75 | } 76 | 77 | private func resolveReject(message: RTVIMessageInbound, resolve: Bool = true) -> RTVIMessageInbound { 78 | DispatchQueue.main.async { 79 | if let index = self.queue.firstIndex(where: { $0.message.id == message.id }) { 80 | let queuedMessage = self.queue[index] 81 | if resolve { 82 | queuedMessage.promise.resolve(value:message) 83 | } else { 84 | if let errorResponse = try? JSONDecoder().decode(ErrorResponse.self, from: Data(message.data!.utf8)) { 85 | queuedMessage.promise.reject(error:BotResponseError(message: "Received error response from backend: \(errorResponse.error)")) 86 | } else { 87 | queuedMessage.promise.reject(error:BotResponseError()) 88 | } 89 | } 90 | self.queue.remove(at: index) 91 | } else { 92 | // unknown messages are just ignored 93 | } 94 | } 95 | return message 96 | 97 | } 98 | 99 | func resolve(message: RTVIMessageInbound) -> RTVIMessageInbound { 100 | return resolveReject(message: message, resolve: true) 101 | } 102 | 103 | func reject(message: RTVIMessageInbound) -> RTVIMessageInbound { 104 | return resolveReject(message: message, resolve: false) 105 | } 106 | 107 | /// Removing the messages that we have not received a response in the specified time 108 | private func gc() { 109 | let currentTime = Date() 110 | DispatchQueue.main.async { 111 | self.queue.removeAll { queuedMessage in 112 | let timeElapsed = currentTime.timeIntervalSince(queuedMessage.timestamp) 113 | if timeElapsed >= self.gcTime { 114 | queuedMessage.promise.reject(error:ResponseTimeoutError()) 115 | return true 116 | } 117 | return false 118 | } 119 | } 120 | } 121 | 122 | private func startGCTimer() { 123 | gcTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in 124 | self?.gc() 125 | } 126 | } 127 | 128 | private func stopGCTimer() { 129 | gcTimer?.invalidate() 130 | gcTimer = nil 131 | } 132 | } 133 | 134 | struct QueuedVoiceMessage { 135 | let message: RTVIMessageOutbound 136 | let timestamp: Date 137 | let promise: Promise 138 | } 139 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/Promise.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Represents an ongoing asynchronous operation. 4 | class Promise { 5 | 6 | public var onResolve: ((Value) -> Void)? = nil 7 | public var onReject: ((Error) -> Void)? = nil 8 | 9 | public func resolve(value: Value) { 10 | self.onResolve?(value) 11 | } 12 | 13 | public func reject(error: Error) { 14 | self.onReject?(error) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Tests/PipecatClientIOSTests/RTVIClientIOSTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | @testable import PipecatClientIOS 3 | 4 | final class RTVIClientIOSTests: XCTestCase { 5 | func testExample() throws { 6 | // XCTest Documentation 7 | // https://developer.apple.com/documentation/xctest 8 | 9 | // Defining Test Cases and Test Methods 10 | // https://developer.apple.com/documentation/xctest/defining_test_cases_and_test_methods 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /pipecat-ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pipecat-ai/pipecat-client-ios/992641fb5f7d1a794ecfc33babb5fe36e2a8ffdd/pipecat-ios.png -------------------------------------------------------------------------------- /scripts/generateDocs.sh: -------------------------------------------------------------------------------- 1 | URL_BASE_PATH=$1 2 | 3 | # clean the temp dir 4 | rm -rf ./tmpDocs 5 | 6 | # create the docs 7 | xcodebuild docbuild -scheme 'PipecatClientIOS' -destination "generic/platform=iOS" -derivedDataPath ./tmpDocs 8 | 9 | # convert the doc archive for static hosting 10 | $(xcrun --find docc) process-archive transform-for-static-hosting \ 11 | ./tmpDocs/Build/Products/Debug-iphoneos/PipecatClientIOS.doccarchive \ 12 | --output-path ./tmpDocs/htmldoc \ 13 | --hosting-base-path $URL_BASE_PATH 14 | # In case we need to change the host path 15 | # more details here: https://www.createwithswift.com/publishing-docc-documention-as-a-static-website-on-github-pages/ 16 | # --hosting-base-path URL_BASE_PATH 17 | 18 | # To access the docs, need to access: /documentation/PipecatClientIOS/ 19 | --------------------------------------------------------------------------------