├── .gitignore ├── pipecat-ios.png ├── Sources └── PipecatClientIOS │ ├── types │ ├── StartBotResult.swift │ ├── voiceMessages │ │ ├── ErrorResponse.swift │ │ ├── BotLLMText.swift │ │ ├── BotTTSText.swift │ │ ├── BotError.swift │ │ └── BotReadyResponse.swift │ ├── PipecatMetricsData.swift │ ├── MediaTrackId.swift │ ├── MediaDeviceId.swift │ ├── ParticipantId.swift │ ├── PipecatMetrics.swift │ ├── ClientMessageData.swift │ ├── Tracks.swift │ ├── LLMFunctionCallResult.swift │ ├── MediaStreamTrack.swift │ ├── MediaDeviceInfo.swift │ ├── Participant.swift │ ├── SendTextOptions.swift │ ├── LLMContextMessage.swift │ ├── ParticipantTracks.swift │ ├── LLMFunctionCallData.swift │ ├── rest │ │ └── ApiRequest.swift │ ├── Transcript.swift │ ├── TransportState.swift │ ├── BotLLMSearchResponse.swift │ └── Value.swift │ ├── transport │ ├── TransportConnectionParams.swift │ ├── Transport.swift │ ├── RTVIMessageOutbound.swift │ └── RTVIMessageInbound.swift │ ├── PipecatClientVersion.swift │ ├── utils │ ├── Promise.swift │ ├── Logger.swift │ └── MessageDispatcher.swift │ ├── PipecatClientOptions.swift │ ├── RTVIError.swift │ ├── PipecatClientDelegate.swift │ └── PipecatClient.swift ├── scripts ├── formatCode.sh └── generateDocs.sh ├── .swiftpm └── xcode │ └── package.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ └── IDEWorkspaceChecks.plist ├── Tests └── PipecatClientIOSTests │ └── RTVIClientIOSTests.swift ├── PipecatClientIOS.podspec ├── Package.swift ├── LICENSE ├── README.md ├── .swift-format ├── .swiftlint └── CHANGELOG.md /.gitignore: -------------------------------------------------------------------------------- 1 | /.swiftpm/xcode/xcuserdata 2 | tmpDocs/ 3 | -------------------------------------------------------------------------------- /pipecat-ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pipecat-ai/pipecat-client-ios/HEAD/pipecat-ios.png -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/StartBotResult.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public protocol StartBotResult: Decodable {} 4 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/TransportConnectionParams.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public protocol TransportConnectionParams: Decodable {} 4 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/ErrorResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ErrorResponse: Codable { 4 | public let error: String 5 | } 6 | -------------------------------------------------------------------------------- /scripts/formatCode.sh: -------------------------------------------------------------------------------- 1 | # Formatting the code 2 | swift-format format Sources -i -r 3 | # Checking for code practices and style conventions 4 | swiftlint --fix Sources 5 | 6 | -------------------------------------------------------------------------------- /.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/PipecatClientVersion.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | extension PipecatClient { 4 | 5 | public static let library = "pipecat-client-ios" 6 | public static let libraryVersion = "1.1.3" 7 | 8 | } 9 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/PipecatMetricsData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Metrics data received from a Pipecat instance. 4 | public struct PipecatMetricsData: Codable { 5 | let processor: String 6 | let value: Double 7 | } 8 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotLLMText.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotLLMText: Codable { 4 | public let text: String 5 | 6 | public init(text: String) { 7 | self.text = text 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotTTSText.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotTTSText: Codable { 4 | public let text: String 5 | 6 | public init(text: String) { 7 | self.text = text 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaTrackId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An identifier for a media track. 4 | public struct MediaTrackId: Hashable, Equatable { 5 | let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaDeviceId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A unique identifier for a media device. 4 | public struct MediaDeviceId: Equatable { 5 | public let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ParticipantId.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A unique identifier for a session participant. 4 | public struct ParticipantId: Equatable { 5 | let id: String 6 | 7 | public init(id: String) { 8 | self.id = id 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/PipecatMetrics.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Metrics received from a Pipecat instance. 4 | public struct PipecatMetrics: Codable { 5 | let processing: [PipecatMetricsData]? 6 | let ttfb: [PipecatMetricsData]? 7 | let characters: [PipecatMetricsData]? 8 | } 9 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ClientMessageData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct ClientMessageData: Codable { 4 | 5 | public let t: String 6 | public let d: Value? 7 | 8 | public init(t: String, d: Value?) { 9 | self.t = t 10 | self.d = d 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotError.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotError: Codable { 4 | public let error: String 5 | public let fatal: Bool? 6 | 7 | init(error: String, fatal: Bool) { 8 | self.error = error 9 | self.fatal = fatal 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/voiceMessages/BotReadyResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct BotReadyData: Codable { 4 | 5 | public let version: String 6 | public let about: Value? 7 | 8 | public init(version: String, about: Value?) { 9 | self.version = version 10 | self.about = about 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Tracks.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Media tracks for the local user and remote bot. 4 | public struct Tracks: Equatable { 5 | public let local: ParticipantTracks 6 | public let bot: ParticipantTracks? 7 | 8 | public init(local: ParticipantTracks, bot: ParticipantTracks?) { 9 | self.local = local 10 | self.bot = bot 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/Promise.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Represents an ongoing asynchronous operation. 4 | class Promise { 5 | 6 | public var onResolve: ((Value) -> Void)? 7 | public var onReject: ((Error) -> Void)? 8 | 9 | public func resolve(value: Value) { 10 | self.onResolve?(value) 11 | } 12 | 13 | public func reject(error: Error) { 14 | self.onReject?(error) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/LLMFunctionCallResult.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct LLMFunctionCallResult: Codable { 4 | let functionName: String 5 | let toolCallID: String 6 | let arguments: Value 7 | let result: Value 8 | 9 | enum CodingKeys: String, CodingKey { 10 | case functionName = "function_name" 11 | case toolCallID = "tool_call_id" 12 | case arguments 13 | case result 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Tests/PipecatClientIOSTests/RTVIClientIOSTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | @testable import PipecatClientIOS 3 | 4 | final class RTVIClientIOSTests: XCTestCase { 5 | func testExample() throws { 6 | // XCTest Documentation 7 | // https://developer.apple.com/documentation/xctest 8 | 9 | // Defining Test Cases and Test Methods 10 | // https://developer.apple.com/documentation/xctest/defining_test_cases_and_test_methods 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaStreamTrack.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct MediaStreamTrack: Hashable, Equatable { 4 | 5 | public enum Kind: String, CaseIterable { 6 | case audio = "audio" 7 | case video = "video" 8 | } 9 | 10 | public let id: MediaTrackId 11 | public let kind: Kind 12 | 13 | public init(id: MediaTrackId, kind: Kind) { 14 | self.id = id 15 | self.kind = kind 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/MediaDeviceInfo.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Information about a media device. 4 | public struct MediaDeviceInfo: Equatable { 5 | 6 | public let id: MediaDeviceId 7 | public let name: String 8 | 9 | public init(id: MediaDeviceId, name: String) { 10 | self.id = id 11 | self.name = name 12 | } 13 | 14 | public static func == (lhs: MediaDeviceInfo, rhs: MediaDeviceInfo) -> Bool { 15 | lhs.id == rhs.id 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Participant.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Information about a session participant. 4 | public struct Participant { 5 | public let id: ParticipantId 6 | public let name: String? 7 | /// True if this participant represents the local user, false otherwise. 8 | public let local: Bool 9 | 10 | public init(id: ParticipantId, name: String?, local: Bool) { 11 | self.id = id 12 | self.name = name 13 | self.local = local 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/SendTextOptions.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct SendTextOptions: Codable { 4 | public let runImmediately: Bool? 5 | public let audioResponse: Bool? 6 | 7 | enum CodingKeys: String, CodingKey { 8 | case audioResponse = "audio_response" 9 | case runImmediately = "run_immediately" 10 | } 11 | 12 | public init(runImmediately: Bool? = true, audioResponse: Bool? = false) { 13 | self.runImmediately = runImmediately 14 | self.audioResponse = audioResponse 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/LLMContextMessage.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct LLMContextMessage: Codable { 4 | public enum Role: String, Codable { 5 | case user 6 | case assistant 7 | } 8 | 9 | public let role: Role 10 | public let content: String 11 | public let runImmediately: Bool? 12 | 13 | enum CodingKeys: String, CodingKey { 14 | case role 15 | case content 16 | case runImmediately = "run_immediately" 17 | } 18 | 19 | public init(role: Role, content: String, runImmediately: Bool? = false) { 20 | self.role = role 21 | self.content = content 22 | self.runImmediately = runImmediately 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/ParticipantTracks.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Media tracks associated with a participant. 4 | public struct ParticipantTracks: Equatable { 5 | public let audio: MediaStreamTrack? 6 | public let video: MediaStreamTrack? 7 | public let screenAudio: MediaStreamTrack? 8 | public let screenVideo: MediaStreamTrack? 9 | 10 | public init( 11 | audio: MediaStreamTrack?, 12 | video: MediaStreamTrack?, 13 | screenAudio: MediaStreamTrack?, 14 | screenVideo: MediaStreamTrack? 15 | ) { 16 | self.audio = audio 17 | self.video = video 18 | self.screenAudio = screenAudio 19 | self.screenVideo = screenVideo 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/LLMFunctionCallData.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct LLMFunctionCallData: Codable { 4 | public let functionName: String 5 | public let toolCallID: String 6 | public let args: Value 7 | 8 | enum CodingKeys: String, CodingKey { 9 | case functionName = "function_name" 10 | case toolCallID = "tool_call_id" 11 | case args 12 | } 13 | 14 | public init(functionName: String, toolCallID: String, args: Value) { 15 | self.functionName = functionName 16 | self.toolCallID = toolCallID 17 | self.args = args 18 | } 19 | } 20 | 21 | public typealias FunctionCallCallback = (LLMFunctionCallData, @escaping (Value) async -> Void) async -> Void 22 | -------------------------------------------------------------------------------- /scripts/generateDocs.sh: -------------------------------------------------------------------------------- 1 | URL_BASE_PATH=$1 2 | 3 | # clean the temp dir 4 | rm -rf ./tmpDocs 5 | 6 | # create the docs 7 | xcodebuild docbuild -scheme 'PipecatClientIOS' -destination "generic/platform=iOS" -derivedDataPath ./tmpDocs 8 | 9 | # convert the doc archive for static hosting 10 | $(xcrun --find docc) process-archive transform-for-static-hosting \ 11 | ./tmpDocs/Build/Products/Debug-iphoneos/PipecatClientIOS.doccarchive \ 12 | --output-path ./tmpDocs/htmldoc \ 13 | --hosting-base-path $URL_BASE_PATH 14 | # In case we need to change the host path 15 | # more details here: https://www.createwithswift.com/publishing-docc-documention-as-a-static-website-on-github-pages/ 16 | # --hosting-base-path URL_BASE_PATH 17 | 18 | # To access the docs, need to access: /documentation/PipecatClientIOS/ 19 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/rest/ApiRequest.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public struct APIRequest: Codable { 4 | public let endpoint: URL 5 | 6 | /// Custom HTTP headers to be sent with the POST request to baseUrl. 7 | public let headers: [[String: String]] 8 | 9 | /// Custom parameters to add to the auth request body. 10 | public let requestData: Value? 11 | 12 | public let timeout: TimeInterval? 13 | 14 | public init( 15 | endpoint: URL, 16 | headers: [[String: String]] = [], 17 | requestData: Value? = nil, 18 | timeout: TimeInterval? = nil 19 | ) { 20 | self.endpoint = endpoint 21 | self.headers = headers 22 | self.requestData = requestData 23 | self.timeout = timeout 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/PipecatClientOptions.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Configuration options when instantiating a VoiceClient. 4 | public struct PipecatClientOptions { 5 | 6 | /// Transport class for media streaming 7 | public let transport: Transport 8 | 9 | /// Enable the user mic input. Defaults to true. 10 | public let enableMic: Bool 11 | 12 | /// Enable user cam input. Defaults to false. 13 | public let enableCam: Bool 14 | 15 | // TODO: need to add support for screen share in the future 16 | // public let enableScreenShare: Bool 17 | 18 | public init(transport: Transport, enableMic: Bool = true, enableCam: Bool = false) { 19 | self.transport = transport 20 | self.enableMic = enableMic 21 | self.enableCam = enableCam 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /PipecatClientIOS.podspec: -------------------------------------------------------------------------------- 1 | Pod::Spec.new do |s| 2 | s.name = 'PipecatClientIOS' 3 | s.version = '1.1.3' 4 | s.summary = 'Pipecat iOS client library.' 5 | s.description = <<-DESC 6 | Pipecat iOS client library that implements the RTVI-AI standard. 7 | DESC 8 | s.homepage = 'https://github.com/pipecat-ai/pipecat-client-ios' 9 | s.documentation_url = "https://docs.pipecat.ai/client/ios/introduction" 10 | s.license = { :type => 'BSD-2', :file => 'LICENSE' } 11 | s.author = { "Daily.co" => "help@daily.co" } 12 | s.source = { :git => 'https://github.com/pipecat-ai/pipecat-client-ios.git', :tag => "1.1.3" } 13 | s.ios.deployment_target = '13.0' 14 | s.source_files = 'Sources/**/*.{swift,h,m}' 15 | s.exclude_files = 'Sources/Exclude' 16 | s.swift_version = '5.5' 17 | end 18 | 19 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.5 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "PipecatClientIOS", 8 | platforms: [ 9 | .iOS(.v13) 10 | ], 11 | products: [ 12 | // Products define the executables and libraries a package produces, making them visible to other packages. 13 | .library( 14 | name: "PipecatClientIOS", 15 | targets: ["PipecatClientIOS"]), 16 | ], 17 | targets: [ 18 | // Targets are the basic building blocks of a package, defining a module or a test suite. 19 | // Targets can depend on other targets in this package and products from dependencies. 20 | .target( 21 | name: "PipecatClientIOS"), 22 | .testTarget( 23 | name: "PipecatClientIOSTests", 24 | dependencies: ["PipecatClientIOS"]), 25 | ] 26 | ) 27 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/Transcript.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A written transcript of some spoken words. 4 | public struct Transcript: Codable { 5 | public let text: String 6 | public let final: Bool? 7 | public let timestamp: String? 8 | public let userId: String? 9 | 10 | enum CodingKeys: String, CodingKey { 11 | case text 12 | case final 13 | case timestamp 14 | case userId = "user_id" 15 | } 16 | 17 | public init(text: String, final: Bool? = false, timestamp: String? = nil, userId: String? = nil) { 18 | self.text = text 19 | self.final = final 20 | self.timestamp = timestamp 21 | self.userId = userId 22 | } 23 | 24 | public init(from decoder: any Decoder) throws { 25 | let container = try decoder.container(keyedBy: CodingKeys.self) 26 | self.text = try container.decode(String.self, forKey: .text) 27 | self.final = try container.decodeIfPresent(Bool.self, forKey: .final) 28 | self.timestamp = try container.decodeIfPresent(String.self, forKey: .timestamp) 29 | self.userId = try container.decodeIfPresent(String.self, forKey: .userId) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/TransportState.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// The current state of the session transport. 4 | public enum TransportState { 5 | case disconnected 6 | case initializing 7 | case initialized 8 | case authenticating 9 | case authenticated 10 | case connecting 11 | case connected 12 | case ready 13 | case disconnecting 14 | case error 15 | } 16 | 17 | extension TransportState { 18 | public var description: String { 19 | switch self { 20 | case .initializing: 21 | return "Initializing" 22 | case .initialized: 23 | return "Initialized" 24 | case .authenticating: 25 | return "Authenticating" 26 | case .authenticated: 27 | return "Authenticated" 28 | case .connecting: 29 | return "Connecting" 30 | case .connected: 31 | return "Connected" 32 | case .ready: 33 | return "Ready" 34 | case .disconnecting: 35 | return "Disconnecting" 36 | case .disconnected: 37 | return "Disconnected" 38 | case .error: 39 | return "Error" 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2025, Daily 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/Transport.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI transport. 4 | @MainActor 5 | public protocol Transport { 6 | init() 7 | var delegate: PipecatClientDelegate? { get set } 8 | var onMessage: ((RTVIMessageInbound) -> Void)? { get set } 9 | 10 | func initialize(options: PipecatClientOptions) 11 | func initDevices() async throws 12 | func release() 13 | func connect(transportParams: TransportConnectionParams?) async throws 14 | func disconnect() async throws 15 | func getAllMics() -> [MediaDeviceInfo] 16 | func getAllCams() -> [MediaDeviceInfo] 17 | func updateMic(micId: MediaDeviceId) async throws 18 | func updateCam(camId: MediaDeviceId) async throws 19 | func selectedMic() -> MediaDeviceInfo? 20 | func selectedCam() -> MediaDeviceInfo? 21 | func enableMic(enable: Bool) async throws 22 | func enableCam(enable: Bool) async throws 23 | func isCamEnabled() -> Bool 24 | func isMicEnabled() -> Bool 25 | func sendMessage(message: RTVIMessageOutbound) throws 26 | func state() -> TransportState 27 | func setState(state: TransportState) 28 | func tracks() -> Tracks? 29 | func transformStartBotResultToConnectionParams(startBotParams: APIRequest, startBotResult: StartBotResult) throws 30 | -> TransportConnectionParams 31 | } 32 | 33 | extension Transport { 34 | public func transformStartBotResultToConnectionParams(startBotParams: APIRequest, startBotResult: StartBotResult) 35 | throws 36 | -> TransportConnectionParams { 37 | if let existingParams = startBotResult as? TransportConnectionParams { 38 | return existingParams 39 | } 40 | throw InvalidTransportParamsError() 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |  pipecat 3 |

4 | 5 | The [Pipecat](https://github.com/pipecat-ai/) project uses [RTVI-AI](https://docs.pipecat.ai/client/introduction), an open standard for Real-Time Voice [and Video] Inference. 6 | 7 | This iOS core library exports a VoiceClient that has no associated transport. 8 | 9 | When building an RTVI application, you should use your transport-specific export (see [here](https://docs.pipecat.ai/client/ios/transports/daily) for available first-party packages.) 10 | The base class has no out-of-the-box bindings included. 11 | 12 | ## Install 13 | 14 | To depend on the client package, you can add this package via Xcode's package manager using the URL of this git repository directly, or you can declare your dependency in your `Package.swift`: 15 | 16 | ```swift 17 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios.git", from: "1.1.3"), 18 | ``` 19 | 20 | and add `"PipecatClientIOS"` to your application/library target, `dependencies`, e.g. like this: 21 | 22 | ```swift 23 | .target(name: "YourApp", dependencies: [ 24 | .product(name: "PipecatClientIOS", package: "pipecat-client-ios") 25 | ], 26 | ``` 27 | 28 | ## References 29 | - [pipecat-client-ios reference docs](https://docs-ios.pipecat.ai/PipecatClientIOS/documentation/pipecatclientios). 30 | - [pipecat-client-ios SDK docs](https://docs.pipecat.ai/client/ios/introduction). 31 | 32 | ## Contributing 33 | 34 | We are welcoming contributions to this project in form of issues and pull request. For questions about RTVI head over to the [Pipecat discord server](https://discord.gg/pipecat) and check the [#rtvi](https://discord.com/channels/1239284677165056021/1265086477964935218) channel. 35 | -------------------------------------------------------------------------------- /.swift-format: -------------------------------------------------------------------------------- 1 | { 2 | "fileScopedDeclarationPrivacy": { 3 | "accessLevel": "private" 4 | }, 5 | "indentation": { 6 | "spaces": 4 7 | }, 8 | "indentConditionalCompilationBlocks": false, 9 | "indentSwitchCaseLabels": false, 10 | "lineBreakAroundMultilineExpressionChainComponents": true, 11 | "lineBreakBeforeControlFlowKeywords": false, 12 | "lineBreakBeforeEachArgument": true, 13 | "lineBreakBeforeEachGenericRequirement": true, 14 | "lineLength": 120, 15 | "maximumBlankLines": 1, 16 | "prioritizeKeepingFunctionOutputTogether": false, 17 | "respectsExistingLineBreaks": true, 18 | "rules": { 19 | "AllPublicDeclarationsHaveDocumentation": false, 20 | "AlwaysUseLowerCamelCase": true, 21 | "AmbiguousTrailingClosureOverload": true, 22 | "BeginDocumentationCommentWithOneLineSummary": false, 23 | "DoNotUseSemicolons": true, 24 | "DontRepeatTypeInStaticProperties": true, 25 | "FileScopedDeclarationPrivacy": true, 26 | "FullyIndirectEnum": true, 27 | "GroupNumericLiterals": true, 28 | "IdentifiersMustBeASCII": true, 29 | "NeverForceUnwrap": false, 30 | "NeverUseForceTry": false, 31 | "NeverUseImplicitlyUnwrappedOptionals": false, 32 | "NoAccessLevelOnExtensionDeclaration": true, 33 | "NoBlockComments": true, 34 | "NoCasesWithOnlyFallthrough": true, 35 | "NoEmptyTrailingClosureParentheses": true, 36 | "NoLabelsInCasePatterns": true, 37 | "NoLeadingUnderscores": false, 38 | "NoParensAroundConditions": true, 39 | "NoVoidReturnOnFunctionSignature": true, 40 | "OneCasePerLine": true, 41 | "OneVariableDeclarationPerLine": true, 42 | "OnlyOneTrailingClosureArgument": true, 43 | "OrderedImports": false, 44 | "ReturnVoidInsteadOfEmptyTuple": true, 45 | "UseLetInEveryBoundCaseVariable": true, 46 | "UseShorthandTypeNames": true, 47 | "UseSingleLinePropertyGetter": true, 48 | "UseSynthesizedInitializer": true, 49 | "UseTripleSlashForDocumentationComments": true, 50 | "ValidateDocumentationComments": false 51 | }, 52 | "tabWidth": 4, 53 | "version": 1 54 | } 55 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/Logger.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import os.log 3 | 4 | /// Log level used for logging by the framework. 5 | public enum LogLevel: UInt8 { 6 | /// A level lower than all log levels. 7 | case off = 0 8 | /// Corresponds to the `.error` log level. 9 | case error = 1 10 | /// Corresponds to the `.warn` log level. 11 | case warn = 2 12 | /// Corresponds to the `.debug` log level. 13 | case debug = 4 14 | /// Corresponds to the `.info` log level. 15 | case info = 3 16 | /// Corresponds to the `.trace` log level. 17 | case trace = 5 18 | } 19 | 20 | extension LogLevel { 21 | fileprivate var logType: OSLogType? { 22 | switch self { 23 | case .off: 24 | return nil 25 | case .error: 26 | return .error 27 | case .warn: 28 | return .default 29 | case .info: 30 | return .info 31 | case .debug: 32 | return .debug 33 | case .trace: 34 | return .debug 35 | } 36 | } 37 | } 38 | 39 | /// Sets the log level for logs produce by the framework. 40 | public func setLogLevel(_ logLevel: LogLevel) { 41 | Logger.shared.setLogLevel(logLevel: logLevel) 42 | } 43 | 44 | internal final class Logger { 45 | fileprivate var level: LogLevel = .warn 46 | 47 | fileprivate let osLog: OSLog = .init(subsystem: "co.daily.rtvi", category: "main") 48 | 49 | internal static let shared: Logger = .init() 50 | 51 | public func setLogLevel(logLevel: LogLevel) { 52 | self.level = logLevel 53 | } 54 | 55 | @inlinable 56 | internal func error(_ message: @autoclosure () -> String) { 57 | self.log(.error, message()) 58 | } 59 | 60 | @inlinable 61 | internal func warn(_ message: @autoclosure () -> String) { 62 | self.log(.warn, message()) 63 | } 64 | 65 | @inlinable 66 | internal func info(_ message: @autoclosure () -> String) { 67 | self.log(.info, message()) 68 | } 69 | 70 | @inlinable 71 | internal func debug(_ message: @autoclosure () -> String) { 72 | self.log(.debug, message()) 73 | } 74 | 75 | @inlinable 76 | internal func trace(_ message: @autoclosure () -> String) { 77 | self.log(.trace, message()) 78 | } 79 | 80 | @inlinable 81 | internal func log(_ level: LogLevel, _ message: @autoclosure () -> String) { 82 | guard self.level.rawValue >= level.rawValue else { 83 | return 84 | } 85 | 86 | guard self.level != .off else { 87 | return 88 | } 89 | 90 | let log = self.osLog 91 | 92 | // The following force-unwrap is okay since we check for `.off` above: 93 | // swiftlint:disable:next force_unwrapping 94 | let type = level.logType! 95 | 96 | os_log("%@", log: log, type: type, message()) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/RTVIMessageOutbound.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI control message sent to the Transport. 4 | public struct RTVIMessageOutbound: Encodable { 5 | 6 | public static let RTVI_PROTOCOL_VERSION = "1.0.0" 7 | public static let RTVI_MESSAGE_LABEL = "rtvi-ai" 8 | 9 | public let id: String 10 | public let label: String 11 | public let type: String 12 | public let data: Value? 13 | 14 | /// Messages from the client to the server. 15 | public enum MessageType { 16 | public static let CLIENT_READY = "client-ready" 17 | public static let DISCONNECT_BOT = "disconnect-bot" 18 | public static let CLIENT_MESSAGE = "client-message" 19 | public static let APPEND_TO_CONTEXT = "append-to-context" 20 | public static let SEND_TEXT = "send-text" 21 | public static let LLM_FUNCTION_CALL_RESULT = "llm-function-call-result" 22 | } 23 | 24 | public init( 25 | id: String = String(UUID().uuidString.prefix(8)), 26 | label: String = RTVI_MESSAGE_LABEL, 27 | type: String, 28 | data: Value? = nil 29 | ) { 30 | self.id = id 31 | self.label = label 32 | self.type = type 33 | self.data = data 34 | } 35 | 36 | public static func clientReady() -> RTVIMessageOutbound { 37 | let data = Value.object([ 38 | "version": .string(RTVI_PROTOCOL_VERSION), 39 | "about": .object([ 40 | "library": .string(PipecatClient.library), 41 | "library_version": .string(PipecatClient.libraryVersion) 42 | ]) 43 | ]) 44 | return RTVIMessageOutbound( 45 | type: RTVIMessageOutbound.MessageType.CLIENT_READY, 46 | data: data 47 | ) 48 | } 49 | 50 | public static func disconnectBot() -> RTVIMessageOutbound { 51 | return RTVIMessageOutbound( 52 | type: RTVIMessageOutbound.MessageType.DISCONNECT_BOT 53 | ) 54 | } 55 | 56 | public static func clientMessage(msgType: String, data: Value? = nil) -> RTVIMessageOutbound { 57 | let data = Value.object(["t": .string(msgType), "d": data]) 58 | return RTVIMessageOutbound( 59 | type: RTVIMessageOutbound.MessageType.CLIENT_MESSAGE, 60 | data: data 61 | ) 62 | } 63 | 64 | public static func appendToContext(msg: LLMContextMessage) throws -> RTVIMessageOutbound { 65 | return RTVIMessageOutbound( 66 | type: RTVIMessageOutbound.MessageType.APPEND_TO_CONTEXT, 67 | data: try msg.convertToRtviValue() 68 | ) 69 | } 70 | 71 | public static func sendText(content: String, options: SendTextOptions? = nil) throws -> RTVIMessageOutbound { 72 | return RTVIMessageOutbound( 73 | type: RTVIMessageOutbound.MessageType.SEND_TEXT, 74 | data: .object([ 75 | "content": .string(content), 76 | "options": try options?.self.convertToRtviValue() 77 | ]) 78 | ) 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/types/BotLLMSearchResponse.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Search entry point from bot LLM search response 4 | public struct SearchEntryPoint: Codable { 5 | public let renderedContent: String? 6 | 7 | enum CodingKeys: String, CodingKey { 8 | case renderedContent = "rendered_content" 9 | } 10 | 11 | public init(renderedContent: String?) { 12 | self.renderedContent = renderedContent 13 | } 14 | } 15 | 16 | /// Web source information from bot LLM search response 17 | public struct WebSource: Codable { 18 | public let uri: String? 19 | public let title: String? 20 | 21 | public init(uri: String?, title: String?) { 22 | self.uri = uri 23 | self.title = title 24 | } 25 | } 26 | 27 | /// Grounding chunk information 28 | public struct GroundingChunk: Codable { 29 | public let web: WebSource? 30 | 31 | public init(web: WebSource?) { 32 | self.web = web 33 | } 34 | } 35 | 36 | /// Grounding segment information 37 | public struct GroundingSegment: Codable { 38 | public let partIndex: Int? 39 | public let startIndex: Int? 40 | public let endIndex: Int? 41 | public let text: String? 42 | 43 | enum CodingKeys: String, CodingKey { 44 | case partIndex = "part_index" 45 | case startIndex = "start_index" 46 | case endIndex = "end_index" 47 | case text 48 | } 49 | 50 | public init(partIndex: Int?, startIndex: Int?, endIndex: Int?, text: String?) { 51 | self.partIndex = partIndex 52 | self.startIndex = startIndex 53 | self.endIndex = endIndex 54 | self.text = text 55 | } 56 | } 57 | 58 | /// Grounding support information 59 | public struct GroundingSupport: Codable { 60 | public let segment: GroundingSegment? 61 | public let groundingChunkIndices: [Int]? 62 | public let confidenceScores: [Double]? 63 | 64 | enum CodingKeys: String, CodingKey { 65 | case segment 66 | case groundingChunkIndices = "grounding_chunk_indices" 67 | case confidenceScores = "confidence_scores" 68 | } 69 | 70 | public init(segment: GroundingSegment?, groundingChunkIndices: [Int]?, confidenceScores: [Double]?) { 71 | self.segment = segment 72 | self.groundingChunkIndices = groundingChunkIndices 73 | self.confidenceScores = confidenceScores 74 | } 75 | } 76 | 77 | /// Bot LLM search response data structure received from the backend 78 | public struct BotLLMSearchResponseData: Codable { 79 | public let searchEntryPoint: SearchEntryPoint? 80 | public let groundingChunks: [GroundingChunk]? 81 | public let groundingSupports: [GroundingSupport]? 82 | public let webSearchQueries: [String]? 83 | 84 | enum CodingKeys: String, CodingKey { 85 | case searchEntryPoint = "search_entry_point" 86 | case groundingChunks = "grounding_chunks" 87 | case groundingSupports = "grounding_supports" 88 | case webSearchQueries = "web_search_queries" 89 | } 90 | 91 | public init( 92 | searchEntryPoint: SearchEntryPoint?, 93 | groundingChunks: [GroundingChunk]?, 94 | groundingSupports: [GroundingSupport]?, 95 | webSearchQueries: [String]? 96 | ) { 97 | self.searchEntryPoint = searchEntryPoint 98 | self.groundingChunks = groundingChunks 99 | self.groundingSupports = groundingSupports 100 | self.webSearchQueries = webSearchQueries 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /.swiftlint: -------------------------------------------------------------------------------- 1 | # By default, SwiftLint uses a set of sensible default rules you can adjust: 2 | 3 | disabled_rules: 4 | - redundant_optional_initialization # nope, consistency/explicitness > brevity! 5 | - empty_enum_arguments # nope, consistency/explicitness > brevity! 6 | - closure_parameter_position 7 | - trailing_comma # just nope, this is a feature, not a bug! 8 | - redundant_string_enum_value # nope, consistency/explicitness > brevity! 9 | - unused_closure_parameter 10 | - opening_brace # nope, since this prevents proper formatting of `where` constraints 11 | - redundant_discardable_let 12 | 13 | opt_in_rules: 14 | - closure_body_length 15 | - closure_end_indentation 16 | - closure_spacing 17 | - collection_alignment 18 | - contains_over_filter_count 19 | - contains_over_filter_is_empty 20 | - contains_over_first_not_nil 21 | - contains_over_range_nil_comparison 22 | - convenience_type 23 | - discouraged_object_literal 24 | - discouraged_optional_collection 25 | - empty_count 26 | - empty_string 27 | - fallthrough 28 | - file_name_no_space 29 | - first_where 30 | - flatmap_over_map_reduce 31 | - force_unwrapping 32 | - function_default_parameter_at_end 33 | - implicit_return 34 | - implicitly_unwrapped_optional 35 | - indentation_width 36 | - joined_default_parameter 37 | - last_where 38 | - legacy_multiple 39 | - legacy_random 40 | - literal_expression_end_indentation 41 | - multiline_function_chains 42 | - no_extension_access_modifier 43 | - operator_usage_whitespace 44 | - private_action 45 | - private_outlet 46 | - redundant_set_access_control 47 | - sorted_first_last 48 | - switch_case_on_newline 49 | - unneeded_parentheses_in_closure_argument 50 | - unowned_variable_capture 51 | - vertical_whitespace_opening_braces 52 | 53 | # Rules run by `swiftlint analyze` (experimental) 54 | analyzer_rules: 55 | - explicit_self 56 | - unused_import 57 | 58 | excluded: 59 | - Daily/DailyTests 60 | - Daily/DailyDataModelTests 61 | 62 | force_cast: warning # implicitly 63 | force_try: 64 | severity: warning # explicitly 65 | 66 | line_length: 67 | warning: 120 68 | error: 150 69 | 70 | type_body_length: 71 | - 300 # warning 72 | - 400 # error 73 | 74 | file_length: 75 | warning: 750 76 | error: 1000 77 | 78 | type_name: 79 | min_length: 4 # only warning 80 | max_length: # warning and error 81 | warning: 50 82 | error: 70 83 | excluded: 84 | - Key 85 | - ID 86 | allowed_symbols: ["_"] # these are allowed in type names 87 | 88 | identifier_name: 89 | min_length: # only min_length 90 | warning: 2 # only warn 91 | excluded: # excluded via string array 92 | - i 93 | - x 94 | - y 95 | - id 96 | 97 | indentation_width: 98 | indentation_width: 4 99 | 100 | custom_rules: 101 | # https://www.swift.org/documentation/api-design-guidelines/#follow-case-conventions 102 | uniform_casing_for_id: 103 | name: "Use id or ID instead of Id" 104 | regex: '(? Value { 95 | // Encode the current object to JSON data 96 | let jsonData = try JSONEncoder().encode(self) 97 | // Decode the JSON data into a Value object 98 | let value = try JSONDecoder().decode(Value.self, from: jsonData) 99 | return value 100 | } 101 | var asString: String { 102 | do { 103 | let jsonData = try JSONEncoder().encode(self) 104 | return String(data: jsonData, encoding: .utf8)! 105 | } catch {} 106 | return "" 107 | } 108 | } 109 | 110 | public extension Value { 111 | public var asObject: [String: Value] { 112 | if case .object(let dict) = self { 113 | return dict 114 | } 115 | return [:] 116 | } 117 | 118 | public var asString: String { 119 | if case .object = self { 120 | do { 121 | let jsonData = try JSONEncoder().encode(self) 122 | return String(data: jsonData, encoding: .utf8)! 123 | } catch {} 124 | } else if case .string(let stringValue) = self { 125 | return stringValue 126 | } 127 | return "" 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/transport/RTVIMessageInbound.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI control message received the by the Transport. 4 | public struct RTVIMessageInbound: Codable { 5 | 6 | public let id: String? 7 | public let label: String? 8 | public let type: String? 9 | public let data: String? 10 | 11 | /// Messages from the server to the client. 12 | public enum MessageType { 13 | /// Bot is connected and ready to receive messages 14 | public static let BOT_READY = "bot-ready" 15 | 16 | /// Server response to client message 17 | public static let SERVER_RESPONSE = "server-response" 18 | 19 | /// Received an error response from the server 20 | public static let ERROR_RESPONSE = "error-response" 21 | 22 | /// Received an error from the server 23 | public static let ERROR = "error" 24 | 25 | /// STT transcript (both local and remote) flagged with partial final or sentence 26 | public static let TRANSCRIPT = "transcript" 27 | 28 | /// STT transcript from the user 29 | public static let USER_TRANSCRIPTION = "user-transcription" 30 | 31 | /// STT transcript from the bot 32 | public static let BOT_TRANSCRIPTION = "bot-transcription" 33 | 34 | /// User started speaking 35 | public static let USER_STARTED_SPEAKING = "user-started-speaking" 36 | 37 | // User stopped speaking 38 | public static let USER_STOPPED_SPEAKING = "user-stopped-speaking" 39 | 40 | // Bot started speaking 41 | public static let BOT_STARTED_SPEAKING = "bot-started-speaking" 42 | 43 | // Bot stopped speaking 44 | public static let BOT_STOPPED_SPEAKING = "bot-stopped-speaking" 45 | 46 | /// Pipecat metrics 47 | public static let METRICS = "metrics" 48 | 49 | /// LLM transcript from the bot 50 | public static let BOT_LLM_TEXT = "bot-llm-text" 51 | /// LLM transcript from the bot has started 52 | public static let BOT_LLM_STARTED = "bot-llm-started" 53 | /// LLM transcript from the bot has stopped 54 | public static let BOT_LLM_STOPPED = "bot-llm-stopped" 55 | 56 | /// TTS transcript from the bot 57 | public static let BOT_TTS_TEXT = "bot-tts-text" 58 | /// LLM transcript from the bot has started 59 | public static let BOT_TTS_STARTED = "bot-tts-started" 60 | /// LLM transcript from the bot has stopped 61 | public static let BOT_TTS_STOPPED = "bot-tts-stopped" 62 | 63 | /// Server message 64 | public static let SERVER_MESSAGE = "server-message" 65 | 66 | /// LLM 67 | public static let LLM_FUNCTION_CALL = "llm-function-call" 68 | 69 | /// Context 70 | public static let APPEND_TO_CONTEXT_RESULT = "append-to-context-result" 71 | 72 | // Bot LLM search response 73 | public static let BOT_LLM_SEARCH_RESPONSE = "bot-llm-search-response" 74 | } 75 | 76 | public init(type: String?, data: String?) { 77 | self.init(type: type, data: data, id: String(UUID().uuidString.prefix(8)), label: "rtvi-ai") 78 | } 79 | 80 | public init(type: String?, data: String?, id: String?, label: String? = "rtvi-ai") { 81 | self.id = id 82 | self.label = label 83 | self.type = type 84 | self.data = data 85 | } 86 | 87 | private enum CodingKeys: String, CodingKey { 88 | case id 89 | case label 90 | case type 91 | case data 92 | } 93 | 94 | public init(from decoder: Decoder) throws { 95 | let container = try decoder.container(keyedBy: CodingKeys.self) 96 | 97 | let type = try container.decode(String.self, forKey: .type) 98 | 99 | let datavalue = try? container.decodeIfPresent(Value.self, forKey: .data) 100 | let data: String? 101 | if datavalue != nil { 102 | data = try? String(data: JSONEncoder().encode(datavalue), encoding: .utf8) 103 | } else { 104 | data = nil 105 | } 106 | 107 | let label = try? container.decodeIfPresent(String.self, forKey: .label) 108 | let id = try? container.decodeIfPresent(String.self, forKey: .id) 109 | 110 | self.init(type: type, data: data, id: id, label: label) 111 | } 112 | 113 | public static func errorMessage(error: String, fatal: Bool = false) -> RTVIMessageInbound { 114 | return RTVIMessageInbound( 115 | type: RTVIMessageInbound.MessageType.ERROR, 116 | data: BotError(error: error, fatal: fatal).asString 117 | ) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/utils/MessageDispatcher.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | enum MessageDispatcherError: Error { 4 | case transportNotConnected 5 | } 6 | 7 | /// Helper class for sending messages to the server and awaiting the response. 8 | class MessageDispatcher { 9 | 10 | private let transport: Transport 11 | 12 | /// How long to wait before resolving the message/ 13 | private var gcTime: TimeInterval 14 | @MainActor 15 | private var queue: [QueuedVoiceMessage] = [] 16 | private var gcTimer: Timer? 17 | 18 | init(transport: Transport) { 19 | self.gcTime = 10.0 // 10 seconds 20 | self.transport = transport 21 | startGCTimer() 22 | } 23 | 24 | deinit { 25 | stopGCTimer() 26 | } 27 | 28 | @MainActor 29 | func dispatch(message: RTVIMessageOutbound) throws -> Promise { 30 | if self.gcTimer == nil { 31 | self.startGCTimer() 32 | } 33 | let promise = Promise() 34 | self.queue.append( 35 | QueuedVoiceMessage( 36 | message: message, 37 | timestamp: Date(), 38 | promise: promise 39 | ) 40 | ) 41 | do { 42 | try self.transport.sendMessage(message: message) 43 | } catch { 44 | Logger.shared.error("Failed to send app message \(error)") 45 | if let index = queue.firstIndex(where: { $0.message.id == message.id }) { 46 | // Removing the item that we have failed to send 47 | self.queue.remove(at: index) 48 | } 49 | throw error 50 | } 51 | return promise 52 | } 53 | 54 | @MainActor 55 | func dispatchAsync(message: RTVIMessageOutbound) async throws -> RTVIMessageInbound { 56 | try await withCheckedThrowingContinuation { continuation in 57 | do { 58 | let promise = try self.dispatch(message: message) 59 | promise.onResolve = { (inboundMessage: RTVIMessageInbound) in 60 | continuation.resume(returning: inboundMessage) 61 | } 62 | promise.onReject = { (error: Error) in 63 | continuation.resume(throwing: error) 64 | } 65 | } catch { 66 | continuation.resume(throwing: error) 67 | } 68 | } 69 | } 70 | 71 | private func resolveReject(message: RTVIMessageInbound, resolve: Bool = true) -> RTVIMessageInbound { 72 | DispatchQueue.main.async { 73 | if let index = self.queue.firstIndex(where: { $0.message.id == message.id }) { 74 | let queuedMessage = self.queue[index] 75 | if resolve { 76 | queuedMessage.promise.resolve(value: message) 77 | } else { 78 | if let errorResponse = try? JSONDecoder().decode(ErrorResponse.self, from: Data(message.data!.utf8)) { 79 | queuedMessage.promise.reject( 80 | error: BotResponseError( 81 | message: "Received error response from backend: \(errorResponse.error)" 82 | ) 83 | ) 84 | } else { 85 | queuedMessage.promise.reject(error: BotResponseError()) 86 | } 87 | } 88 | self.queue.remove(at: index) 89 | } else { 90 | // unknown messages are just ignored 91 | } 92 | } 93 | return message 94 | 95 | } 96 | 97 | func resolve(message: RTVIMessageInbound) -> RTVIMessageInbound { 98 | return resolveReject(message: message, resolve: true) 99 | } 100 | 101 | func reject(message: RTVIMessageInbound) -> RTVIMessageInbound { 102 | return resolveReject(message: message, resolve: false) 103 | } 104 | 105 | func disconnect() { 106 | self.stopGCTimer() 107 | DispatchQueue.main.async { 108 | self.queue.removeAll() 109 | } 110 | } 111 | 112 | /// Removing the messages that we have not received a response in the specified time 113 | private func gc() { 114 | let currentTime = Date() 115 | DispatchQueue.main.async { 116 | self.queue.removeAll { queuedMessage in 117 | let timeElapsed = currentTime.timeIntervalSince(queuedMessage.timestamp) 118 | if timeElapsed >= self.gcTime { 119 | queuedMessage.promise.reject(error: ResponseTimeoutError()) 120 | return true 121 | } 122 | return false 123 | } 124 | } 125 | } 126 | 127 | private func startGCTimer() { 128 | gcTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in 129 | self?.gc() 130 | } 131 | } 132 | 133 | private func stopGCTimer() { 134 | gcTimer?.invalidate() 135 | gcTimer = nil 136 | } 137 | } 138 | 139 | struct QueuedVoiceMessage { 140 | let message: RTVIMessageOutbound 141 | let timestamp: Date 142 | let promise: Promise 143 | } 144 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/RTVIError.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// A protocol representing a base error occurring during an operation. 4 | public protocol RTVIError: Error { 5 | /// A human-readable description of the error. 6 | var message: String { get } 7 | 8 | /// If the error was caused by another error, this value is set. 9 | var underlyingError: Error? { get } 10 | } 11 | 12 | extension RTVIError { 13 | public var underlyingError: Error? { return nil } 14 | 15 | /// Provides a detailed description of the error, including any underlying error. 16 | public var localizedDescription: String { 17 | if let underlyingError = self.underlyingError as? RTVIError { 18 | // Finding the root cause 19 | var rootCauseError: RTVIError = underlyingError 20 | while let underlyingError = rootCauseError.underlyingError as? RTVIError { 21 | rootCauseError = underlyingError 22 | } 23 | return "\(message) \(rootCauseError.localizedDescription)" 24 | } else { 25 | return message 26 | } 27 | } 28 | } 29 | 30 | public struct BotAlreadyStartedError: RTVIError { 31 | public let message: String = 32 | "Pipecat client has already been started. Please call disconnect() before starting again." 33 | public let underlyingError: Error? 34 | 35 | public init(underlyingError: Error? = nil) { 36 | self.underlyingError = underlyingError 37 | } 38 | } 39 | 40 | /// Invalid or malformed auth bundle provided to Transport. 41 | public struct InvalidTransportParamsError: RTVIError { 42 | public let message: String = "Invalid or malformed transport params provided to Transport." 43 | public let underlyingError: Error? 44 | 45 | public init(underlyingError: Error? = nil) { 46 | self.underlyingError = underlyingError 47 | } 48 | } 49 | 50 | /// Failed to fetch the authentication bundle from the RTVI backend. 51 | public struct HttpError: RTVIError { 52 | public let message: String 53 | public let underlyingError: Error? 54 | 55 | public init(message: String, underlyingError: Error? = nil) { 56 | self.underlyingError = underlyingError 57 | self.message = message 58 | } 59 | } 60 | 61 | /// Failed to fetch the auth bundle. 62 | public struct StartBotError: RTVIError { 63 | public let message: String = "Failed to start the bot using the specified endpoint." 64 | public let underlyingError: Error? 65 | 66 | public init(underlyingError: Error? = nil) { 67 | self.underlyingError = underlyingError 68 | } 69 | } 70 | 71 | /// Bot is not ready yet. 72 | public struct BotNotReadyError: RTVIError { 73 | public let message: String = "Bot is not ready yet." 74 | public let underlyingError: Error? 75 | 76 | public init(underlyingError: Error? = nil) { 77 | self.underlyingError = underlyingError 78 | } 79 | } 80 | 81 | /// Received error response from backend. 82 | public struct BotResponseError: RTVIError { 83 | public let message: String 84 | public let underlyingError: Error? 85 | 86 | public init(message: String = "Received error response from backend.", underlyingError: Error? = nil) { 87 | self.message = message 88 | self.underlyingError = underlyingError 89 | } 90 | } 91 | 92 | /// The operation timed out before it could complete. 93 | public struct ResponseTimeoutError: RTVIError { 94 | public let message: String = "The operation timed out before it could complete." 95 | public let underlyingError: Error? 96 | 97 | public init(underlyingError: Error? = nil) { 98 | self.underlyingError = underlyingError 99 | } 100 | } 101 | 102 | /// A feature is not supported by the current implementation or source. 103 | public struct UnsupportedFeatureError: RTVIError { 104 | public let message: String 105 | public let underlyingError: Error? 106 | public let feature: String 107 | 108 | public init( 109 | feature: String, 110 | source: String? = nil, 111 | message customMessage: String? = nil, 112 | underlyingError: Error? = nil 113 | ) { 114 | self.feature = feature 115 | self.underlyingError = underlyingError 116 | 117 | var msg = "\(feature) not supported" 118 | if let source = source { 119 | msg = "\(source) does not support \(feature)" 120 | } 121 | 122 | if let customMessage = customMessage { 123 | msg += ": \(customMessage)" 124 | } 125 | 126 | self.message = msg 127 | } 128 | } 129 | 130 | /// Received an error response when trying to execute the function. 131 | public struct AsyncExecutionError: RTVIError { 132 | public let message: String 133 | public let underlyingError: Error? 134 | 135 | public init(functionName: String, underlyingError: Error? = nil) { 136 | self.message = "Received an error response when trying to execute the function \(functionName)." 137 | self.underlyingError = underlyingError 138 | } 139 | } 140 | 141 | /// An unknown error occurred.. 142 | public struct OtherError: RTVIError { 143 | public let message: String 144 | public let underlyingError: Error? 145 | 146 | public init(message: String, underlyingError: Error? = nil) { 147 | self.message = message 148 | self.underlyingError = underlyingError 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 1.1.3 — 2025-12-11 2 | 3 | ### Fixed 4 | 5 | - Fixed memory management in message handling to ensure proper deallocation of `PipecatClient` instances. 6 | 7 | # 1.1.2 — 2025-10-28 8 | 9 | ### Added 10 | 11 | - Added default implementations for all optional `Transport` protocol methods to ensure backward compatibility. 12 | 13 | # 1.1.1 — 2025-10-23 14 | 15 | ### Added 16 | 17 | - Allowing the transports to transform `StartBotResult` into `TransportConnectionParams` when using startBotAndConnect. 18 | 19 | # 1.1.0 — 2025-10-14 20 | 21 | ### Added 22 | 23 | - Added new `sendText()` method to support the new RTVI `send-text` event. 24 | - Note: This is a replacement for the current `appendToContext()` method and changes the default of `run_immediately` to `True`. 25 | - New delegate function `onBotStarted`. 26 | 27 | ### Deprecated 28 | 29 | - Deprecated `appendToContext()` in lieu of the new `sendText()` method. This sets a standard for future methods like `sendImage()`. 30 | 31 | # 1.0.1 - 2025-09-19 32 | 33 | ### Added 34 | - Added support for lower versions of Xcode (16.2 and below) 35 | 36 | # 1.0.0 - 2025-08-25 37 | 38 | ### Changed 39 | - Protocol Updates: 40 | - `client-ready` and `bot-ready` messages now include: 41 | - `version`: Specifies the RTVI protocol version. 42 | `about`: Provides metadata about the client platform and environment. 43 | - Deprecated and removed: 44 | - Action-related messages and methods (e.g., `action()`, `describeActions()`). 45 | - Service configuration messages and related methods (e.g., `getConfig()`, `updateConfig()`). 46 | - Helper-related utilities (e.g., `RTVIClientHelper`, `LLMHelper`). 47 | - Introduced new messaging types: 48 | - `client-message`: For sending messages from client to server. 49 | - `append-to-context`: To append data to the LLM context. 50 | - `disconnect-bot`: For disconnecting the bot 51 | - `server-response`: For receiving responses from the server. 52 | - `bot-llm-search-response`: For receiving the llm search response. 53 | 54 | - PipecatClient Enhancements: 55 | - Constructor no longer accepts `params` for pipeline or endpoint configuration. 56 | - `connect()` method now accepts a set of parameters defined by the transport in use. 57 | 58 | ### Removed 59 | - All action-related methods, events and types: 60 | - `action()`, `describeActions()`, `onActionsAvailable`, etc. 61 | - All configuration-related methods, events and types: 62 | - `getConfig()`, `updateConfig()`, `describeConfig()`, `onConfigUpdated`, `onConfigDescribed`, etc. 63 | - All helper-related methods, types, and files: 64 | - `RTVIClientHelper`, `registerHelper`, `LLMHelper`, etc. 65 | - `transportExpiry()` method. 66 | 67 | ### Added 68 | - `appendToContext()`: Ability to append data to the LLM context. 69 | - `sendClientRequest()`: Send a message and wait for a response. 70 | - `sendClientMessage()`: Sends a one-way message to the bot without expecting a response. 71 | - `disconnectBot()`: Sends a disconnect signal to the bot while maintaining the transport connection. 72 | - `registerFunctionCallHandler`: Registers a function call handler for a specific function name. 73 | - `unregisterFunctionCallHandler`: Unregisters a function call handler for a specific function name. 74 | - `unregisterAllFunctionCallHandlers`: Unregisters all function call handlers. 75 | - `startBot()`: Fetches the POST endpoint for kicking off a bot process and optionally returning the connection parameters required by the transport. 76 | - `startBotAndConnect()`: Calls both `startBot()` and `connect()`, passing any data returned from the `startBot()` endpoint to `connect()` as transport parameters. 77 | 78 | # 0.3.6 - 2025-06-11 79 | 80 | ### Added 81 | 82 | - New delegate function `onServerMessage`. 83 | 84 | # 0.3.5 - 2025-04-02 85 | 86 | ### Added 87 | 88 | - Triggering `onBotStartedSpeaking` and `onBotStoppedSpeaking` based on the RTVI events that we receive from Pipecat. 89 | 90 | # 0.3.4 - 2025-03-20 91 | 92 | ### Added 93 | 94 | - Made changes to enable transports to handle function calling. 95 | 96 | # 0.3.3 - 2025-03-19 97 | 98 | ### Added 99 | 100 | - Added improvements to support new transports to send transcription events. 101 | 102 | # 0.3.2 - 2025-01-15 103 | 104 | ### Changed 105 | 106 | - Refactored the `PipecatClientIOS` target to enable publishing to CocoaPods. 107 | - **Breaking Change:** Replace imports of `RTVIClientIOS` with `PipecatClientIOS`. 108 | 109 | # 0.3.1 - 2025-01-03 110 | 111 | ### Added 112 | 113 | - Added improvements to support the `GeminiLiveWebSocket` transport. 114 | 115 | ## 0.3.0 - 2024-12-10 116 | 117 | ### Changed 118 | 119 | - Renamed the package from `RTVIClientIOS` to `PipecatClientIOS`. 120 | 121 | ## 0.2.0 - 2024-10-10 122 | 123 | - Adding support for the HTTP action. 124 | - Renamed: 125 | - `VoiceClient` to `RTVIClient` 126 | - `VoiceClientOptions` to `RTVIClientOptions` 127 | - `VoiceClientDelegate` to `RTVIClientDelete` 128 | - `VoiceError` to `RTVIError` 129 | - `VoiceClientHelper` to `RTVIClientHelper` 130 | - `FailedToFetchAuthBundle` to `HttpError` 131 | - `RTVIClient()` constructor parameter changes 132 | - `options` is now mandatory 133 | - `baseUrl` has been moved to `options.params.baseUrl` 134 | - `baseUrl` and `endpoints` are now separate, and the endpoint names are appended to the `baseUrl` 135 | - Moved `RTVIClientOptions.config` to `RTVIClientOptions.params.config` 136 | - Moved `RTVIClientOptions.customHeaders` to `RTVIClientOptions.params.headers` 137 | - Moved `RTVIClientOptions.customBodyParams` to `RTVIClientOptions.params.requestData` 138 | - `TransportState` changes 139 | - Removed `Idle` state, replaced with `Disconnected` 140 | - Added `Disconnecting` state 141 | - Added callbacks 142 | - `onBotLLMText()` 143 | - `onBotTTSText()` 144 | - `onStorageItemStored()` 145 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/PipecatClientDelegate.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// Callbacks invoked when changes occur in the voice session. 4 | public protocol PipecatClientDelegate: AnyObject { 5 | /// Invoked when the underlying transport has connected. 6 | func onConnected() 7 | 8 | /// Invoked when the underlying transport has disconnected. 9 | func onDisconnected() 10 | 11 | /// Invoked when an error occurs. 12 | func onError(message: RTVIMessageInbound) 13 | 14 | /// Invoked when the session state has changed. 15 | func onTransportStateChanged(state: TransportState) 16 | 17 | /// Invoked when the bot has connected to the session. 18 | func onBotConnected(participant: Participant) 19 | 20 | /// Invoked when the bot process has started. 21 | func onBotStarted(botResponse: Any) 22 | 23 | /// Invoked when the bot has indicated it is ready for commands. 24 | func onBotReady(botReadyData: BotReadyData) 25 | 26 | /// Invoked when the bot has disconnected from the session. 27 | func onBotDisconnected(participant: Participant) 28 | 29 | /// Invoked when session metrics are received. 30 | func onMetrics(data: PipecatMetrics) 31 | 32 | /// Invoked when a message from the backend is received which was not handled by the VoiceClient or a registered helper. 33 | func onServerMessage(data: Any) 34 | 35 | /// Invoked when a message error occurs. 36 | func onMessageError(message: RTVIMessageInbound) 37 | 38 | /// Invoked when a participant has joined the session. 39 | func onParticipantJoined(participant: Participant) 40 | 41 | /// Invoked when a participant has left the session. 42 | func onParticipantLeft(participant: Participant) 43 | 44 | /// Invoked when the list of available cameras has changed. 45 | func onAvailableCamsUpdated(cams: [MediaDeviceInfo]) 46 | 47 | /// Invoked when the list of available microphones has updated. 48 | func onAvailableMicsUpdated(mics: [MediaDeviceInfo]) 49 | 50 | /// Invoked when the list of available speakers has updated. 51 | func onAvailableSpeakersUpdated(speakers: [MediaDeviceInfo]) 52 | 53 | /// Invoked when the selected cam has changed. 54 | func onCamUpdated(cam: MediaDeviceInfo?) 55 | 56 | /// Invoked when the selected microphone has changed. 57 | func onMicUpdated(mic: MediaDeviceInfo?) 58 | 59 | /// Invoked when the selected speaker has changed. 60 | func onSpeakerUpdated(speaker: MediaDeviceInfo?) 61 | 62 | /// Invoked when a track starts. 63 | func onTrackStarted(track: MediaStreamTrack, participant: Participant?) 64 | 65 | /// Invoked when a track stops. 66 | func onTrackStopped(track: MediaStreamTrack, participant: Participant?) 67 | 68 | /// Invoked when a screen track starts. 69 | func onScreenTrackStarted(track: MediaStreamTrack, participant: Participant?) 70 | 71 | /// Invoked when a screen track stops. 72 | func onScreenTrackStopped(track: MediaStreamTrack, participant: Participant?) 73 | 74 | /// Invoked when a screen share error occurs. 75 | func onScreenShareError(errorMessage: String) 76 | 77 | /// Invoked regularly with the volume of the locally captured audio. 78 | func onLocalAudioLevel(level: Float) 79 | 80 | /// Invoked regularly with the audio volume of each remote participant. 81 | func onRemoteAudioLevel(level: Float, participant: Participant) 82 | 83 | /// Invoked when the bot starts talking. 84 | func onBotStartedSpeaking() 85 | 86 | /// Invoked when the bot stops talking. 87 | func onBotStoppedSpeaking() 88 | 89 | /// Invoked when the local user starts talking. 90 | func onUserStartedSpeaking() 91 | 92 | /// Invoked when the local user stops talking. 93 | func onUserStoppedSpeaking() 94 | 95 | /// Invoked when user transcript data is available. 96 | func onUserTranscript(data: Transcript) 97 | 98 | /// Invoked when bot transcript data is available. 99 | func onBotTranscript(data: BotLLMText) 100 | 101 | /// Invoked when received the bot transcription from the LLM. 102 | func onBotLlmText(data: BotLLMText) 103 | 104 | /// Invoked when the bot LLM text has started. 105 | func onBotLlmStarted() 106 | 107 | /// Invoked when the bot LLM text has stopped. 108 | func onBotLlmStopped() 109 | 110 | /// Invoked when text is spoken by the bot. 111 | func onBotTtsText(data: BotTTSText) 112 | 113 | /// Invoked when the bot TTS text has started. 114 | func onBotTtsStarted() 115 | 116 | /// Invoked when the bot TTS text has stopped. 117 | func onBotTtsStopped() 118 | 119 | /// Invoked when we receive a bot LLM search response. 120 | func onBotLlmSearchResponse(data: BotLLMSearchResponseData) 121 | 122 | /// Invoked when the LLM attempts to invoke a function. The provided callback must be provided with a return value. 123 | func onLLMFunctionCall(functionCallData: LLMFunctionCallData, onResult: ((Value) async -> Void)) async 124 | } 125 | 126 | extension PipecatClientDelegate { 127 | public func onConnected() {} 128 | public func onDisconnected() {} 129 | public func onError(message: RTVIMessageInbound) {} 130 | public func onTransportStateChanged(state: TransportState) {} 131 | public func onBotConnected(participant: Participant) {} 132 | public func onBotReady(botReadyData: BotReadyData) {} 133 | public func onBotStarted(botResponse: Any) {} 134 | public func onBotDisconnected(participant: Participant) {} 135 | public func onMetrics(data: PipecatMetrics) {} 136 | public func onServerMessage(data: Any) {} 137 | public func onMessageError(message: RTVIMessageInbound) {} 138 | public func onParticipantJoined(participant: Participant) {} 139 | public func onParticipantLeft(participant: Participant) {} 140 | public func onAvailableCamsUpdated(cams: [MediaDeviceInfo]) {} 141 | public func onAvailableMicsUpdated(mics: [MediaDeviceInfo]) {} 142 | public func onAvailableSpeakersUpdated(speakers: [MediaDeviceInfo]) {} 143 | public func onCamUpdated(cam: MediaDeviceInfo?) {} 144 | public func onMicUpdated(mic: MediaDeviceInfo?) {} 145 | public func onSpeakerUpdated(speaker: MediaDeviceInfo?) {} 146 | public func onTrackStarted(track: MediaStreamTrack, participant: Participant?) {} 147 | public func onTrackStopped(track: MediaStreamTrack, participant: Participant?) {} 148 | public func onScreenTrackStarted(track: MediaStreamTrack, participant: Participant?) {} 149 | public func onScreenTrackStopped(track: MediaStreamTrack, participant: Participant?) {} 150 | public func onScreenShareError(errorMessage: String) {} 151 | public func onLocalAudioLevel(level: Float) {} 152 | public func onRemoteAudioLevel(level: Float, participant: Participant) {} 153 | public func onBotStartedSpeaking() {} 154 | public func onBotStoppedSpeaking() {} 155 | public func onUserStartedSpeaking() {} 156 | public func onUserStoppedSpeaking() {} 157 | public func onUserTranscript(data: Transcript) {} 158 | public func onBotTranscript(data: BotLLMText) {} 159 | public func onBotLlmText(data: BotLLMText) {} 160 | public func onBotLlmStarted() {} 161 | public func onBotLlmStopped() {} 162 | public func onBotTtsText(data: BotTTSText) {} 163 | public func onBotTtsStarted() {} 164 | public func onBotTtsStopped() {} 165 | public func onBotLlmSearchResponse(data: BotLLMSearchResponseData) {} 166 | public func onLLMFunctionCall(functionCallData: LLMFunctionCallData, onResult: ((Value) async -> Void)) async {} 167 | } 168 | -------------------------------------------------------------------------------- /Sources/PipecatClientIOS/PipecatClient.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | /// An RTVI client that connects to an RTVI backend and handles bidirectional audio and video communication. 4 | /// 5 | /// `PipecatClient` provides a high-level interface for establishing connections, managing media devices, 6 | /// and handling real-time communication with AI bots. It supports both async/await and completion handler patterns 7 | /// for maximum flexibility in different use cases. 8 | @MainActor 9 | open class PipecatClient { 10 | 11 | private let options: PipecatClientOptions 12 | public private(set) var transport: Transport 13 | 14 | private let messageDispatcher: MessageDispatcher 15 | 16 | private var devicesInitialized: Bool = false 17 | private var disconnectRequested: Bool = false 18 | 19 | private var functionCallCallbacks: [String: FunctionCallCallback] = [:] 20 | 21 | private lazy var onMessage: (RTVIMessageInbound) -> Void = { [weak self] (voiceMessage: RTVIMessageInbound) in 22 | guard let self else { return } 23 | guard let type = voiceMessage.type else { 24 | // Ignoring the message, it doesn't have a type 25 | return 26 | } 27 | Logger.shared.debug("Received voice message \(voiceMessage)") 28 | switch type { 29 | case RTVIMessageInbound.MessageType.BOT_READY: 30 | self.transport.setState(state: .ready) 31 | if let botReadyData = try? JSONDecoder().decode(BotReadyData.self, from: Data(voiceMessage.data!.utf8)) { 32 | Logger.shared.info("Bot ready: \(botReadyData)") 33 | self.delegate?.onBotReady(botReadyData: botReadyData) 34 | } 35 | case RTVIMessageInbound.MessageType.USER_TRANSCRIPTION: 36 | if let transcript = try? JSONDecoder().decode(Transcript.self, from: Data(voiceMessage.data!.utf8)) { 37 | self.delegate?.onUserTranscript(data: transcript) 38 | } 39 | case RTVIMessageInbound.MessageType.BOT_TRANSCRIPTION: 40 | if let transcript = try? JSONDecoder().decode(BotLLMText.self, from: Data(voiceMessage.data!.utf8)) { 41 | self.delegate?.onBotTranscript(data: transcript) 42 | } 43 | case RTVIMessageInbound.MessageType.BOT_LLM_STARTED: 44 | self.delegate?.onBotLlmStarted() 45 | case RTVIMessageInbound.MessageType.BOT_LLM_TEXT: 46 | if let botLLMText = try? JSONDecoder().decode(BotLLMText.self, from: Data(voiceMessage.data!.utf8)) { 47 | self.delegate?.onBotLlmText(data: botLLMText) 48 | } 49 | case RTVIMessageInbound.MessageType.BOT_LLM_STOPPED: 50 | self.delegate?.onBotLlmStopped() 51 | case RTVIMessageInbound.MessageType.BOT_TTS_STARTED: 52 | self.delegate?.onBotTtsStarted() 53 | case RTVIMessageInbound.MessageType.BOT_TTS_TEXT: 54 | if let botTTSText = try? JSONDecoder().decode(BotTTSText.self, from: Data(voiceMessage.data!.utf8)) { 55 | self.delegate?.onBotTtsText(data: botTTSText) 56 | } 57 | case RTVIMessageInbound.MessageType.BOT_TTS_STOPPED: 58 | self.delegate?.onBotTtsStopped() 59 | case RTVIMessageInbound.MessageType.SERVER_MESSAGE, RTVIMessageInbound.MessageType.APPEND_TO_CONTEXT_RESULT: 60 | if let storedData = try? JSONDecoder().decode(Value.self, from: Data(voiceMessage.data!.utf8)) { 61 | self.delegate?.onServerMessage(data: storedData) 62 | } 63 | case RTVIMessageInbound.MessageType.METRICS: 64 | if let metricsData = try? JSONDecoder() 65 | .decode(PipecatMetrics.self, from: Data(voiceMessage.data!.utf8)) { 66 | self.delegate?.onMetrics(data: metricsData) 67 | } 68 | case RTVIMessageInbound.MessageType.USER_STARTED_SPEAKING: 69 | self.delegate?.onUserStartedSpeaking() 70 | case RTVIMessageInbound.MessageType.USER_STOPPED_SPEAKING: 71 | self.delegate?.onUserStoppedSpeaking() 72 | case RTVIMessageInbound.MessageType.BOT_STARTED_SPEAKING: 73 | self.delegate?.onBotStartedSpeaking() 74 | case RTVIMessageInbound.MessageType.BOT_STOPPED_SPEAKING: 75 | self.delegate?.onBotStoppedSpeaking() 76 | case RTVIMessageInbound.MessageType.SERVER_RESPONSE: 77 | _ = self.messageDispatcher.resolve(message: voiceMessage) 78 | case RTVIMessageInbound.MessageType.ERROR_RESPONSE: 79 | Logger.shared.warn("RECEIVED ON ERROR_RESPONSE \(voiceMessage)") 80 | _ = self.messageDispatcher.reject(message: voiceMessage) 81 | self.delegate?.onMessageError(message: voiceMessage) 82 | case RTVIMessageInbound.MessageType.ERROR: 83 | Logger.shared.warn("RECEIVED ON ERROR \(voiceMessage)") 84 | _ = self.messageDispatcher.reject(message: voiceMessage) 85 | self.delegate?.onError(message: voiceMessage) 86 | if let botError = try? JSONDecoder().decode(BotError.self, from: Data(voiceMessage.data!.utf8)) { 87 | let errorMessage = "Received an error from the Bot: \(botError.error)" 88 | if botError.fatal ?? false { 89 | self.disconnect(completion: nil) 90 | } 91 | } 92 | case RTVIMessageInbound.MessageType.LLM_FUNCTION_CALL: 93 | if let functionCallData = try? JSONDecoder() 94 | .decode(LLMFunctionCallData.self, from: Data(voiceMessage.data!.utf8)) { 95 | Task { 96 | // Check if we have a registered handler for this function 97 | if let registeredCallback = self.functionCallCallbacks[functionCallData.functionName] { 98 | // Use the registered callback 99 | await registeredCallback(functionCallData) { result in 100 | let resultData = try? await LLMFunctionCallResult( 101 | functionName: functionCallData.functionName, 102 | toolCallID: functionCallData.toolCallID, 103 | arguments: functionCallData.args, 104 | result: result 105 | ) 106 | .convertToRtviValue() 107 | let resultMessage = RTVIMessageOutbound( 108 | type: RTVIMessageOutbound.MessageType.LLM_FUNCTION_CALL_RESULT, 109 | data: resultData 110 | ) 111 | self.sendMessage(msg: resultMessage) { result in 112 | if case .failure(let error) = result { 113 | Logger.shared.error("Failing to send app result message \(error)") 114 | } 115 | } 116 | } 117 | } 118 | await self.delegate? 119 | .onLLMFunctionCall(functionCallData: functionCallData) { result in 120 | let resultData = try? await LLMFunctionCallResult( 121 | functionName: functionCallData.functionName, 122 | toolCallID: functionCallData.toolCallID, 123 | arguments: functionCallData.args, 124 | result: result 125 | ) 126 | .convertToRtviValue() 127 | let resultMessage = RTVIMessageOutbound( 128 | type: RTVIMessageOutbound.MessageType.LLM_FUNCTION_CALL_RESULT, 129 | data: resultData 130 | ) 131 | self.sendMessage(msg: resultMessage) { result in 132 | if case .failure(let error) = result { 133 | Logger.shared.error("Failing to send app result message \(error)") 134 | } 135 | } 136 | } 137 | } 138 | } 139 | 140 | case RTVIMessageInbound.MessageType.BOT_LLM_SEARCH_RESPONSE: 141 | if let searchResponseData = try? JSONDecoder() 142 | .decode(BotLLMSearchResponseData.self, from: Data(voiceMessage.data!.utf8)) { 143 | self.delegate?.onBotLlmSearchResponse(data: searchResponseData) 144 | } 145 | default: 146 | Logger.shared.debug("[Pipecat Client] Unrecognized message type: \(type), message: \(voiceMessage)") 147 | } 148 | } 149 | 150 | /// The delegate object that receives PipecatClient events and callbacks. 151 | /// 152 | /// Set this property to an object conforming to `PipecatClientDelegate` to receive 153 | /// notifications about connection state changes, transcription events, bot responses, 154 | /// and other real-time events during the session. 155 | /// 156 | /// - Note: The delegate is held with a weak reference to prevent retain cycles. 157 | private weak var _delegate: PipecatClientDelegate? 158 | public weak var delegate: PipecatClientDelegate? { 159 | get { 160 | return _delegate 161 | } 162 | set { 163 | _delegate = newValue 164 | self.transport.delegate = _delegate 165 | } 166 | } 167 | 168 | /// Initializes a new PipecatClient instance with the specified configuration options. 169 | /// 170 | /// - Parameter options: Configuration options that specify the transport layer, 171 | /// media settings, and other client behaviors. 172 | public init(options: PipecatClientOptions) { 173 | Logger.shared.info("Initializing Pipecat Client iOS version \(PipecatClient.libraryVersion)") 174 | self.options = options 175 | self.transport = options.transport 176 | self.messageDispatcher = MessageDispatcher.init(transport: transport) 177 | self.transport.onMessage = self.onMessage 178 | self.transport.initialize(options: options) 179 | } 180 | 181 | /// Initializes local media devices such as camera and microphone (completion-based). 182 | /// 183 | /// This method prepares the device's audio and video hardware for use in the session. 184 | /// Call this method before attempting to connect to ensure proper media device availability. 185 | /// 186 | /// - Parameter completion: A closure called when device initialization completes. 187 | /// Contains a `Result` indicating success or failure. 188 | /// 189 | /// - Note: If devices are already initialized, this method returns immediately without error. 190 | public func initDevices(completion: ((Result) -> Void)?) { 191 | Task { 192 | do { 193 | try await self.initDevices() 194 | completion?(.success(())) 195 | } catch { 196 | completion?(.failure(AsyncExecutionError(functionName: "initDevices", underlyingError: error))) 197 | } 198 | } 199 | } 200 | 201 | /// Sets the logging level for the PipecatClient and its components. 202 | /// 203 | /// Use this method to control the verbosity of log messages during development and debugging. 204 | /// 205 | /// - Parameter logLevel: The desired log level (e.g., `.debug`, `.info`, `.warn`, `.error`). 206 | public func setLogLevel(logLevel: LogLevel) { 207 | Logger.shared.setLogLevel(logLevel: logLevel) 208 | } 209 | 210 | /// Initializes local media devices such as camera and microphone (async/await). 211 | /// 212 | /// This method prepares the device's audio and video hardware for use in the session. 213 | /// Call this method before attempting to connect to ensure proper media device availability. 214 | /// 215 | /// - Throws: An error if device initialization fails due to permissions or hardware issues. 216 | /// 217 | /// - Note: If devices are already initialized, this method returns immediately without error. 218 | public func initDevices() async throws { 219 | if self.devicesInitialized { 220 | // There is nothing to do in this case 221 | return 222 | } 223 | try await self.transport.initDevices() 224 | self.devicesInitialized = true 225 | } 226 | 227 | private func fetchStartBot(startBotParams: APIRequest) async throws -> T { 228 | var request = URLRequest(url: startBotParams.endpoint) 229 | request.httpMethod = "POST" 230 | request.setValue("application/json", forHTTPHeaderField: "Content-Type") 231 | 232 | // Adding the custom headers if they have been provided 233 | for header in startBotParams.headers { 234 | for (key, value) in header { 235 | request.setValue(value, forHTTPHeaderField: key) 236 | } 237 | } 238 | 239 | do { 240 | if let customBodyParams = startBotParams.requestData { 241 | request.httpBody = try JSONEncoder().encode(startBotParams.requestData) 242 | } 243 | 244 | Logger.shared.debug( 245 | "Fetching from \(String(data: request.httpBody!, encoding: .utf8) ?? "")" 246 | ) 247 | 248 | // Create a custom URLSession configuration with the timeout from APIRequest 249 | let urlSession: URLSession 250 | if let startTimeout = startBotParams.timeout { 251 | let config = URLSessionConfiguration.default 252 | config.timeoutIntervalForRequest = startTimeout 253 | config.timeoutIntervalForResource = startTimeout 254 | urlSession = URLSession(configuration: config) 255 | } else { 256 | urlSession = URLSession.shared 257 | } 258 | 259 | let (data, response) = try await urlSession.data(for: request) 260 | 261 | guard let httpResponse = response as? HTTPURLResponse, 262 | httpResponse.statusCode >= 200 && httpResponse.statusCode <= 299 263 | else { 264 | let errorMessage = String(data: data, encoding: .utf8) ?? "Unknown error" 265 | let message = "Error fetching: \(errorMessage)" 266 | Logger.shared.error(message) 267 | throw HttpError(message: message) 268 | } 269 | 270 | return try JSONDecoder().decode(T.self, from: data) 271 | } catch { 272 | Logger.shared.error(error.localizedDescription) 273 | throw HttpError(message: "Error fetching.", underlyingError: error) 274 | } 275 | } 276 | 277 | private func bailIfDisconnected() -> Bool { 278 | if self.disconnectRequested { 279 | if self.transport.state() != .disconnecting && self.transport.state() != .disconnected { 280 | self.disconnect(completion: nil) 281 | } 282 | return true 283 | } 284 | return false 285 | } 286 | 287 | /// Initiates bot and retrieves connection parameters (async/await). 288 | /// 289 | /// This method sends a POST request to the specified endpoint to request a new bot to run 290 | /// and obtain the necessary parameters for establishing a transport connection. 291 | /// 292 | /// - Parameter startBotParams: API request configuration including endpoint URL, headers, and request data. 293 | /// - Returns: Transport connection parameters of the specified generic type. 294 | /// - Throws: `BotAlreadyStartedError` if a session is already active, or `StartBotError` for other failures. 295 | /// 296 | /// - Note: The client must be in a disconnected state to call this method. 297 | public func startBot(startBotParams: APIRequest) async throws -> T { 298 | if self.transport.state() == .authenticating || self.transport.state() == .connecting 299 | || self.transport.state() == .connected || self.transport.state() == .ready { 300 | throw BotAlreadyStartedError() 301 | } 302 | do { 303 | self.transport.setState(state: .authenticating) 304 | 305 | // Send POST request to start the bot 306 | let startBotResult: T = try await fetchStartBot(startBotParams: startBotParams) 307 | self.transport.setState(state: .authenticated) 308 | self.delegate?.onBotStarted(botResponse: startBotResult) 309 | return startBotResult 310 | } catch { 311 | self.disconnect(completion: nil) 312 | self.transport.setState(state: .disconnected) 313 | throw StartBotError(underlyingError: error) 314 | } 315 | } 316 | 317 | /// Initiates bot and retrieves connection parameters (completion-based). 318 | /// 319 | /// This method sends a POST request to the specified endpoint to request a new bot to run 320 | /// and obtain the necessary parameters for establishing a transport connection. 321 | /// 322 | /// - Parameters: 323 | /// - startBotParams: API request configuration including endpoint URL, headers, and request data. 324 | /// - completion: A closure called when the operation completes, containing the result or error. 325 | /// 326 | /// - Note: The client must be in a disconnected state to call this method. 327 | public func startBot( 328 | startBotParams: APIRequest, 329 | completion: ((Result) -> Void)? 330 | ) { 331 | Task { 332 | do { 333 | let startBotResult: T = try await self.startBot(startBotParams: startBotParams) 334 | completion?(.success((startBotResult))) 335 | } catch { 336 | completion?(.failure(AsyncExecutionError(functionName: "start", underlyingError: error))) 337 | } 338 | } 339 | } 340 | 341 | /// Establishes a transport connection with the bot service (async/await). 342 | /// 343 | /// This method creates the underlying transport connection using the provided 344 | /// connection parameters and waits for the bot to signal readiness. 345 | /// 346 | /// - Parameter transportParams: Connection parameters obtained from `startBot()`. 347 | /// 348 | /// - Note: Devices will be automatically initialized if not already done. 349 | public func connect(transportParams: TransportConnectionParams) async throws { 350 | if self.transport.state() == .authenticating || self.transport.state() == .connecting 351 | || self.transport.state() == .connected || self.transport.state() == .ready { 352 | throw BotAlreadyStartedError() 353 | } 354 | do { 355 | self.disconnectRequested = false 356 | if !self.devicesInitialized { 357 | try await self.initDevices() 358 | } 359 | 360 | if self.bailIfDisconnected() { 361 | return 362 | } 363 | 364 | try await self.transport.connect(transportParams: transportParams) 365 | 366 | if self.bailIfDisconnected() { 367 | return 368 | } 369 | } catch { 370 | self.disconnect(completion: nil) 371 | self.transport.setState(state: .disconnected) 372 | throw StartBotError(underlyingError: error) 373 | } 374 | } 375 | 376 | /// Establishes a transport connection with the bot service (completion-based). 377 | /// 378 | /// This method creates the underlying transport connection using the provided 379 | /// connection parameters and waits for the bot to signal readiness. 380 | /// 381 | /// - Parameters: 382 | /// - transportParams: Connection parameters obtained from `startBot()`. 383 | /// - completion: A closure called when the connection attempt completes. 384 | /// 385 | /// - Note: Devices will be automatically initialized if not already done. 386 | public func connect( 387 | transportParams: TransportConnectionParams, 388 | completion: ((Result) -> Void)? 389 | ) { 390 | Task { 391 | do { 392 | try await self.connect(transportParams: transportParams) 393 | completion?(.success(())) 394 | } catch { 395 | completion?(.failure(AsyncExecutionError(functionName: "connect", underlyingError: error))) 396 | } 397 | } 398 | } 399 | 400 | /// Performs bot start request and connection in a single operation (async/await). 401 | /// 402 | /// This convenience method combines `startBot()` and `connect()` into a single call, 403 | /// handling the complete flow from authentication to established connection. 404 | /// 405 | /// - Parameter startBotParams: API request configuration for bot authentication. 406 | /// - Returns: The transport connection parameters used for the connection. 407 | /// - Throws: Various errors related to authentication or connection failures. 408 | public func startBotAndConnect(startBotParams: APIRequest) async throws -> T { 409 | let startBotResult: T = await try self.startBot(startBotParams: startBotParams) 410 | let transportParams = try self.transport.transformStartBotResultToConnectionParams( 411 | startBotParams: startBotParams, 412 | startBotResult: startBotResult 413 | ) 414 | await try self.connect(transportParams: transportParams) 415 | return startBotResult 416 | } 417 | 418 | /// Performs bot start request and connection in a single operation (completion-based). 419 | /// 420 | /// This convenience method combines `startBot()` and `connect()` into a single call, 421 | /// handling the complete flow from authentication to established connection. 422 | /// 423 | /// - Parameters: 424 | /// - startBotParams: API request configuration for bot authentication. 425 | /// - completion: A closure called when the operation completes with the result. 426 | public func startBotAndConnect( 427 | startBotParams: APIRequest, 428 | completion: ((Result) -> Void)? 429 | ) { 430 | Task { 431 | do { 432 | let transportParams: T = try await self.startBotAndConnect(startBotParams: startBotParams) 433 | completion?(.success((transportParams))) 434 | } catch { 435 | completion?(.failure(AsyncExecutionError(functionName: "startBotAndConnect", underlyingError: error))) 436 | } 437 | } 438 | } 439 | 440 | /// Disconnects from the active RTVI session (async/await). 441 | /// 442 | /// This method gracefully closes the transport connection, cleans up resources, 443 | /// and transitions the client to a disconnected state. 444 | /// 445 | /// - Throws: An error if the disconnection process fails. 446 | /// 447 | /// - Note: This method is safe to call multiple times and when already disconnected. 448 | public func disconnect() async throws { 449 | self.transport.setState(state: .disconnecting) 450 | self.disconnectRequested = true 451 | try await self.transport.disconnect() 452 | self.messageDispatcher.disconnect() 453 | } 454 | 455 | /// Disconnects from the active RTVI session (completion-based). 456 | /// 457 | /// This method gracefully closes the transport connection, cleans up resources, 458 | /// and transitions the client to a disconnected state. 459 | /// 460 | /// - Parameter completion: A closure called when the disconnection completes. 461 | /// 462 | /// - Note: This method is safe to call multiple times and when already disconnected. 463 | public func disconnect(completion: ((Result) -> Void)?) { 464 | Task { 465 | do { 466 | try await self.disconnect() 467 | completion?(.success(())) 468 | } catch { 469 | completion?(.failure(AsyncExecutionError(functionName: "disconnect", underlyingError: error))) 470 | } 471 | } 472 | } 473 | 474 | /// The current connection state of the transport layer. 475 | public var state: TransportState { 476 | self.transport.state() 477 | } 478 | 479 | /// Returns `true` if the client is connected and ready for communication. 480 | /// 481 | /// This is a convenience method that checks if the current state is either 482 | /// `.connected` or `.ready`, indicating an active session. 483 | /// 484 | /// - Returns: `true` if connected, `false` otherwise. 485 | public func connected() -> Bool { 486 | return [.connected, .ready].contains(self.transport.state()) 487 | } 488 | 489 | /// The version string of the PipecatClient library. 490 | /// 491 | /// Use this property for debugging, logging, or displaying version information in your app. 492 | public var version: String { 493 | PipecatClient.libraryVersion 494 | } 495 | 496 | /// Returns a list of all available audio input devices. 497 | /// 498 | /// Query this property to present microphone selection options to users 499 | /// or to programmatically select specific audio input devices. 500 | /// 501 | /// - Returns: An array of `MediaDeviceInfo` objects representing available microphones. 502 | public func getAllMics() -> [MediaDeviceInfo] { 503 | return self.transport.getAllMics() 504 | } 505 | 506 | /// Returns a list of all available video input devices. 507 | /// 508 | /// Query this property to present camera selection options to users 509 | /// or to programmatically select specific video input devices. 510 | /// 511 | /// - Returns: An array of `MediaDeviceInfo` objects representing available cameras. 512 | public func getAllCams() -> [MediaDeviceInfo] { 513 | return self.transport.getAllCams() 514 | } 515 | 516 | /// Returns a list of all available audio output devices. 517 | /// 518 | /// - Returns: An array of `MediaDeviceInfo` objects representing available speakers. 519 | /// 520 | /// - Note: On mobile devices, microphone and speaker may be detected as the same device. 521 | public func getAllSpeakers() -> [MediaDeviceInfo] { 522 | // On mobile devices, the microphone and speaker are detected as the same audio device. 523 | self.transport.getAllMics() 524 | } 525 | 526 | /// The currently selected audio input device. 527 | /// 528 | /// This property returns the microphone currently being used for audio capture, 529 | /// or `nil` if no microphone is selected or available. 530 | public var selectedMic: MediaDeviceInfo? { 531 | return self.transport.selectedMic() 532 | } 533 | 534 | /// The currently selected video input device. 535 | /// 536 | /// This property returns the camera currently being used for video capture, 537 | /// or `nil` if no camera is selected or available. 538 | public var selectedCam: MediaDeviceInfo? { 539 | return self.transport.selectedCam() 540 | } 541 | 542 | /// The currently selected audio output device. 543 | /// 544 | /// This property returns the speaker currently being used for audio output, 545 | /// or `nil` if no speaker is selected or available. 546 | /// 547 | /// - Note: On mobile devices, this returns the same device as `selectedMic`. 548 | public var selectedSpeaker: MediaDeviceInfo? { 549 | // On mobile devices, the microphone and speaker are detected as the same audio device. 550 | return self.transport.selectedMic() 551 | } 552 | 553 | /// Switches to the specified audio input device (async/await). 554 | /// 555 | /// Use this method to programmatically change the active microphone during a session. 556 | /// 557 | /// - Parameter micId: The unique identifier of the desired microphone device. 558 | /// - Throws: An error if the device switch fails or the device is not available. 559 | public func updateMic(micId: MediaDeviceId) async throws { 560 | try await self.transport.updateMic(micId: micId) 561 | } 562 | 563 | /// Switches to the specified audio input device (completion-based). 564 | /// 565 | /// Use this method to programmatically change the active microphone during a session. 566 | /// 567 | /// - Parameters: 568 | /// - micId: The unique identifier of the desired microphone device. 569 | /// - completion: A closure called when the device switch completes. 570 | public func updateMic(micId: MediaDeviceId, completion: ((Result) -> Void)?) { 571 | Task { 572 | do { 573 | try await self.updateMic(micId: micId) 574 | completion?(.success(())) 575 | } catch { 576 | completion?(.failure(AsyncExecutionError(functionName: "updateMic", underlyingError: error))) 577 | } 578 | } 579 | } 580 | 581 | /// Switches to the specified video input device (async/await). 582 | /// 583 | /// Use this method to programmatically change the active camera during a session. 584 | /// 585 | /// - Parameter camId: The unique identifier of the desired camera device. 586 | /// - Throws: An error if the device switch fails or the device is not available. 587 | public func updateCam(camId: MediaDeviceId) async throws { 588 | try await self.transport.updateCam(camId: camId) 589 | } 590 | 591 | /// Switches to the specified video input device (completion-based). 592 | /// 593 | /// Use this method to programmatically change the active camera during a session. 594 | /// 595 | /// - Parameters: 596 | /// - camId: The unique identifier of the desired camera device. 597 | /// - completion: A closure called when the device switch completes. 598 | public func updateCam(camId: MediaDeviceId, completion: ((Result) -> Void)?) { 599 | Task { 600 | do { 601 | try await self.updateCam(camId: camId) 602 | completion?(.success(())) 603 | } catch { 604 | completion?(.failure(AsyncExecutionError(functionName: "updateCam", underlyingError: error))) 605 | } 606 | } 607 | } 608 | 609 | /// Switches to the specified audio output device (async/await). 610 | /// 611 | /// Use this method to programmatically change the active speaker during a session. 612 | /// 613 | /// - Parameter speakerId: The unique identifier of the desired speaker device. 614 | /// - Throws: An error if the device switch fails or the device is not available. 615 | /// 616 | /// - Note: On mobile devices, this affects the same device as `updateMic()`. 617 | public func updateSpeaker(speakerId: MediaDeviceId) async throws { 618 | // On mobile devices, the microphone and speaker are detected as the same audio device. 619 | try await self.transport.updateMic(micId: speakerId) 620 | } 621 | 622 | /// Switches to the specified audio output device (completion-based). 623 | /// 624 | /// Use this method to programmatically change the active speaker during a session. 625 | /// 626 | /// - Parameters: 627 | /// - speakerId: The unique identifier of the desired speaker device. 628 | /// - completion: A closure called when the device switch completes. 629 | /// 630 | /// - Note: On mobile devices, this affects the same device as `updateMic()`. 631 | public func updateSpeaker(speakerId: MediaDeviceId, completion: ((Result) -> Void)?) { 632 | Task { 633 | do { 634 | try await self.updateSpeaker(speakerId: speakerId) 635 | completion?(.success(())) 636 | } catch { 637 | completion?(.failure(AsyncExecutionError(functionName: "updateSpeaker", underlyingError: error))) 638 | } 639 | } 640 | } 641 | 642 | /// Enables or disables the audio input device (async/await). 643 | /// 644 | /// Use this method to mute/unmute the microphone during a session without changing the selected device. 645 | /// 646 | /// - Parameter enable: `true` to enable (unmute) the microphone, `false` to disable (mute). 647 | /// - Throws: An error if the operation fails. 648 | public func enableMic(enable: Bool) async throws { 649 | try await self.transport.enableMic(enable: enable) 650 | } 651 | 652 | /// Enables or disables the audio input device (completion-based). 653 | /// 654 | /// Use this method to mute/unmute the microphone during a session without changing the selected device. 655 | /// 656 | /// - Parameters: 657 | /// - enable: `true` to enable (unmute) the microphone, `false` to disable (mute). 658 | /// - completion: A closure called when the operation completes. 659 | public func enableMic(enable: Bool, completion: ((Result) -> Void)?) { 660 | Task { 661 | do { 662 | try await self.enableMic(enable: enable) 663 | completion?(.success(())) 664 | } catch { 665 | completion?(.failure(AsyncExecutionError(functionName: "enableMic", underlyingError: error))) 666 | } 667 | } 668 | } 669 | 670 | /// Enables or disables the video input device (async/await). 671 | /// 672 | /// Use this method to show/hide video during a session without changing the selected camera. 673 | /// 674 | /// - Parameter enable: `true` to enable (show) the camera, `false` to disable (hide). 675 | /// - Throws: An error if the operation fails. 676 | public func enableCam(enable: Bool) async throws { 677 | try await self.transport.enableCam(enable: enable) 678 | } 679 | 680 | /// Enables or disables the video input device (completion-based). 681 | /// 682 | /// Use this method to show/hide video during a session without changing the selected camera. 683 | /// 684 | /// - Parameters: 685 | /// - enable: `true` to enable (show) the camera, `false` to disable (hide). 686 | /// - completion: A closure called when the operation completes. 687 | public func enableCam(enable: Bool, completion: ((Result) -> Void)?) { 688 | Task { 689 | do { 690 | try await self.enableCam(enable: enable) 691 | completion?(.success(())) 692 | } catch { 693 | completion?(.failure(AsyncExecutionError(functionName: "enableCam", underlyingError: error))) 694 | } 695 | } 696 | } 697 | 698 | /// Indicates whether the microphone is currently enabled (unmuted). 699 | /// 700 | /// - Returns: `true` if the microphone is enabled and capturing audio, `false` if muted. 701 | public var isMicEnabled: Bool { 702 | self.transport.isMicEnabled() 703 | } 704 | 705 | /// Indicates whether the camera is currently enabled (showing video). 706 | /// 707 | /// - Returns: `true` if the camera is enabled and capturing video, `false` if hidden. 708 | public var isCamEnabled: Bool { 709 | self.transport.isCamEnabled() 710 | } 711 | 712 | // TODO: need to add support for screen share in the future 713 | /// Enables or disables the video input device. 714 | /*public func enableScreenShare(enable: Bool) async throws { 715 | } 716 | public var isScreenShareEnabled: Bool { 717 | }*/ 718 | 719 | /// Returns the current media tracks for all participants in the session. 720 | /// 721 | /// This property provides access to the audio and video streams for both local and remote participants. 722 | /// Use this to implement custom UI components for displaying participant media. 723 | /// 724 | /// - Returns: A `Tracks` object containing local and remote media track information, or `nil` if not connected. 725 | var tracks: Tracks? { 726 | return self.transport.tracks() 727 | } 728 | 729 | /// Releases all resources and cleans up the PipecatClient instance. 730 | /// 731 | /// Call this method when you're finished with the client to ensure proper cleanup 732 | /// of media devices, network connections, and other system resources. 733 | /// 734 | /// - Important: After calling this method, the client instance should not be used further. 735 | public func release() { 736 | self.transport.release() 737 | } 738 | 739 | func assertReady() throws { 740 | if self.state != .ready { 741 | throw BotNotReadyError() 742 | } 743 | } 744 | 745 | // ------ Messages 746 | func sendMessage(msg: RTVIMessageOutbound) throws { 747 | try self.transport.sendMessage(message: msg) 748 | } 749 | 750 | func sendMessage(msg: RTVIMessageOutbound, completion: ((Result) -> Void)?) { 751 | Task { 752 | do { 753 | try await self.sendMessage(msg: msg) 754 | completion?(.success(())) 755 | } catch { 756 | completion?(.failure(AsyncExecutionError(functionName: "sendMessage", underlyingError: error))) 757 | } 758 | } 759 | } 760 | 761 | func dispatchMessage(message: RTVIMessageOutbound) async throws -> T { 762 | let voiceMessageResponse = try await self.messageDispatcher.dispatchAsync(message: message) 763 | return try JSONDecoder().decode(T.self, from: Data(voiceMessageResponse.data!.utf8)) 764 | } 765 | 766 | /// Sends a one-way message to the bot without expecting a response. 767 | /// 768 | /// Use this method to send fire-and-forget messages or notifications to the bot. 769 | /// 770 | /// - Parameters: 771 | /// - msgType: A string identifier for the message type. 772 | /// - data: Optional message payload data. 773 | /// - Throws: `BotNotReadyError` if the bot is not ready, or other transport-related errors. 774 | public func sendClientMessage(msgType: String, data: Value? = nil) throws { 775 | try self.assertReady() 776 | try self.sendMessage(msg: .clientMessage(msgType: msgType, data: data)) 777 | } 778 | 779 | /// Sends a request message to the bot and waits for a response (async/await). 780 | /// 781 | /// Use this method for request-response communication patterns with the bot. 782 | /// 783 | /// - Parameters: 784 | /// - msgType: A string identifier for the request type. 785 | /// - data: Optional request payload data. 786 | /// - Returns: The bot's response as `ClientMessageData`. 787 | /// - Throws: `BotNotReadyError` if the bot is not ready, or other communication errors. 788 | public func sendClientRequest(msgType: String, data: Value? = nil) async throws -> ClientMessageData { 789 | try self.assertReady() 790 | return try await self.dispatchMessage( 791 | message: .clientMessage(msgType: msgType, data: data) 792 | ) 793 | } 794 | 795 | /// Sends a request message to the bot and waits for a response (completion-based). 796 | /// 797 | /// Use this method for request-response communication patterns with the bot. 798 | /// 799 | /// - Parameters: 800 | /// - msgType: A string identifier for the request type. 801 | /// - data: Optional request payload data. 802 | /// - completion: A closure called when the response is received or an error occurs. 803 | public func sendClientRequest( 804 | msgType: String, 805 | data: Value? = nil, 806 | completion: ((Result) -> Void)? 807 | ) { 808 | Task { 809 | do { 810 | let response = try await self.sendClientRequest(msgType: msgType, data: data) 811 | completion?(.success((response))) 812 | } catch { 813 | completion?(.failure(AsyncExecutionError(functionName: "sendClientRequest", underlyingError: error))) 814 | } 815 | } 816 | } 817 | 818 | /// Appends a message to the bot's LLM conversation context (async/await). 819 | /// 820 | /// This method programmatically adds a message to the Large Language Model's conversation history, 821 | /// allowing you to inject user context, assistant responses, or other relevant 822 | /// information that will influence the bot's subsequent responses. 823 | /// 824 | /// The context message becomes part of the LLM's memory for the current session and will be 825 | /// considered when generating future responses. 826 | /// 827 | /// - Parameter message: An `LLMContextMessage` containing the role (user, assistant), 828 | /// content to add to the conversation context and a flag for whether the bot should respond immediately. 829 | /// - Throws: 830 | /// - `BotNotReadyError` if the bot is not in a ready state to accept context updates 831 | /// - Communication errors if the message fails to send or receive a response 832 | /// 833 | /// - Important: The bot must be in a `.ready` state for this method to succeed. 834 | /// - Note: Context messages persist only for the current session and are cleared when disconnecting. 835 | @available(*, deprecated, message: "Use sendText() instead. This method will be removed in a future version.") 836 | public func appendToContext(message: LLMContextMessage) async throws { 837 | try self.assertReady() 838 | try self.sendMessage(msg: .appendToContext(msg: message)) 839 | } 840 | 841 | /// Appends a message to the bot's LLM conversation context (completion-based). 842 | /// 843 | /// This method programmatically adds a message to the Large Language Model's conversation history, 844 | /// allowing you to inject user context, assistant responses, or other relevant 845 | /// information that will influence the bot's subsequent responses. 846 | /// 847 | /// The context message becomes part of the LLM's memory for the current session and will be 848 | /// considered when generating future responses. 849 | /// 850 | /// - Parameters: 851 | /// - message: An `LLMContextMessage` containing the role (user, assistant), 852 | /// content to add to the conversation context and a flag for whether the bot should respond immediately. 853 | /// - completion: A closure called when the context update operation completes. 854 | /// Contains a `Result` with either the bot's acknowledgment 855 | /// response or an error describing what went wrong. 856 | /// 857 | /// - Important: The bot must be in a `.ready` state for this method to succeed. 858 | /// - Note: Context messages persist only for the current session and are cleared when disconnecting. 859 | @available(*, deprecated, message: "Use sendText() instead. This method will be removed in a future version.") 860 | public func appendToContext( 861 | message: LLMContextMessage, 862 | completion: ((Result) -> Void)? 863 | ) { 864 | Task { 865 | do { 866 | try await self.appendToContext(message: message) 867 | completion?(.success(())) 868 | } catch { 869 | completion?(.failure(AsyncExecutionError(functionName: "appendToContext", underlyingError: error))) 870 | } 871 | } 872 | } 873 | 874 | /// Sends a text message to the bot for processing. 875 | /// 876 | /// This method sends a text message directly to the bot, which will be processed by the 877 | /// Large Language Model and may generate a spoken response. Unlike `appendToContext()`, 878 | /// this method defaults to `run_immediately = true`, meaning the bot will process and 879 | /// respond to the message immediately. 880 | /// 881 | /// - Parameters: 882 | /// - content: The text content to send to the bot for processing. 883 | /// - options: Optional `SendTextOptions` to customize the message behavior. 884 | /// - Throws: 885 | /// - `BotNotReadyError` if the bot is not in a ready state to accept messages 886 | /// - Communication errors if the message fails to send 887 | /// 888 | /// - Important: The bot must be in a `.ready` state for this method to succeed. 889 | /// - Note: This is the preferred method for sending text messages to the bot. 890 | public func sendText(content: String, options: SendTextOptions? = nil) throws { 891 | try self.assertReady() 892 | try self.sendMessage(msg: .sendText(content: content, options: options)) 893 | } 894 | 895 | /// Sends a disconnect signal to the bot while maintaining the transport connection. 896 | /// 897 | /// This method instructs the bot to gracefully end the current conversation session 898 | /// and clean up its internal state, but keeps the underlying transport connection 899 | /// (WebRTC, WebSocket, etc.) active. This is different from `disconnect()` which 900 | /// closes the entire connection. 901 | /// 902 | /// - Throws: 903 | /// - `BotNotReadyError` if the bot is not in a ready state to accept the disconnect command 904 | /// - Transport errors if the disconnect message fails to send 905 | /// 906 | /// - Important: The bot must be in a `.ready` state for this method to succeed. 907 | /// - Note: This method sends a fire-and-forget message and does not wait for acknowledgment. 908 | /// The bot state change will be reflected through delegate callbacks. 909 | /// - SeeAlso: `disconnect()` for closing the entire transport connection. 910 | public func disconnectBot() throws { 911 | try self.assertReady() 912 | try self.sendMessage( 913 | msg: .disconnectBot() 914 | ) 915 | } 916 | 917 | /// Registers a function call handler for a specific function name. 918 | /// 919 | /// When the bot calls a function with the specified name, the registered callback 920 | /// will be invoked instead of the delegate's `onLLMFunctionCall` method. 921 | /// 922 | /// - Parameters: 923 | /// - functionName: The name of the function to handle. 924 | /// - callback: The callback to invoke when this function is called. 925 | public func registerFunctionCallHandler( 926 | functionName: String, 927 | callback: @escaping FunctionCallCallback 928 | ) { 929 | functionCallCallbacks[functionName] = callback 930 | } 931 | 932 | /// Unregisters a function call handler for a specific function name. 933 | /// 934 | /// After calling this method, function calls with the specified name will 935 | /// be handled by the delegate's `onLLMFunctionCall` method instead. 936 | /// 937 | /// - Parameter functionName: The name of the function to unregister. 938 | public func unregisterFunctionCallHandler(functionName: String) { 939 | functionCallCallbacks.removeValue(forKey: functionName) 940 | } 941 | 942 | /// Unregisters all function call handlers. 943 | /// 944 | /// After calling this method, all function calls will be handled by 945 | /// the delegate's `onLLMFunctionCall` method. 946 | public func unregisterAllFunctionCallHandlers() { 947 | functionCallCallbacks.removeAll() 948 | } 949 | 950 | } 951 | --------------------------------------------------------------------------------