├── .gitignore ├── .prettierrc ├── README.md ├── client ├── android │ ├── api-reference.mdx │ ├── introduction.mdx │ └── transports │ │ ├── daily.mdx │ │ ├── gemini-websocket.mdx │ │ └── openai-webrtc.mdx ├── c++ │ ├── api-reference.mdx │ ├── introduction.mdx │ └── transport.mdx ├── introduction.mdx ├── ios │ ├── api-reference.mdx │ ├── introduction.mdx │ └── transports │ │ ├── daily.mdx │ │ ├── gemini-websocket.mdx │ │ └── openai-webrtc.mdx ├── js │ ├── api-reference │ │ ├── actions.mdx │ │ ├── callbacks.mdx │ │ ├── client-constructor.mdx │ │ ├── client-methods.mdx │ │ ├── configuration.mdx │ │ ├── errors.mdx │ │ ├── messages.mdx │ │ └── services.mdx │ ├── helpers │ │ ├── introduction.mdx │ │ └── llm.mdx │ ├── introduction.mdx │ └── transports │ │ ├── daily.mdx │ │ ├── gemini.mdx │ │ ├── openai-webrtc.mdx │ │ ├── realtime.mdx │ │ ├── small-webrtc.mdx │ │ └── transport.mdx ├── react-native │ ├── api-reference.mdx │ └── introduction.mdx └── react │ ├── components.mdx │ ├── hooks.mdx │ └── introduction.mdx ├── examples.mdx ├── favicon.svg ├── getting-started ├── core-concepts.mdx ├── installation.mdx ├── next-steps.mdx ├── overview.mdx └── quickstart.mdx ├── guides ├── deployment │ ├── cerebrium.mdx │ ├── fly.mdx │ ├── images │ │ ├── deployment-1.png │ │ ├── deployment-fly.png │ │ └── modal.jpg │ ├── modal.mdx │ ├── overview.mdx │ ├── pattern.mdx │ └── pipecat-cloud.mdx ├── features │ ├── gemini-multimodal-live.mdx │ ├── krisp.mdx │ ├── metrics.mdx │ ├── openai-audio-models-and-apis.mdx │ └── pipecat-flows.mdx ├── fundamentals │ ├── detecting-user-idle.mdx │ ├── end-pipeline.mdx │ ├── function-calling.mdx │ ├── recording-audio.mdx │ ├── recording-transcripts.mdx │ └── user-input-muting.mdx ├── introduction.mdx └── telephony │ ├── daily-webrtc.mdx │ ├── dialout.mdx │ ├── images │ └── twilio-webhook-setup.png │ ├── overview.mdx │ ├── twilio-daily-webrtc.mdx │ └── twilio-websockets.mdx ├── images ├── allow-krisp.png ├── architecture-1.png ├── architecture-2.png ├── async-frame-processor.svg ├── async-parallel-pipeline.svg ├── checks-passed.png ├── food-ordering-flow.png ├── gemini-client-final.png ├── hero-dark.svg ├── hero-light.svg ├── krisp-portal.png ├── open-anyway-krisp.png ├── openai-cascade.jpg ├── openai-s2s.jpg ├── openai-twilio.png ├── simple-pipeline.svg ├── sync-frame-processor.svg └── sync-parallel-pipeline.svg ├── logo ├── dark.svg ├── light.svg ├── pipecat-logo.png ├── pipecat-logo.svg └── pipecat-social.png ├── mint.json ├── pipecat-docs.png ├── server ├── base-classes │ ├── introduction.mdx │ ├── llm.mdx │ ├── mcp │ │ └── mcp.mdx │ ├── media.mdx │ ├── speech.mdx │ └── transport.mdx ├── frameworks │ ├── flows │ │ └── pipecat-flows.mdx │ └── rtvi │ │ ├── introduction.mdx │ │ ├── rtvi-observer.mdx │ │ └── rtvi-processor.mdx ├── introduction.mdx ├── links │ └── server-reference.mdx ├── pipeline │ ├── heartbeats.mdx │ ├── parallel-pipeline.mdx │ ├── pipeline-idle-detection.mdx │ ├── pipeline-params.mdx │ └── pipeline-task.mdx ├── services │ ├── analytics │ │ └── sentry.mdx │ ├── image-generation │ │ ├── fal.mdx │ │ ├── google-imagen.mdx │ │ └── openai.mdx │ ├── llm │ │ ├── anthropic.mdx │ │ ├── aws.mdx │ │ ├── azure.mdx │ │ ├── cerebras.mdx │ │ ├── deepseek.mdx │ │ ├── fireworks.mdx │ │ ├── gemini.mdx │ │ ├── google-vertex.mdx │ │ ├── grok.mdx │ │ ├── groq.mdx │ │ ├── nim.mdx │ │ ├── ollama.mdx │ │ ├── openai.mdx │ │ ├── openpipe.mdx │ │ ├── openrouter.mdx │ │ ├── perplexity.mdx │ │ ├── qwen.mdx │ │ └── together.mdx │ ├── memory │ │ └── mem0.mdx │ ├── s2s │ │ ├── aws.mdx │ │ ├── gemini.mdx │ │ └── openai.mdx │ ├── stt │ │ ├── assemblyai.mdx │ │ ├── aws.mdx │ │ ├── azure.mdx │ │ ├── deepgram.mdx │ │ ├── fal.mdx │ │ ├── gladia.mdx │ │ ├── google.mdx │ │ ├── groq.mdx │ │ ├── openai.mdx │ │ ├── riva.mdx │ │ ├── ultravox.mdx │ │ └── whisper.mdx │ ├── supported-services.mdx │ ├── transport │ │ ├── daily.mdx │ │ ├── fastapi-websocket.mdx │ │ ├── small-webrtc.mdx │ │ └── websocket-server.mdx │ ├── tts │ │ ├── aws.mdx │ │ ├── azure.mdx │ │ ├── cartesia.mdx │ │ ├── deepgram.mdx │ │ ├── elevenlabs.mdx │ │ ├── fish.mdx │ │ ├── google.mdx │ │ ├── groq.mdx │ │ ├── lmnt.mdx │ │ ├── minimax.mdx │ │ ├── neuphonic.mdx │ │ ├── openai.mdx │ │ ├── piper.mdx │ │ ├── playht.mdx │ │ ├── rime.mdx │ │ ├── riva.mdx │ │ ├── sarvam.mdx │ │ └── xtts.mdx │ ├── video │ │ ├── simli.mdx │ │ └── tavus.mdx │ └── vision │ │ └── moondream.mdx └── utilities │ ├── audio │ ├── audio-recording.mdx │ ├── koala-filter.mdx │ ├── krisp-filter.mdx │ ├── noisereduce-filter.mdx │ ├── silero-vad-analyzer.mdx │ └── soundfile-mixer.mdx │ ├── daily │ └── rest-helpers.mdx │ ├── dtmf-aggregator.mdx │ ├── filters │ ├── frame-filter.mdx │ ├── function-filter.mdx │ ├── identify-filter.mdx │ ├── null-filter.mdx │ ├── stt-mute.mdx │ ├── wake-check-filter.mdx │ └── wake-notifier-filter.mdx │ ├── frame │ └── producer-consumer.mdx │ ├── interruption-strategies.mdx │ ├── observers │ ├── debug-observer.mdx │ ├── llm-observer.mdx │ ├── observer-pattern.mdx │ ├── transcription-observer.mdx │ └── turn-tracking-observer.mdx │ ├── opentelemetry.mdx │ ├── serializers │ ├── introduction.mdx │ ├── plivo.mdx │ ├── telnyx.mdx │ └── twilio.mdx │ ├── smart-turn │ ├── fal-smart-turn.mdx │ ├── local-coreml-smart-turn.mdx │ └── smart-turn-overview.mdx │ ├── text │ ├── markdown-text-filter.mdx │ └── pattern-pair-aggregator.mdx │ ├── transcript-processor.mdx │ └── user-idle-processor.mdx └── snippets └── snippet-intro.mdx /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "tabWidth": 2, 4 | "useTabs": false, 5 | "singleQuote": false 6 | } 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | pipecat 3 |

4 | 5 | Welcome to the Pipecat documentation repository! This project contains the official [documentation](https://docs.pipecat.ai)] for the [Pipecat](https://github.com/pipecat-ai/pipecat) open-source project. 6 | 7 | This repository is deployed on [docs.pipecat.ai](https://docs.pipecat.ai). 8 | 9 | ## What is Pipecat 10 | 11 | `pipecat` is a framework for building voice (and multimodal) conversational agents. Things like personal coaches, meeting assistants, [storytelling toys for kids](https://storytelling-chatbot.fly.dev/), customer support bots, [intake flows](https://www.youtube.com/watch?v=lDevgsp9vn0), and snarky social companions. 12 | 13 | ## Documentation Structure 14 | 15 | This repository is dedicated to maintaining up-to-date, high-quality documentation to support users and contributors. Here you’ll find: 16 | 17 | - **User Guides**: Step-by-step instructions to get started with Pipecat. 18 | - **API Documentation**: Detailed API references. 19 | - **Tutorials**: Hands-on tutorials to help you automate your workflows. 20 | 21 | ## Contributing to the Documentation 22 | 23 | We welcome contributions of all kinds! Whether you're fixing a typo, adding a new section, or improving the readability of the existing content, your help is appreciated. Follow these steps to get involved: 24 | 25 | 1. **Fork this repository**: Start by forking the Pipecat Documentation repository to your GitHub account. 26 | 27 | 2. **Clone the repository**: Clone your forked repository to your local machine. 28 | ```bash 29 | git clone https://github.com/your-username/docs 30 | ``` 31 | 3. **Create a branch**: For your contribution, create a new branch. 32 | ```bash 33 | git checkout -b your-branch-name 34 | ``` 35 | 4. **Make your changes**: Edit or add files as necessary. 36 | 5. **Test your changes**: Ensure that your changes look correct and follow the style guide. Refer to **Development** section to test portal locally. 37 | 6. **Commit your changes**: Once you're satisfied with your changes, commit them with a meaningful message. 38 | 39 | ```bash 40 | git commit -m "Description of your changes" 41 | ``` 42 | 43 | 7. **Push your changes**: Push your branch to your forked repository. 44 | 45 | ```bash 46 | git push origin your-branch-name 47 | ``` 48 | 49 | 9. **Submit a Pull Request (PR)**: Open a PR from your forked repository to the main branch of this repo. Describe the changes you've made clearly. 50 | 51 | Our maintainers will review your PR, and once everything is good, your contributions will be merged! 52 | 53 | ### Development 54 | 55 | The documentation is using [Mintlify](https://mintlify.com/) to render beautifuly. Mintlify comes with a set of [components](https://mintlify.com/docs/content/components/) to help you write better more interactive documentation. 56 | 57 | Install the [Mintlify CLI](https://www.npmjs.com/package/mintlify) to preview the documentation changes locally. To install, use the following command 58 | 59 | ``` 60 | npm i -g mintlify 61 | ``` 62 | 63 | Run the following command at the root of your documentation (where mint.json is) 64 | 65 | ``` 66 | mintlify dev 67 | ``` 68 | 69 | Open `https://localhost:3000` in your browser and check your changes. 70 | 71 | #### Troubleshooting 72 | 73 | - Mintlify dev isn't running - Run `mintlify install` it'll re-install dependencies. 74 | - Page loads as a 404 - Make sure you are running in a folder with `mint.json` 75 | -------------------------------------------------------------------------------- /client/android/api-reference.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API Reference" 3 | url: "https://docs-android.rtvi.ai/" 4 | --- 5 | -------------------------------------------------------------------------------- /client/android/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "SDK Introduction" 3 | description: "Build Android applications with Pipecat's Kotlin client library" 4 | --- 5 | 6 | The Pipecat Android SDK provides a Kotlin implementation for building voice and multimodal AI applications on Android. It handles: 7 | 8 | - Real-time audio and video streaming 9 | - Bot communication and state management 10 | - Media device handling 11 | - Configuration management 12 | - Event handling 13 | 14 | ## Installation 15 | 16 | Add the dependency for your chosen transport to your `build.gradle` file. For example, to use the Daily transport: 17 | 18 | ```gradle 19 | implementation "ai.pipecat:daily-transport:0.3.3" 20 | ``` 21 | 22 | ## Example 23 | 24 | Here's a simple example using Daily as the transport layer. Note that the `clientConfig` is optional and depends 25 | on what is required by the bot backend. 26 | 27 | ```kotlin 28 | val clientConfig = listOf( 29 | ServiceConfig( 30 | service = "llm", 31 | options = listOf( 32 | Option("model", "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"), 33 | Option("messages", Value.Array( 34 | Value.Object( 35 | "role" to Value.Str("system"), 36 | "content" to Value.Str("You are a helpful assistant.") 37 | ) 38 | )) 39 | ) 40 | ), 41 | ServiceConfig( 42 | service = "tts", 43 | options = listOf( 44 | Option("voice", "79a125e8-cd45-4c13-8a67-188112f4dd22") 45 | ) 46 | ) 47 | ) 48 | 49 | val callbacks = object : RTVIEventCallbacks() { 50 | override fun onBackendError(message: String) { 51 | Log.e(TAG, "Error from backend: $message") 52 | } 53 | } 54 | 55 | val options = RTVIClientOptions( 56 | services = listOf(ServiceRegistration("llm", "together"), ServiceRegistration("tts", "cartesia")), 57 | params = RTVIClientParams(baseUrl = "", config = clientConfig) 58 | ) 59 | 60 | val client = RTVIClient(DailyTransport.Factory(context), callbacks, options) 61 | client.connect().await() // Using Coroutines 62 | 63 | // Or using callbacks: 64 | // client.start().withCallback { /* handle completion */ } 65 | ``` 66 | 67 | ## Documentation 68 | 69 | 70 | 75 | Complete SDK API documentation 76 | 77 | 78 | 79 | WebRTC implementation using Daily 80 | 81 | 82 | -------------------------------------------------------------------------------- /client/android/transports/daily.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Daily WebRTC Transport" 3 | description: "WebRTC implementation for Android using Daily" 4 | --- 5 | 6 | The Daily transport implementation enables real-time audio and video communication in your Pipecat Android applications using [Daily's](https://www.daily.co/) WebRTC infrastructure. 7 | 8 | ## Installation 9 | 10 | Add the Daily transport dependency to your `build.gradle`: 11 | 12 | ```gradle 13 | implementation "ai.pipecat:daily-transport:0.3.3" 14 | ``` 15 | 16 | ## Usage 17 | 18 | Create a client using the Daily transport: 19 | 20 | ```kotlin 21 | val callbacks = object : RTVIEventCallbacks() { 22 | override fun onBackendError(message: String) { 23 | Log.e(TAG, "Error from backend: $message") 24 | } 25 | } 26 | 27 | val options = RTVIClientOptions( 28 | services = listOf(ServiceRegistration("llm", "together"), ServiceRegistration("tts", "cartesia")), 29 | params = RTVIClientParams(baseUrl = "", config = clientConfig) 30 | ) 31 | 32 | val client = RTVIClient(DailyTransport.Factory(context), callbacks, options) 33 | client.connect().await() 34 | ``` 35 | 36 | ## Configuration 37 | 38 | Your server endpoint should return Daily-specific configuration: 39 | 40 | ```json 41 | { 42 | "url": "https://your-domain.daily.co/room-name", 43 | "token": "your-daily-token" 44 | } 45 | ``` 46 | 47 | ## Resources 48 | 49 | 50 | 56 | Simple Chatbot Demo 57 | 58 | 59 | 65 | Client Transports 66 | 67 | 68 | 69 | 74 | Complete API documentation for the Daily transport implementation 75 | -------------------------------------------------------------------------------- /client/android/transports/gemini-websocket.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Gemini Live Websocket Transport" 3 | description: "Websocket implementation for Android using Gemini" 4 | --- 5 | 6 | The Gemini Live Websocket transport implementation enables real-time audio communication with the Gemini Multimodal Live service, using a direct websocket connection. 7 | 8 | 9 | Transports of this type are designed primarily for development and testing 10 | purposes. For production applications, you will need to build a server 11 | component with a server-friendly transport, like the 12 | [DailyTransport](./daily), to securely handle API keys. 13 | 14 | 15 | ## Installation 16 | 17 | Add the transport dependency to your `build.gradle`: 18 | 19 | ```gradle 20 | implementation "ai.pipecat:gemini-live-websocket-transport:0.3.3" 21 | ``` 22 | 23 | ## Usage 24 | 25 | Create a client: 26 | 27 | ```kotlin 28 | val transport = GeminiLiveWebsocketTransport.Factory(context) 29 | 30 | val options = RTVIClientOptions( 31 | params = RTVIClientParams( 32 | baseUrl = null, 33 | config = GeminiLiveWebsocketTransport.buildConfig( 34 | apiKey = "", 35 | generationConfig = Value.Object( 36 | "speech_config" to Value.Object( 37 | "voice_config" to Value.Object( 38 | "prebuilt_voice_config" to Value.Object( 39 | "voice_name" to Value.Str("Puck") 40 | ) 41 | ) 42 | ) 43 | ), 44 | initialUserMessage = "How tall is the Eiffel Tower?" 45 | ) 46 | ) 47 | ) 48 | 49 | val client = RTVIClient(transport, callbacks, options) 50 | 51 | client.start().withCallback { 52 | // ... 53 | } 54 | ``` 55 | 56 | ## Resources 57 | 58 | 59 | 65 | Simple Chatbot Demo 66 | 67 | 68 | 74 | Client Transports 75 | 76 | 77 | 78 | 83 | Complete API documentation for the Pipecat Android client. 84 | 85 | -------------------------------------------------------------------------------- /client/android/transports/openai-webrtc.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "OpenAI Realtime WebRTC Transport" 3 | description: "WebRTC implementation for Android using OpenAI" 4 | --- 5 | 6 | The OpenAI Realtime WebRTC transport implementation enables real-time audio communication with the OpenAI Realtime service, using a direct WebRTC connection. 7 | 8 | ## Installation 9 | 10 | Add the transport dependency to your `build.gradle`: 11 | 12 | ```gradle 13 | implementation "ai.pipecat:openai-realtime-webrtc-transport:0.3.3" 14 | ``` 15 | 16 | ## Usage 17 | 18 | Create a client: 19 | 20 | ```kotlin 21 | val transport = OpenAIRealtimeWebRTCTransport.Factory(context) 22 | 23 | val options = RTVIClientOptions( 24 | params = RTVIClientParams( 25 | baseUrl = null, 26 | config = OpenAIRealtimeWebRTCTransport.buildConfig( 27 | apiKey = apiKey, 28 | initialMessages = listOf( 29 | LLMContextMessage(role = "user", content = "How tall is the Eiffel Tower?") 30 | ), 31 | initialConfig = OpenAIRealtimeSessionConfig( 32 | voice = "ballad", 33 | turnDetection = Value.Object("type" to Value.Str("semantic_vad")), 34 | inputAudioNoiseReduction = Value.Object("type" to Value.Str("near_field")), 35 | inputAudioTranscription = Value.Object("model" to Value.Str("gpt-4o-transcribe")) 36 | ) 37 | ) 38 | ) 39 | ) 40 | 41 | val client = RTVIClient(transport, callbacks, options) 42 | 43 | client.start().withCallback { 44 | // ... 45 | } 46 | ``` 47 | 48 | ## Resources 49 | 50 | 51 | 57 | Simple Chatbot Demo 58 | 59 | 60 | 66 | Client Transports 67 | 68 | 69 | 70 | 75 | Complete API documentation for the Pipecat Android client. 76 | 77 | -------------------------------------------------------------------------------- /client/c++/api-reference.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API Reference" 3 | url: "https://github.com/pipecat-ai/pipecat-client-cxx" 4 | --- 5 | -------------------------------------------------------------------------------- /client/c++/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "SDK Introduction" 3 | description: "Build native applications with Pipecat’s C++ client library" 4 | --- 5 | 6 | The Pipecat C++ SDK provides a native implementation for building voice and multimodal AI applications. It supports: 7 | 8 | - Linux (`x86_64` and `aarch64`) 9 | - macOS (`aarch64`) 10 | - Windows (`x86_64`) 11 | 12 | ## Dependencies 13 | 14 | ### libcurl 15 | 16 | The SDK uses [libcurl](https://curl.se/libcurl/) for HTTP requests. 17 | 18 | 19 | 20 | 21 | ```bash 22 | sudo apt-get install libcurl4-openssl-dev 23 | ``` 24 | 25 | 26 | 27 | 28 | On macOS `libcurl` is already included so there is nothing to install. 29 | 30 | 31 | 32 | 33 | On Windows we use [vcpkg](https://vcpkg.io/en/) to install dependencies. You 34 | need to set it up following one of the 35 | [tutorials](https://learn.microsoft.com/en-us/vcpkg/get_started/get-started). 36 | 37 | The `libcurl` dependency will be automatically downloaded when building. 38 | 39 | 40 | 41 | 42 | ## Installation 43 | 44 | Build the SDK using CMake: 45 | 46 | 47 | 48 | ```bash 49 | cmake . -G Ninja -Bbuild -DCMAKE_BUILD_TYPE=Release 50 | ninja -C build 51 | ``` 52 | 53 | 54 | 55 | ```bash 56 | # Initialize Visual Studio environment 57 | "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Auxiliary\Build\vcvarsall.bat" amd64 58 | # Configure and build 59 | cmake . -Bbuild --preset vcpkg 60 | cmake --buildbuild --config Release 61 | ``` 62 | 63 | 64 | 65 | 66 | ### Cross-compilation 67 | 68 | For Linux aarch64: 69 | 70 | ```bash 71 | cmake . -G Ninja -Bbuild -DCMAKE_TOOLCHAIN_FILE=aarch64-linux-toolchain.cmake -DCMAKE_BUILD_TYPE=Release 72 | ninja -C build 73 | ``` 74 | 75 | ## Documentation 76 | 77 | 78 | 83 | Complete SDK API documentation 84 | 85 | 86 | 87 | WebRTC implementation using Daily 88 | 89 | 90 | -------------------------------------------------------------------------------- /client/c++/transport.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Daily WebRTC Transport" 3 | description: "WebRTC implementation for C++ using Daily" 4 | --- 5 | 6 | The Daily transport implementation enables real-time audio and video communication in your Pipecat C++ applications using [Daily's](https://www.daily.co/) WebRTC infrastructure. 7 | 8 | ## Dependencies 9 | 10 | ### Daily Core C++ SDK 11 | 12 | Download the [Daily Core C++ SDK](https://github.com/daily-co/daily-core-sdk) from the [available releases](https://github.com/daily-co/daily-core-sdk/releases) for your platform and set: 13 | 14 | ```bash 15 | export DAILY_CORE_PATH=/path/to/daily-core-sdk 16 | ``` 17 | 18 | ### Pipecat C++ SDK 19 | 20 | Build the base [Pipecat C++ SDK](https://github.com/pipecat-ai/pipecat-client-cxx-daily) first and set: 21 | 22 | ```bash 23 | export PIPECAT_SDK_PATH=/path/to/pipecat-client-cxx 24 | ``` 25 | 26 | ## Building 27 | 28 | First, set a few environment variables: 29 | 30 | ```bash 31 | PIPECAT_SDK_PATH=/path/to/pipecat-client-cxx 32 | DAILY_CORE_PATH=/path/to/daily-core-sdk 33 | ``` 34 | 35 | Then, build the project: 36 | 37 | 38 | 39 | 40 | ```bash 41 | cmake . -G Ninja -Bbuild -DCMAKE_BUILD_TYPE=Release 42 | ninja -C build 43 | ``` 44 | 45 | 46 | 47 | 48 | ```bash 49 | # Initialize Visual Studio environment 50 | "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Auxiliary\Build\vcvarsall.bat" amd64 51 | 52 | # Configure and build 53 | cmake . -Bbuild --preset vcpkg 54 | cmake --build build --config Release 55 | ``` 56 | 57 | 58 | 59 | 60 | ## Examples 61 | 62 | 63 | 68 | Simple C++ implementation example 69 | 70 | 71 | {" "} 72 | 73 | 78 | C++ client with PortAudio support 79 | 80 | 81 | 86 | Example Node.js proxy implementation 87 | 88 | 89 | -------------------------------------------------------------------------------- /client/ios/api-reference.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API Reference" 3 | url: "https://docs-ios.pipecat.ai/" 4 | --- 5 | -------------------------------------------------------------------------------- /client/ios/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "SDK Introduction" 3 | description: "Build iOS applications with Pipecat’s Swift client library" 4 | --- 5 | 6 | The Pipecat iOS SDK provides a Swift implementation for building voice and multimodal AI applications on iOS. It handles: 7 | 8 | - Real-time audio streaming 9 | - Bot communication and state management 10 | - Media device handling 11 | - Configuration management 12 | - Event handling 13 | 14 | ## Installation 15 | 16 | Add the SDK to your project using Swift Package Manager: 17 | 18 | ```swift 19 | // Core SDK 20 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios.git", from: "0.3.0"), 21 | 22 | // Daily transport implementation 23 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios-daily.git", from: "0.3.0"), 24 | ``` 25 | 26 | Then add the dependencies to your target: 27 | 28 | ```swift 29 | .target(name: "YourApp", dependencies: [ 30 | .product(name: "PipecatClientIOS", package: "pipecat-client-ios") 31 | .product(name: "PipecatClientIOSDaily", package: "pipecat-client-ios-daily") 32 | ]), 33 | ``` 34 | 35 | ## Example 36 | 37 | Here's a simple example using Daily as the transport layer: 38 | 39 | ```swift 40 | import PipecatClientIOS 41 | import PipecatClientIOSDaily 42 | 43 | let clientConfig = [ 44 | ServiceConfig( 45 | service: "llm", 46 | options: [ 47 | Option(name: "model", value: .string("meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo")), 48 | Option(name: "messages", value: .array([ 49 | .object([ 50 | "role" : .string("system"), 51 | "content": .string("You are a helpful assistant.") 52 | ]) 53 | ])) 54 | ] 55 | ), 56 | ServiceConfig( 57 | service: "tts", 58 | options: [ 59 | Option(name: "voice", value: .string("79a125e8-cd45-4c13-8a67-188112f4dd22")) 60 | ] 61 | ) 62 | ] 63 | 64 | let options = RTVIClientOptions.init( 65 | enableMic: true, 66 | params: RTVIClientParams( 67 | baseUrl: $PIPECAT_API_URL, 68 | config: clientConfig 69 | ) 70 | ) 71 | 72 | let client = RTVIClient.init( 73 | transport: DailyTransport.init(options: configOptions), 74 | options: configOptions 75 | ) 76 | try await client.start() 77 | ``` 78 | 79 | ## Documentation 80 | 81 | 82 | 87 | Complete SDK API documentation 88 | 89 | 95 | Pipecat Client iOS 96 | 97 | 98 | 99 | 105 | Simple Chatbot Demo 106 | 107 | 112 | WebRTC implementation using Daily 113 | 114 | 115 | -------------------------------------------------------------------------------- /client/ios/transports/daily.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Daily WebRTC Transport" 3 | description: "WebRTC implementation for iOS using Daily" 4 | --- 5 | 6 | The Daily transport implementation enables real-time audio and video communication in your Pipecat iOS applications using [Daily's](https://www.daily.co/) WebRTC infrastructure. 7 | 8 | ## Installation 9 | 10 | Add the Daily transport package to your project: 11 | 12 | ```swift 13 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios-daily.git", from: "0.3.0") 14 | 15 | // Add to your target dependencies 16 | .target(name: "YourApp", dependencies: [ 17 | .product(name: "PipecatClientIOSDaily", package: "pipecat-client-ios-daily") 18 | ]) 19 | ``` 20 | 21 | ## Usage 22 | 23 | Create a client using the Daily transport: 24 | 25 | ```swift 26 | import PipecatClientIOS 27 | import PipecatClientIOSDaily 28 | 29 | let configOptions = RTVIClientOptions.init( 30 | enableMic: true, 31 | params: RTVIClientParams( 32 | baseUrl: $PIPECAT_API_URL 33 | ) 34 | ) 35 | 36 | let client = RTVIClient.init( 37 | transport: DailyTransport.init(options: configOptions), 38 | options: configOptions 39 | ) 40 | 41 | try await client.start() 42 | ``` 43 | 44 | ## Configuration 45 | 46 | Your server endpoint should return Daily-specific configuration: 47 | 48 | ```swift 49 | // Example server response 50 | { 51 | "url": "https://your-domain.daily.co/room-name", 52 | "token": "your-daily-token" 53 | } 54 | ``` 55 | 56 | ## API Reference 57 | 58 | 59 | 65 | Simple Chatbot Demo 66 | 67 | 68 | 74 | Daily Transport 75 | 76 | 77 | 82 | Complete API documentation for the Daily transport implementation 83 | 84 | -------------------------------------------------------------------------------- /client/ios/transports/gemini-websocket.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Gemini Live Websocket Transport" 3 | description: "Websocket implementation for iOS using Gemini" 4 | --- 5 | 6 | The Gemini Live Websocket transport implementation enables real-time audio communication with the Gemini Multimodal Live service, using a direct websocket connection. 7 | 8 | 9 | Transports of this type are designed primarily for development and testing 10 | purposes. For production applications, you will need to build a server 11 | component with a server-friendly transport, like the 12 | [DailyTransport](./daily), to securely handle API keys. 13 | 14 | 15 | ## Installation 16 | 17 | Add the Gemini transport package to your project: 18 | 19 | ```swift 20 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios-gemini-live-websocket.git", from: "0.3.1"), 21 | 22 | // Add to your target dependencies 23 | .target(name: "YourApp", dependencies: [ 24 | .product(name: "PipecatClientIOSGeminiLiveWebSocket", package: "pipecat-client-ios-gemini-live-websocket") 25 | ], 26 | ``` 27 | 28 | ## Usage 29 | 30 | Create a client: 31 | 32 | ```swift 33 | let options: RTVIClientOptions = .init( 34 | params: .init(config: [ 35 | .init( 36 | service: "llm", 37 | options: [ 38 | .init(name: "api_key", value: .string("")), 39 | .init(name: "initial_messages", value: .array([ 40 | .object([ 41 | "role": .string("user"), // "user" | "system" 42 | "content": .string("I need your help planning my next vacation.") 43 | ]) 44 | ])), 45 | .init(name: "generation_config", value: .object([ 46 | "speech_config": .object([ 47 | "voice_config": .object([ 48 | "prebuilt_voice_config": .object([ 49 | "voice_name": .string("Puck") // "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede" 50 | ]) 51 | ]) 52 | ]) 53 | ])) 54 | ] 55 | ) 56 | ]) 57 | ) 58 | 59 | let client = GeminiLiveWebSocketVoiceClient(options: options) 60 | 61 | try await client.start() 62 | ``` 63 | 64 | ## API Reference 65 | 66 | 67 | 73 | Simple Chatbot Gemini Demo 74 | 75 | 76 | 82 | iOS Gemini Live WebSocket 83 | 84 | 85 | 90 | Complete API documentation for the Gemini transport implementation 91 | 92 | -------------------------------------------------------------------------------- /client/ios/transports/openai-webrtc.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "OpenAIRealTimeWebRTCTransport" 3 | --- 4 | 5 | ## Overview 6 | 7 | The OpenAI Realtime WebRTC transport implementation enables real-time audio communication directly with the [OpenAI Realtime API using WebRTC](https://platform.openai.com/docs/guides/realtime-webrtc) voice-to-voice service. 8 | It handles media device management, audio/video streams, and state management for the connection. 9 | 10 | ## Installation 11 | 12 | Add the OpenAI transport package to your project: 13 | 14 | ```swift 15 | .package(url: "https://github.com/pipecat-ai/pipecat-client-ios-openai-realtime.git", from: "0.0.1"), 16 | 17 | // Add to your target dependencies 18 | .target(name: "YourApp", dependencies: [ 19 | .product(name: "PipecatClientIOSOpenAIRealtimeWebrtc", package: "pipecat-client-ios-openai-realtime") 20 | ], 21 | ``` 22 | 23 | ## Usage 24 | 25 | Create a client: 26 | 27 | ```swift 28 | let rtviClientOptions = RTVIClientOptions.init( 29 | enableMic: currentSettings.enableMic, 30 | enableCam: false, 31 | params: .init(config: [ 32 | .init( 33 | service: "llm", 34 | options: [ 35 | .init(name: "api_key", value: .string(openaiAPIKey)), 36 | .init(name: "initial_messages", value: .array([ 37 | .object([ 38 | "role": .string("user"), // "user" | "system" 39 | "content": .string("Start by introducing yourself.") 40 | ]) 41 | ])), 42 | .init(name: "session_config", value: .object([ 43 | "instructions": .string("You are Chatbot, a friendly and helpful assistant who provides useful information, including weather updates."), 44 | "voice": .string("echo"), 45 | "input_audio_noise_reduction": .object([ 46 | "type": .string("near_field") 47 | ]), 48 | "turn_detection": .object([ 49 | "type": .string("semantic_vad") 50 | ]) 51 | ])), 52 | ] 53 | ) 54 | ]) 55 | ) 56 | self.rtviClientIOS = RTVIClient.init( 57 | transport: OpenAIRealtimeTransport.init(options: rtviClientOptions), 58 | options: rtviClientOptions 59 | ) 60 | try await rtviClientIOS.start() 61 | ``` 62 | 63 | 64 | Currently, invalid session configurations will result in the OpenAI connection 65 | being failed. 66 | 67 | 68 | ## API Reference 69 | 70 | 71 | 77 | Simple Chatbot OpenAI Demo 78 | 79 | 80 | 86 | iOS OpenAI Realtime WebRTC 87 | 88 | 89 | 94 | Complete API documentation for the OpenAI transport implementation 95 | 96 | -------------------------------------------------------------------------------- /client/js/api-reference/configuration.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Configuration" 3 | description: "Passing service configuration values to a bot." 4 | --- 5 | 6 | Pipecat bots expose [services](./services) and service configuration options to clients. Your client config can be set when initializing your bot or at runtime. The configuration follows the RTVI standard for consistency and compatibility. 7 | 8 | A typical bot config, in JSON, might look like this: 9 | 10 | ```json 11 | [ 12 | { 13 | "service": "vad", 14 | "options": [{ "name": "params", "value": { "stop_secs": 3.0 } }] 15 | }, 16 | { 17 | "service": "tts", 18 | "options": [ 19 | { "name": "voice", "value": "79a125e8-cd45-4c13-8a67-188112f4dd22" } 20 | ] 21 | }, 22 | { 23 | "service": "llm", 24 | "options": [ 25 | { 26 | "name": "model", 27 | "value": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo" 28 | }, 29 | { 30 | "name": "initial_messages", 31 | "value": [ 32 | { 33 | "role": "system", 34 | "content": `You are a assistant called ExampleBot. You can ask me anything. 35 | Keep responses brief and legible. 36 | Your responses will converted to audio. Please do not include any special characters in your response other than '!' or '?'. 37 | Start by briefly introducing yourself.` 38 | } 39 | ] 40 | }, 41 | { "name": "run_on_config", "value": true } 42 | ] 43 | } 44 | ] 45 | ``` 46 | 47 | ## Client-side configuration 48 | 49 | You can pass a config into the `RTVIClient` params properties on the [constructor](./client-constructor). 50 | Passing a config from the client is optional. A bot will always start with a default config if no config is passed from the client. 51 | Some RTVI implementations may also choose to ignore configs passed from the client, for security or other reasons. 52 | 53 | ### Working with the RTVI config array 54 | 55 | RTVI config is defined as a list because order matters; configurations are applied sequentially on your bot pipeline. 56 | For example, to configure a TTS service with a new voice and an LLM service with new prompting, specify the TTS first to ensure the voice is applied before the prompting. 57 | This ordering principle also applies to service options, ensuring deterministic outcomes for all RTVI implementations. 58 | 59 | RTVI clients instances expose various methods for working with config arrays which you can read about [here](./client-methods#bot-configuration). 60 | 61 | ## Server-side configuration 62 | 63 | Platforms implementing RTVI on the server side will generally provide a method for passing a config into a `connect` endpoint. It's a good practice to use the same config format for both client-side and server-side configuration, though of course this choice is left up to the implementor of the server-side APIs. 64 | 65 | ### Setting service API keys 66 | 67 | It's important to note that API keys should never be included in configuration messages from or to clients. Clients shouldn't have access to API keys at all. 68 | 69 | Platforms implementing RTVI should use a separate mechanism for passing API keys to a bot. A typical approach is to connect to a bot with a larger, "meta config" that includes API keys, a list of services the bot should instantiate, the client-visible bot configuration, and perhaps other fields like the maximum session duration. 70 | 71 | For example: 72 | 73 | ```javascript 74 | const bot_connect_rest_api_payload = { 75 | api_keys: api_keys_map_for_env 76 | max_duration: duration_in_seconds 77 | services: [{ llm: "together", tts: "cartesia" }] 78 | config: config_passed_from_client 79 | }; 80 | ``` 81 | -------------------------------------------------------------------------------- /client/js/api-reference/errors.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Errors" 3 | --- 4 | 5 | 6 | Base `RTVIClient` error type. `status:number`: unique identifier (or HTTP code 7 | if applicable) `message:string`: explanation of error 8 | 9 | 10 | 11 | Bot did not enter a ready state within the specified `timeout` constructor 12 | param. 13 | 14 | 15 | 16 | Invalid response from the provided `params.baseUrl` route. This may be due to the base URL being unavailable, or failure to parse the provided configuration. 17 | Any error text returned by the base URL endpoint will be referenced in the `message` property. 18 | 19 | - `status:number` returned HTTP status code 20 | - `message:string` Verbose error message (if provided via `info` response) 21 | - `error:string` Error type (defaults to `invalid-request-error`) 22 | 23 | 24 | 25 | 26 | Transport was not able to connect. Check the auth bundle returned by the 27 | baseUrl is valid. 28 | 29 | 30 | 31 | Client attempted an action or method that requires the bot to be in a ready 32 | state. You must call `connect` first and wait for the bot to be ready. 33 | 34 | 35 | 36 | Bot was unable to parse provided configuration properties. 37 | 38 | -------------------------------------------------------------------------------- /client/js/api-reference/messages.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Messages" 3 | --- 4 | 5 | The Pipecat JavaScript client can send and receive arbitrary data messages to a bot using the RTVI standard for message handling. 6 | 7 | Messages differ from [actions](./actions) in that they are not associated with a specific service and do not return a Promise. 8 | 9 | Messages are used by a client for passing an instruction to a bot that typically results in a [callback or event](./callbacks). 10 | 11 | Examples of messages include: 12 | 13 | - `updateConfig` 14 | - `describeConfig` 15 | - `describeActions` 16 | - `disconnectBot` 17 | 18 | Developers can build their own messages and create handlers to respond to messages received from a bot either by extending `RTVIClient` with their own class or by building a [helper](./helpers/introduction). 19 | 20 | ### sendMessage(message: RTVIMessage) 21 | 22 | This is a "fire and forget" function to send an arbitrary message to a bot. It does not wait for a response, so nothing is returned. 23 | 24 | Most clients will not use this function directly. They will use [actions](./actions) and [helper methods](./helpers), instead. 25 | 26 | It is up to the transport layer to pack and unpack messages, send messages over the wire, and deliver messages to the SDK so that events can be emitted. 27 | 28 | ### Anatomy of a message 29 | 30 | ```typescript 31 | { 32 | label: "rtvi-ai", 33 | type: "EVENT_TYPE", 34 | data: { EVENT_TYPE_DATA } 35 | } 36 | 37 | // example message dispatch 38 | rtviClient.sendMessage({ 39 | label: "rtvi-ai", 40 | type: "myMessage", 41 | data: { 42 | "hello": "world" 43 | } 44 | }); 45 | ``` 46 | -------------------------------------------------------------------------------- /client/js/helpers/llm.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "LLM Helper" 3 | --- 4 | 5 | The Pipecat JavaScript client includes an LLM helper for common language model tasks and workflows. 6 | 7 | ## Using the LLM helper 8 | 9 | ```typescript 10 | import { RTVIClient, LLMHelper } from "@pipecat-ai/client-js"; 11 | 12 | const rtviClient = new RTVIClient({ 13 | // ... 14 | }); 15 | const llmHelper = new LLMHelper({ 16 | callbacks: { 17 | // ... 18 | }, 19 | }); 20 | rtviClient.registerHelper("llm", llmHelper); 21 | ``` 22 | 23 | ## Actions 24 | 25 | All of the below are abstracted [actions](/v02/api-reference/actions). 26 | 27 | As with all actions, they can be awaited or chained with `.then()`. 28 | 29 | If the bot is unable to process the action, it will trigger the `onMessageError` callback and `MessageError` event. 30 | 31 | ### getContext() 32 | 33 | `llm:get_context` 34 | 35 | Retrieve LLM context from bot. Returns `Promise` 36 | 37 | ```typescript 38 | const llmHelper = rtviClient.getHelper("llm") as LLMHelper; 39 | const ctx = await llmHelper.getContext(); 40 | 41 | // > { messages?: LLMContextMessage[]; tools?: []; } 42 | ``` 43 | 44 | ### setContext() 45 | 46 | `llm:set_context` 47 | 48 | Replaces the current LLM context with the provided one. Returns `Promise`. 49 | 50 | 51 | LLMContext option to set. 52 | 53 | 54 | 55 | Interrupt the current conversation and apply the new context immediately. 56 | 57 | 58 | ```typescript 59 | const llmHelper = rtviClient.getHelper("llm") as LLMHelper; 60 | await llmHelper.setContext( 61 | { 62 | messages: [ 63 | { 64 | role: "system", 65 | content: "Your are a helpful assistant", 66 | }, 67 | { 68 | role: "user", 69 | content: "Tell me a joke", 70 | }, 71 | ], 72 | }, 73 | false | true 74 | ); 75 | 76 | // > true | false (if error) 77 | ``` 78 | 79 | ### appendToMessages() 80 | 81 | `llm:append_to_messages` 82 | 83 | Append a new message to the existing context. Returns `Promise`. 84 | 85 | 86 | New message to apply to the context. 87 | 88 | 89 | 90 | Apply the new message immediately, or wait until the current turn concludes. 91 | 92 | 93 | ```typescript 94 | const llmHelper = rtviClient.getHelper("llm") as LLMHelper; 95 | await llmHelper.appendToMessages( 96 | { 97 | role: "user", 98 | content: "Tell me a joke", 99 | }, 100 | false | true 101 | ); 102 | 103 | // > true | false (if error) 104 | ``` 105 | 106 | ## Callbacks and events 107 | 108 | ```typescript 109 | onLLMJsonCompletion: (jsonString: string) => void; 110 | onLLMFunctionCall: (func: LLMFunctionCallData) => void; 111 | onLLMFunctionCallStart: (functionName: string) => void; 112 | onLLMMessage: (message: LLMContextMessage) => void; 113 | ``` 114 | -------------------------------------------------------------------------------- /client/js/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "SDK Introduction" 3 | description: "Build web applications with Pipecat’s JavaScript client library" 4 | --- 5 | 6 | The Pipecat JavaScript SDK provides a lightweight client implementation that handles: 7 | 8 | - Device and media stream management 9 | - Managing bot configuration 10 | - Sending actions to the bot 11 | - Handling bot messages and responses 12 | - Managing session state and errors 13 | 14 | ## Installation 15 | 16 | Install the SDK and a transport implementation (e.g. Daily for WebRTC): 17 | 18 | ```bash 19 | npm install @pipecat-ai/client-js 20 | npm install @pipecat-ai/daily-transport 21 | ``` 22 | 23 | ## Example 24 | 25 | Here's a simple example using Daily as the transport layer: 26 | 27 | ```javascript 28 | import { RTVIClient } from "@pipecat-ai/client-js"; 29 | import { DailyTransport } from "@pipecat-ai/daily-transport"; 30 | 31 | // Handle incoming audio from the bot 32 | function handleBotAudio(track, participant) { 33 | if (participant.local || track.kind !== "audio") return; 34 | 35 | const audioElement = document.createElement("audio"); 36 | audioElement.srcObject = new MediaStream([track]); 37 | document.body.appendChild(audioElement); 38 | audioElement.play(); 39 | } 40 | 41 | // Create and configure the client 42 | const rtviClient = new RTVIClient({ 43 | params: { 44 | baseUrl: process.env.PIPECAT_API_URL || "/api", 45 | }, 46 | transport: new DailyTransport(), 47 | enableMic: true, 48 | callbacks: { 49 | onTrackStart: handleBotAudio, 50 | }, 51 | }); 52 | 53 | // Connect to your bot 54 | rtviClient.connect(); 55 | ``` 56 | 57 | ## Explore the SDK 58 | 59 | 60 | 65 | Configure your client instance with transport and callbacks 66 | 67 | 72 | Core methods for interacting with your bot 73 | 74 | 75 | Detailed documentation of all available APIs 76 | 77 | 78 | Utility functions for common operations 79 | 80 | 81 | 82 | The Pipecat JavaScript SDK implements the [RTVI standard](/client/introduction#about-rtvi) for real-time AI inference, ensuring compatibility with any RTVI-compatible server and transport layer. 83 | -------------------------------------------------------------------------------- /client/js/transports/gemini.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "GeminiLiveWebSocketTransport" 3 | --- 4 | 5 | ## Overview 6 | 7 | The `GeminiLiveWebsocketTransport` class extends the [`RealTimeWebsocketTransport`](./realtime) to implement a fully functional [RTVI `Transport`](./transport). It provides a framework for implementing real-time communication directly with the [Gemini Multimodal Live](https://ai.google.dev/api/multimodal-live) service. `RealTimeWebsocketTransport` handles media device management, audio/video streams, and state management for the connection. 8 | 9 | 10 | Transports of this type are designed primarily for development and testing 11 | purposes. For production applications, you will need to build a server 12 | component with a server-friendly transport, like the 13 | [DailyTransport](./daily), to securely handle API keys. 14 | 15 | 16 | ## Usage 17 | 18 | ### Basic Setup 19 | 20 | ```javascript 21 | import { GeminiLiveWebsocketTransport, GeminiLLMServiceOptions } from '@pipecat-ai/gemini-live-websocket-transport'; 22 | import { RTVIClientOptions } from '@pipecat-ai/client-js'; 23 | 24 | const options: GeminiLLMServiceOptions = { 25 | api_key: 'YOUR_API_KEY', 26 | generation_config: { 27 | temperature: 0.7, 28 | maxOutput_tokens: 1000 29 | } 30 | }; 31 | 32 | const transport = new GeminiLiveWebsocketTransport(options); 33 | let RTVIConfig: RTVIClientOptions = { 34 | transport, 35 | ... 36 | }; 37 | 38 | ``` 39 | 40 | ### Configuration Options 41 | 42 | ```typescript 43 | interface GeminiLLMServiceOptions { 44 | api_key: string; // Required: Your Gemini API key 45 | initial_messages?: Array<{ 46 | // Optional: Initial conversation context 47 | content: string; 48 | role: string; 49 | }>; 50 | generation_config?: { 51 | // Optional: Generation parameters 52 | candidate_count?: number; 53 | max_output_tokens?: number; 54 | temperature?: number; 55 | top_p?: number; 56 | top_k?: number; 57 | presence_penalty?: number; 58 | frequency_penalty?: number; 59 | response_modalities?: string; 60 | speech_config?: { 61 | voice_config?: { 62 | prebuilt_voice_config?: { 63 | voice_name: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede"; 64 | }; 65 | }; 66 | }; 67 | }; 68 | } 69 | ``` 70 | 71 | ### Sending Messages 72 | 73 | ```javascript 74 | // at setup time... 75 | llmHelper = new LLMHelper({}); 76 | rtviClient.registerHelper("llm", llmHelper); 77 | // the 'llm' name in this call above isn't used. 78 | //that value is specific to working with a pipecat pipeline 79 | 80 | // at time of sending message... 81 | // Send text prompt message 82 | llmHelper.appendToMessages({ role: "user", content: 'Hello OpenAI!' }); 83 | ``` 84 | 85 | ### Handling Events 86 | 87 | The transport implements the various [RTVI event handlers](https://docs.pipecat.ai/client/js/api-reference/callbacks). Check out the docs or samples for more info. 88 | 89 | ## More Information 90 | 91 | 92 | 98 | Gemini MultiModal Live Basic Demo 99 | 100 | 101 | 107 | `GeminiLiveWebsocketTransport` 108 | 109 | 110 | 116 | `@pipecat-ai/realtime-websocket-transport` 117 | 118 | -------------------------------------------------------------------------------- /client/js/transports/realtime.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "RealTimeWebSocketTransport" 3 | --- 4 | 5 | ## Overview 6 | 7 | The Pipecat client SDKs offer multiple transport packages for real-time communication with a Voice-to-Voice LLM servers. The RealTimeWebSocketTransport provides an abstraction for establishing direct connections to LLM servers and handling media streaming. 8 | 9 | 10 | Transports of this type are designed primarily for development and testing 11 | purposes. For production applications, you will need to build a server 12 | component with a server-friendly transport, like the 13 | [DailyTransport](./daily), to securely handle API keys. 14 | 15 | 16 | ## Transports 17 | 18 | | Transport | Service | Package | 19 | | ---------------------------- | ------------------------------------------------------------------- | --------------------------------------------- | 20 | | GeminiLiveWebsocketTransport | [Gemini MultiModal Live](https://ai.google.dev/api/multimodal-live) | `@pipecat-ai/gemini-live-websocket-transport` | 21 | 22 | ## More Information 23 | 24 | 30 | `RealTimeWebSocketTransport` 31 | 32 | 33 | 39 | `@pipecat-ai/realtime-websocket-transport` 40 | 41 | -------------------------------------------------------------------------------- /client/js/transports/transport.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Transport Overview" 3 | sidebarTitle: "Overview" 4 | --- 5 | 6 | Transports are the means by which Pipecat clients communicate with their bot services. They handle both message exchange between client and server and real-time media transport. Pipecat implements the RTVI standard for these communications. 7 | 8 | ## Transport lifecycle 9 | 10 | Your transport instance is constructed when you call `connect()` on your client instance. 11 | 12 | ```typescript 13 | import { RTVIClient } from "@pipecat-ai/client-js"; 14 | import { DailyTransport } from "@pipecat-ai/daily-transport"; 15 | 16 | const rtviClient = new RTVIClient({ 17 | transport: new DailyTransport(), 18 | ... 19 | }); 20 | 21 | await rtviClient.connect(); // Transport instance is created here 22 | await rtviClient.disconnect(); // Transport instance is destroyed here 23 | ``` 24 | 25 | ## Transport states 26 | 27 | `TransportState` 28 | 29 | Your transport instance goes through a series of states during its lifecycle. These states are: 30 | 31 | 32 | 33 | Transport is idle and has not yet been initialized (default state). 34 | 35 | 36 | Transport is being initialized. Typically in response to a 37 | `rtviClient.initDevices()` call, where the transport is being set up in 38 | order to enumerate local media devices. 39 | 40 | 41 | Transport has been initialized and is ready to connect. This state is 42 | typically reached after a successful `rtviClient.initDevices()` call. 43 | 44 | 45 | Your client has called `rtviClient.connect()` and is waiting for a response 46 | from your server containing 'auth bundle' credentials (such as a session URL 47 | and token.) 48 | 49 | 50 | Transport has received the 'auth bundle' and is connecting to the server. 51 | 52 | 53 | Transport has successfully connected to the session and is awaiting a 54 | client-ready signal (indicated audio and video tracks are ready to be sent.) 55 | 56 | Transport is ready and the session can begin. 57 | 58 | Transport is disconnecting from the session. 59 | 60 | An error occurred during the transport lifecycle. 61 | 62 | 63 | You can access the current transport state via `rtviClient.state`, or by defining a callback or event: 64 | 65 | ```typescript 66 | // Callback 67 | const rtviClient = new RTVIClient({ 68 | transport: new DailyTransport(), 69 | callbacks: { 70 | onTransportStateChange: (state) => { 71 | console.log(state); 72 | } 73 | //... 74 | }); 75 | 76 | // Event 77 | rtviClient.on(RTVIEvent.TransportStateChanged, (e) => console.log(e)); 78 | 79 | // Client getter 80 | console.log(rtviClient.state); // Disconnected 81 | ``` 82 | -------------------------------------------------------------------------------- /client/react-native/api-reference.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "API Reference" 3 | description: "API reference for the Pipecat React Native SDK" 4 | --- 5 | 6 | 7 | The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless integration with React Native applications. 8 | For detailed information, please reference to the [Javascript SDK docs](/client/js/api-reference/client-constructor). 9 | 10 | **Just ensure you use the appropriate transport layer for React Native.** 11 | 12 | -------------------------------------------------------------------------------- /client/react-native/introduction.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "SDK Introduction" 3 | description: "Build React Native applications with Pipecat's React Native client library" 4 | --- 5 | 6 | The Pipecat React Native SDK leverages the [Pipecat JavaScript SDK](/client/js/introduction) to provide seamless integration for React Native applications. 7 | Since the JavaScript SDK is designed to work across both web and React Native platforms, the core functionalities remain the same: 8 | - Device and media stream management 9 | - Managing bot configuration 10 | - Sending actions to the bot 11 | - Handling bot messages and responses 12 | - Managing session state and errors 13 | 14 | The primary difference lies in the transport layer, which is tailored to support the unique requirements of the React Native environment. 15 | 16 | For example, when using the SDK with React Native, you would install `RNDailyTransport` instead of `DailyTransport`. 17 | 18 | ## Installation 19 | 20 | Install the SDK and a transport implementation (e.g. Daily for WebRTC): 21 | 22 | ```bash 23 | npm i @pipecat-ai/react-native-daily-transport 24 | npm i @daily-co/react-native-daily-js@^0.70.0 25 | npm i @daily-co/react-native-webrtc@^118.0.3-daily.2 26 | npm i @react-native-async-storage/async-storage@^1.23.1 27 | npm i react-native-background-timer@^2.4.1 28 | npm i react-native-get-random-values@^1.11.0 29 | ``` 30 | 31 | Installing `@pipecat-ai/react-native-daily-transport` automatically includes the corresponding version of the JavaScript SDK. 32 | 33 | If you are using Expo, you will also need to add the following dependencies: 34 | 35 | ```bash 36 | npm i @config-plugins/react-native-webrtc@^10.0.0 37 | npm i @daily-co/config-plugin-rn-daily-js@0.0.7 38 | ``` 39 | 40 | ## Requirements 41 | 42 | This package introduces some constraints on what OS/SDK versions your project can support: 43 | 44 | - iOS: Deployment target >= 13 45 | - Android: `minSdkVersion` >= 24 46 | 47 | ## Quick start 48 | 49 | Here's a simple example using Daily as the transport layer: 50 | 51 | ```tsx 52 | import { RNDailyTransport } from '@pipecat-ai/react-native-daily-transport'; 53 | import { RTVIClient } from '@pipecat-ai/client-js'; 54 | 55 | // Create and configure the client 56 | let voiceClient = new RTVIClient({ 57 | params: { 58 | baseUrl: process.env.PIPECAT_API_URL || "/api", 59 | }, 60 | transport: new RNDailyTransport(), 61 | enableMic: true 62 | }); 63 | 64 | // Connect to your bot 65 | await voiceClient.connect(); 66 | ``` 67 | 68 | > You can find a basic working example [here](https://github.com/pipecat-ai/pipecat-client-react-native-daily-transport/tree/main/example) 69 | > and a more comprehensive example [here](https://github.com/daily-demos/daily-bots-react-native-demo/). 70 | 71 | ## Explore the SDK 72 | 73 | The Pipecat React Native SDK leverages the Pipecat JavaScript SDK for seamless integration with React Native applications. For detailed information, refer to our JavaScript documentation. 74 | 75 | > Just ensure you use the appropriate transport layer for React Native. 76 | 77 | 78 | 83 | Configure your client instance with transport and callbacks 84 | 85 | 90 | Core methods for interacting with your bot 91 | 92 | 93 | Detailed documentation of all available APIs 94 | 95 | 96 | Utility functions for common operations 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /client/react/components.mdx: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Components" 3 | description: "Ready-to-use React components for Pipecat applications" 4 | --- 5 | 6 | The Pipecat React SDK provides several components for handling audio, video, and visualization in your application. 7 | 8 | ## RTVIClientProvider 9 | 10 | The root component for providing Pipecat client context to your application. 11 | 12 | ### Props 13 | 14 | - `client` (RTVIClient, required): A singleton instance of RTVIClient 15 | 16 | ```jsx 17 | 18 | {/* Child components */} 19 | 20 | ``` 21 | 22 | ## RTVIClientAudio 23 | 24 | Creates a new `