├── .gitignore ├── .parcelrc ├── .prettierrc ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── client-js ├── README.md ├── eslint.config.mjs ├── package.json ├── src │ ├── actions.ts │ ├── client.ts │ ├── decorators.ts │ ├── errors.ts │ ├── events.ts │ ├── helpers │ │ ├── index.ts │ │ └── llm.ts │ ├── index.ts │ ├── logger.ts │ ├── messages.ts │ └── transport.ts ├── tests │ ├── client.spec.ts │ └── stubs │ │ └── transport.ts └── tsconfig.json ├── client-react ├── README.md ├── eslint.config.mjs ├── package.json ├── src │ ├── RTVIClientAudio.tsx │ ├── RTVIClientCamToggle.tsx │ ├── RTVIClientMicToggle.tsx │ ├── RTVIClientProvider.tsx │ ├── RTVIClientVideo.tsx │ ├── VoiceVisualizer.tsx │ ├── index.ts │ ├── useMergedRef.ts │ ├── useRTVIClient.ts │ ├── useRTVIClientCamControl.ts │ ├── useRTVIClientEvent.ts │ ├── useRTVIClientMediaDevices.ts │ ├── useRTVIClientMediaTrack.ts │ ├── useRTVIClientMicControl.ts │ └── useRTVIClientTransportState.ts └── tsconfig.json ├── package-lock.json ├── package.json ├── pipecat-js.png ├── pipecat-react.png └── pipecat-web.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | .parcel-cache 15 | 16 | .env 17 | 18 | # Editor directories and files 19 | .vscode/* 20 | !.vscode/extensions.json 21 | .idea 22 | .DS_Store 23 | *.suo 24 | *.ntvs* 25 | *.njsproj 26 | *.sln 27 | *.sw? 28 | -------------------------------------------------------------------------------- /.parcelrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@parcel/config-default", 3 | "transformers": { 4 | "*.{ts,tsx}": [ 5 | "@parcel/transformer-typescript-tsc" 6 | ] 7 | } 8 | } -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "tabWidth": 2, 4 | "useTabs": false, 5 | "singleQuote": false 6 | } 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to **Pipecat Client Web** will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [0.3.5] - 2025-03-20 11 | 12 | - Added a getter to the client for getting the underlying transport. The transport that is returned is safe-guarded so that calls like `connect`/`disconnect` which should be called by the client are rejected. 13 | 14 | - Added missing support for handling fatal transport errors. Transports should call the onError callback with a data field `fatal: true` to trigger an client disconnect. 15 | 16 | - Improved action types for cleaner handling 17 | 18 | ## [0.3.4] - 2025-03-13 19 | 20 | ### Changes 21 | 22 | - Changed transport initialization to only occur in the RTVIClient constructor and not in its `disconnect`. This allows Transports to better reason about and be in control of what happens in both the initialize and disconnect methods. 23 | 24 | ## [0.3.3] - 2025-02-28 25 | 26 | ### Added 27 | 28 | - Added an `onBotLlmSearchResponse` callback and `BotLlmSearchResponse` event to correspond with `RTVIBotLLMSearchResponseMessage`. 29 | 30 | - Added an `onServerMessage` callback and `ServerMessage` event to correspond with `RTVIServerMessage`. 31 | 32 | ## [0.3.2] - 2024-12-16 33 | 34 | ### Added 35 | 36 | - Screen media sharing methods implemented: 37 | - Added `startScreenShare` and `stopScreenShare` methods to `RTVIClient` and `Transport`. 38 | - Added `isSharingScreen` getter to `RTVIClient` and `Transport`. 39 | 40 | ### Changes 41 | 42 | - `baseUrl` and `endpoints` are now optional parameters in the `RTVIClient` constructor (`RTVIClientParams`), allowing developers to connect directly to a transport without requiring a handshake auth bundle. 43 | - Note: Most transport services require an API key for secure operation, and setting these keys dangerously on the client is not recommended for production. This change intends to simplify testing and local developement where running a server-side connect method can be cumbersome. 44 | 45 | ## [0.3.1] - 2024-12-10 46 | 47 | ### Fixed 48 | 49 | - Incorrect package.lock version resulted in types being omitted from the previous build. This has been fixed (0.3.0 has been unpublished). 50 | 51 | ## [0.3.0] - 2024-12-10 52 | 53 | ### Changed 54 | 55 | - The RTVI client web libraries are now part of Pipecat. Repo and directory names have changed from RTVI to Pipecat. 56 | - Package names have also been updated: 57 | - `realtime-ai` is now `@pipecat-ai/client-js` 58 | - `realtime-ai-react` is now `@pipecat-ai/client-react` 59 | 60 | Please update your imports to the new package names. 61 | 62 | ## [0.2.3] - 2024-12-09 63 | 64 | ### Fixed 65 | 66 | - Added initial callback for `onSpeakerUpdated` event 67 | - Fixed event name typo - `TrackedStopped` should be `TrackStopped` 68 | 69 | ## [0.2.2] - 2024-11-12 70 | 71 | ### Added 72 | 73 | - `disconnectBot()` method added to `RTVIClient` that disconnects the bot from the session, but keeps the session alive for the connected user. 74 | - `logger` singleton added and used within all `RTVIClient` files. This aims to provide more granular control over console output - with verbose logging enabled, RTVI can be a little noisy. 75 | - `setLogLevel` method added to `RTVIClient` to allow developers to set the log level. 76 | - `disconnectBot()` method added to `RTVIMessage` to allow developers to tell the bot to leave a session. 77 | 78 | ## [0.2.1] - 2024-10-28 79 | 80 | ### Removed 81 | 82 | - `realtime-ai-daily` has been moved to [@daily-co/realtime-ai-daily](https://github.com/daily-co/realtime-ai-daily) to align to being a provider agnostic codebase. The last release for the Daily transport package is `0.2.0`, which is still available with `npm install realtime-ai-daily` (https://www.npmjs.com/package/realtime-ai-daily). Please update your project imports to the new package install. 83 | 84 | ### Changed 85 | 86 | - `onBotText` callback renamed to `onBotLlmText` for consistency. `onBotText` has been marked as deprecated. 87 | - `onUserText` callback and events removed as unused. 88 | - `onBotLlmText` callback correctly accepts a `text:BotLLMTextData` typed parameter. 89 | - `onBotTranscript` callback correctly accepts a `text:BotLLMTextData` typed parameter (previously `TranscriptData`) 90 | - `botLlmStarted`, `botLlmStopped`, `botTtsStarted`, `botTtsStopped`, `onBotStartedSpeaking` and `onBotStoppedSpeaking` pass no parameters. Previously, these callbacks were given a participant object which was unused. 91 | - `TTSTextData` type renamed to `BotTTSTextData` for consistency. 92 | 93 | ### Fixed 94 | 95 | - `endpoints` is redefined as a partial, meaning you no longer receive a linting error when you only want to override a single endpoint. 96 | 97 | ## [0.2.0] - 2024-09-13 98 | 99 | RTVI 0.2.0 removes client-side configuration, ensuring that state management is handled exclusively by the bot or the developer’s application logic. Clients no longer maintain an internal config array that can be modified outside of a ready state. Developers who require stateful configuration before a session starts should implement it independently. 100 | 101 | This change reinforces a key design principle of the RTVI standard: the bot should always be the single source of truth for configuration, and RTVI clients should remain stateless. 102 | 103 | Additionally, this release expands action capabilities, enabling disconnected action dispatch, and renames key classes and types from` VoiceClientX` to `RTVIClientX`. Where possible, we have left deprecated aliases to maintain backward compatibility. 104 | 105 | ### Added 106 | 107 | - `params` client constructor option, a partial object that will be sent as JSON stringified body params at `connect()` to your hosted endpoint. If you want to declare initial configuration in your client, or specify start services on the client, you can declare them here. 108 | - baseUrl: string; 109 | - headers?: Headers; 110 | - endpoints?: connect | action; 111 | - requestData?: object; 112 | - config?: RTVIClientConfigOption[]; 113 | - Any additional request params for all fetch requests, e.g. `[key: string]: unknown;` 114 | - `endpoints` (as part of `params`) declares two default endpoints that are appended to your `baseUrl`. `connect/` (start a realtime bot session) and `/action` (for disconnect actions). 115 | - `onConfig` and `RTVIEvent.Config` callback & event added, triggered by `getConfig` voice message. 116 | - `@transportReady` decorator added to methods that should only be called at runtime. Note: decorator support required several Parcel configuration changes and additional dev dependencies. 117 | - `@getIfTransportInState` getter decorator added to getter methods that should only be called in a specified transport state. 118 | - `rtvi_client_version` is now sent as a body parameter to the `connect` fetch request, enabling bot <> client compatibility checks. 119 | - `action()` will now function when in a disconnected state. When not connected, this method expects a HTTP streamed response from the `action` endpoint declared in your params. 120 | - New callbacks and events: 121 | - `onBotTtsText` Bot TTS text output 122 | - `onBotTtsStarted` Bot TTS response starts 123 | - `onBotTtsStopped` Bot TTS response stops 124 | - `onBotText` Streaming chunk/word, directly after LLM 125 | - `onBotLlmStarted` Bot LLM response starts 126 | - `onBotLlmStopped` Bot LLM response stops 127 | - `onUserText` Aggregated user text which is sent to LLM 128 | - `onStorageItemStored` Item was stored to storage 129 | 130 | ### Changed 131 | 132 | - `start()` has been renamed to `connect()`. 133 | - Client no longer expects a `services` map as a constructor param (note: remains in place but flagged as deprecated.) If you want to pass a services map to your endpoint, please use `params`. 134 | - `customHeaders` has been renamed to `headers`. 135 | - Config getter and setter methods (`getConfig` and `updateConfig`) are only supported at runtime. 136 | - `updateConfig` promise is typed to `Promise` (previously `unknown` to support offline updates.) 137 | - `getConfig` promise is typed to `Promise` (previously `unknown` to support offline updates.) 138 | - `services` getter and setter methods have been deprecated. 139 | - `getServiceOptionsFromConfig`, `getServiceOptionValueFromConfig`, `setConfigOptions` and `setServiceOptionInConfig` are now async to support `getConfig` at runtime and accept an optional `config` param for working with local config arrays. 140 | - `registerHelper` no longer checks for a registered service and instead relies on string matching. 141 | - LLM Helper `getContext()` now accepts optional `config` param for working with local configs. 142 | - `customAuthHandler` updated to receive `startParams` as second dependency. 143 | - jest tests updated to reflect changes. 144 | - `VoiceClientOptions` is now `RTVIClientOptions`. 145 | - `VoiceClientConfigOption` is now `RTVIClientConfigOption`. 146 | - `VoiceEvent` is now `RTVIEvent`. 147 | 148 | ### Fixed 149 | 150 | - `RTVIMessageType.CONFIG` message now correctly calls `onConfig` and `RTIEvents.Config`. 151 | 152 | ### Deprecated 153 | 154 | - `getBotConfig` has been renamed to `getConfig` to match the bot action name / for consistency. 155 | - voiceClient.config getter is deprecated. 156 | - `config` and `services` constructor params should now be set inside of `params` and are optional. 157 | - `customBodyParams` and `customHeaders` have been marked as deprecated. Use `params` instead. 158 | 159 | ### Removed 160 | 161 | - `RTVIClient.partialToConfig` removed (unused) 162 | - `nanoid` dependency removed. 163 | 164 | ## [0.1.10] - 2024-09-06 165 | 166 | - LLMContextMessage content not types to `unknown` to support broader LLM use-cases. 167 | 168 | ## [0.1.9] - 2024-09-04 169 | 170 | ### Changed 171 | 172 | - `voiceClient.action()` now returns a new type `VoiceMessageActionResponse` that aligns to RTVI's action response shape. Dispatching an action is the same as dispatching a `VoiceMessage` except the messageDispatcher will type the response accordingly. `action-response` will resolve or reject as a `VoiceMessageActionResponse`, whereas any other message type is typed as a `VoiceMessage`. This change makes it less verbose to handle action responses, where the `data` blob will always contain a `result` property. 173 | - LLM Helper `getContext` returns a valid promise return type (`Promise`). 174 | - LLMHelper `getContext` resolves with the action result (not the data object). 175 | - LLMHelper `setContext` returns a valid promise return type (`Promise`). 176 | - LLMHelper `setContext` resolves with the action result boolean (not the data object). 177 | - LLMHelper `appendToMessages` returns a valid promise return type (`Promise`). 178 | - LLMHelper `appendToMessages` resolves with the action result boolean (not the data object). 179 | 180 | ### Fixed 181 | 182 | - `customAuthHandler` is now provided with the timeout object, allowing developers to manually clear it (if set) in response to their custom auth logic. 183 | - `getServiceOptionsFromConfig` returns `unknown | undefined` when a service option is not found in the config definition. 184 | - `getServiceOptionsValueFromConfig` returns `unknown | undefined` when a service option or value is not found in the config definition. 185 | - `getServiceOptionValueFromConfig` returns a deep clone of the value, to avoid nested references. 186 | - `VoiceMessageType.CONFIG_AVAILABLE` resolves the dispatched action, allowing `describeConfig()` to be awaited. 187 | - `VoiceMessageType.ACTIONS_AVAILABLE` resolves the dispatched action, allowing `describeActions()` to be awaited. 188 | 189 | ### Added 190 | 191 | - Action dispatch tests 192 | 193 | ## [0.1.8] - 2024-09-02 194 | 195 | ### Fixed 196 | 197 | - `getServiceOptionsFromConfig` and `getServiceOptionValueFromConfig` return a deep clone of property to avoid references in returned values. 198 | - LLM Helper `getContext` now returns a new instance of context when not in ready state. 199 | 200 | ### Changed 201 | 202 | - `updateConfig` now calls the `onConfigUpdated` callback (and event) when not in ready state. 203 | 204 | ## [0.1.7] - 2024-08-28 205 | 206 | ### Fixed 207 | 208 | - All config mutation methods (getServiceOptionsFromConfig, getServiceOptionValueFromConfig, setServiceOptionInConfig) now work when not in a ready state. 209 | 210 | ### Added 211 | 212 | - New config method: `getServiceOptionValueFromConfig`. Returns value of config service option with passed service key and option name. 213 | - setServiceOptionInConfig now accepts either one or many ConfigOption arguments (and will set or update all) 214 | - setServiceOptionInConfig now accepts an optional 'config' param, which it will use over the default VoiceClient config. Useful if you want to mutate an existing config option across multiple services before calling `updateConfig`. 215 | - New config method `setConfigOptions` updates multiple service options by running each item through `setServiceOptionInConfig`. 216 | 217 | ### Fixed 218 | 219 | - "@daily-co/daily-js" should not be included in the `rtvi-client-js` package.json. This dependency is only necessary for `rtvi-client-js-daily`. 220 | - Jest unit tests added for config manipulation within `rtvi-client-js` (`yarn run test`) 221 | 222 | ## [0.1.6] - 2024-08-26 223 | 224 | ### Fixed 225 | 226 | - `getServiceOptionsFromConfig` should return a new object, not an instance of the config. This prevents methods like `setContext` from mutating local config unintentionally. 227 | 228 | ## [0.1.5] - 2024-08-19 229 | 230 | ### Added 231 | 232 | - Client now sends a `client-ready` message once it receives a track start event from the transport. This avoids scenarios where the bot starts speaking too soon, before the client has had a change to subscribe to the audio track. 233 | 234 | ## [0.1.4] - 2024-08-19 235 | 236 | ### Added 237 | 238 | - VoiceClientVideo component added to `rtvi-client-react` for rendering local or remote video tracks 239 | - partialToConfig voice client method that returns a new VoiceClientConfigOption[] from provided partial. Does not update config. 240 | 241 | ### Fixed 242 | 243 | - Fixes an issue when re-creating a DailyVoiceClient. Doing so will no longer result in throwing an error. Note: Simultaneous DailyVoiceClient instances is not supported. Creating a new DailyVoiceClient will invalidate any pre-existing ones. 244 | 245 | ## [0.1.3] - 2024-08-17 246 | 247 | ### Added 248 | 249 | - `setServiceOptionsInConfig` Returns mutated / merged config for specified key and service config option 250 | - Voice client constructor `customBodyParams:object`. Add custom request parameters to send with the POST request to baseUrl 251 | - Set voice client services object (when client has not yet connected) 252 | 253 | ### Fixed 254 | 255 | - Pass timeout to customAuthHandler 256 | 257 | ## [0.1.2] - 2024-08-16 258 | 259 | - API refactor to align to RTVI 0.1 260 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing to Pipecat Client Web 2 | 3 | We welcome contributions of all kinds! Your help is appreciated. Follow these steps to get involved: 4 | 5 | 1. **Fork this repository**: Start by forking the Pipecat Client Web repository to your GitHub account. 6 | 7 | 2. **Clone the repository**: Clone your forked repository to your local machine. 8 | ```bash 9 | git clone https://github.com/your-username/pipecat-client-web 10 | ``` 11 | 3. **Create a branch**: For your contribution, create a new branch. 12 | ```bash 13 | git checkout -b your-branch-name 14 | ``` 15 | 4. **Make your changes**: Edit or add files as necessary. 16 | 5. **Test your changes**: Ensure that your changes look correct and follow the style set in the codebase. 17 | 6. **Commit your changes**: Once you're satisfied with your changes, commit them with a meaningful message. 18 | 19 | ```bash 20 | git commit -m "Description of your changes" 21 | ``` 22 | 23 | 7. **Push your changes**: Push your branch to your forked repository. 24 | 25 | ```bash 26 | git push origin your-branch-name 27 | ``` 28 | 29 | 9. **Submit a Pull Request (PR)**: Open a PR from your forked repository to the main branch of this repo. 30 | > Important: Describe the changes you've made clearly! 31 | 32 | Our maintainers will review your PR, and once everything is good, your contributions will be merged! 33 | 34 | # Contributor Covenant Code of Conduct 35 | 36 | ## Our Pledge 37 | 38 | We as members, contributors, and leaders pledge to make participation in our 39 | community a harassment-free experience for everyone, regardless of age, body 40 | size, visible or invisible disability, ethnicity, sex characteristics, gender 41 | identity and expression, level of experience, education, socio-economic status, 42 | nationality, personal appearance, race, caste, color, religion, or sexual 43 | identity and orientation. 44 | 45 | We pledge to act and interact in ways that contribute to an open, welcoming, 46 | diverse, inclusive, and healthy community. 47 | 48 | ## Our Standards 49 | 50 | Examples of behavior that contributes to a positive environment for our 51 | community include: 52 | 53 | - Demonstrating empathy and kindness toward other people 54 | - Being respectful of differing opinions, viewpoints, and experiences 55 | - Giving and gracefully accepting constructive feedback 56 | - Accepting responsibility and apologizing to those affected by our mistakes, 57 | and learning from the experience 58 | - Focusing on what is best not just for us as individuals, but for the overall 59 | community 60 | 61 | Examples of unacceptable behavior include: 62 | 63 | - The use of sexualized language or imagery, and sexual attention or advances of 64 | any kind 65 | - Trolling, insulting or derogatory comments, and personal or political attacks 66 | - Public or private harassment 67 | - Publishing others' private information, such as a physical or email address, 68 | without their explicit permission 69 | - Other conduct which could reasonably be considered inappropriate in a 70 | professional setting 71 | 72 | ## Enforcement Responsibilities 73 | 74 | Community leaders are responsible for clarifying and enforcing our standards of 75 | acceptable behavior and will take appropriate and fair corrective action in 76 | response to any behavior that they deem inappropriate, threatening, offensive, 77 | or harmful. 78 | 79 | Community leaders have the right and responsibility to remove, edit, or reject 80 | comments, commits, code, wiki edits, issues, and other contributions that are 81 | not aligned to this Code of Conduct, and will communicate reasons for moderation 82 | decisions when appropriate. 83 | 84 | ## Scope 85 | 86 | This Code of Conduct applies within all community spaces, and also applies when 87 | an individual is officially representing the community in public spaces. 88 | Examples of representing our community include using an official email address, 89 | posting via an official social media account, or acting as an appointed 90 | representative at an online or offline event. 91 | 92 | ## Enforcement 93 | 94 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 95 | reported to the community leaders responsible for enforcement at pipecat-ai@daily.co. 96 | All complaints will be reviewed and investigated promptly and fairly. 97 | 98 | All community leaders are obligated to respect the privacy and security of the 99 | reporter of any incident. 100 | 101 | ## Enforcement Guidelines 102 | 103 | Community leaders will follow these Community Impact Guidelines in determining 104 | the consequences for any action they deem in violation of this Code of Conduct: 105 | 106 | ### 1. Correction 107 | 108 | **Community Impact**: Use of inappropriate language or other behavior deemed 109 | unprofessional or unwelcome in the community. 110 | 111 | **Consequence**: A private, written warning from community leaders, providing 112 | clarity around the nature of the violation and an explanation of why the 113 | behavior was inappropriate. A public apology may be requested. 114 | 115 | ### 2. Warning 116 | 117 | **Community Impact**: A violation through a single incident or series of 118 | actions. 119 | 120 | **Consequence**: A warning with consequences for continued behavior. No 121 | interaction with the people involved, including unsolicited interaction with 122 | those enforcing the Code of Conduct, for a specified period of time. This 123 | includes avoiding interactions in community spaces as well as external channels 124 | like social media. Violating these terms may lead to a temporary or permanent 125 | ban. 126 | 127 | ### 3. Temporary Ban 128 | 129 | **Community Impact**: A serious violation of community standards, including 130 | sustained inappropriate behavior. 131 | 132 | **Consequence**: A temporary ban from any sort of interaction or public 133 | communication with the community for a specified period of time. No public or 134 | private interaction with the people involved, including unsolicited interaction 135 | with those enforcing the Code of Conduct, is allowed during this period. 136 | Violating these terms may lead to a permanent ban. 137 | 138 | ### 4. Permanent Ban 139 | 140 | **Community Impact**: Demonstrating a pattern of violation of community 141 | standards, including sustained inappropriate behavior, harassment of an 142 | individual, or aggression toward or disparagement of classes of individuals. 143 | 144 | **Consequence**: A permanent ban from any sort of public interaction within the 145 | community. 146 | 147 | ## Attribution 148 | 149 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 150 | version 2.1, available at 151 | [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. 152 | 153 | Community Impact Guidelines were inspired by 154 | [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. 155 | 156 | For answers to common questions about this code of conduct, see the FAQ at 157 | [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at 158 | [https://www.contributor-covenant.org/translations][translations]. 159 | 160 | [homepage]: https://www.contributor-covenant.org 161 | [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html 162 | [Mozilla CoC]: https://github.com/mozilla/diversity 163 | [FAQ]: https://www.contributor-covenant.org/faq 164 | [translations]: https://www.contributor-covenant.org/translations 165 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2024, Daily 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 19 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | pipecat client web 3 |

4 | 5 | [![Docs](https://img.shields.io/badge/documentation-blue)](https://docs.pipecat.ai/client/introduction) 6 | ![NPM Version](https://img.shields.io/npm/v/@pipecat-ai/client-js) 7 | 8 | The official web client SDK for [Pipecat](https://github.com/pipecat-ai/pipecat), an open source Python framework for building voice and multimodal AI applications. 9 | 10 | ## Overview 11 | 12 | This monorepo contains two packages: 13 | 14 | - `client-js`: JavaScript/TypeScript SDK for connecting to and communicating with Pipecat servers 15 | - `client-react`: React components and hooks for building Pipecat applications 16 | 17 | The SDK handles: 18 | 19 | - Device and media stream management 20 | - Managing bot configuration 21 | - Sending generic actions to the bot 22 | - Handling bot messages and responses 23 | - Managing session state and errors 24 | 25 | To connect to a bot, you will need both this SDK and a transport implementation. 26 | 27 | It’s also recommended for you to stand up your own server-side endpoints to handle authentication, and passing your bot process secrets (such as service API keys, etc) that would otherwise be compromised on the client. 28 | 29 | The entry point for creating a client can be found via: 30 | 31 | - [Pipecat JS](/client-js/) `@pipecat-ai/client-js` 32 | 33 | React context, hooks and components: 34 | 35 | - [Pipecat React](/client-react/) `@pipecat-ai/client-react` 36 | 37 | **Transport packages:** 38 | 39 | For connected use-cases, you must pass a transport instance to the constructor for your chosen protocol or provider. 40 | 41 | For example, if you were looking to use WebRTC as a transport layer, you may use a provider like [Daily](https://daily.co). In this scenario, you’d construct a transport instance and pass it to the client accordingly: 42 | 43 | ```ts 44 | import { RTVIClient } from "@pipecat-ai/client-js"; 45 | import { DailyTransport } from "@pipecat-ai/daily-transport"; 46 | 47 | const DailyTransport = new DailyTransport(); 48 | const rtviClient = new RTVIClient({ 49 | transport: DailyTransport, 50 | }); 51 | ``` 52 | 53 | All Pipecat SDKs require a media transport for sending and receiving audio and video data over the Internet. Pipecat Web does not include any transport capabilities out of the box, so you will need to install the package for your chosen provider. 54 | 55 | All transport packages (such as `DailyTransport`) extend from the Transport base class. You can extend this class if you are looking to implement your own or add additional functionality. 56 | 57 | ## Install 58 | 59 | Install the Pipecat JS client library 60 | 61 | ```bash 62 | npm install @pipecat-ai/client-js 63 | ``` 64 | 65 | Optionally, install the React client library 66 | 67 | ```bash 68 | npm install @pipecat-ai/client-react 69 | ``` 70 | 71 | Lastly, install a transport layer, like Daily 72 | 73 | ```bash 74 | npm install @pipecat-ai/daily-transport 75 | ``` 76 | 77 | ## Quickstart 78 | 79 | To connect to a bot, you will need both this SDK and a transport implementation. 80 | 81 | It’s also recommended for you to stand up your own server-side endpoints to handle authentication, and passing your bot process secrets (such as service API keys, etc) that would otherwise be compromised on the client. 82 | 83 | #### Starter projects: 84 | 85 | Creating and starting a session with RTVI Web (using Daily as transport): 86 | 87 | ```typescript 88 | import { RTVIClient, RTVIEvent, RTVIMessage } from "@pipecat-ai/client-js"; 89 | import { DailyTransport } from "@pipecat-ai/daily-transport"; 90 | 91 | const DailyTransport = new DailyTransport(); 92 | 93 | const rtviClient = new RTVIClient({ 94 | transport: DailyTransport, 95 | params: { 96 | baseUrl: "https://your-server-side-url", 97 | services: { 98 | llm: "together", 99 | tts: "cartesia", 100 | }, 101 | config: [ 102 | { 103 | service: "tts", 104 | options: [ 105 | { name: "voice", value: "79a125e8-cd45-4c13-8a67-188112f4dd22" }, 106 | ], 107 | }, 108 | { 109 | service: "llm", 110 | options: [ 111 | { 112 | name: "model", 113 | value: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", 114 | }, 115 | { 116 | name: "messages", 117 | value: [ 118 | { 119 | role: "system", 120 | content: 121 | "You are a assistant called ExampleBot. You can ask me anything. Keep responses brief and legible. Your responses will be converted to audio, so please avoid using any special characters except '!' or '?'.", 122 | }, 123 | ], 124 | }, 125 | ], 126 | }, 127 | ], 128 | }, 129 | enableMic: true, 130 | enableCam: false, 131 | timeout: 15 * 1000, 132 | callbacks: { 133 | onConnected: () => { 134 | console.log("[CALLBACK] User connected"); 135 | }, 136 | onDisconnected: () => { 137 | console.log("[CALLBACK] User disconnected"); 138 | }, 139 | onTransportStateChanged: (state: string) => { 140 | console.log("[CALLBACK] State change:", state); 141 | }, 142 | onBotConnected: () => { 143 | console.log("[CALLBACK] Bot connected"); 144 | }, 145 | onBotDisconnected: () => { 146 | console.log("[CALLBACK] Bot disconnected"); 147 | }, 148 | onBotReady: () => { 149 | console.log("[CALLBACK] Bot ready to chat!"); 150 | }, 151 | }, 152 | }); 153 | 154 | try { 155 | await rtviClient.connect(); 156 | } catch (e) { 157 | console.error(e.message); 158 | } 159 | 160 | // Events 161 | rtviClient.on(RTVIEvent.TransportStateChanged, (state) => { 162 | console.log("[EVENT] Transport state change:", state); 163 | }); 164 | rtviClient.on(RTVIEvent.BotReady, () => { 165 | console.log("[EVENT] Bot is ready"); 166 | }); 167 | rtviClient.on(RTVIEvent.Connected, () => { 168 | console.log("[EVENT] User connected"); 169 | }); 170 | rtviClient.on(RTVIEvent.Disconnected, () => { 171 | console.log("[EVENT] User disconnected"); 172 | }); 173 | ``` 174 | 175 | ## Documentation 176 | 177 | Pipecat Client Web implements a client instance that: 178 | 179 | - Facilitates web requests to an endpoint you create. 180 | - Dispatches single-turn actions to a HTTP bot service when disconnected. 181 | - Provides methods that handle the connectivity state and realtime interaction with your bot service. 182 | - Manages media transport (such as audio and video). 183 | - Provides callbacks and events for handling bot messages and actions. 184 | - Optionally configures your AI services and pipeline. 185 | 186 | Docs and API reference can be found at https://docs.pipecat.ai/client/introduction. 187 | 188 | ## Hack on the framework 189 | 190 | Install a provider transport 191 | 192 | ```bash 193 | yarn 194 | yarn workspace @pipecat-ai/client-js build 195 | ``` 196 | 197 | Watch for file changes: 198 | 199 | ```bash 200 | yarn workspace @pipecat-ai/client-js run dev 201 | ``` 202 | 203 | ## Contributing 204 | 205 | We welcome contributions from the community! Whether you're fixing bugs, improving documentation, or adding new features, here's how you can help: 206 | 207 | - **Found a bug?** Open an [issue](https://github.com/pipecat-ai/pipecat-client-web/issues) 208 | - **Have a feature idea?** Start a [discussion](https://discord.gg/pipecat) 209 | - **Want to contribute code?** Check our [CONTRIBUTING.md](CONTRIBUTING.md) guide 210 | - **Documentation improvements?** [Docs](https://github.com/pipecat-ai/docs) PRs are always welcome 211 | 212 | Before submitting a pull request, please check existing issues and PRs to avoid duplicates. 213 | 214 | We aim to review all contributions promptly and provide constructive feedback to help get your changes merged. 215 | 216 | ## Getting help 217 | 218 | ➡️ [Join our Discord](https://discord.gg/pipecat) 219 | 220 | ➡️ [Read the docs](https://docs.pipecat.ai) 221 | 222 | ➡️ [Reach us on X](https://x.com/pipecat_ai) 223 | -------------------------------------------------------------------------------- /client-js/README.md: -------------------------------------------------------------------------------- 1 |

2 | pipecat js 3 |

4 | 5 | [![Docs](https://img.shields.io/badge/documentation-blue)](https://docs.pipecat.ai/client/introduction) 6 | ![NPM Version](https://img.shields.io/npm/v/@pipecat-ai/client-js) 7 | 8 | ## Install 9 | 10 | ```bash 11 | yarn add @pipecat-ai/client-js 12 | # or 13 | npm install @pipecat-ai/client-js 14 | ``` 15 | 16 | ## Quick Start 17 | 18 | Instantiate a `RTVIClient` instance, wire up the bot's audio, and start the conversation: 19 | 20 | ```ts 21 | import { RTVIEvent, RTVIMessage, RTVIClient } from "@pipecat-ai/client-js"; 22 | import { DailyTransport } from "@pipecat-ai/daily-transport"; 23 | 24 | const dailyTransport = new DailyTransport(); 25 | 26 | const rtviClient = new RTVIClient({ 27 | transport: dailyTransport, 28 | params: { 29 | baseUrl: "https://your-connect-end-point-here", 30 | services: { 31 | llm: "together", 32 | tts: "cartesia", 33 | }, 34 | config: [ 35 | { 36 | service: "tts", 37 | options: [ 38 | { name: "voice", value: "79a125e8-cd45-4c13-8a67-188112f4dd22" }, 39 | ], 40 | }, 41 | { 42 | service: "llm", 43 | options: [ 44 | { 45 | name: "model", 46 | value: "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", 47 | }, 48 | { 49 | name: "messages", 50 | value: [ 51 | { 52 | role: "system", 53 | content: 54 | "You are a assistant called ExampleBot. You can ask me anything. Keep responses brief and legible. Your responses will be converted to audio, so please avoid using any special characters except '!' or '?'.", 55 | }, 56 | ], 57 | }, 58 | ], 59 | }, 60 | ], 61 | }, 62 | enableMic: true, 63 | enableCam: false, 64 | timeout: 15 * 1000, 65 | callbacks: { 66 | onConnected: () => { 67 | console.log("[CALLBACK] User connected"); 68 | }, 69 | onDisconnected: () => { 70 | console.log("[CALLBACK] User disconnected"); 71 | }, 72 | onTransportStateChanged: (state: string) => { 73 | console.log("[CALLBACK] State change:", state); 74 | }, 75 | onBotConnected: () => { 76 | console.log("[CALLBACK] Bot connected"); 77 | }, 78 | onBotDisconnected: () => { 79 | console.log("[CALLBACK] Bot disconnected"); 80 | }, 81 | onBotReady: () => { 82 | console.log("[CALLBACK] Bot ready to chat!"); 83 | }, 84 | }, 85 | }); 86 | 87 | try { 88 | await rtviClient.connect(); 89 | } catch (e) { 90 | console.error(e.message); 91 | } 92 | 93 | // Events 94 | rtviClient.on(RTVIEvent.TransportStateChanged, (state) => { 95 | console.log("[EVENT] Transport state change:", state); 96 | }); 97 | rtviClient.on(RTVIEvent.BotReady, () => { 98 | console.log("[EVENT] Bot is ready"); 99 | }); 100 | rtviClient.on(RTVIEvent.Connected, () => { 101 | console.log("[EVENT] User connected"); 102 | }); 103 | rtviClient.on(RTVIEvent.Disconnected, () => { 104 | console.log("[EVENT] User disconnected"); 105 | }); 106 | ``` 107 | 108 | ## API 109 | 110 | Please see API reference [here](https://docs.pipecat.ai/client/reference/js/introduction). 111 | -------------------------------------------------------------------------------- /client-js/eslint.config.mjs: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import jsRecommended from "@eslint/js"; 8 | import tsRecommended from "@typescript-eslint/eslint-plugin"; 9 | import tsParser from "@typescript-eslint/parser"; 10 | import simpleImportSort from "eslint-plugin-simple-import-sort"; 11 | import globals from "globals"; 12 | 13 | export default [ 14 | { 15 | files: ["**/*.{js,mjs,cjs,ts}"], 16 | 17 | languageOptions: { 18 | parser: tsParser, 19 | globals: globals.browser, 20 | }, 21 | 22 | plugins: { 23 | "simple-import-sort": simpleImportSort, 24 | "@typescript-eslint": tsRecommended, 25 | }, 26 | 27 | rules: { 28 | "simple-import-sort/imports": "error", 29 | "simple-import-sort/exports": "error", 30 | ...jsRecommended.configs.recommended.rules, 31 | ...tsRecommended.configs.recommended.rules, 32 | }, 33 | }, 34 | ]; 35 | -------------------------------------------------------------------------------- /client-js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@pipecat-ai/client-js", 3 | "version": "0.3.5", 4 | "license": "BSD-2-Clause", 5 | "main": "dist/index.js", 6 | "module": "dist/index.module.js", 7 | "types": "dist/index.d.ts", 8 | "source": "src/index.ts", 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/pipecat-ai/pipecat-client-web.git" 12 | }, 13 | "files": [ 14 | "dist", 15 | "package.json", 16 | "README.md" 17 | ], 18 | "scripts": { 19 | "build": "jest --silent && parcel build --no-cache", 20 | "dev": "parcel watch", 21 | "lint": "eslint src/ --report-unused-disable-directives --max-warnings 0", 22 | "test": "jest" 23 | }, 24 | "jest": { 25 | "preset": "ts-jest", 26 | "testEnvironment": "node" 27 | }, 28 | "devDependencies": { 29 | "@jest/globals": "^29.7.0", 30 | "@types/clone-deep": "^4.0.4", 31 | "@types/jest": "^29.5.12", 32 | "@types/uuid": "^10.0.0", 33 | "eslint": "^9.11.1", 34 | "eslint-config-prettier": "^9.1.0", 35 | "eslint-plugin-simple-import-sort": "^12.1.1", 36 | "jest": "^29.7.0", 37 | "ts-jest": "^29.2.5" 38 | }, 39 | "dependencies": { 40 | "@types/events": "^3.0.3", 41 | "clone-deep": "^4.0.1", 42 | "events": "^3.3.0", 43 | "typed-emitter": "^2.1.0", 44 | "uuid": "^10.0.0" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /client-js/src/actions.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { logger, RTVIClientParams, RTVIError } from "."; 8 | import { RTVIActionRequest, RTVIActionResponse } from "./messages"; 9 | 10 | export async function httpActionGenerator( 11 | actionUrl: string, 12 | action: RTVIActionRequest, 13 | params: RTVIClientParams, 14 | handleResponse: (response: RTVIActionResponse) => void 15 | ): Promise { 16 | try { 17 | logger.debug("[RTVI] Fetch action", actionUrl, action); 18 | 19 | const headers = new Headers({ 20 | ...Object.fromEntries((params.headers ?? new Headers()).entries()), 21 | }); 22 | 23 | if (!headers.has("Content-Type")) { 24 | headers.set("Content-Type", "application/json"); 25 | } 26 | headers.set("Cache-Control", "no-cache"); 27 | headers.set("Connection", "keep-alive"); 28 | 29 | // Perform the fetch request 30 | const response = await fetch(actionUrl, { 31 | method: "POST", 32 | headers, 33 | body: JSON.stringify({ ...params.requestData, actions: [action] }), 34 | }); 35 | 36 | // Check the response content type 37 | const contentType = response.headers.get("content-type"); 38 | 39 | // Handle non-ok response status 40 | if (!response.ok) { 41 | const errorMessage = await response.text(); 42 | throw new RTVIError( 43 | `Failed to resolve action: ${errorMessage}`, 44 | response.status 45 | ); 46 | } 47 | 48 | if (response.body && contentType?.includes("text/event-stream")) { 49 | // Parse streamed responses 50 | const reader = response.body 51 | .pipeThrough(new TextDecoderStream()) 52 | .getReader(); 53 | 54 | let buffer = ""; 55 | 56 | while (true) { 57 | const { value, done } = await reader.read(); 58 | if (done) break; 59 | 60 | buffer += value; 61 | 62 | let boundary = buffer.indexOf("\n\n"); 63 | while (boundary !== -1) { 64 | const message = buffer.slice(0, boundary); 65 | buffer = buffer.slice(boundary + 2); 66 | 67 | // Split on the first ":" to extract the JSON part 68 | const lines = message.split("\n"); 69 | let encodedData = ""; 70 | for (const line of lines) { 71 | const colonIndex = line.indexOf(":"); 72 | if (colonIndex !== -1) { 73 | encodedData += line.slice(colonIndex + 1).trim(); 74 | } 75 | } 76 | 77 | try { 78 | const jsonData = atob(encodedData); 79 | const parsedData = JSON.parse(jsonData); 80 | handleResponse(parsedData); 81 | } catch (error) { 82 | logger.error("[RTVI] Failed to parse JSON:", error); 83 | throw error; 84 | } 85 | 86 | boundary = buffer.indexOf("\n\n"); 87 | } 88 | } 89 | } else { 90 | // For regular non-streamed responses, parse and handle the data as JSON 91 | const data = await response.json(); 92 | handleResponse(data); 93 | } 94 | } catch (error) { 95 | logger.error("[RTVI] Error during fetch:", error); 96 | throw error; 97 | } 98 | } 99 | /* 100 | //@TODO: implement abortController when mode changes / bad things happen 101 | export async function dispatchAction( 102 | this: RTVIClient, 103 | action: RTVIActionRequest 104 | ): Promise { 105 | const promise = new Promise((resolve, reject) => { 106 | (async () => { 107 | if (this.connected) { 108 | return this._messageDispatcher.dispatch(action); 109 | } else { 110 | const actionUrl = this.constructUrl("action"); 111 | try { 112 | const result = await httpActionGenerator( 113 | actionUrl, 114 | action, 115 | this.params, 116 | (response) => { 117 | this.handleMessage(response); 118 | } 119 | ); 120 | resolve(result); 121 | } catch (error) { 122 | reject(error); 123 | } 124 | } 125 | })(); 126 | }); 127 | 128 | return promise as Promise; 129 | } 130 | */ 131 | -------------------------------------------------------------------------------- /client-js/src/client.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import cloneDeep from "clone-deep"; 8 | import EventEmitter from "events"; 9 | import TypedEmitter from "typed-emitter"; 10 | 11 | import packageJson from "../package.json"; 12 | import { getIfTransportInState, transportReady } from "./decorators"; 13 | import * as RTVIErrors from "./errors"; 14 | import { RTVIEvent, RTVIEvents } from "./events"; 15 | import { RTVIClientHelper, RTVIClientHelpers } from "./helpers"; 16 | import { logger, LogLevel } from "./logger"; 17 | import { 18 | BotLLMSearchResponseData, 19 | BotLLMTextData, 20 | BotReadyData, 21 | BotTTSTextData, 22 | ConfigData, 23 | ErrorData, 24 | MessageDispatcher, 25 | PipecatMetricsData, 26 | RTVIActionRequest, 27 | RTVIActionRequestData, 28 | RTVIActionResponse, 29 | RTVIMessage, 30 | RTVIMessageType, 31 | StorageItemStoredData, 32 | TranscriptData, 33 | } from "./messages"; 34 | import { 35 | Participant, 36 | Tracks, 37 | Transport, 38 | TransportState, 39 | TransportWrapper, 40 | } from "./transport"; 41 | 42 | export type ConfigOption = { 43 | name: string; 44 | value: unknown; 45 | }; 46 | 47 | export type RTVIClientConfigOption = { 48 | service: string; 49 | options: ConfigOption[]; 50 | }; 51 | 52 | export type RTVIURLEndpoints = "connect" | "action"; 53 | 54 | const defaultEndpoints: Record = { 55 | connect: "/connect", 56 | action: "/action", 57 | }; 58 | 59 | export type RTVIClientParams = { 60 | baseUrl?: string | null; 61 | } & Partial<{ 62 | headers?: Headers; 63 | endpoints: Partial>; 64 | requestData?: object; 65 | config?: RTVIClientConfigOption[]; 66 | }> & { 67 | [key: string]: unknown; 68 | }; 69 | 70 | export interface RTVIClientOptions { 71 | /** 72 | * Parameters passed as JSON stringified body params to all endpoints (e.g. connect) 73 | */ 74 | params: RTVIClientParams; 75 | 76 | /** 77 | * Transport class for media streaming 78 | */ 79 | transport: Transport; 80 | 81 | /** 82 | * Optional callback methods for RTVI events 83 | */ 84 | callbacks?: RTVIEventCallbacks; 85 | 86 | /** 87 | * Handshake timeout 88 | * 89 | * How long should the client wait for the bot ready event (when authenticating / requesting an agent) 90 | * Defaults to no timeout (undefined) 91 | */ 92 | timeout?: number; 93 | 94 | /** 95 | * Enable user mic input 96 | * 97 | * Default to true 98 | */ 99 | enableMic?: boolean; 100 | 101 | /** 102 | * Enable user cam input 103 | * 104 | * Default to false 105 | */ 106 | enableCam?: boolean; 107 | 108 | /** 109 | * Custom start method handler for retrieving auth bundle for transport 110 | * @param baseUrl 111 | * @param params 112 | * @param timeout 113 | * @param abortController 114 | * @returns Promise 115 | */ 116 | customConnectHandler?: ( 117 | params: RTVIClientParams, 118 | timeout: ReturnType | undefined, 119 | abortController: AbortController 120 | ) => Promise; 121 | } 122 | 123 | export type RTVIEventCallbacks = Partial<{ 124 | onGenericMessage: (data: unknown) => void; 125 | onMessageError: (message: RTVIMessage) => void; 126 | onError: (message: RTVIMessage) => void; 127 | onConnected: () => void; 128 | onDisconnected: () => void; 129 | onTransportStateChanged: (state: TransportState) => void; 130 | onConfig: (config: RTVIClientConfigOption[]) => void; 131 | onConfigDescribe: (configDescription: unknown) => void; 132 | onActionsAvailable: (actions: unknown) => void; 133 | onBotConnected: (participant: Participant) => void; 134 | onBotReady: (botReadyData: BotReadyData) => void; 135 | onBotDisconnected: (participant: Participant) => void; 136 | onParticipantJoined: (participant: Participant) => void; 137 | onParticipantLeft: (participant: Participant) => void; 138 | onMetrics: (data: PipecatMetricsData) => void; 139 | 140 | onAvailableCamsUpdated: (cams: MediaDeviceInfo[]) => void; 141 | onAvailableMicsUpdated: (mics: MediaDeviceInfo[]) => void; 142 | onAvailableSpeakersUpdated: (speakers: MediaDeviceInfo[]) => void; 143 | onCamUpdated: (cam: MediaDeviceInfo) => void; 144 | onMicUpdated: (mic: MediaDeviceInfo) => void; 145 | onSpeakerUpdated: (speaker: MediaDeviceInfo) => void; 146 | onTrackStarted: (track: MediaStreamTrack, participant?: Participant) => void; 147 | onTrackStopped: (track: MediaStreamTrack, participant?: Participant) => void; 148 | onScreenTrackStarted: ( 149 | track: MediaStreamTrack, 150 | participant?: Participant 151 | ) => void; 152 | onScreenTrackStopped: ( 153 | track: MediaStreamTrack, 154 | participant?: Participant 155 | ) => void; 156 | onScreenShareError: (errorMessage: string) => void; 157 | onLocalAudioLevel: (level: number) => void; 158 | onRemoteAudioLevel: (level: number, participant: Participant) => void; 159 | 160 | onBotStartedSpeaking: () => void; 161 | onBotStoppedSpeaking: () => void; 162 | onUserStartedSpeaking: () => void; 163 | onUserStoppedSpeaking: () => void; 164 | onUserTranscript: (data: TranscriptData) => void; 165 | onBotTranscript: (data: BotLLMTextData) => void; 166 | 167 | onBotLlmText: (data: BotLLMTextData) => void; 168 | onBotLlmStarted: () => void; 169 | onBotLlmStopped: () => void; 170 | onBotTtsText: (data: BotTTSTextData) => void; 171 | onBotTtsStarted: () => void; 172 | onBotTtsStopped: () => void; 173 | onBotLlmSearchResponse: (data: BotLLMSearchResponseData) => void; 174 | 175 | onStorageItemStored: (data: StorageItemStoredData) => void; 176 | 177 | onServerMessage: (data: any) => void; 178 | }>; 179 | 180 | abstract class RTVIEventEmitter extends (EventEmitter as unknown as new () => TypedEmitter) {} 181 | 182 | export class RTVIClient extends RTVIEventEmitter { 183 | public params: RTVIClientParams; 184 | protected _options: RTVIClientOptions; 185 | private _abortController: AbortController | undefined; 186 | private _handshakeTimeout: ReturnType | undefined; 187 | private _helpers: RTVIClientHelpers; 188 | private _startResolve: ((value: unknown) => void) | undefined; 189 | protected _transport: Transport; 190 | protected _transportWrapper: TransportWrapper; 191 | protected declare _messageDispatcher: MessageDispatcher; 192 | 193 | constructor(options: RTVIClientOptions) { 194 | super(); 195 | 196 | this.params = { 197 | ...options.params, 198 | endpoints: { 199 | ...defaultEndpoints, 200 | ...(options.params.endpoints ?? {}), 201 | }, 202 | }; 203 | 204 | this._helpers = {}; 205 | this._transport = options.transport; 206 | this._transportWrapper = new TransportWrapper(this._transport); 207 | 208 | // Wrap transport callbacks with event triggers 209 | // This allows for either functional callbacks or .on / .off event listeners 210 | const wrappedCallbacks: RTVIEventCallbacks = { 211 | ...options.callbacks, 212 | onMessageError: (message: RTVIMessage) => { 213 | options?.callbacks?.onMessageError?.(message); 214 | this.emit(RTVIEvent.MessageError, message); 215 | }, 216 | onError: (message: RTVIMessage) => { 217 | options?.callbacks?.onError?.(message); 218 | try { 219 | this.emit(RTVIEvent.Error, message); 220 | } catch (e) { 221 | logger.debug("Could not emit error", message); 222 | } 223 | const data = message.data as ErrorData; 224 | if (data?.fatal) { 225 | logger.error("Fatal error reported. Disconnecting..."); 226 | this.disconnect(); 227 | } 228 | }, 229 | onConnected: () => { 230 | options?.callbacks?.onConnected?.(); 231 | this.emit(RTVIEvent.Connected); 232 | }, 233 | onDisconnected: () => { 234 | options?.callbacks?.onDisconnected?.(); 235 | this.emit(RTVIEvent.Disconnected); 236 | }, 237 | onTransportStateChanged: (state: TransportState) => { 238 | options?.callbacks?.onTransportStateChanged?.(state); 239 | this.emit(RTVIEvent.TransportStateChanged, state); 240 | }, 241 | onConfig: (config: RTVIClientConfigOption[]) => { 242 | options?.callbacks?.onConfig?.(config); 243 | this.emit(RTVIEvent.Config, config); 244 | }, 245 | onConfigDescribe: (configDescription: unknown) => { 246 | options?.callbacks?.onConfigDescribe?.(configDescription); 247 | this.emit(RTVIEvent.ConfigDescribe, configDescription); 248 | }, 249 | onActionsAvailable: (actionsAvailable: unknown) => { 250 | options?.callbacks?.onActionsAvailable?.(actionsAvailable); 251 | this.emit(RTVIEvent.ActionsAvailable, actionsAvailable); 252 | }, 253 | onParticipantJoined: (p) => { 254 | options?.callbacks?.onParticipantJoined?.(p); 255 | this.emit(RTVIEvent.ParticipantConnected, p); 256 | }, 257 | onParticipantLeft: (p) => { 258 | options?.callbacks?.onParticipantLeft?.(p); 259 | this.emit(RTVIEvent.ParticipantLeft, p); 260 | }, 261 | onTrackStarted: (track, p) => { 262 | options?.callbacks?.onTrackStarted?.(track, p); 263 | this.emit(RTVIEvent.TrackStarted, track, p); 264 | }, 265 | onTrackStopped: (track, p) => { 266 | options?.callbacks?.onTrackStopped?.(track, p); 267 | this.emit(RTVIEvent.TrackStopped, track, p); 268 | }, 269 | onScreenTrackStarted: (track, p) => { 270 | options?.callbacks?.onScreenTrackStarted?.(track, p); 271 | this.emit(RTVIEvent.ScreenTrackStarted, track, p); 272 | }, 273 | onScreenTrackStopped: (track, p) => { 274 | options?.callbacks?.onScreenTrackStopped?.(track, p); 275 | this.emit(RTVIEvent.ScreenTrackStopped, track, p); 276 | }, 277 | onScreenShareError: (errorMessage) => { 278 | options?.callbacks?.onScreenShareError?.(errorMessage); 279 | this.emit(RTVIEvent.ScreenShareError, errorMessage); 280 | }, 281 | onAvailableCamsUpdated: (cams) => { 282 | options?.callbacks?.onAvailableCamsUpdated?.(cams); 283 | this.emit(RTVIEvent.AvailableCamsUpdated, cams); 284 | }, 285 | onAvailableMicsUpdated: (mics) => { 286 | options?.callbacks?.onAvailableMicsUpdated?.(mics); 287 | this.emit(RTVIEvent.AvailableMicsUpdated, mics); 288 | }, 289 | onAvailableSpeakersUpdated: (speakers) => { 290 | options?.callbacks?.onAvailableSpeakersUpdated?.(speakers); 291 | this.emit(RTVIEvent.AvailableSpeakersUpdated, speakers); 292 | }, 293 | onCamUpdated: (cam) => { 294 | options?.callbacks?.onCamUpdated?.(cam); 295 | this.emit(RTVIEvent.CamUpdated, cam); 296 | }, 297 | onMicUpdated: (mic) => { 298 | options?.callbacks?.onMicUpdated?.(mic); 299 | this.emit(RTVIEvent.MicUpdated, mic); 300 | }, 301 | onSpeakerUpdated: (speaker) => { 302 | options?.callbacks?.onSpeakerUpdated?.(speaker); 303 | this.emit(RTVIEvent.SpeakerUpdated, speaker); 304 | }, 305 | onBotConnected: (p) => { 306 | options?.callbacks?.onBotConnected?.(p); 307 | this.emit(RTVIEvent.BotConnected, p); 308 | }, 309 | onBotReady: (botReadyData: BotReadyData) => { 310 | options?.callbacks?.onBotReady?.(botReadyData); 311 | this.emit(RTVIEvent.BotReady, botReadyData); 312 | }, 313 | onBotDisconnected: (p) => { 314 | options?.callbacks?.onBotDisconnected?.(p); 315 | this.emit(RTVIEvent.BotDisconnected, p); 316 | }, 317 | onBotStartedSpeaking: () => { 318 | options?.callbacks?.onBotStartedSpeaking?.(); 319 | this.emit(RTVIEvent.BotStartedSpeaking); 320 | }, 321 | onBotStoppedSpeaking: () => { 322 | options?.callbacks?.onBotStoppedSpeaking?.(); 323 | this.emit(RTVIEvent.BotStoppedSpeaking); 324 | }, 325 | onRemoteAudioLevel: (level, p) => { 326 | options?.callbacks?.onRemoteAudioLevel?.(level, p); 327 | this.emit(RTVIEvent.RemoteAudioLevel, level, p); 328 | }, 329 | onUserStartedSpeaking: () => { 330 | options?.callbacks?.onUserStartedSpeaking?.(); 331 | this.emit(RTVIEvent.UserStartedSpeaking); 332 | }, 333 | onUserStoppedSpeaking: () => { 334 | options?.callbacks?.onUserStoppedSpeaking?.(); 335 | this.emit(RTVIEvent.UserStoppedSpeaking); 336 | }, 337 | onLocalAudioLevel: (level) => { 338 | options?.callbacks?.onLocalAudioLevel?.(level); 339 | this.emit(RTVIEvent.LocalAudioLevel, level); 340 | }, 341 | onUserTranscript: (data) => { 342 | options?.callbacks?.onUserTranscript?.(data); 343 | this.emit(RTVIEvent.UserTranscript, data); 344 | }, 345 | onBotTranscript: (text) => { 346 | options?.callbacks?.onBotTranscript?.(text); 347 | this.emit(RTVIEvent.BotTranscript, text); 348 | }, 349 | onBotLlmText: (text) => { 350 | options?.callbacks?.onBotLlmText?.(text); 351 | this.emit(RTVIEvent.BotLlmText, text); 352 | }, 353 | onBotLlmStarted: () => { 354 | options?.callbacks?.onBotLlmStarted?.(); 355 | this.emit(RTVIEvent.BotLlmStarted); 356 | }, 357 | onBotLlmStopped: () => { 358 | options?.callbacks?.onBotLlmStopped?.(); 359 | this.emit(RTVIEvent.BotLlmStopped); 360 | }, 361 | onBotTtsText: (text) => { 362 | options?.callbacks?.onBotTtsText?.(text); 363 | this.emit(RTVIEvent.BotTtsText, text); 364 | }, 365 | onBotTtsStarted: () => { 366 | options?.callbacks?.onBotTtsStarted?.(); 367 | this.emit(RTVIEvent.BotTtsStarted); 368 | }, 369 | onBotTtsStopped: () => { 370 | options?.callbacks?.onBotTtsStopped?.(); 371 | this.emit(RTVIEvent.BotTtsStopped); 372 | }, 373 | onStorageItemStored: (data) => { 374 | options?.callbacks?.onStorageItemStored?.(data); 375 | this.emit(RTVIEvent.StorageItemStored, data); 376 | }, 377 | }; 378 | 379 | // Update options to reference wrapped callbacks and config defaults 380 | this._options = { 381 | ...options, 382 | callbacks: wrappedCallbacks, 383 | enableMic: options.enableMic ?? true, 384 | enableCam: options.enableCam ?? false, 385 | }; 386 | 387 | // Instantiate the transport class and bind message handler 388 | this._initialize(); 389 | 390 | // Get package version number 391 | logger.debug("[RTVI Client] Initialized", this.version); 392 | } 393 | 394 | public constructUrl(endpoint: RTVIURLEndpoints): string { 395 | if (!this.params.baseUrl) { 396 | throw new RTVIErrors.RTVIError( 397 | "Base URL not set. Please set rtviClient.params.baseUrl" 398 | ); 399 | } 400 | const baseUrl = this.params.baseUrl.replace(/\/+$/, ""); 401 | return baseUrl + (this.params.endpoints?.[endpoint] ?? ""); 402 | } 403 | 404 | public setLogLevel(level: LogLevel) { 405 | logger.setLevel(level); 406 | } 407 | 408 | // ------ Transport methods 409 | 410 | /** 411 | * Initialize local media devices 412 | */ 413 | public async initDevices() { 414 | logger.debug("[RTVI Client] Initializing devices..."); 415 | await this._transport.initDevices(); 416 | } 417 | 418 | /** 419 | * Connect the voice client session with chosen transport 420 | * Call async (await) to handle errors 421 | */ 422 | public async connect(): Promise { 423 | if ( 424 | ["authenticating", "connecting", "connected", "ready"].includes( 425 | this._transport.state 426 | ) 427 | ) { 428 | throw new RTVIErrors.RTVIError( 429 | "Voice client has already been started. Please call disconnect() before starting again." 430 | ); 431 | } 432 | 433 | this._abortController = new AbortController(); 434 | 435 | // Establish transport session and await bot ready signal 436 | return new Promise((resolve, reject) => { 437 | (async () => { 438 | this._startResolve = resolve; 439 | 440 | if (this._transport.state === "disconnected") { 441 | await this._transport.initDevices(); 442 | } 443 | 444 | this._transport.state = "authenticating"; 445 | 446 | // Set a timer for the bot to enter a ready state, otherwise abort the attempt 447 | if (this._options.timeout) { 448 | this._handshakeTimeout = setTimeout(async () => { 449 | this._abortController?.abort(); 450 | await this.disconnect(); 451 | this._transport.state = "error"; 452 | reject(new RTVIErrors.ConnectionTimeoutError()); 453 | }, this._options.timeout); 454 | } 455 | 456 | let authBundle: unknown; 457 | const customConnectHandler = this._options.customConnectHandler; 458 | 459 | logger.debug("[RTVI Client] Start params", this.params); 460 | 461 | this.params = { 462 | ...this.params, 463 | requestData: { 464 | ...this.params.requestData, 465 | rtvi_client_version: this.version, 466 | }, 467 | }; 468 | 469 | if (!this.params.baseUrl && !this.params.endpoints?.connect) { 470 | // If baseUrl and endpoints.connect are not set, bypass the handshake and connect directly 471 | // This is useful with transports that do not require service side auth, especially in local development 472 | // Note: this is not recommended for production use, see [docs link] 473 | logger.debug( 474 | "[RTVI Client] Connecting directly (skipping handshake / auth)..." 475 | ); 476 | clearTimeout(this._handshakeTimeout); 477 | } else { 478 | const connectUrl = this.constructUrl("connect"); 479 | 480 | logger.debug("[RTVI Client] Connecting...", connectUrl); 481 | logger.debug("[RTVI Client] Start params", this.params); 482 | 483 | try { 484 | if (customConnectHandler) { 485 | authBundle = await customConnectHandler( 486 | this.params, 487 | this._handshakeTimeout, 488 | this._abortController! 489 | ); 490 | } else { 491 | authBundle = await fetch(connectUrl, { 492 | method: "POST", 493 | mode: "cors", 494 | headers: new Headers({ 495 | "Content-Type": "application/json", 496 | ...Object.fromEntries( 497 | (this.params.headers ?? new Headers()).entries() 498 | ), 499 | }), 500 | body: JSON.stringify({ 501 | config: this.params.config, 502 | ...(this.params.services 503 | ? { services: this.params.services } 504 | : {}), // @deprecated - pass services through request data 505 | ...this.params.requestData, 506 | }), 507 | signal: this._abortController?.signal, 508 | }).then((res) => { 509 | clearTimeout(this._handshakeTimeout); 510 | 511 | if (res.ok) { 512 | return res.json(); 513 | } 514 | 515 | return Promise.reject(res); 516 | }); 517 | } 518 | } catch (e) { 519 | clearTimeout(this._handshakeTimeout); 520 | // Handle errors if the request was not aborted 521 | if (this._abortController?.signal.aborted) { 522 | return; 523 | } 524 | this._transport.state = "error"; 525 | if (e instanceof Response) { 526 | const errorResp = await e.json(); 527 | reject( 528 | new RTVIErrors.StartBotError( 529 | errorResp.info ?? errorResp.detail ?? e.statusText, 530 | e.status 531 | ) 532 | ); 533 | } else { 534 | reject(new RTVIErrors.StartBotError()); 535 | } 536 | return; 537 | } 538 | 539 | logger.debug("[RTVI Client] Auth bundle received", authBundle); 540 | } 541 | try { 542 | await this._transport.connect( 543 | authBundle, 544 | this._abortController as AbortController 545 | ); 546 | await this._transport.sendReadyMessage(); 547 | } catch (e) { 548 | clearTimeout(this._handshakeTimeout); 549 | this.disconnect(); 550 | reject(e); 551 | return; 552 | } 553 | })(); 554 | }); 555 | } 556 | 557 | /** 558 | * Disconnect the voice client from the transport 559 | * Reset / reinitialize transport and abort any pending requests 560 | */ 561 | public async disconnect(): Promise { 562 | if (this._abortController) { 563 | this._abortController.abort(); 564 | } 565 | 566 | clearTimeout(this._handshakeTimeout); 567 | 568 | await this._transport.disconnect(); 569 | 570 | this._messageDispatcher = new MessageDispatcher(this); 571 | } 572 | 573 | private _initialize() { 574 | this._transport.initialize(this._options, this.handleMessage.bind(this)); 575 | 576 | // Create a new message dispatch queue for async message handling 577 | this._messageDispatcher = new MessageDispatcher(this); 578 | } 579 | 580 | /** 581 | * Get the current state of the transport 582 | */ 583 | public get connected(): boolean { 584 | return ["connected", "ready"].includes(this._transport.state); 585 | } 586 | 587 | public get transport(): Transport { 588 | return this._transportWrapper.proxy; 589 | } 590 | 591 | public get state(): TransportState { 592 | return this._transport.state; 593 | } 594 | 595 | public get version(): string { 596 | return packageJson.version; 597 | } 598 | 599 | // ------ Device methods 600 | 601 | public async getAllMics(): Promise { 602 | return await this._transport.getAllMics(); 603 | } 604 | 605 | public async getAllCams(): Promise { 606 | return await this._transport.getAllCams(); 607 | } 608 | 609 | public async getAllSpeakers(): Promise { 610 | return await this._transport.getAllSpeakers(); 611 | } 612 | 613 | public get selectedMic() { 614 | return this._transport.selectedMic; 615 | } 616 | 617 | public get selectedCam() { 618 | return this._transport.selectedCam; 619 | } 620 | 621 | public get selectedSpeaker() { 622 | return this._transport.selectedSpeaker; 623 | } 624 | 625 | public updateMic(micId: string) { 626 | this._transport.updateMic(micId); 627 | } 628 | 629 | public updateCam(camId: string) { 630 | this._transport.updateCam(camId); 631 | } 632 | 633 | public updateSpeaker(speakerId: string) { 634 | this._transport.updateSpeaker(speakerId); 635 | } 636 | 637 | public enableMic(enable: boolean) { 638 | this._transport.enableMic(enable); 639 | } 640 | 641 | public get isMicEnabled(): boolean { 642 | return this._transport.isMicEnabled; 643 | } 644 | 645 | public enableCam(enable: boolean) { 646 | this._transport.enableCam(enable); 647 | } 648 | 649 | public get isCamEnabled(): boolean { 650 | return this._transport.isCamEnabled; 651 | } 652 | 653 | public tracks(): Tracks { 654 | return this._transport.tracks(); 655 | } 656 | 657 | public enableScreenShare(enable: boolean) { 658 | return this._transport.enableScreenShare(enable); 659 | } 660 | 661 | public get isSharingScreen(): boolean { 662 | return this._transport.isSharingScreen; 663 | } 664 | 665 | // ------ Config methods 666 | 667 | /** 668 | * Request the bot to send the current configuration 669 | * @returns Promise - Promise that resolves with the bot's configuration 670 | */ 671 | @transportReady 672 | public async getConfig(): Promise { 673 | const configMsg = await this._messageDispatcher.dispatch( 674 | RTVIMessage.getBotConfig() 675 | ); 676 | return (configMsg.data as ConfigData).config as RTVIClientConfigOption[]; 677 | } 678 | 679 | /** 680 | * Update pipeline and services 681 | * @param config - RTVIClientConfigOption[] partial object with the new configuration 682 | * @param interrupt - boolean flag to interrupt the current pipeline, or wait until the next turn 683 | * @returns Promise - Promise that resolves with the updated configuration 684 | */ 685 | @transportReady 686 | public async updateConfig( 687 | config: RTVIClientConfigOption[], 688 | interrupt: boolean = false 689 | ): Promise { 690 | logger.debug("[RTVI Client] Updating config", config); 691 | // Only send the partial config if the bot is ready to prevent 692 | // potential racing conditions whilst pipeline is instantiating 693 | return this._messageDispatcher.dispatch( 694 | RTVIMessage.updateConfig(config, interrupt) 695 | ); 696 | } 697 | 698 | /** 699 | * Request bot describe the current configuration options 700 | * @returns Promise - Promise that resolves with the bot's configuration description 701 | */ 702 | @transportReady 703 | public async describeConfig(): Promise { 704 | return this._messageDispatcher.dispatch(RTVIMessage.describeConfig()); 705 | } 706 | 707 | /** 708 | * Returns configuration options for specified service key 709 | * @param serviceKey - Service name to get options for (e.g. "llm") 710 | * @param config? - Optional RTVIClientConfigOption[] to query (vs. using remote config) 711 | * @returns RTVIClientConfigOption | undefined - Configuration options array for the service with specified key or undefined 712 | */ 713 | public async getServiceOptionsFromConfig( 714 | serviceKey: string, 715 | config?: RTVIClientConfigOption[] 716 | ): Promise { 717 | if (!config && this.state !== "ready") { 718 | throw new RTVIErrors.BotNotReadyError( 719 | "getServiceOptionsFromConfig called without config array before bot is ready" 720 | ); 721 | } 722 | 723 | return Promise.resolve().then(async () => { 724 | // Check if we have registered service with name service 725 | if (!serviceKey) { 726 | logger.debug("Target service name is required"); 727 | return undefined; 728 | } 729 | 730 | const passedConfig: RTVIClientConfigOption[] = 731 | config ?? (await this.getConfig()); 732 | 733 | // Find matching service name in the config and update the messages 734 | const configServiceKey = passedConfig.find( 735 | (config: RTVIClientConfigOption) => config.service === serviceKey 736 | ); 737 | 738 | if (!configServiceKey) { 739 | logger.debug( 740 | "No service with name " + serviceKey + " not found in config" 741 | ); 742 | return undefined; 743 | } 744 | 745 | // Return a new object, as to not mutate existing state 746 | return configServiceKey; 747 | }); 748 | } 749 | 750 | /** 751 | * Returns configuration option value (unknown) for specified service key and option name 752 | * @param serviceKey - Service name to get options for (e.g. "llm") 753 | * @optional option Name of option return from the config (e.g. "model") 754 | * @returns Promise - Service configuration option value or undefined 755 | */ 756 | public async getServiceOptionValueFromConfig( 757 | serviceKey: string, 758 | option: string, 759 | config?: RTVIClientConfigOption[] 760 | ): Promise { 761 | const configServiceKey: RTVIClientConfigOption | undefined = 762 | await this.getServiceOptionsFromConfig(serviceKey, config); 763 | 764 | if (!configServiceKey) { 765 | logger.debug("Service with name " + serviceKey + " not found in config"); 766 | return undefined; 767 | } 768 | 769 | // Find matching option key in the service config 770 | const optionValue: ConfigOption | undefined = configServiceKey.options.find( 771 | (o: ConfigOption) => o.name === option 772 | ); 773 | 774 | return optionValue ? (optionValue as ConfigOption).value : undefined; 775 | } 776 | 777 | private _updateOrAddOption( 778 | existingOptions: ConfigOption[], 779 | newOption: ConfigOption 780 | ): ConfigOption[] { 781 | const existingOptionIndex = existingOptions.findIndex( 782 | (item) => item.name === newOption.name 783 | ); 784 | if (existingOptionIndex !== -1) { 785 | // Update existing option 786 | return existingOptions.map((item, index) => 787 | index === existingOptionIndex 788 | ? { ...item, value: newOption.value } 789 | : item 790 | ); 791 | } else { 792 | // Add new option 793 | return [ 794 | ...existingOptions, 795 | { name: newOption.name, value: newOption.value }, 796 | ]; 797 | } 798 | } 799 | 800 | /** 801 | * Returns config with updated option(s) for specified service key and option name 802 | * Note: does not update current config, only returns a new object (call updateConfig to apply changes) 803 | * @param serviceKey - Service name to get options for (e.g. "llm") 804 | * @param option - Service name to get options for (e.g. "model") 805 | * @param config - Optional RTVIClientConfigOption[] to update (vs. using current config) 806 | * @returns Promise - Configuration options array with updated option(s) or undefined 807 | */ 808 | public async setServiceOptionInConfig( 809 | serviceKey: string, 810 | option: ConfigOption | ConfigOption[], 811 | config?: RTVIClientConfigOption[] 812 | ): Promise { 813 | const newConfig: RTVIClientConfigOption[] = cloneDeep( 814 | config ?? (await this.getConfig()) 815 | ); 816 | 817 | const serviceOptions = await this.getServiceOptionsFromConfig( 818 | serviceKey, 819 | newConfig 820 | ); 821 | 822 | if (!serviceOptions) { 823 | logger.debug( 824 | "Service with name '" + serviceKey + "' not found in config" 825 | ); 826 | return newConfig; 827 | } 828 | 829 | const optionsArray = Array.isArray(option) ? option : [option]; 830 | 831 | for (const opt of optionsArray) { 832 | const existingItem = newConfig.find( 833 | (item) => item.service === serviceKey 834 | ); 835 | const updatedOptions = existingItem 836 | ? this._updateOrAddOption(existingItem.options, opt) 837 | : [{ name: opt.name, value: opt.value }]; 838 | 839 | if (existingItem) { 840 | existingItem.options = updatedOptions; 841 | } else { 842 | newConfig.push({ service: serviceKey, options: updatedOptions }); 843 | } 844 | } 845 | 846 | return newConfig; 847 | } 848 | 849 | /** 850 | * Returns config object with updated properties from passed array. 851 | * @param configOptions - Array of RTVIClientConfigOption[] to update 852 | * @param config? - Optional RTVIClientConfigOption[] to update (vs. using current config) 853 | * @returns Promise - Configuration options 854 | */ 855 | public async setConfigOptions( 856 | configOptions: RTVIClientConfigOption[], 857 | config?: RTVIClientConfigOption[] 858 | ): Promise { 859 | let accumulator: RTVIClientConfigOption[] = cloneDeep( 860 | config ?? (await this.getConfig()) 861 | ); 862 | 863 | for (const configOption of configOptions) { 864 | accumulator = 865 | (await this.setServiceOptionInConfig( 866 | configOption.service, 867 | configOption.options, 868 | accumulator 869 | )) || accumulator; 870 | } 871 | return accumulator; 872 | } 873 | 874 | // ------ Actions 875 | 876 | /** 877 | * Dispatch an action message to the bot or http single-turn endpoint 878 | */ 879 | public async action( 880 | action: RTVIActionRequestData 881 | ): Promise { 882 | return this._messageDispatcher.dispatchAction( 883 | new RTVIActionRequest(action), 884 | this.handleMessage.bind(this) 885 | ); 886 | } 887 | 888 | /** 889 | * Describe available / registered actions the bot has 890 | * @returns Promise - Promise that resolves with the bot's actions 891 | */ 892 | @transportReady 893 | public async describeActions(): Promise { 894 | return this._messageDispatcher.dispatch(RTVIMessage.describeActions()); 895 | } 896 | 897 | // ------ Transport methods 898 | 899 | /** 900 | * Get the session expiry time for the transport session (if applicable) 901 | * @returns number - Expiry time in milliseconds 902 | */ 903 | @getIfTransportInState("connected", "ready") 904 | public get transportExpiry(): number | undefined { 905 | return this._transport.expiry; 906 | } 907 | 908 | // ------ Messages 909 | 910 | /** 911 | * Directly send a message to the bot via the transport 912 | * @param message - RTVIMessage object to send 913 | */ 914 | @transportReady 915 | public sendMessage(message: RTVIMessage): void { 916 | this._transport.sendMessage(message); 917 | } 918 | 919 | /** 920 | * Disconnects the bot, but keeps the session alive 921 | */ 922 | @transportReady 923 | public disconnectBot(): void { 924 | this._transport.sendMessage( 925 | new RTVIMessage(RTVIMessageType.DISCONNECT_BOT, {}) 926 | ); 927 | } 928 | 929 | protected handleMessage(ev: RTVIMessage): void { 930 | logger.debug("[RTVI Message]", ev); 931 | 932 | switch (ev.type) { 933 | case RTVIMessageType.BOT_READY: 934 | clearTimeout(this._handshakeTimeout); 935 | this._startResolve?.(ev.data as BotReadyData); 936 | this._options.callbacks?.onBotReady?.(ev.data as BotReadyData); 937 | break; 938 | case RTVIMessageType.CONFIG_AVAILABLE: { 939 | this._messageDispatcher.resolve(ev); 940 | this._options.callbacks?.onConfigDescribe?.(ev.data); 941 | break; 942 | } 943 | case RTVIMessageType.CONFIG: { 944 | const resp = this._messageDispatcher.resolve(ev); 945 | this._options.callbacks?.onConfig?.((resp.data as ConfigData).config); 946 | break; 947 | } 948 | case RTVIMessageType.ACTIONS_AVAILABLE: { 949 | this._messageDispatcher.resolve(ev); 950 | this._options.callbacks?.onActionsAvailable?.(ev.data); 951 | break; 952 | } 953 | case RTVIMessageType.ACTION_RESPONSE: { 954 | this._messageDispatcher.resolve(ev); 955 | break; 956 | } 957 | case RTVIMessageType.ERROR_RESPONSE: { 958 | const resp = this._messageDispatcher.reject(ev); 959 | this._options.callbacks?.onMessageError?.(resp as RTVIMessage); 960 | break; 961 | } 962 | case RTVIMessageType.ERROR: 963 | this._options.callbacks?.onError?.(ev); 964 | break; 965 | case RTVIMessageType.USER_STARTED_SPEAKING: 966 | this._options.callbacks?.onUserStartedSpeaking?.(); 967 | break; 968 | case RTVIMessageType.USER_STOPPED_SPEAKING: 969 | this._options.callbacks?.onUserStoppedSpeaking?.(); 970 | break; 971 | case RTVIMessageType.BOT_STARTED_SPEAKING: 972 | this._options.callbacks?.onBotStartedSpeaking?.(); 973 | break; 974 | case RTVIMessageType.BOT_STOPPED_SPEAKING: 975 | this._options.callbacks?.onBotStoppedSpeaking?.(); 976 | break; 977 | case RTVIMessageType.USER_TRANSCRIPTION: { 978 | const TranscriptData = ev.data as TranscriptData; 979 | this._options.callbacks?.onUserTranscript?.(TranscriptData); 980 | break; 981 | } 982 | case RTVIMessageType.BOT_TRANSCRIPTION: { 983 | this._options.callbacks?.onBotTranscript?.(ev.data as BotLLMTextData); 984 | break; 985 | } 986 | case RTVIMessageType.BOT_LLM_TEXT: 987 | this._options.callbacks?.onBotLlmText?.(ev.data as BotLLMTextData); 988 | break; 989 | case RTVIMessageType.BOT_LLM_STARTED: 990 | this._options.callbacks?.onBotLlmStarted?.(); 991 | break; 992 | case RTVIMessageType.BOT_LLM_STOPPED: 993 | this._options.callbacks?.onBotLlmStopped?.(); 994 | break; 995 | case RTVIMessageType.BOT_TTS_TEXT: 996 | this._options.callbacks?.onBotTtsText?.(ev.data as BotTTSTextData); 997 | break; 998 | case RTVIMessageType.BOT_TTS_STARTED: 999 | this._options.callbacks?.onBotTtsStarted?.(); 1000 | break; 1001 | case RTVIMessageType.BOT_TTS_STOPPED: 1002 | this._options.callbacks?.onBotTtsStopped?.(); 1003 | break; 1004 | case RTVIMessageType.BOT_LLM_SEARCH_RESPONSE: 1005 | this._options.callbacks?.onBotLlmSearchResponse?.( 1006 | ev.data as BotLLMSearchResponseData 1007 | ); 1008 | this.emit( 1009 | RTVIEvent.BotLlmSearchResponse, 1010 | ev.data as BotLLMSearchResponseData 1011 | ); 1012 | break; 1013 | case RTVIMessageType.METRICS: 1014 | this.emit(RTVIEvent.Metrics, ev.data as PipecatMetricsData); 1015 | this._options.callbacks?.onMetrics?.(ev.data as PipecatMetricsData); 1016 | break; 1017 | case RTVIMessageType.STORAGE_ITEM_STORED: 1018 | this._options.callbacks?.onStorageItemStored?.( 1019 | ev.data as StorageItemStoredData 1020 | ); 1021 | break; 1022 | case RTVIMessageType.SERVER_MESSAGE: { 1023 | this._options.callbacks?.onServerMessage?.(ev.data); 1024 | this.emit(RTVIEvent.ServerMessage, ev.data); 1025 | break; 1026 | } 1027 | default: { 1028 | let match: boolean = false; 1029 | // Pass message to registered helpers 1030 | for (const helper of Object.values( 1031 | this._helpers 1032 | ) as RTVIClientHelper[]) { 1033 | if (helper.getMessageTypes().includes(ev.type)) { 1034 | match = true; 1035 | helper.handleMessage(ev); 1036 | } 1037 | } 1038 | if (!match) { 1039 | this._options.callbacks?.onGenericMessage?.(ev.data); 1040 | } 1041 | } 1042 | } 1043 | } 1044 | 1045 | // ------ Helpers 1046 | 1047 | /** 1048 | * Register a new helper to the client 1049 | * This (optionally) provides a way to reference helpers directly 1050 | * from the client and use the event dispatcher 1051 | * @param service - Target service for this helper 1052 | * @param helper - Helper instance 1053 | * @returns RTVIClientHelper - Registered helper instance 1054 | */ 1055 | public registerHelper( 1056 | service: string, 1057 | helper: RTVIClientHelper 1058 | ): RTVIClientHelper { 1059 | if (this._helpers[service]) { 1060 | throw new Error(`Helper with name '${service}' already registered`); 1061 | } 1062 | 1063 | // Check helper is instance of RTVIClientHelper 1064 | if (!(helper instanceof RTVIClientHelper)) { 1065 | throw new Error(`Helper must be an instance of RTVIClientHelper`); 1066 | } 1067 | 1068 | helper.service = service; 1069 | helper.client = this; 1070 | 1071 | this._helpers[service] = helper; 1072 | 1073 | return this._helpers[service]; 1074 | } 1075 | 1076 | public getHelper(service: string): T | undefined { 1077 | const helper = this._helpers[service]; 1078 | if (!helper) { 1079 | logger.debug(`Helper targeting service '${service}' not found`); 1080 | return undefined; 1081 | } 1082 | return helper as T; 1083 | } 1084 | 1085 | public unregisterHelper(service: string) { 1086 | if (!this._helpers[service]) { 1087 | return; 1088 | } 1089 | delete this._helpers[service]; 1090 | } 1091 | } 1092 | -------------------------------------------------------------------------------- /client-js/src/decorators.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { RTVIClient } from "."; 8 | import { BotNotReadyError } from "./errors"; 9 | 10 | export function transportReady( 11 | _target: T, 12 | propertyKey: string | symbol, 13 | descriptor: PropertyDescriptor 14 | ): PropertyDescriptor | void { 15 | const originalMethod = descriptor.value; 16 | 17 | descriptor.value = function (this: T, ...args: unknown[]) { 18 | if (this.state === "ready") { 19 | return originalMethod.apply(this, args); 20 | } else { 21 | throw new BotNotReadyError( 22 | `Attempt to call ${propertyKey.toString()} when transport not in ready state. Await connect() first.` 23 | ); 24 | } 25 | }; 26 | 27 | return descriptor; 28 | } 29 | export function transportInState(states: string[]) { 30 | return function ( 31 | _target: T, 32 | propertyKey: string | symbol, 33 | descriptor: PropertyDescriptor 34 | ): PropertyDescriptor | void { 35 | const originalMethod = descriptor.value; 36 | 37 | descriptor.get = function (this: T, ...args: unknown[]) { 38 | if (states.includes(this.state)) { 39 | return originalMethod.apply(this, args); 40 | } else { 41 | throw new BotNotReadyError( 42 | `Attempt to call ${propertyKey.toString()} when transport not in ${states}.` 43 | ); 44 | } 45 | }; 46 | 47 | return descriptor; 48 | }; 49 | } 50 | 51 | export function getIfTransportInState( 52 | ...states: string[] 53 | ) { 54 | states = ["ready", ...states]; 55 | 56 | return function ( 57 | _target: T, 58 | propertyKey: string | symbol, 59 | descriptor: PropertyDescriptor 60 | ): PropertyDescriptor | void { 61 | const originalGetter = descriptor.get; 62 | 63 | descriptor.get = function (this: T) { 64 | if (states.includes(this.state)) { 65 | return originalGetter?.apply(this); 66 | } else { 67 | throw new BotNotReadyError( 68 | `Attempt to call ${propertyKey.toString()} when transport not in ${states}. Await connect() first.` 69 | ); 70 | } 71 | }; 72 | 73 | return descriptor; 74 | }; 75 | } 76 | -------------------------------------------------------------------------------- /client-js/src/errors.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | export class RTVIError extends Error { 8 | readonly status: number | undefined; 9 | 10 | constructor(message?: string, status?: number | undefined) { 11 | super(message); 12 | this.status = status; 13 | } 14 | } 15 | 16 | export class ConnectionTimeoutError extends RTVIError { 17 | constructor(message?: string | undefined) { 18 | super( 19 | message ?? 20 | "Bot did not enter ready state within the specified timeout period." 21 | ); 22 | } 23 | } 24 | 25 | export class StartBotError extends RTVIError { 26 | readonly error: string = "invalid-request-error"; 27 | constructor(message?: string | undefined, status?: number) { 28 | super( 29 | message ?? `Failed to connect / invalid auth bundle from base url`, 30 | status ?? 500 31 | ); 32 | } 33 | } 34 | 35 | export class TransportStartError extends RTVIError { 36 | constructor(message?: string | undefined) { 37 | super(message ?? "Unable to connect to transport"); 38 | } 39 | } 40 | 41 | export class BotNotReadyError extends RTVIError { 42 | constructor(message?: string | undefined) { 43 | super( 44 | message ?? 45 | "Attempt to call action on transport when not in 'ready' state." 46 | ); 47 | } 48 | } 49 | 50 | export class ConfigUpdateError extends RTVIError { 51 | override readonly status = 400; 52 | constructor(message?: string | undefined) { 53 | super(message ?? "Unable to update configuration"); 54 | } 55 | } 56 | 57 | export class ActionEndpointNotSetError extends RTVIError { 58 | constructor(message?: string | undefined) { 59 | super(message ?? "Action endpoint is not set"); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /client-js/src/events.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { RTVIClientConfigOption } from "."; 8 | import { LLMFunctionCallData } from "./helpers/llm"; 9 | import { 10 | BotLLMTextData, 11 | BotLLMSearchResponseData, 12 | BotReadyData, 13 | BotTTSTextData, 14 | PipecatMetricsData, 15 | RTVIMessage, 16 | StorageItemStoredData, 17 | TranscriptData, 18 | } from "./messages"; 19 | import { Participant, TransportState } from "./transport"; 20 | 21 | export enum RTVIEvent { 22 | MessageError = "messageError", 23 | Error = "error", 24 | 25 | Connected = "connected", 26 | Disconnected = "disconnected", 27 | TransportStateChanged = "transportStateChanged", 28 | 29 | Config = "config", 30 | ConfigDescribe = "configDescribe", 31 | ActionsAvailable = "actionsAvailable", 32 | 33 | ParticipantConnected = "participantConnected", 34 | ParticipantLeft = "participantLeft", 35 | TrackStarted = "trackStarted", 36 | TrackStopped = "trackStopped", 37 | ScreenTrackStarted = "screenTrackStarted", 38 | ScreenTrackStopped = "screenTrackStopped", 39 | ScreenShareError = "screenShareError", 40 | 41 | AvailableCamsUpdated = "availableCamsUpdated", 42 | AvailableMicsUpdated = "availableMicsUpdated", 43 | AvailableSpeakersUpdated = "availableSpeakersUpdated", 44 | CamUpdated = "camUpdated", 45 | MicUpdated = "micUpdated", 46 | SpeakerUpdated = "speakerUpdated", 47 | 48 | BotConnected = "botConnected", 49 | BotReady = "botReady", 50 | BotDisconnected = "botDisconnected", 51 | BotStartedSpeaking = "botStartedSpeaking", 52 | BotStoppedSpeaking = "botStoppedSpeaking", 53 | RemoteAudioLevel = "remoteAudioLevel", 54 | 55 | UserStartedSpeaking = "userStartedSpeaking", 56 | UserStoppedSpeaking = "userStoppedSpeaking", 57 | LocalAudioLevel = "localAudioLevel", 58 | 59 | Metrics = "metrics", 60 | 61 | UserTranscript = "userTranscript", 62 | BotTranscript = "botTranscript", 63 | 64 | BotLlmText = "botLlmText", 65 | BotLlmStarted = "botLlmStarted", 66 | BotLlmStopped = "botLlmStopped", 67 | 68 | BotTtsText = "botTtsText", 69 | BotTtsStarted = "botTtsStarted", 70 | BotTtsStopped = "botTtsStopped", 71 | 72 | LLMFunctionCall = "llmFunctionCall", 73 | LLMFunctionCallStart = "llmFunctionCallStart", 74 | LLMJsonCompletion = "llmJsonCompletion", 75 | 76 | StorageItemStored = "storageItemStored", 77 | 78 | BotLlmSearchResponse = "botLlmSearchResponse", 79 | ServerMessage = "serverMessage", 80 | } 81 | 82 | export type RTVIEvents = Partial<{ 83 | connected: () => void; 84 | disconnected: () => void; 85 | transportStateChanged: (state: TransportState) => void; 86 | 87 | config: (config: RTVIClientConfigOption[]) => void; 88 | configUpdated: (config: RTVIClientConfigOption[]) => void; 89 | configDescribe: (configDescription: unknown) => void; 90 | actionsAvailable: (actions: unknown) => void; 91 | 92 | participantConnected: (participant: Participant) => void; 93 | participantLeft: (participant: Participant) => void; 94 | trackStarted: (track: MediaStreamTrack, participant?: Participant) => void; 95 | trackStopped: (track: MediaStreamTrack, participant?: Participant) => void; 96 | screenTrackStarted: (track: MediaStreamTrack, p?: Participant) => void; 97 | screenTrackStopped: (track: MediaStreamTrack, p?: Participant) => void; 98 | screenShareError: (errorMessage: string) => void; 99 | 100 | availableCamsUpdated: (cams: MediaDeviceInfo[]) => void; 101 | availableMicsUpdated: (mics: MediaDeviceInfo[]) => void; 102 | availableSpeakersUpdated: (speakers: MediaDeviceInfo[]) => void; 103 | camUpdated: (cam: MediaDeviceInfo) => void; 104 | micUpdated: (mic: MediaDeviceInfo) => void; 105 | speakerUpdated: (speaker: MediaDeviceInfo) => void; 106 | 107 | botReady: (botData: BotReadyData) => void; 108 | botConnected: (participant: Participant) => void; 109 | botDisconnected: (participant: Participant) => void; 110 | botStartedSpeaking: () => void; 111 | botStoppedSpeaking: () => void; 112 | remoteAudioLevel: (level: number, p: Participant) => void; 113 | 114 | userStartedSpeaking: () => void; 115 | userStoppedSpeaking: () => void; 116 | localAudioLevel: (level: number) => void; 117 | 118 | metrics: (data: PipecatMetricsData) => void; 119 | 120 | userTranscript: (data: TranscriptData) => void; 121 | botTranscript: (data: BotLLMTextData) => void; 122 | 123 | botLlmText: (data: BotLLMTextData) => void; 124 | botLlmStarted: () => void; 125 | botLlmStopped: () => void; 126 | 127 | botTtsText: (data: BotTTSTextData) => void; 128 | botTtsStarted: () => void; 129 | botTtsStopped: () => void; 130 | 131 | error: (message: RTVIMessage) => void; 132 | messageError: (message: RTVIMessage) => void; 133 | 134 | llmFunctionCall: (func: LLMFunctionCallData) => void; 135 | llmFunctionCallStart: (functionName: string) => void; 136 | llmJsonCompletion: (data: string) => void; 137 | 138 | storageItemStored: (data: StorageItemStoredData) => void; 139 | 140 | botLlmSearchResponse: (data: BotLLMSearchResponseData) => void; 141 | serverMessage: (data: any) => void; 142 | }>; 143 | 144 | export type RTVIEventHandler = E extends keyof RTVIEvents 145 | ? RTVIEvents[E] 146 | : never; 147 | -------------------------------------------------------------------------------- /client-js/src/helpers/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { RTVIClient } from "../client"; 8 | import { RTVIMessage } from "../messages"; 9 | 10 | export type RTVIClientHelpers = Partial>; 11 | 12 | export type RTVIClientHelperCallbacks = Partial; 13 | 14 | export interface RTVIClientHelperOptions { 15 | /** 16 | * Callback methods for events / messages 17 | */ 18 | callbacks?: RTVIClientHelperCallbacks; 19 | } 20 | 21 | export abstract class RTVIClientHelper { 22 | protected _options: RTVIClientHelperOptions; 23 | protected declare _client: RTVIClient; 24 | protected declare _service: string; 25 | 26 | constructor(options: RTVIClientHelperOptions) { 27 | this._options = options; 28 | } 29 | 30 | public abstract handleMessage(ev: RTVIMessage): void; 31 | public abstract getMessageTypes(): string[]; 32 | public set client(client: RTVIClient) { 33 | this._client = client; 34 | } 35 | public set service(service: string) { 36 | this._service = service; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /client-js/src/helpers/llm.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import * as RTVIErrors from "./../errors"; 8 | import { RTVIEvent } from "./../events"; 9 | import { 10 | RTVIActionRequestData, 11 | RTVIActionResponse, 12 | RTVIMessage, 13 | } from "./../messages"; 14 | import { RTVIClientHelper, RTVIClientHelperOptions } from "."; 15 | 16 | // --- Types 17 | 18 | export type LLMFunctionCallData = { 19 | function_name: string; 20 | tool_call_id: string; 21 | args: unknown; 22 | result?: unknown; 23 | }; 24 | 25 | export type LLMContextMessage = { 26 | role: string; 27 | content: unknown; 28 | }; 29 | 30 | export type LLMContext = Partial<{ 31 | messages?: LLMContextMessage[]; 32 | tools?: []; 33 | }>; 34 | 35 | export type FunctionCallParams = { 36 | functionName: string; 37 | arguments: unknown; 38 | }; 39 | 40 | export type FunctionCallCallback = (fn: FunctionCallParams) => Promise; 41 | 42 | // --- Message types 43 | export enum LLMMessageType { 44 | LLM_FUNCTION_CALL = "llm-function-call", 45 | LLM_FUNCTION_CALL_START = "llm-function-call-start", 46 | LLM_FUNCTION_CALL_RESULT = "llm-function-call-result", 47 | LLM_JSON_COMPLETION = "llm-json-completion", 48 | } 49 | 50 | export enum LLMActionType { 51 | APPEND_TO_MESSAGES = "append_to_messages", 52 | GET_CONTEXT = "get_context", 53 | SET_CONTEXT = "set_context", 54 | RUN = "run", 55 | } 56 | 57 | // --- Callbacks 58 | export type LLMHelperCallbacks = Partial<{ 59 | onLLMJsonCompletion: (jsonString: string) => void; 60 | onLLMFunctionCall: (func: LLMFunctionCallData) => void; 61 | onLLMFunctionCallStart: (functionName: string) => void; 62 | onLLMMessage: (message: LLMContextMessage) => void; 63 | }>; 64 | 65 | // --- Interface and class 66 | export interface LLMHelperOptions extends RTVIClientHelperOptions { 67 | callbacks?: LLMHelperCallbacks; 68 | } 69 | 70 | export class LLMHelper extends RTVIClientHelper { 71 | protected declare _options: LLMHelperOptions; 72 | private _functionCallCallback: FunctionCallCallback | null; 73 | 74 | constructor(options: LLMHelperOptions) { 75 | super(options); 76 | 77 | this._functionCallCallback = null; 78 | } 79 | 80 | public getMessageTypes(): string[] { 81 | return Object.values(LLMMessageType) as string[]; 82 | } 83 | 84 | // --- Actions 85 | 86 | /** 87 | * Retrieve the bot's current LLM context. 88 | * @returns Promise 89 | */ 90 | public async getContext(): Promise { 91 | if (this._client.state !== "ready") { 92 | throw new RTVIErrors.BotNotReadyError( 93 | "getContext called while transport not in ready state" 94 | ); 95 | } 96 | const actionResponseMsg: RTVIActionResponse = await this._client.action({ 97 | service: this._service, 98 | action: LLMActionType.GET_CONTEXT, 99 | } as RTVIActionRequestData); 100 | return actionResponseMsg.data.result as LLMContext; 101 | } 102 | 103 | /** 104 | * Update the bot's LLM context. 105 | * If this is called while the transport is not in the ready state, the local context will be updated 106 | * @param context LLMContext - The new context 107 | * @param interrupt boolean - Whether to interrupt the bot, or wait until it has finished speaking 108 | * @returns Promise 109 | */ 110 | 111 | public async setContext( 112 | context: LLMContext, 113 | interrupt: boolean = false 114 | ): Promise { 115 | if (this._client.state !== "ready") { 116 | throw new RTVIErrors.BotNotReadyError( 117 | "setContext called while transport not in ready state" 118 | ); 119 | } 120 | 121 | const actionResponse: RTVIActionResponse = (await this._client.action({ 122 | service: this._service, 123 | action: LLMActionType.SET_CONTEXT, 124 | arguments: [ 125 | { 126 | name: "messages", 127 | value: context.messages, 128 | }, 129 | { 130 | name: "interrupt", 131 | value: interrupt, 132 | }, 133 | ], 134 | } as RTVIActionRequestData)) as RTVIActionResponse; 135 | 136 | return !!actionResponse.data.result; 137 | } 138 | 139 | /** 140 | * Append a new message to the LLM context. 141 | * If this is called while the transport is not in the ready state, the local context will be updated 142 | * @param context LLMContextMessage 143 | * @param runImmediately boolean - wait until pipeline is idle before running 144 | * @returns boolean 145 | */ 146 | 147 | public async appendToMessages( 148 | message: LLMContextMessage, 149 | runImmediately: boolean = false 150 | ): Promise { 151 | if (this._client.state !== "ready") { 152 | throw new RTVIErrors.BotNotReadyError( 153 | "setContext called while transport not in ready state" 154 | ); 155 | } 156 | 157 | const actionResponse = (await this._client.action({ 158 | service: this._service, 159 | action: LLMActionType.APPEND_TO_MESSAGES, 160 | arguments: [ 161 | { 162 | name: "messages", 163 | value: [message], 164 | }, 165 | { 166 | name: "run_immediately", 167 | value: runImmediately, 168 | }, 169 | ], 170 | } as RTVIActionRequestData)) as RTVIActionResponse; 171 | return !!actionResponse.data.result; 172 | } 173 | 174 | /** 175 | * Run the bot's current LLM context. 176 | * Useful when appending messages to the context without runImmediately set to true. 177 | * Will do nothing if the bot is not in the ready state. 178 | * @param interrupt boolean - Whether to interrupt the bot, or wait until it has finished speaking 179 | * @returns Promise 180 | */ 181 | public async run(interrupt: boolean = false): Promise { 182 | if (this._client.state !== "ready") { 183 | return; 184 | } 185 | 186 | return this._client.action({ 187 | service: this._service, 188 | action: LLMActionType.RUN, 189 | arguments: [ 190 | { 191 | name: "interrupt", 192 | value: interrupt, 193 | }, 194 | ], 195 | } as RTVIActionRequestData); 196 | } 197 | 198 | // --- Handlers 199 | 200 | /** 201 | * If the LLM wants to call a function, RTVI will invoke the callback defined 202 | * here. Whatever the callback returns will be sent to the LLM as the function result. 203 | * @param callback 204 | * @returns void 205 | */ 206 | public handleFunctionCall(callback: FunctionCallCallback): void { 207 | this._functionCallCallback = callback; 208 | } 209 | 210 | public handleMessage(ev: RTVIMessage): void { 211 | switch (ev.type) { 212 | case LLMMessageType.LLM_JSON_COMPLETION: 213 | this._options.callbacks?.onLLMJsonCompletion?.(ev.data as string); 214 | this._client.emit(RTVIEvent.LLMJsonCompletion, ev.data as string); 215 | break; 216 | case LLMMessageType.LLM_FUNCTION_CALL: { 217 | const d = ev.data as LLMFunctionCallData; 218 | this._options.callbacks?.onLLMFunctionCall?.( 219 | ev.data as LLMFunctionCallData 220 | ); 221 | this._client.emit( 222 | RTVIEvent.LLMFunctionCall, 223 | ev.data as LLMFunctionCallData 224 | ); 225 | if (this._functionCallCallback) { 226 | const fn = { 227 | functionName: d.function_name, 228 | arguments: d.args, 229 | }; 230 | if (this._client.state === "ready") { 231 | this._functionCallCallback(fn).then((result) => { 232 | this._client.sendMessage( 233 | new RTVIMessage(LLMMessageType.LLM_FUNCTION_CALL_RESULT, { 234 | function_name: d.function_name, 235 | tool_call_id: d.tool_call_id, 236 | arguments: d.args, 237 | result, 238 | }) 239 | ); 240 | }); 241 | } else { 242 | throw new RTVIErrors.BotNotReadyError( 243 | "Attempted to send a function call result from bot while transport not in ready state" 244 | ); 245 | } 246 | } 247 | break; 248 | } 249 | case LLMMessageType.LLM_FUNCTION_CALL_START: { 250 | const e = ev.data as LLMFunctionCallData; 251 | this._options.callbacks?.onLLMFunctionCallStart?.( 252 | e.function_name as string 253 | ); 254 | this._client.emit(RTVIEvent.LLMFunctionCallStart, e.function_name); 255 | break; 256 | } 257 | } 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /client-js/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | export * from "./actions"; 8 | export * from "./client"; 9 | export * from "./errors"; 10 | export * from "./events"; 11 | export * from "./helpers"; 12 | export * from "./helpers/llm"; 13 | export * from "./logger"; 14 | export * from "./messages"; 15 | export * from "./transport"; 16 | -------------------------------------------------------------------------------- /client-js/src/logger.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | export enum LogLevel { 8 | NONE = 0, 9 | ERROR = 1, 10 | WARN = 2, 11 | INFO = 3, 12 | DEBUG = 4, 13 | } 14 | 15 | class Logger { 16 | private static instance: Logger; 17 | private level: LogLevel = LogLevel.DEBUG; 18 | 19 | private constructor() {} 20 | 21 | static getInstance(): Logger { 22 | if (!Logger.instance) { 23 | Logger.instance = new Logger(); 24 | } 25 | return Logger.instance; 26 | } 27 | 28 | setLevel(level: LogLevel) { 29 | this.level = level; 30 | } 31 | 32 | debug(...args: unknown[]) { 33 | if (this.level >= LogLevel.DEBUG) { 34 | console.debug(...args); 35 | } 36 | } 37 | 38 | info(...args: unknown[]) { 39 | if (this.level >= LogLevel.INFO) { 40 | console.info(...args); 41 | } 42 | } 43 | 44 | warn(...args: unknown[]) { 45 | if (this.level >= LogLevel.WARN) { 46 | console.warn(...args); 47 | } 48 | } 49 | 50 | error(...args: unknown[]) { 51 | if (this.level >= LogLevel.ERROR) { 52 | console.error(...args); 53 | } 54 | } 55 | } 56 | 57 | export const logger = Logger.getInstance(); 58 | 59 | export type ILogger = Logger; 60 | -------------------------------------------------------------------------------- /client-js/src/messages.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { v4 as uuidv4 } from "uuid"; 8 | 9 | import { httpActionGenerator } from "./actions"; 10 | import { RTVIClient, RTVIClientConfigOption } from "./client"; 11 | import { ActionEndpointNotSetError } from "./errors"; 12 | import { logger } from "./logger"; 13 | 14 | export const RTVI_MESSAGE_LABEL = "rtvi-ai"; 15 | 16 | export enum RTVIMessageType { 17 | // Outbound 18 | CLIENT_READY = "client-ready", 19 | UPDATE_CONFIG = "update-config", 20 | GET_CONFIG = "get-config", 21 | DESCRIBE_CONFIG = "describe-config", 22 | DESCRIBE_ACTIONS = "describe-actions", 23 | DISCONNECT_BOT = "disconnect-bot", 24 | ACTION = "action", 25 | 26 | // Inbound 27 | BOT_READY = "bot-ready", // Bot is connected and ready to receive messages 28 | ERROR = "error", // Bot initialization error 29 | ERROR_RESPONSE = "error-response", // Error response from the bot in response to an action 30 | CONFIG = "config", // Bot configuration 31 | CONFIG_AVAILABLE = "config-available", // Configuration options available on the bot 32 | CONFIG_ERROR = "config-error", // Configuration options have changed failed 33 | ACTIONS_AVAILABLE = "actions-available", // Actions available on the bot 34 | ACTION_RESPONSE = "action-response", // Action response from the bot 35 | METRICS = "metrics", // RTVI reporting metrics 36 | USER_TRANSCRIPTION = "user-transcription", // Local user speech to text transcription (partials and finals) 37 | BOT_TRANSCRIPTION = "bot-transcription", // Bot full text transcription (sentence aggregated) 38 | USER_STARTED_SPEAKING = "user-started-speaking", // User started speaking 39 | USER_STOPPED_SPEAKING = "user-stopped-speaking", // User stopped speaking 40 | BOT_STARTED_SPEAKING = "bot-started-speaking", // Bot started speaking 41 | BOT_STOPPED_SPEAKING = "bot-stopped-speaking", // Bot stopped speaking 42 | // Service-specific 43 | USER_LLM_TEXT = "user-llm-text", // Aggregated user input text which is sent to LLM 44 | BOT_LLM_TEXT = "bot-llm-text", // Streamed token returned by the LLM 45 | BOT_LLM_STARTED = "bot-llm-started", // Bot LLM inference starts 46 | BOT_LLM_STOPPED = "bot-llm-stopped", // Bot LLM inference stops 47 | BOT_TTS_TEXT = "bot-tts-text", // Bot TTS text output (streamed word as it is spoken) 48 | BOT_TTS_STARTED = "bot-tts-started", // Bot TTS response starts 49 | BOT_TTS_STOPPED = "bot-tts-stopped", // Bot TTS response stops 50 | BOT_LLM_SEARCH_RESPONSE = "bot-llm-search-response", // Bot LLM search response 51 | // Storage 52 | STORAGE_ITEM_STORED = "storage-item-stored", // Item was stored to configured storage, if applicable 53 | // Server-to-client messages 54 | SERVER_MESSAGE = "server-message", 55 | } 56 | 57 | // ----- Message Data Types 58 | 59 | export type ConfigData = { 60 | config: RTVIClientConfigOption[]; 61 | }; 62 | 63 | export type BotReadyData = { 64 | config: RTVIClientConfigOption[]; 65 | version: string; 66 | }; 67 | 68 | export type ErrorData = { 69 | message: string; 70 | fatal: boolean; 71 | }; 72 | 73 | export type PipecatMetricData = { 74 | processor: string; 75 | value: number; 76 | }; 77 | 78 | export type PipecatMetricsData = { 79 | processing?: PipecatMetricData[]; 80 | ttfb?: PipecatMetricData[]; 81 | characters?: PipecatMetricData[]; 82 | }; 83 | 84 | export type TranscriptData = { 85 | text: string; 86 | final: boolean; 87 | timestamp: string; 88 | user_id: string; 89 | }; 90 | 91 | export type BotLLMTextData = { 92 | text: string; 93 | }; 94 | 95 | export type BotTTSTextData = { 96 | text: string; 97 | }; 98 | 99 | export type StorageItemStoredData = { 100 | action: string; 101 | items: unknown; 102 | }; 103 | 104 | export type LLMSearchResult = { 105 | text: string; 106 | confidence: number[]; 107 | }; 108 | 109 | export type LLMSearchOrigin = { 110 | site_uri?: string; 111 | site_title?: string; 112 | results: LLMSearchResult[]; 113 | }; 114 | 115 | export type BotLLMSearchResponseData = { 116 | search_result?: string; 117 | rendered_content?: string; 118 | origins: LLMSearchOrigin[]; 119 | }; 120 | 121 | export type ServerMessageData = { 122 | data: any; 123 | }; 124 | 125 | // ----- Message Classes 126 | 127 | export type RTVIMessageActionResponse = { 128 | id: string; 129 | label: string; 130 | type: string; 131 | data: { result: unknown }; 132 | }; 133 | 134 | export class RTVIMessage { 135 | id: string; 136 | label: string = RTVI_MESSAGE_LABEL; 137 | type: string; 138 | data: unknown; 139 | 140 | constructor(type: string, data: unknown, id?: string) { 141 | this.type = type; 142 | this.data = data; 143 | this.id = id || uuidv4().slice(0, 8); 144 | } 145 | 146 | // Outbound message types 147 | static clientReady(): RTVIMessage { 148 | return new RTVIMessage(RTVIMessageType.CLIENT_READY, {}); 149 | } 150 | 151 | static updateConfig( 152 | config: RTVIClientConfigOption[], 153 | interrupt: boolean = false 154 | ): RTVIMessage { 155 | return new RTVIMessage(RTVIMessageType.UPDATE_CONFIG, { 156 | config, 157 | interrupt, 158 | }); 159 | } 160 | 161 | static describeConfig(): RTVIMessage { 162 | return new RTVIMessage(RTVIMessageType.DESCRIBE_CONFIG, {}); 163 | } 164 | 165 | static getBotConfig(): RTVIMessage { 166 | return new RTVIMessage(RTVIMessageType.GET_CONFIG, {}); 167 | } 168 | 169 | static describeActions(): RTVIMessage { 170 | return new RTVIMessage(RTVIMessageType.DESCRIBE_ACTIONS, {}); 171 | } 172 | 173 | static disconnectBot(): RTVIMessage { 174 | return new RTVIMessage(RTVIMessageType.DISCONNECT_BOT, {}); 175 | } 176 | 177 | static error(message: string, fatal = false): RTVIMessage { 178 | return new RTVIMessage(RTVIMessageType.ERROR, { message, fatal }); 179 | } 180 | } 181 | 182 | // ----- Action Types 183 | 184 | export type RTVIActionRequestData = { 185 | service: string; 186 | action: string; 187 | arguments?: { name: string; value: unknown }[]; 188 | }; 189 | 190 | export class RTVIActionRequest extends RTVIMessage { 191 | constructor(data: RTVIActionRequestData) { 192 | super(RTVIMessageType.ACTION, data); 193 | } 194 | } 195 | 196 | export type RTVIActionResponse = { 197 | id: string; 198 | label: string; 199 | type: string; 200 | data: { result: unknown }; 201 | }; 202 | 203 | // ----- Message Dispatcher 204 | 205 | interface QueuedRTVIMessage { 206 | message: RTVIMessage; 207 | timestamp: number; 208 | resolve: (value: unknown) => void; 209 | reject: (reason?: unknown) => void; 210 | } 211 | 212 | export class MessageDispatcher { 213 | private _client: RTVIClient; 214 | private _gcTime: number; 215 | private _queue = new Array(); 216 | 217 | constructor(client: RTVIClient) { 218 | this._gcTime = 10000; // How long to wait before resolving the message 219 | this._queue = []; 220 | this._client = client; 221 | } 222 | 223 | public dispatch(message: RTVIMessage): Promise { 224 | const promise = new Promise((resolve, reject) => { 225 | this._queue.push({ 226 | message, 227 | timestamp: Date.now(), 228 | resolve, 229 | reject, 230 | }); 231 | }); 232 | 233 | logger.debug("[MessageDispatcher] dispatch", message); 234 | 235 | this._client.sendMessage(message); 236 | 237 | this._gc(); 238 | 239 | return promise as Promise; 240 | } 241 | 242 | public async dispatchAction( 243 | action: RTVIActionRequest, 244 | onMessage: (message: RTVIMessage) => void 245 | ): Promise { 246 | const promise = new Promise((resolve, reject) => { 247 | this._queue.push({ 248 | message: action, 249 | timestamp: Date.now(), 250 | resolve, 251 | reject, 252 | }); 253 | }); 254 | 255 | logger.debug("[MessageDispatcher] action", action); 256 | 257 | if (this._client.connected) { 258 | // Send message to transport when connected 259 | this._client.sendMessage(action); 260 | } else { 261 | if (!this._client.params.endpoints?.action) { 262 | logger.error( 263 | "[MessageDispatcher] Action endpoint is required when dispatching action in disconnected state" 264 | ); 265 | throw new ActionEndpointNotSetError(); 266 | } 267 | const actionUrl = this._client.constructUrl("action"); 268 | 269 | try { 270 | // Dispatch action via HTTP when disconnected 271 | await httpActionGenerator( 272 | actionUrl, 273 | action, 274 | this._client.params, 275 | (response: RTVIActionResponse) => { 276 | onMessage(response); 277 | } 278 | ); 279 | // On HTTP success (resolve), send `action` message (for callbacks) 280 | } catch (e) { 281 | onMessage( 282 | new RTVIMessage( 283 | RTVIMessageType.ERROR_RESPONSE, 284 | `Action endpoint '${actionUrl}' returned an error response`, 285 | action.id 286 | ) 287 | ); 288 | } 289 | } 290 | 291 | this._gc(); 292 | 293 | return promise as Promise; 294 | } 295 | 296 | private _resolveReject( 297 | message: RTVIMessage, 298 | resolve: boolean = true 299 | ): RTVIMessage { 300 | const queuedMessage = this._queue.find( 301 | (msg) => msg.message.id === message.id 302 | ); 303 | 304 | if (queuedMessage) { 305 | if (resolve) { 306 | logger.debug("[MessageDispatcher] Resolve", message); 307 | queuedMessage.resolve( 308 | message.type === RTVIMessageType.ACTION_RESPONSE 309 | ? (message as RTVIMessageActionResponse) 310 | : (message as RTVIMessage) 311 | ); 312 | } else { 313 | logger.debug("[MessageDispatcher] Reject", message); 314 | queuedMessage.reject(message as RTVIMessage); 315 | } 316 | // Remove message from queue 317 | this._queue = this._queue.filter((msg) => msg.message.id !== message.id); 318 | logger.debug("[MessageDispatcher] Queue", this._queue); 319 | } 320 | 321 | return message; 322 | } 323 | 324 | public resolve(message: RTVIMessage): RTVIMessage { 325 | return this._resolveReject(message, true); 326 | } 327 | 328 | public reject(message: RTVIMessage): RTVIMessage { 329 | return this._resolveReject(message, false); 330 | } 331 | 332 | private _gc() { 333 | this._queue = this._queue.filter((msg) => { 334 | return Date.now() - msg.timestamp < this._gcTime; 335 | }); 336 | logger.debug("[MessageDispatcher] GC", this._queue); 337 | } 338 | } 339 | -------------------------------------------------------------------------------- /client-js/src/transport.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { RTVIClientOptions, RTVIEventCallbacks } from "./client"; 8 | import { RTVIMessage } from "./messages"; 9 | 10 | export type TransportState = 11 | | "disconnected" 12 | | "initializing" 13 | | "initialized" 14 | | "authenticating" 15 | | "connecting" 16 | | "connected" 17 | | "ready" 18 | | "disconnecting" 19 | | "error"; 20 | 21 | export type Participant = { 22 | id: string; 23 | name: string; 24 | local: boolean; 25 | }; 26 | 27 | export type Tracks = { 28 | local: { 29 | audio?: MediaStreamTrack; 30 | video?: MediaStreamTrack; 31 | screenAudio?: MediaStreamTrack; 32 | screenVideo?: MediaStreamTrack; 33 | }; 34 | bot?: { 35 | audio?: MediaStreamTrack; 36 | screenAudio?: undefined; 37 | screenVideo?: undefined; 38 | video?: MediaStreamTrack; 39 | }; 40 | }; 41 | 42 | export abstract class Transport { 43 | protected declare _options: RTVIClientOptions; 44 | protected declare _onMessage: (ev: RTVIMessage) => void; 45 | protected declare _callbacks: RTVIEventCallbacks; 46 | protected _state: TransportState = "disconnected"; 47 | protected _expiry?: number = undefined; 48 | 49 | constructor() {} 50 | 51 | abstract initialize( 52 | options: RTVIClientOptions, 53 | messageHandler: (ev: RTVIMessage) => void 54 | ): void; 55 | 56 | abstract initDevices(): Promise; 57 | 58 | abstract connect( 59 | authBundle: unknown, 60 | abortController: AbortController 61 | ): Promise; 62 | abstract disconnect(): Promise; 63 | abstract sendReadyMessage(): void; 64 | 65 | abstract getAllMics(): Promise; 66 | abstract getAllCams(): Promise; 67 | abstract getAllSpeakers(): Promise; 68 | 69 | abstract updateMic(micId: string): void; 70 | abstract updateCam(camId: string): void; 71 | abstract updateSpeaker(speakerId: string): void; 72 | 73 | abstract get selectedMic(): MediaDeviceInfo | Record; 74 | abstract get selectedCam(): MediaDeviceInfo | Record; 75 | abstract get selectedSpeaker(): MediaDeviceInfo | Record; 76 | 77 | abstract enableMic(enable: boolean): void; 78 | abstract enableCam(enable: boolean): void; 79 | abstract enableScreenShare(enable: boolean): void; 80 | abstract get isCamEnabled(): boolean; 81 | abstract get isMicEnabled(): boolean; 82 | abstract get isSharingScreen(): boolean; 83 | 84 | abstract sendMessage(message: RTVIMessage): void; 85 | 86 | abstract get state(): TransportState; 87 | abstract set state(state: TransportState); 88 | 89 | get expiry(): number | undefined { 90 | return this._expiry; 91 | } 92 | 93 | abstract tracks(): Tracks; 94 | } 95 | 96 | export class TransportWrapper { 97 | private _transport: Transport; 98 | private _proxy: Transport; 99 | 100 | constructor(transport: Transport) { 101 | this._transport = transport; 102 | this._proxy = new Proxy(this._transport, { 103 | get: (target, prop, receiver) => { 104 | if (typeof target[prop as keyof Transport] === "function") { 105 | let errMsg; 106 | switch (String(prop)) { 107 | // Disable methods that modify the lifecycle of the call. These operations 108 | // should be performed via the RTVI client in order to keep state in sync. 109 | case "initialize": 110 | errMsg = `Direct calls to initialize() are disabled and used internally by the RTVIClient.`; 111 | break; 112 | case "initDevices": 113 | errMsg = `Direct calls to initDevices() are disabled. Please use the RTVIClient.initDevices() wrapper or let RTVIClient.connect() call it for you.`; 114 | break; 115 | case "sendReadyMessage": 116 | errMsg = `Direct calls to sendReadyMessage() are disabled and used internally by the RTVIClient.`; 117 | break; 118 | case "connect": 119 | errMsg = `Direct calls to connect() are disabled. Please use the RTVIClient.connect() wrapper.`; 120 | break; 121 | case "disconnect": 122 | errMsg = `Direct calls to disconnect() are disabled. Please use the RTVIClient.disconnect() wrapper.`; 123 | break; 124 | } 125 | if (errMsg) { 126 | return () => { 127 | throw new Error(errMsg); 128 | }; 129 | } 130 | // Forward other method calls 131 | return (...args: any[]) => { 132 | return (target[prop as keyof Transport] as Function)(...args); 133 | }; 134 | } 135 | // Forward property access 136 | return Reflect.get(target, prop, receiver); 137 | }, 138 | }); 139 | } 140 | 141 | get proxy(): Transport { 142 | return this._proxy; 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /client-js/tests/client.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { beforeEach, describe, expect, test } from "@jest/globals"; 8 | 9 | import { 10 | ActionEndpointNotSetError, 11 | BotNotReadyError, 12 | LLMHelper, 13 | RTVIClient, 14 | type RTVIClientConfigOption, 15 | RTVIClientOptions, 16 | RTVIEvent, 17 | } from "../src/"; 18 | import { TransportStub } from "./stubs/transport"; 19 | 20 | const exampleServices = { 21 | tts: "tts", 22 | llm: "llm", 23 | vad: "vad", 24 | }; 25 | 26 | const exampleConfig: RTVIClientConfigOption[] = [ 27 | { service: "vad", options: [{ name: "params", value: { stop_secs: 0.8 } }] }, 28 | { 29 | service: "tts", 30 | options: [{ name: "voice", value: "VoiceABC" }], 31 | }, 32 | { 33 | service: "llm", 34 | options: [ 35 | { name: "model", value: "ModelABC" }, 36 | { 37 | name: "initial_messages", 38 | value: [ 39 | { 40 | role: "system", 41 | content: 42 | "You are a assistant called ExampleBot. You can ask me anything.", 43 | }, 44 | ], 45 | }, 46 | { name: "run_on_config", value: true }, 47 | ], 48 | }, 49 | ]; 50 | 51 | describe("RTVIClient Methods", () => { 52 | let client: RTVIClient; 53 | 54 | beforeEach(() => { 55 | const transport = new TransportStub(); 56 | const args = { 57 | params: { 58 | baseUrl: "/", 59 | services: exampleServices, 60 | config: exampleConfig, 61 | }, 62 | transport: transport, 63 | customConnectHandler: () => Promise.resolve(), 64 | }; 65 | client = new RTVIClient(args as RTVIClientOptions); 66 | }); 67 | 68 | test("connect() and disconnect()", async () => { 69 | const stateChanges: string[] = []; 70 | const mockStateChangeHandler = (newState: string) => { 71 | stateChanges.push(newState); 72 | }; 73 | client.on(RTVIEvent.TransportStateChanged, mockStateChangeHandler); 74 | 75 | expect(client.connected).toBe(false); 76 | 77 | await client.connect(); 78 | 79 | expect(client.connected).toBe(true); 80 | expect(client.state === "ready").toBe(true); 81 | 82 | await client.disconnect(); 83 | 84 | expect(client.connected).toBe(false); 85 | expect(client.state).toBe("disconnected"); 86 | 87 | expect(stateChanges).toEqual([ 88 | "initializing", 89 | "initialized", 90 | "authenticating", 91 | "connecting", 92 | "connected", 93 | "ready", 94 | "disconnecting", 95 | "disconnected", 96 | ]); 97 | }); 98 | 99 | test("initDevices() sets initialized state", async () => { 100 | const stateChanges: string[] = []; 101 | const mockStateChangeHandler = (newState: string) => { 102 | stateChanges.push(newState); 103 | }; 104 | client.on(RTVIEvent.TransportStateChanged, mockStateChangeHandler); 105 | 106 | await client.initDevices(); 107 | 108 | expect(client.state === "initialized").toBe(true); 109 | 110 | expect(stateChanges).toEqual(["initializing", "initialized"]); 111 | }); 112 | 113 | test("Endpoints should have defaults", () => { 114 | const connectUrl = client.constructUrl("connect"); 115 | const disconnectedActionsUrl = client.constructUrl("action"); 116 | 117 | expect(connectUrl).toEqual("/connect"); 118 | expect(disconnectedActionsUrl).toEqual("/action"); 119 | }); 120 | 121 | test("Base URL and connect endpoint should should be nullable", async () => { 122 | const stateChanges: string[] = []; 123 | const mockStateChangeHandler = (newState: string) => { 124 | stateChanges.push(newState); 125 | }; 126 | client.on(RTVIEvent.TransportStateChanged, mockStateChangeHandler); 127 | client.params.baseUrl = ""; 128 | client.params.endpoints = { 129 | connect: null, 130 | }; 131 | await client.connect(); 132 | expect(client.state === "ready").toBe(true); 133 | expect(stateChanges).toEqual([ 134 | "initializing", 135 | "initialized", 136 | "authenticating", 137 | "connecting", 138 | "connected", 139 | "ready", 140 | ]); 141 | }); 142 | 143 | test("Connect endpoint should be nullable with base URL", async () => { 144 | client.params.baseUrl = "/test"; 145 | client.params.endpoints = { 146 | connect: null, 147 | }; 148 | await client.connect(); 149 | const connectUrl = client.constructUrl("connect"); 150 | expect(connectUrl).toEqual("/test"); 151 | await client.disconnect(); 152 | }); 153 | 154 | test("Client should throw an error when action endpoint is not set in disconnected state", async () => { 155 | await client.disconnect(); 156 | 157 | client.params.endpoints = { 158 | action: null, 159 | }; 160 | 161 | await expect( 162 | client.action({ service: "llm", action: "test" }) 163 | ).rejects.toThrow(ActionEndpointNotSetError); 164 | }); 165 | 166 | test("transportExpiry should throw an error when not in connected state", () => { 167 | expect(() => client.transportExpiry).toThrowError(BotNotReadyError); 168 | }); 169 | 170 | test("transportExpiry should return value when in connected state", async () => { 171 | await client.connect(); 172 | expect(client.transportExpiry).toBeUndefined(); 173 | }); 174 | 175 | test("registerHelper should register a new helper with the specified name", async () => { 176 | const llmHelper = new LLMHelper({ callbacks: {} }); 177 | client.registerHelper("llm", llmHelper); 178 | expect(client.getHelper("llm")).not.toBeUndefined(); 179 | client.unregisterHelper("llm"); 180 | expect(client.getHelper("llm")).toBeUndefined(); 181 | }); 182 | 183 | test("enableScreenShare should enable screen share", async () => { 184 | await client.connect(); 185 | client.enableScreenShare(true); 186 | expect(client.isSharingScreen).toBe(true); 187 | }); 188 | }); 189 | -------------------------------------------------------------------------------- /client-js/tests/stubs/transport.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024, Daily. 3 | * 4 | * SPDX-License-Identifier: BSD-2-Clause 5 | */ 6 | 7 | import { 8 | RTVIClientOptions, 9 | RTVIMessage, 10 | RTVIMessageType, 11 | Tracks, 12 | Transport, 13 | TransportStartError, 14 | TransportState, 15 | } from "../../src"; 16 | 17 | class mockState { 18 | public isSharingScreen = false; 19 | } 20 | 21 | export class TransportStub extends Transport { 22 | private _mockState: mockState; 23 | 24 | constructor() { 25 | super(); 26 | this._mockState = new mockState(); 27 | } 28 | 29 | public initDevices(): Promise { 30 | return new Promise((resolve) => { 31 | this.state = "initializing"; 32 | setTimeout(() => { 33 | this.state = "initialized"; 34 | resolve(); 35 | }, 100); 36 | }); 37 | } 38 | 39 | public initialize( 40 | options: RTVIClientOptions, 41 | messageHandler: (ev: RTVIMessage) => void 42 | ): void { 43 | this._onMessage = messageHandler; 44 | this._callbacks = options.callbacks ?? {}; 45 | 46 | this.state = "disconnected"; 47 | } 48 | 49 | public async connect(authBundle: boolean = true): Promise { 50 | return new Promise((resolve) => { 51 | this.state = "connecting"; 52 | 53 | if (!authBundle) { 54 | this.state = "error"; 55 | throw new TransportStartError(); 56 | } 57 | 58 | setTimeout(() => { 59 | this.state = "connected"; 60 | resolve(); 61 | }, 100); 62 | }); 63 | } 64 | 65 | public async disconnect(): Promise { 66 | return new Promise((resolve) => { 67 | this.state = "disconnecting"; 68 | setTimeout(() => { 69 | this.state = "disconnected"; 70 | resolve(); 71 | }, 100); 72 | }); 73 | } 74 | 75 | async sendReadyMessage(): Promise { 76 | return new Promise((resolve) => { 77 | (async () => { 78 | this.state = "ready"; 79 | 80 | resolve(); 81 | 82 | this._onMessage({ 83 | label: "rtvi-ai", 84 | id: "123", 85 | type: RTVIMessageType.BOT_READY, 86 | data: {}, 87 | } as RTVIMessage); 88 | })(); 89 | }); 90 | } 91 | 92 | public getAllMics(): Promise { 93 | return Promise.resolve([]); 94 | } 95 | public getAllCams(): Promise { 96 | return Promise.resolve([]); 97 | } 98 | public getAllSpeakers(): Promise { 99 | return Promise.resolve([]); 100 | } 101 | 102 | public updateMic(micId: string): void { 103 | console.log(micId); 104 | return; 105 | } 106 | public updateCam(camId: string): void { 107 | console.log(camId); 108 | return; 109 | } 110 | public updateSpeaker(speakerId: string): void { 111 | console.log(speakerId); 112 | return; 113 | } 114 | 115 | public get selectedMic(): MediaDeviceInfo | Record { 116 | return {}; 117 | } 118 | public get selectedCam(): MediaDeviceInfo | Record { 119 | return {}; 120 | } 121 | public get selectedSpeaker(): MediaDeviceInfo | Record { 122 | return {}; 123 | } 124 | 125 | public enableMic(enable: boolean): void { 126 | console.log(enable); 127 | return; 128 | } 129 | public enableCam(enable: boolean): void { 130 | console.log(enable); 131 | return; 132 | } 133 | public enableScreenShare(enable: boolean): void { 134 | this._mockState.isSharingScreen = enable; 135 | return; 136 | } 137 | 138 | public get isCamEnabled(): boolean { 139 | return true; 140 | } 141 | public get isMicEnabled(): boolean { 142 | return true; 143 | } 144 | public get isSharingScreen(): boolean { 145 | return this._mockState.isSharingScreen; 146 | } 147 | 148 | public sendMessage(message: RTVIMessage) { 149 | if (message.type === RTVIMessageType.ACTION) { 150 | this._onMessage({ 151 | type: RTVIMessageType.ACTION_RESPONSE, 152 | id: "123", 153 | label: "rtvi-ai", 154 | data: { 155 | result: true, 156 | }, 157 | }); 158 | } else { 159 | // Mock the response from the server 160 | console.log("[STUB] message.type:", message.type); 161 | 162 | switch (message.type) { 163 | case RTVIMessageType.UPDATE_CONFIG: 164 | this._onMessage({ 165 | ...message, 166 | type: RTVIMessageType.CONFIG, 167 | }); 168 | break; 169 | case RTVIMessageType.GET_CONFIG: 170 | this._onMessage({ 171 | ...message, 172 | data: {}, 173 | type: RTVIMessageType.CONFIG, 174 | }); 175 | break; 176 | default: 177 | this._onMessage(message); 178 | } 179 | } 180 | return true; 181 | } 182 | 183 | public get state(): TransportState { 184 | return this._state; 185 | } 186 | 187 | private set state(state: TransportState) { 188 | if (this._state === state) return; 189 | 190 | this._state = state; 191 | this._callbacks.onTransportStateChanged?.(state); 192 | } 193 | 194 | get expiry(): number | undefined { 195 | return this._expiry; 196 | } 197 | 198 | public tracks(): Tracks { 199 | return { local: { audio: undefined, video: undefined } }; 200 | } 201 | } 202 | 203 | export default TransportStub; 204 | -------------------------------------------------------------------------------- /client-js/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "ESNext", 5 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 6 | "skipLibCheck": true, 7 | "jsx": "preserve", 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "noEmit": true, 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "moduleDetection": "force", 16 | "esModuleInterop": true, 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "noFallthroughCasesInSwitch": true, 23 | 24 | "experimentalDecorators": true, 25 | }, 26 | "include": ["src"] 27 | } 28 | -------------------------------------------------------------------------------- /client-react/README.md: -------------------------------------------------------------------------------- 1 |

2 | pipecat react 3 |

4 | 5 | [![Docs](https://img.shields.io/badge/documentation-blue)](https://docs.pipecat.ai/client/introduction) 6 | ![NPM Version](https://img.shields.io/npm/v/@pipecat-ai/client-react) 7 | 8 | ## Install 9 | 10 | ```bash 11 | npm install @pipecat-ai/client-js @pipecat-ai/client-react 12 | ``` 13 | 14 | ## Quick Start 15 | 16 | Instantiate an `RTVIClient` instance and pass it down to the `RTVIClientProvider`. Render the `` component to have audio output setup automatically. 17 | 18 | ```tsx 19 | import { RTVIClient } from "@pipecat-ai/client-js"; 20 | import { RTVIClientAudio, RTVIClientProvider } from "@pipecat-ai/client-react"; 21 | 22 | const client = new RTVIClient({ 23 | baseUrl: "https://rtvi.pipecat.bot", 24 | enableMic: true, 25 | }); 26 | 27 | render( 28 | 29 | 30 | 31 | 32 | ); 33 | ``` 34 | 35 | We recommend starting the voiceClient from a click of a button, so here's a minimal implementation of `` to get started: 36 | 37 | ```tsx 38 | import { useRTVIClient } from "@pipecat-ai/client-react"; 39 | 40 | const MyApp = () => { 41 | const client = useRTVIClient(); 42 | return ; 43 | }; 44 | ``` 45 | 46 | ## Components 47 | 48 | ### RTVIClientProvider 49 | 50 | The root component for providing RTVI client context to your application. 51 | 52 | #### Props 53 | 54 | - `client` (RTVIClient, required): A singleton instance of RTVIClient. 55 | 56 | ```jsx 57 | 58 | {/* Child components */} 59 | 60 | ``` 61 | 62 | ### RTVIClientAudio 63 | 64 | Creates a new `