├── .env ├── .gcloudignore ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── app.yaml ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html └── robots.txt ├── readme └── thumbnail.png ├── src ├── App.scss ├── App.test.tsx ├── App.tsx ├── components │ ├── altair │ │ └── Altair.tsx │ ├── audio-pulse │ │ ├── AudioPulse.tsx │ │ └── audio-pulse.scss │ ├── control-tray │ │ ├── ControlTray.tsx │ │ └── control-tray.scss │ ├── logger │ │ ├── Logger.tsx │ │ ├── logger.scss │ │ └── mock-logs.ts │ ├── settings-dialog │ │ ├── ResponseModalitySelector.tsx │ │ ├── SettingsDialog.tsx │ │ ├── VoiceSelector.tsx │ │ └── settings-dialog.scss │ └── side-panel │ │ ├── SidePanel.tsx │ │ ├── react-select.scss │ │ └── side-panel.scss ├── contexts │ └── LiveAPIContext.tsx ├── hooks │ ├── use-live-api.ts │ ├── use-media-stream-mux.ts │ ├── use-screen-capture.ts │ └── use-webcam.ts ├── index.css ├── index.tsx ├── lib │ ├── audio-recorder.ts │ ├── audio-streamer.ts │ ├── audioworklet-registry.ts │ ├── genai-live-client.ts │ ├── store-logger.ts │ ├── utils.ts │ └── worklets │ │ ├── audio-processing.ts │ │ └── vol-meter.ts ├── react-app-env.d.ts ├── reportWebVitals.ts ├── setupTests.ts └── types.ts └── tsconfig.json /.env: -------------------------------------------------------------------------------- 1 | # create your own API KEY at https://aistudio.google.com/apikey 2 | #REACT_APP_GEMINI_API_KEY='' 3 | -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | # Ignore everything except app.yaml and the build directory 2 | * 3 | !app.yaml 4 | !build 5 | !build/** -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | We'd love to accept your patches and contributions to this project. 4 | 5 | ## Before you begin 6 | 7 | ### Sign our Contributor License Agreement 8 | 9 | Contributions to this project must be accompanied by a 10 | [Contributor License Agreement](https://cla.developers.google.com/about) (CLA). 11 | You (or your employer) retain the copyright to your contribution; this simply 12 | gives us permission to use and redistribute your contributions as part of the 13 | project. 14 | 15 | If you or your current employer have already signed the Google CLA (even if it 16 | was for a different project), you probably don't need to do it again. 17 | 18 | Visit to see your current agreements or to 19 | sign a new one. 20 | 21 | ### Review our community guidelines 22 | 23 | This project follows 24 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 25 | 26 | ## Contribution process 27 | 28 | ### Code reviews 29 | 30 | All submissions, including submissions by project members, require review. We 31 | use GitHub pull requests for this purpose. Consult 32 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 33 | information on using pull requests. 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Live API - Web Console 2 | 3 | This repository contains a react-based starter app for using the [Live API](<[https://ai.google.dev/gemini-api](https://ai.google.dev/api/multimodal-live)>) over a websocket. It provides modules for streaming audio playback, recording user media such as from a microphone, webcam or screen capture as well as a unified log view to aid in development of your application. 4 | 5 | [![Live API Demo](readme/thumbnail.png)](https://www.youtube.com/watch?v=J_q7JY1XxFE) 6 | 7 | Watch the demo of the Live API [here](https://www.youtube.com/watch?v=J_q7JY1XxFE). 8 | 9 | ## Usage 10 | 11 | To get started, [create a free Gemini API key](https://aistudio.google.com/apikey) and add it to the `.env` file. Then: 12 | 13 | ``` 14 | $ npm install && npm start 15 | ``` 16 | 17 | We have provided several example applications on other branches of this repository: 18 | 19 | - [demos/GenExplainer](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genexplainer) 20 | - [demos/GenWeather](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genweather) 21 | - [demos/GenList](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genlist) 22 | 23 | ## Example 24 | 25 | Below is an example of an entire application that will use Google Search grounding and then render graphs using [vega-embed](https://github.com/vega/vega-embed): 26 | 27 | ```typescript 28 | import { type FunctionDeclaration, SchemaType } from "@google/generative-ai"; 29 | import { useEffect, useRef, useState, memo } from "react"; 30 | import vegaEmbed from "vega-embed"; 31 | import { useLiveAPIContext } from "../../contexts/LiveAPIContext"; 32 | 33 | export const declaration: FunctionDeclaration = { 34 | name: "render_altair", 35 | description: "Displays an altair graph in json format.", 36 | parameters: { 37 | type: SchemaType.OBJECT, 38 | properties: { 39 | json_graph: { 40 | type: SchemaType.STRING, 41 | description: 42 | "JSON STRING representation of the graph to render. Must be a string, not a json object", 43 | }, 44 | }, 45 | required: ["json_graph"], 46 | }, 47 | }; 48 | 49 | export function Altair() { 50 | const [jsonString, setJSONString] = useState(""); 51 | const { client, setConfig } = useLiveAPIContext(); 52 | 53 | useEffect(() => { 54 | setConfig({ 55 | model: "models/gemini-2.0-flash-exp", 56 | systemInstruction: { 57 | parts: [ 58 | { 59 | text: 'You are my helpful assistant. Any time I ask you for a graph call the "render_altair" function I have provided you. Dont ask for additional information just make your best judgement.', 60 | }, 61 | ], 62 | }, 63 | tools: [{ googleSearch: {} }, { functionDeclarations: [declaration] }], 64 | }); 65 | }, [setConfig]); 66 | 67 | useEffect(() => { 68 | const onToolCall = (toolCall: ToolCall) => { 69 | console.log(`got toolcall`, toolCall); 70 | const fc = toolCall.functionCalls.find( 71 | (fc) => fc.name === declaration.name 72 | ); 73 | if (fc) { 74 | const str = (fc.args as any).json_graph; 75 | setJSONString(str); 76 | } 77 | }; 78 | client.on("toolcall", onToolCall); 79 | return () => { 80 | client.off("toolcall", onToolCall); 81 | }; 82 | }, [client]); 83 | 84 | const embedRef = useRef(null); 85 | 86 | useEffect(() => { 87 | if (embedRef.current && jsonString) { 88 | vegaEmbed(embedRef.current, JSON.parse(jsonString)); 89 | } 90 | }, [embedRef, jsonString]); 91 | return
; 92 | } 93 | ``` 94 | 95 | ## development 96 | 97 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 98 | Project consists of: 99 | 100 | - an Event-emitting websocket-client to ease communication between the websocket and the front-end 101 | - communication layer for processing audio in and out 102 | - a boilerplate view for starting to build your apps and view logs 103 | 104 | ## Available Scripts 105 | 106 | In the project directory, you can run: 107 | 108 | ### `npm start` 109 | 110 | Runs the app in the development mode.\ 111 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 112 | 113 | The page will reload if you make edits.\ 114 | You will also see any lint errors in the console. 115 | 116 | ### `npm run build` 117 | 118 | Builds the app for production to the `build` folder.\ 119 | It correctly bundles React in production mode and optimizes the build for the best performance. 120 | 121 | The build is minified and the filenames include the hashes.\ 122 | Your app is ready to be deployed! 123 | 124 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 125 | 126 | _This is an experiment showcasing the Live API, not an official Google product. We’ll do our best to support and maintain this experiment but your mileage may vary. We encourage open sourcing projects as a way of learning from each other. Please respect our and other creators' rights, including copyright and trademark rights when present, when sharing these works and creating derivative work. If you want more info on Google's policy, you can find that [here](https://developers.google.com/terms/site-policies)._ 127 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | runtime: nodejs20 16 | env: standard 17 | 18 | handlers: 19 | # serve static files 20 | - url: /(.*\..+)$ 21 | static_files: build/\1 22 | upload: build/(.*\..+)$ 23 | 24 | # Catch all handler to index.html 25 | - url: /.* 26 | static_files: build/index.html 27 | secure: always 28 | redirect_http_response_code: 301 29 | upload: buid/index.html 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multimodal-live-api-web-console", 3 | "version": "0.1.0", 4 | "dependencies": { 5 | "@google/genai": "^0.14.0", 6 | "classnames": "^2.5.1", 7 | "dotenv-flow": "^4.1.0", 8 | "eventemitter3": "^5.0.1", 9 | "lodash": "^4.17.21", 10 | "react": "^18.3.1", 11 | "react-dom": "^18.3.1", 12 | "react-icons": "^5.3.0", 13 | "react-scripts": "5.0.1", 14 | "react-select": "^5.8.3", 15 | "react-syntax-highlighter": "^15.6.1", 16 | "sass": "^1.80.6", 17 | "vega": "^5.30.0", 18 | "vega-embed": "^6.29.0", 19 | "vega-lite": "^5.22.0", 20 | "web-vitals": "^2.1.4", 21 | "zustand": "^5.0.1" 22 | }, 23 | "scripts": { 24 | "start-https": "HTTPS=true react-scripts start", 25 | "start": "react-scripts start", 26 | "build": "react-scripts build", 27 | "test": "react-scripts test", 28 | "eject": "react-scripts eject" 29 | }, 30 | "eslintConfig": { 31 | "extends": [ 32 | "react-app", 33 | "react-app/jest" 34 | ] 35 | }, 36 | "browserslist": { 37 | "production": [ 38 | ">0.2%", 39 | "not dead", 40 | "not op_mini all" 41 | ], 42 | "development": [ 43 | "last 1 chrome version", 44 | "last 1 firefox version", 45 | "last 1 safari version" 46 | ] 47 | }, 48 | "devDependencies": { 49 | "@testing-library/jest-dom": "^5.17.0", 50 | "@testing-library/react": "^13.4.0", 51 | "@testing-library/user-event": "^13.5.0", 52 | "@types/jest": "^27.5.2", 53 | "@types/lodash": "^4.17.13", 54 | "@types/node": "^16.18.119", 55 | "@types/react": "^18.3.12", 56 | "@types/react-dom": "^18.3.1", 57 | "@types/react-syntax-highlighter": "^15.5.13", 58 | "ts-node": "^10.9.2", 59 | "typescript": "^5.6.3" 60 | }, 61 | "overrides": { 62 | "typescript": "^5.6.3" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-gemini/live-api-web-console/e4de0d367dfc22fd9b9f0fd80bd86ecfea4e8b57/public/favicon.ico -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | 18 | 22 | 23 | 32 | Multimodal Live - Console 33 | 34 | 35 | 36 | 37 |
38 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /readme/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-gemini/live-api-web-console/e4de0d367dfc22fd9b9f0fd80bd86ecfea4e8b57/readme/thumbnail.png -------------------------------------------------------------------------------- /src/App.scss: -------------------------------------------------------------------------------- 1 | :root { 2 | --text: white; 3 | --gray-200: #b4b8bb; 4 | --gray-300: #80868b; 5 | --gray-500: #5f6368; 6 | --gray-600: #444444; 7 | --gray-700: #202020; 8 | --gray-800: #171717; 9 | --gray-900: #111111; 10 | --gray-1000: #0a0a0a; 11 | --border-stroke: #444444; 12 | --accent-blue: rgb(161, 228, 242); 13 | --accent-blue-active-bg: #001233; 14 | --accent-blue-active: #98beff; 15 | --accent-blue-headers: #448dff; 16 | --accent-green: rgb(168, 218, 181); 17 | 18 | --midnight-blue: rgb(0, 18, 51); 19 | --blue-30: #99beff; 20 | 21 | --accent-red: #ff4600; 22 | 23 | --background: var(--gray-900); 24 | --color: var(--text); 25 | 26 | scrollbar-color: var(--gray-600) var(--gray-900); 27 | scrollbar-width: thin; 28 | 29 | --font-family: "Space Mono", monospace; 30 | 31 | /* */ 32 | --Neutral-00: #000; 33 | --Neutral-5: #181a1b; 34 | --Neutral-10: #1c1f21; 35 | --Neutral-15: #232729; 36 | --Neutral-20: #2a2f31; 37 | --Neutral-30: #404547; 38 | --Neutral-50: #707577; 39 | --Neutral-60: #888d8f; 40 | --Neutral-80: #c3c6c7; 41 | --Neutral-90: #e1e2e3; 42 | 43 | --Green-500: #0d9c53; 44 | --Green-700: #025022; 45 | 46 | --Blue-400: #80c1ff; 47 | --Blue-500: #1f94ff; 48 | --Blue-800: #0f3557; 49 | 50 | --Red-400: #ff9c7a; 51 | --Red-500: #ff4600; 52 | --Red-600: #e03c00; 53 | --Red-700: #bd3000; 54 | } 55 | 56 | body { 57 | font-family: "Space Mono", monospace; 58 | background: var(--Neutral-30); 59 | } 60 | 61 | .material-symbols-outlined { 62 | &.filled { 63 | font-variation-settings: 64 | "FILL" 1, 65 | "wght" 400, 66 | "GRAD" 0, 67 | "opsz" 24; 68 | } 69 | } 70 | 71 | .space-mono-regular { 72 | font-family: "Space Mono", monospace; 73 | font-weight: 400; 74 | font-style: normal; 75 | } 76 | 77 | .space-mono-bold { 78 | font-family: "Space Mono", monospace; 79 | font-weight: 700; 80 | font-style: normal; 81 | } 82 | 83 | .space-mono-regular-italic { 84 | font-family: "Space Mono", monospace; 85 | font-weight: 400; 86 | font-style: italic; 87 | } 88 | 89 | .space-mono-bold-italic { 90 | font-family: "Space Mono", monospace; 91 | font-weight: 700; 92 | font-style: italic; 93 | } 94 | 95 | .hidden { 96 | display: none; 97 | } 98 | 99 | .flex { 100 | display: flex; 101 | } 102 | 103 | .h-screen-full { 104 | height: 100vh; 105 | } 106 | 107 | .w-screen-full { 108 | width: 100vw; 109 | } 110 | 111 | .flex-col { 112 | flex-direction: column; 113 | } 114 | 115 | @media (prefers-reduced-motion: no-preference) { 116 | } 117 | 118 | .streaming-console { 119 | background: var(--Neutral-15); 120 | color: var(--gray-300); 121 | display: flex; 122 | height: 100vh; 123 | width: 100vw; 124 | 125 | a, 126 | a:visited, 127 | a:active { 128 | color: var(--gray-300); 129 | } 130 | 131 | .disabled { 132 | pointer-events: none; 133 | 134 | > * { 135 | pointer-events: none; 136 | } 137 | } 138 | 139 | main { 140 | position: relative; 141 | display: flex; 142 | flex-direction: column; 143 | align-items: center; 144 | justify-content: center; 145 | flex-grow: 1; 146 | gap: 1rem; 147 | max-width: 100%; 148 | overflow: hidden; 149 | } 150 | 151 | .main-app-area { 152 | display: flex; 153 | flex: 1; 154 | align-items: center; 155 | justify-content: center; 156 | } 157 | 158 | .function-call { 159 | position: absolute; 160 | top: 0; 161 | width: 100%; 162 | height: 50%; 163 | overflow-y: auto; 164 | } 165 | } 166 | 167 | /* video player */ 168 | .stream { 169 | flex-grow: 1; 170 | max-width: 90%; 171 | border-radius: 32px; 172 | max-height: fit-content; 173 | } 174 | -------------------------------------------------------------------------------- /src/App.test.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import React from 'react'; 18 | import { render, screen } from '@testing-library/react'; 19 | import App from './App'; 20 | 21 | test('renders learn react link', () => { 22 | render(); 23 | const linkElement = screen.getByText(/learn react/i); 24 | expect(linkElement).toBeInTheDocument(); 25 | }); 26 | -------------------------------------------------------------------------------- /src/App.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { useRef, useState } from "react"; 18 | import "./App.scss"; 19 | import { LiveAPIProvider } from "./contexts/LiveAPIContext"; 20 | import SidePanel from "./components/side-panel/SidePanel"; 21 | import { Altair } from "./components/altair/Altair"; 22 | import ControlTray from "./components/control-tray/ControlTray"; 23 | import cn from "classnames"; 24 | import { LiveClientOptions } from "./types"; 25 | 26 | const API_KEY = process.env.REACT_APP_GEMINI_API_KEY as string; 27 | if (typeof API_KEY !== "string") { 28 | throw new Error("set REACT_APP_GEMINI_API_KEY in .env"); 29 | } 30 | 31 | const apiOptions: LiveClientOptions = { 32 | apiKey: API_KEY, 33 | }; 34 | 35 | function App() { 36 | // this video reference is used for displaying the active stream, whether that is the webcam or screen capture 37 | // feel free to style as you see fit 38 | const videoRef = useRef(null); 39 | // either the screen capture, the video or null, if null we hide it 40 | const [videoStream, setVideoStream] = useState(null); 41 | 42 | return ( 43 |
44 | 45 |
46 | 47 |
48 |
49 | {/* APP goes here */} 50 | 51 |
60 | 61 | 67 | {/* put your own buttons here */} 68 | 69 |
70 |
71 | 72 |
73 | ); 74 | } 75 | 76 | export default App; 77 | -------------------------------------------------------------------------------- /src/components/altair/Altair.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import { useEffect, useRef, useState, memo } from "react"; 17 | import vegaEmbed from "vega-embed"; 18 | import { useLiveAPIContext } from "../../contexts/LiveAPIContext"; 19 | import { 20 | FunctionDeclaration, 21 | LiveServerToolCall, 22 | Modality, 23 | Type, 24 | } from "@google/genai"; 25 | 26 | const declaration: FunctionDeclaration = { 27 | name: "render_altair", 28 | description: "Displays an altair graph in json format.", 29 | parameters: { 30 | type: Type.OBJECT, 31 | properties: { 32 | json_graph: { 33 | type: Type.STRING, 34 | description: 35 | "JSON STRING representation of the graph to render. Must be a string, not a json object", 36 | }, 37 | }, 38 | required: ["json_graph"], 39 | }, 40 | }; 41 | 42 | function AltairComponent() { 43 | const [jsonString, setJSONString] = useState(""); 44 | const { client, setConfig, setModel } = useLiveAPIContext(); 45 | 46 | useEffect(() => { 47 | setModel("models/gemini-2.0-flash-exp"); 48 | setConfig({ 49 | responseModalities: [Modality.AUDIO], 50 | speechConfig: { 51 | voiceConfig: { prebuiltVoiceConfig: { voiceName: "Aoede" } }, 52 | }, 53 | systemInstruction: { 54 | parts: [ 55 | { 56 | text: 'You are my helpful assistant. Any time I ask you for a graph call the "render_altair" function I have provided you. Dont ask for additional information just make your best judgement.', 57 | }, 58 | ], 59 | }, 60 | tools: [ 61 | // there is a free-tier quota for search 62 | { googleSearch: {} }, 63 | { functionDeclarations: [declaration] }, 64 | ], 65 | }); 66 | }, [setConfig, setModel]); 67 | 68 | useEffect(() => { 69 | const onToolCall = (toolCall: LiveServerToolCall) => { 70 | if (!toolCall.functionCalls) { 71 | return; 72 | } 73 | const fc = toolCall.functionCalls.find( 74 | (fc) => fc.name === declaration.name 75 | ); 76 | if (fc) { 77 | const str = (fc.args as any).json_graph; 78 | setJSONString(str); 79 | } 80 | // send data for the response of your tool call 81 | // in this case Im just saying it was successful 82 | if (toolCall.functionCalls.length) { 83 | setTimeout( 84 | () => 85 | client.sendToolResponse({ 86 | functionResponses: toolCall.functionCalls?.map((fc) => ({ 87 | response: { output: { success: true } }, 88 | id: fc.id, 89 | name: fc.name, 90 | })), 91 | }), 92 | 200 93 | ); 94 | } 95 | }; 96 | client.on("toolcall", onToolCall); 97 | return () => { 98 | client.off("toolcall", onToolCall); 99 | }; 100 | }, [client]); 101 | 102 | const embedRef = useRef(null); 103 | 104 | useEffect(() => { 105 | if (embedRef.current && jsonString) { 106 | console.log("jsonString", jsonString); 107 | vegaEmbed(embedRef.current, JSON.parse(jsonString)); 108 | } 109 | }, [embedRef, jsonString]); 110 | return
; 111 | } 112 | 113 | export const Altair = memo(AltairComponent); 114 | -------------------------------------------------------------------------------- /src/components/audio-pulse/AudioPulse.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import "./audio-pulse.scss"; 18 | import React from "react"; 19 | import { useEffect, useRef } from "react"; 20 | import c from "classnames"; 21 | 22 | const lineCount = 3; 23 | 24 | export type AudioPulseProps = { 25 | active: boolean; 26 | volume: number; 27 | hover?: boolean; 28 | }; 29 | 30 | export default function AudioPulse({ active, volume, hover }: AudioPulseProps) { 31 | const lines = useRef([]); 32 | 33 | useEffect(() => { 34 | let timeout: number | null = null; 35 | const update = () => { 36 | lines.current.forEach( 37 | (line, i) => 38 | (line.style.height = `${Math.min( 39 | 24, 40 | 4 + volume * (i === 1 ? 400 : 60), 41 | )}px`), 42 | ); 43 | timeout = window.setTimeout(update, 100); 44 | }; 45 | 46 | update(); 47 | 48 | return () => clearTimeout((timeout as number)!); 49 | }, [volume]); 50 | 51 | return ( 52 |
53 | {Array(lineCount) 54 | .fill(null) 55 | .map((_, i) => ( 56 |
(lines.current[i] = el!)} 59 | style={{ animationDelay: `${i * 133}ms` }} 60 | /> 61 | ))} 62 |
63 | ); 64 | } 65 | -------------------------------------------------------------------------------- /src/components/audio-pulse/audio-pulse.scss: -------------------------------------------------------------------------------- 1 | .audioPulse { 2 | display: flex; 3 | width: 24px; 4 | justify-content: space-evenly; 5 | align-items: center; 6 | transition: all 0.5s; 7 | 8 | & > div { 9 | background-color: var(--Neutral-30); 10 | border-radius: 1000px; 11 | width: 4px; 12 | min-height: 4px; 13 | border-radius: 1000px; 14 | transition: height 0.1s; 15 | } 16 | 17 | &.hover > div { 18 | animation: hover 1.4s infinite alternate ease-in-out; 19 | } 20 | 21 | height: 4px; 22 | transition: opacity 0.333s; 23 | 24 | &.active { 25 | opacity: 1; 26 | 27 | & > div { 28 | background-color: var(--Neutral-80); 29 | } 30 | } 31 | } 32 | 33 | @keyframes hover { 34 | from { 35 | transform: translateY(0); 36 | } 37 | 38 | to { 39 | transform: translateY(-3.5px); 40 | } 41 | } 42 | 43 | @keyframes pulse { 44 | from { 45 | scale: 1 1; 46 | } 47 | 48 | to { 49 | scale: 1.2 1.2; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/components/control-tray/ControlTray.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import cn from "classnames"; 18 | 19 | import { memo, ReactNode, RefObject, useEffect, useRef, useState } from "react"; 20 | import { useLiveAPIContext } from "../../contexts/LiveAPIContext"; 21 | import { UseMediaStreamResult } from "../../hooks/use-media-stream-mux"; 22 | import { useScreenCapture } from "../../hooks/use-screen-capture"; 23 | import { useWebcam } from "../../hooks/use-webcam"; 24 | import { AudioRecorder } from "../../lib/audio-recorder"; 25 | import AudioPulse from "../audio-pulse/AudioPulse"; 26 | import "./control-tray.scss"; 27 | import SettingsDialog from "../settings-dialog/SettingsDialog"; 28 | 29 | export type ControlTrayProps = { 30 | videoRef: RefObject; 31 | children?: ReactNode; 32 | supportsVideo: boolean; 33 | onVideoStreamChange?: (stream: MediaStream | null) => void; 34 | enableEditingSettings?: boolean; 35 | }; 36 | 37 | type MediaStreamButtonProps = { 38 | isStreaming: boolean; 39 | onIcon: string; 40 | offIcon: string; 41 | start: () => Promise; 42 | stop: () => any; 43 | }; 44 | 45 | /** 46 | * button used for triggering webcam or screen-capture 47 | */ 48 | const MediaStreamButton = memo( 49 | ({ isStreaming, onIcon, offIcon, start, stop }: MediaStreamButtonProps) => 50 | isStreaming ? ( 51 | 54 | ) : ( 55 | 58 | ) 59 | ); 60 | 61 | function ControlTray({ 62 | videoRef, 63 | children, 64 | onVideoStreamChange = () => {}, 65 | supportsVideo, 66 | enableEditingSettings, 67 | }: ControlTrayProps) { 68 | const videoStreams = [useWebcam(), useScreenCapture()]; 69 | const [activeVideoStream, setActiveVideoStream] = 70 | useState(null); 71 | const [webcam, screenCapture] = videoStreams; 72 | const [inVolume, setInVolume] = useState(0); 73 | const [audioRecorder] = useState(() => new AudioRecorder()); 74 | const [muted, setMuted] = useState(false); 75 | const renderCanvasRef = useRef(null); 76 | const connectButtonRef = useRef(null); 77 | 78 | const { client, connected, connect, disconnect, volume } = 79 | useLiveAPIContext(); 80 | 81 | useEffect(() => { 82 | if (!connected && connectButtonRef.current) { 83 | connectButtonRef.current.focus(); 84 | } 85 | }, [connected]); 86 | useEffect(() => { 87 | document.documentElement.style.setProperty( 88 | "--volume", 89 | `${Math.max(5, Math.min(inVolume * 200, 8))}px` 90 | ); 91 | }, [inVolume]); 92 | 93 | useEffect(() => { 94 | const onData = (base64: string) => { 95 | client.sendRealtimeInput([ 96 | { 97 | mimeType: "audio/pcm;rate=16000", 98 | data: base64, 99 | }, 100 | ]); 101 | }; 102 | if (connected && !muted && audioRecorder) { 103 | audioRecorder.on("data", onData).on("volume", setInVolume).start(); 104 | } else { 105 | audioRecorder.stop(); 106 | } 107 | return () => { 108 | audioRecorder.off("data", onData).off("volume", setInVolume); 109 | }; 110 | }, [connected, client, muted, audioRecorder]); 111 | 112 | useEffect(() => { 113 | if (videoRef.current) { 114 | videoRef.current.srcObject = activeVideoStream; 115 | } 116 | 117 | let timeoutId = -1; 118 | 119 | function sendVideoFrame() { 120 | const video = videoRef.current; 121 | const canvas = renderCanvasRef.current; 122 | 123 | if (!video || !canvas) { 124 | return; 125 | } 126 | 127 | const ctx = canvas.getContext("2d")!; 128 | canvas.width = video.videoWidth * 0.25; 129 | canvas.height = video.videoHeight * 0.25; 130 | if (canvas.width + canvas.height > 0) { 131 | ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height); 132 | const base64 = canvas.toDataURL("image/jpeg", 1.0); 133 | const data = base64.slice(base64.indexOf(",") + 1, Infinity); 134 | client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]); 135 | } 136 | if (connected) { 137 | timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5); 138 | } 139 | } 140 | if (connected && activeVideoStream !== null) { 141 | requestAnimationFrame(sendVideoFrame); 142 | } 143 | return () => { 144 | clearTimeout(timeoutId); 145 | }; 146 | }, [connected, activeVideoStream, client, videoRef]); 147 | 148 | //handler for swapping from one video-stream to the next 149 | const changeStreams = (next?: UseMediaStreamResult) => async () => { 150 | if (next) { 151 | const mediaStream = await next.start(); 152 | setActiveVideoStream(mediaStream); 153 | onVideoStreamChange(mediaStream); 154 | } else { 155 | setActiveVideoStream(null); 156 | onVideoStreamChange(null); 157 | } 158 | 159 | videoStreams.filter((msr) => msr !== next).forEach((msr) => msr.stop()); 160 | }; 161 | 162 | return ( 163 |
164 | 165 | 201 | 202 |
203 |
204 | 213 |
214 | Streaming 215 |
216 | {enableEditingSettings ? : ""} 217 |
218 | ); 219 | } 220 | 221 | export default memo(ControlTray); 222 | -------------------------------------------------------------------------------- /src/components/control-tray/control-tray.scss: -------------------------------------------------------------------------------- 1 | .action-button { 2 | display: flex; 3 | align-items: center; 4 | justify-content: center; 5 | background: var(--Neutral-20); 6 | color: var(--Neutral-60); 7 | font-size: 1.25rem; 8 | line-height: 1.75rem; 9 | text-transform: lowercase; 10 | cursor: pointer; 11 | animation: opacity-pulse 3s ease-in infinite; 12 | transition: all 0.2s ease-in-out; 13 | width: 48px; 14 | height: 48px; 15 | border-radius: 18px; 16 | border: 1px solid rgba(0, 0, 0, 0); 17 | user-select: none; 18 | cursor: pointer; 19 | 20 | &:focus { 21 | border: 2px solid var(--Neutral-20); 22 | outline: 2px solid var(--Neutral-80); 23 | } 24 | 25 | &.outlined { 26 | background: var(--Neutral-2); 27 | border: 1px solid var(--Neutral-20); 28 | } 29 | 30 | .no-action { 31 | pointer-events: none; 32 | } 33 | 34 | &:hover { 35 | background: rgba(0, 0, 0, 0); 36 | border: 1px solid var(--Neutral-20); 37 | } 38 | 39 | &.connected { 40 | background: var(--Blue-800); 41 | color: var(--Blue-500); 42 | 43 | &:hover { 44 | border: 1px solid var(--Blue-500); 45 | } 46 | } 47 | } 48 | 49 | @property --volume { 50 | syntax: "length"; 51 | inherit: false; 52 | initial-value: 0px; 53 | } 54 | 55 | .disabled .mic-button, 56 | .mic-button.disabled { 57 | &:before { 58 | background: rgba(0, 0, 0, 0); 59 | } 60 | } 61 | 62 | .mic-button { 63 | position: relative; 64 | background-color: var(--accent-red); 65 | z-index: 1; 66 | color: black; 67 | transition: all 0.2s ease-in; 68 | 69 | &:focus { 70 | border: 2px solid var(--Neutral-20); 71 | outline: 2px solid var(--Red-500); 72 | } 73 | 74 | &:hover { 75 | background-color: var(--Red-400); 76 | } 77 | 78 | &:before { 79 | position: absolute; 80 | z-index: -1; 81 | top: calc(var(--volume) * -1); 82 | left: calc(var(--volume) * -1); 83 | display: block; 84 | content: ""; 85 | opacity: 0.35; 86 | background-color: var(--Red-500); 87 | width: calc(100% + var(--volume) * 2); 88 | height: calc(100% + var(--volume) * 2); 89 | border-radius: 24px; 90 | transition: all 0.02s ease-in-out; 91 | } 92 | } 93 | 94 | .connect-toggle { 95 | &:focus { 96 | border: 2px solid var(--Neutral-20); 97 | outline: 2px solid var(--Neutral-80); 98 | } 99 | 100 | &:not(.connected) { 101 | background-color: var(--Blue-500); 102 | color: var(--Neutral-5); 103 | } 104 | } 105 | 106 | .control-tray { 107 | position: absolute; 108 | bottom: 0; 109 | left: 50%; 110 | transform: translate(-50%, 0); 111 | display: inline-flex; 112 | justify-content: center; 113 | align-items: flex-start; 114 | gap: 8px; 115 | padding-bottom: 18px; 116 | 117 | .disabled .action-button, 118 | .action-button.disabled { 119 | background: rgba(0, 0, 0, 0); 120 | border: 1px solid var(--Neutral-30, #404547); 121 | color: var(--Neutral-30); 122 | } 123 | 124 | .connection-container { 125 | display: flex; 126 | flex-direction: column; 127 | justify-content: center; 128 | align-items: center; 129 | gap: 4px; 130 | 131 | .connection-button-container { 132 | border-radius: 27px; 133 | border: 1px solid var(--Neutral-30); 134 | background: var(--Neutral-5); 135 | padding: 10px; 136 | } 137 | 138 | .text-indicator { 139 | font-size: 11px; 140 | color: var(--Blue-500); 141 | user-select: none; 142 | } 143 | 144 | &:not(.connected) { 145 | .text-indicator { 146 | opacity: 0; 147 | } 148 | } 149 | } 150 | } 151 | 152 | .actions-nav { 153 | background: var(--Neutral-5); 154 | border: 1px solid var(--Neutral-30); 155 | border-radius: 27px; 156 | display: inline-flex; 157 | gap: 12px; 158 | align-items: center; 159 | overflow: clip; 160 | padding: 10px; 161 | 162 | transition: all 0.6s ease-in; 163 | 164 | &>* { 165 | display: flex; 166 | align-items: center; 167 | flex-direction: column; 168 | gap: 1rem; 169 | } 170 | } 171 | 172 | @keyframes opacity-pulse { 173 | 0% { 174 | opacity: 0.9; 175 | } 176 | 177 | 50% { 178 | opacity: 1; 179 | } 180 | 181 | 100% { 182 | opacity: 0.9; 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/components/logger/Logger.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import "./logger.scss"; 18 | 19 | import cn from "classnames"; 20 | import { memo, ReactNode } from "react"; 21 | import { useLoggerStore } from "../../lib/store-logger"; 22 | import SyntaxHighlighter from "react-syntax-highlighter"; 23 | import { vs2015 as dark } from "react-syntax-highlighter/dist/esm/styles/hljs"; 24 | import { 25 | ClientContentLog as ClientContentLogType, 26 | StreamingLog, 27 | } from "../../types"; 28 | import { 29 | Content, 30 | LiveClientToolResponse, 31 | LiveServerContent, 32 | LiveServerToolCall, 33 | LiveServerToolCallCancellation, 34 | Part, 35 | } from "@google/genai"; 36 | 37 | const formatTime = (d: Date) => d.toLocaleTimeString().slice(0, -3); 38 | 39 | const LogEntry = memo( 40 | ({ 41 | log, 42 | MessageComponent, 43 | }: { 44 | log: StreamingLog; 45 | MessageComponent: ({ 46 | message, 47 | }: { 48 | message: StreamingLog["message"]; 49 | }) => ReactNode; 50 | }): JSX.Element => ( 51 |
  • 61 | {formatTime(log.date)} 62 | {log.type} 63 | 64 | 65 | 66 | {log.count && {log.count}} 67 |
    • 68 | ) 69 | ); 70 | 71 | const PlainTextMessage = ({ 72 | message, 73 | }: { 74 | message: StreamingLog["message"]; 75 | }) => {message as string}; 76 | 77 | type Message = { message: StreamingLog["message"] }; 78 | 79 | const AnyMessage = ({ message }: Message) => ( 80 | 
    {JSON.stringify(message, null, "  ")}
    81 | ); 82 | 83 | function tryParseCodeExecutionResult(output: string) { 84 | try { 85 | const json = JSON.parse(output); 86 | return JSON.stringify(json, null, " "); 87 | } catch (e) { 88 | return output; 89 | } 90 | } 91 | 92 | const RenderPart = memo(({ part }: { part: Part }) => { 93 | if (part.text && part.text.length) { 94 | return

    {part.text}

    ; 95 | } 96 | if (part.executableCode) { 97 | return ( 98 |
    99 |
    executableCode: {part.executableCode.language}
    100 | 104 | {part.executableCode!.code!} 105 | 106 |
    107 | ); 108 | } 109 | if (part.codeExecutionResult) { 110 | return ( 111 |
    112 |
    codeExecutionResult: {part.codeExecutionResult!.outcome}
    113 | 114 | {tryParseCodeExecutionResult(part.codeExecutionResult!.output!)} 115 | 116 |
    117 | ); 118 | } 119 | if (part.inlineData) { 120 | return ( 121 |
    122 |
    Inline Data: {part.inlineData?.mimeType}
    123 |
    124 | ); 125 | } 126 | return
     
    ; 127 | }); 128 | 129 | const ClientContentLog = memo(({ message }: Message) => { 130 | const { turns, turnComplete } = message as ClientContentLogType; 131 | const textParts = turns.filter((part) => !(part.text && part.text === "\n")); 132 | return ( 133 |
    134 |

    User

    135 |
    136 | {textParts.map((part, j) => ( 137 | 138 | ))} 139 |
    140 | {!turnComplete ? turnComplete: false : ""} 141 |
    142 | ); 143 | }); 144 | 145 | const ToolCallLog = memo(({ message }: Message) => { 146 | const { toolCall } = message as { toolCall: LiveServerToolCall }; 147 | return ( 148 |
    149 | {toolCall.functionCalls?.map((fc, i) => ( 150 |
    151 |
    Function call: {fc.name}
    152 | 153 | {JSON.stringify(fc, null, " ")} 154 | 155 |
    156 | ))} 157 |
    158 | ); 159 | }); 160 | 161 | const ToolCallCancellationLog = ({ message }: Message): JSX.Element => ( 162 |
    163 | 164 | {" "} 165 | ids:{" "} 166 | {( 167 | message as { toolCallCancellation: LiveServerToolCallCancellation } 168 | ).toolCallCancellation.ids?.map((id) => ( 169 | 170 | "{id}" 171 | 172 | ))} 173 | 174 |
    175 | ); 176 | 177 | const ToolResponseLog = memo( 178 | ({ message }: Message): JSX.Element => ( 179 |
    180 | {(message as LiveClientToolResponse).functionResponses?.map((fc) => ( 181 |
    182 |
    Function Response: {fc.id}
    183 | 184 | {JSON.stringify(fc.response, null, " ")} 185 | 186 |
    187 | ))} 188 |
    189 | ) 190 | ); 191 | 192 | const ModelTurnLog = ({ message }: Message): JSX.Element => { 193 | const serverContent = (message as { serverContent: LiveServerContent }) 194 | .serverContent; 195 | const { modelTurn } = serverContent as { modelTurn: Content }; 196 | const { parts } = modelTurn; 197 | 198 | return ( 199 |
    200 |

    Model

    201 | {parts 202 | ?.filter((part) => !(part.text && part.text === "\n")) 203 | .map((part, j) => ( 204 | 205 | ))} 206 |
    207 | ); 208 | }; 209 | 210 | const CustomPlainTextLog = (msg: string) => () => 211 | ; 212 | 213 | export type LoggerFilterType = "conversations" | "tools" | "none"; 214 | 215 | export type LoggerProps = { 216 | filter: LoggerFilterType; 217 | }; 218 | 219 | const filters: Record boolean> = { 220 | tools: (log: StreamingLog) => 221 | typeof log.message === "object" && 222 | ("toolCall" in log.message || 223 | "functionResponses" in log.message || 224 | "toolCallCancellation" in log.message), 225 | conversations: (log: StreamingLog) => 226 | typeof log.message === "object" && 227 | (("turns" in log.message && "turnComplete" in log.message) || 228 | "serverContent" in log.message), 229 | none: () => true, 230 | }; 231 | 232 | const component = (log: StreamingLog) => { 233 | if (typeof log.message === "string") { 234 | return PlainTextMessage; 235 | } 236 | if ("turns" in log.message && "turnComplete" in log.message) { 237 | return ClientContentLog; 238 | } 239 | if ("toolCall" in log.message) { 240 | return ToolCallLog; 241 | } 242 | if ("toolCallCancellation" in log.message) { 243 | return ToolCallCancellationLog; 244 | } 245 | if ("functionResponses" in log.message) { 246 | return ToolResponseLog; 247 | } 248 | if ("serverContent" in log.message) { 249 | const { serverContent } = log.message; 250 | if (serverContent?.interrupted) { 251 | return CustomPlainTextLog("interrupted"); 252 | } 253 | if (serverContent?.turnComplete) { 254 | return CustomPlainTextLog("turnComplete"); 255 | } 256 | if (serverContent && "modelTurn" in serverContent) { 257 | return ModelTurnLog; 258 | } 259 | } 260 | return AnyMessage; 261 | }; 262 | 263 | export default function Logger({ filter = "none" }: LoggerProps) { 264 | const { logs } = useLoggerStore(); 265 | 266 | const filterFn = filters[filter]; 267 | 268 | return ( 269 |
    270 |
      271 | {logs.filter(filterFn).map((log, key) => { 272 | return ( 273 | 274 | ); 275 | })} 276 |
    277 |
    278 | ); 279 | } 280 | -------------------------------------------------------------------------------- /src/components/logger/logger.scss: -------------------------------------------------------------------------------- 1 | .logger { 2 | color: var(--gray-300); 3 | width: 100%; 4 | max-width: 100%; 5 | display: block; 6 | 7 | .logger-list { 8 | padding: 0 0px 0 25px; 9 | overflow-x: hidden; 10 | width: calc(100% - 45px); 11 | } 12 | 13 | .user h4 { 14 | color: var(--Green-500); 15 | } 16 | 17 | .model h4 { 18 | color: var(--Blue-500); 19 | } 20 | 21 | .rich-log { 22 | display: flex; 23 | justify-content: center; 24 | gap: 4px; 25 | 26 | pre { 27 | overflow-x: auto; 28 | } 29 | 30 | display: block; 31 | 32 | h4 { 33 | font-size: 14px; 34 | text-transform: uppercase; 35 | padding: 8px 0; 36 | margin: 0; 37 | } 38 | 39 | h5 { 40 | margin: 0; 41 | padding-bottom: 8px; 42 | border-bottom: 1px solid var(--Neutral-20); 43 | } 44 | 45 | .part { 46 | background: var(--Neutral-5); 47 | padding: 14px; 48 | margin-bottom: 4px; 49 | color: var(--Neutral-90); 50 | border-radius: 8px; 51 | } 52 | } 53 | 54 | .plain-log { 55 | &>* { 56 | padding-right: 4px; 57 | } 58 | } 59 | 60 | .inline-code:not(:last-child) { 61 | font-style: italic; 62 | 63 | &::after { 64 | content: ", "; 65 | } 66 | } 67 | } 68 | 69 | .logger li { 70 | display: block; 71 | padding: 8px 0; 72 | color: var(--Neutral-50, #707577); 73 | font-family: "Space Mono"; 74 | font-size: 14px; 75 | font-style: normal; 76 | font-weight: 400; 77 | line-height: normal; 78 | } 79 | 80 | .logger li .timestamp { 81 | width: 70px; 82 | flex-grow: 0; 83 | flex-shrink: 0; 84 | color: var(--Neutral-50); 85 | } 86 | 87 | .logger li .source { 88 | flex-shrink: 0; 89 | font-weight: bold; 90 | } 91 | 92 | .logger li.source-server, 93 | .logger li.receive { 94 | color: var(--Blue-500); 95 | } 96 | 97 | .logger li.source-client, 98 | .logger li.send:not(.source-server) { 99 | color: var(--Green-500); 100 | } 101 | 102 | .logger li .count { 103 | background-color: var(--Neutral-5); 104 | font-size: x-small; 105 | padding: 0em 0.6em; 106 | padding: 0.3em 0.5em; 107 | line-height: 1em; 108 | vertical-align: middle; 109 | border-radius: 8px; 110 | color: var(--Blue-500); 111 | } 112 | 113 | .logger li .message { 114 | flex-grow: 1; 115 | color: var(--Neutral-50); 116 | } 117 | -------------------------------------------------------------------------------- /src/components/logger/mock-logs.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * this module is just mock data, intended to make it easier to develop and style the logger 19 | */ 20 | import type { StreamingLog } from "../../types"; 21 | 22 | const soundLogs = (n: number): StreamingLog[] => 23 | new Array(n).fill(0).map( 24 | (): StreamingLog => ({ 25 | date: new Date(), 26 | type: "server.audio", 27 | message: "buffer (11250)", 28 | }) 29 | ); 30 | // 31 | const realtimeLogs = (n: number): StreamingLog[] => 32 | new Array(n).fill(0).map( 33 | (): StreamingLog => ({ 34 | date: new Date(), 35 | type: "client.realtimeInput", 36 | message: "audio", 37 | }) 38 | ); 39 | 40 | export const mockLogs: StreamingLog[] = [ 41 | { 42 | date: new Date(), 43 | type: "client.open", 44 | message: "connected", 45 | }, 46 | { date: new Date(), type: "receive", message: "setupComplete" }, 47 | ...realtimeLogs(10), 48 | ...soundLogs(10), 49 | { 50 | date: new Date(), 51 | type: "receive.content", 52 | message: { 53 | serverContent: { 54 | interrupted: true, 55 | }, 56 | }, 57 | }, 58 | { 59 | date: new Date(), 60 | type: "receive.content", 61 | message: { 62 | serverContent: { 63 | turnComplete: true, 64 | }, 65 | }, 66 | }, 67 | //this one is just a string 68 | // { 69 | // date: new Date(), 70 | // type: "server.send", 71 | // message: { 72 | // serverContent: { 73 | // turnComplete: true, 74 | // }, 75 | // }, 76 | // }, 77 | ...realtimeLogs(10), 78 | ...soundLogs(20), 79 | { 80 | date: new Date(), 81 | type: "receive.content", 82 | message: { 83 | serverContent: { 84 | modelTurn: { 85 | parts: [{ text: "Hey its text" }, { text: "more" }], 86 | }, 87 | }, 88 | }, 89 | }, 90 | { 91 | date: new Date(), 92 | type: "client.send", 93 | message: { 94 | turns: [ 95 | { 96 | text: "How much wood could a woodchuck chuck if a woodchuck could chuck wood", 97 | }, 98 | { 99 | text: "more text", 100 | }, 101 | ], 102 | turnComplete: false, 103 | }, 104 | }, 105 | { 106 | date: new Date(), 107 | type: "server.toolCall", 108 | message: { 109 | toolCall: { 110 | functionCalls: [ 111 | { 112 | id: "akadjlasdfla-askls", 113 | name: "take_photo", 114 | args: {}, 115 | }, 116 | { 117 | id: "akldjsjskldsj-102", 118 | name: "move_camera", 119 | args: { x: 20, y: 4 }, 120 | }, 121 | ], 122 | }, 123 | }, 124 | }, 125 | { 126 | date: new Date(), 127 | type: "server.toolCallCancellation", 128 | message: { 129 | toolCallCancellation: { 130 | ids: ["akladfjadslfk", "adkafsdljfsdk"], 131 | }, 132 | }, 133 | }, 134 | { 135 | date: new Date(), 136 | type: "client.toolResponse", 137 | message: { 138 | functionResponses: [ 139 | { 140 | response: { success: true }, 141 | id: "akslaj-10102", 142 | }, 143 | ], 144 | }, 145 | }, 146 | { 147 | date: new Date(), 148 | type: "receive.serverContent", 149 | message: "interrupted", 150 | }, 151 | { 152 | date: new Date(), 153 | type: "receive.serverContent", 154 | message: "turnComplete", 155 | }, 156 | ]; 157 | -------------------------------------------------------------------------------- /src/components/settings-dialog/ResponseModalitySelector.tsx: -------------------------------------------------------------------------------- 1 | import { useCallback, useState } from "react"; 2 | import Select from "react-select"; 3 | import { useLiveAPIContext } from "../../contexts/LiveAPIContext"; 4 | import { Modality } from "@google/genai"; 5 | 6 | const responseOptions = [ 7 | { value: "audio", label: "audio" }, 8 | { value: "text", label: "text" }, 9 | ]; 10 | 11 | export default function ResponseModalitySelector() { 12 | const { config, setConfig } = useLiveAPIContext(); 13 | 14 | const [selectedOption, setSelectedOption] = useState<{ 15 | value: string; 16 | label: string; 17 | } | null>(responseOptions[0]); 18 | 19 | const updateConfig = useCallback( 20 | (modality: "audio" | "text") => { 21 | setConfig({ 22 | ...config, 23 | responseModalities: [ 24 | modality === "audio" ? Modality.AUDIO : Modality.TEXT, 25 | ], 26 | }); 27 | }, 28 | [config, setConfig] 29 | ); 30 | 31 | return ( 32 |
    33 | 34 | 144 | 149 | Type something... 150 | 151 | 152 | 158 |
    159 |
    160 |
    161 | ); 162 | } 163 | -------------------------------------------------------------------------------- /src/components/side-panel/react-select.scss: -------------------------------------------------------------------------------- 1 | .react-select { 2 | background: var(--Neutral-20); 3 | color: var(--Neutral-90); 4 | width: 193px; 5 | height: 30px; 6 | 7 | .react-select__single-value { 8 | color: var(--Neutral-90); 9 | } 10 | 11 | .react-select__menu { 12 | background: var(--Neutral-20); 13 | color: var(--Neutral-90); 14 | } 15 | 16 | .react-select__option { 17 | } 18 | 19 | .react-select__value-container { 20 | } 21 | 22 | .react-select__indicators { 23 | } 24 | 25 | .react-select__option:hover, 26 | .react-select__option:focus, 27 | .react-select_option:focus-within { 28 | background: var(--Neutral-30); 29 | } 30 | 31 | .react-select__option--is-focused: { 32 | background: var(--Neutral-30); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/components/side-panel/side-panel.scss: -------------------------------------------------------------------------------- 1 | .side-panel { 2 | background: var(--Neutral-00); 3 | width: 40px; 4 | /* when closed */ 5 | display: flex; 6 | flex-direction: column; 7 | height: 100vh; 8 | transition: all 0.2s ease-in; 9 | font-family: Arial, sans-serif; 10 | border-right: 1px solid var(--gray-600); 11 | color: var(--Neutral-90, #e1e2e3); 12 | font-family: var(--font-family); 13 | font-size: 13px; 14 | font-style: normal; 15 | font-weight: 400; 16 | line-height: 160%; 17 | /* 20.8px */ 18 | 19 | .hidden { 20 | display: none !important; 21 | } 22 | 23 | &.open { 24 | .top { 25 | h2 { 26 | left: 0%; 27 | display: block; 28 | opacity: 1; 29 | } 30 | } 31 | } 32 | 33 | .top { 34 | display: flex; 35 | width: calc(100% - 45px); 36 | justify-content: space-between; 37 | align-items: center; 38 | padding: 12px 20px 12px 25px; 39 | border-bottom: 1px solid var(--Neutral-20); 40 | 41 | h2 { 42 | position: relative; 43 | color: var(--Neutral-90, #e1e2e3); 44 | font-family: "Google Sans"; 45 | font-size: 21px; 46 | font-style: normal; 47 | font-weight: 500; 48 | line-height: 16px; 49 | /* 100% */ 50 | 51 | opacity: 0; 52 | display: none; 53 | left: -100%; 54 | transition: 55 | opacity 0.2s ease-in, 56 | left 0.2s ease-in, 57 | display 0.2s ease-in; 58 | transition-behavior: allow-discrete; 59 | 60 | @starting-style { 61 | left: 0%; 62 | opacity: 1; 63 | } 64 | } 65 | } 66 | 67 | .opener { 68 | height: 30px; 69 | transition: transform 0.2s ease-in; 70 | } 71 | 72 | &:not(.open) { 73 | .side-panel-container { 74 | opacity: 0; 75 | display: none; 76 | transition: all 0.2s ease-in allow-discrete; 77 | transition-delay: 0.1s; 78 | } 79 | 80 | .indicators .streaming-indicator { 81 | width: 30px; 82 | opacity: 0; 83 | } 84 | 85 | .opener { 86 | transform: translate(-50%, 0); 87 | } 88 | 89 | .input-container { 90 | opacity: 0; 91 | display: none; 92 | transition: all 0.2s ease-in allow-discrete; 93 | } 94 | } 95 | 96 | .indicators { 97 | display: flex; 98 | padding: 24px 25px; 99 | justify-content: flex-end; 100 | gap: 21px; 101 | 102 | .streaming-indicator { 103 | user-select: none; 104 | border-radius: 4px; 105 | border: 1px solid var(--Neutral-20, #2a2f31); 106 | background: var(--Neutral-10, #1c1f21); 107 | display: flex; 108 | width: 136px; 109 | height: 30px; 110 | padding-left: 4px; 111 | justify-content: center; 112 | align-items: center; 113 | gap: 6px; 114 | flex-shrink: 0; 115 | text-align: center; 116 | font-family: "Space Mono"; 117 | font-size: 14px; 118 | font-style: normal; 119 | font-weight: 400; 120 | line-height: normal; 121 | transition: width 0.2s ease-in; 122 | 123 | &.connected { 124 | color: var(--Blue-500, #0d9c53); 125 | } 126 | } 127 | } 128 | 129 | .side-panel-container { 130 | align-self: flex-end; 131 | width: 400px; 132 | flex-grow: 1; 133 | overflow-x: hidden; 134 | overflow-y: auto; 135 | /*scrollbar-gutter: stable both-edges;*/ 136 | } 137 | 138 | .input-container { 139 | height: 50px; 140 | flex-grow: 0; 141 | flex-shrink: 0; 142 | border-top: 1px solid var(--Neutral-20); 143 | padding: 14px 25px; 144 | overflow: hidden; 145 | 146 | .input-content { 147 | position: relative; 148 | background: var(--Neutral-10); 149 | border: 1px solid var(--Neutral-15); 150 | height: 22px; 151 | border-radius: 10px; 152 | padding: 11px 18px; 153 | 154 | .send-button { 155 | position: absolute; 156 | top: 50%; 157 | right: 0; 158 | transform: translate(0, -50%); 159 | background: none; 160 | border: 0; 161 | color: var(--Neutral-20); 162 | cursor: pointer; 163 | transition: color 0.1s ease-in; 164 | z-index: 2; 165 | 166 | &:hover { 167 | color: var(--Neutral-60); 168 | } 169 | } 170 | 171 | .input-area { 172 | background: none; 173 | color: var(--Neutral-90); 174 | field-sizing: content; 175 | position: absolute; 176 | top: 0; 177 | left: 0; 178 | z-index: 2; 179 | display: inline-block; 180 | width: calc(100% - 72px); 181 | max-height: 20px; 182 | outline: none; 183 | --webkit-box-flex: 1; 184 | flex: 1; 185 | word-break: break-word; 186 | overflow: auto; 187 | padding: 14px 18px; 188 | border: 0; 189 | resize: none; 190 | } 191 | 192 | .input-content-placeholder { 193 | position: absolute; 194 | left: 0; 195 | top: 0; 196 | display: flex; 197 | align-items: center; 198 | z-index: 1; 199 | height: 100%; 200 | width: 100%; 201 | pointer-events: none; 202 | user-select: none; 203 | padding: 0px 18px; 204 | white-space: pre-wrap; 205 | } 206 | } 207 | } 208 | } 209 | 210 | .side-panel.open { 211 | width: 400px; 212 | height: 100vh; 213 | } 214 | 215 | .side-panel-responses, 216 | .side-panel-requests { 217 | flex-grow: 1; 218 | flex-shrink: 1; 219 | overflow-x: hidden; 220 | overflow-y: auto; 221 | width: 100%; 222 | display: block; 223 | margin-left: 8px; 224 | } 225 | 226 | .top { 227 | width: 100%; 228 | flex-grow: 0; 229 | flex-shrink: 0; 230 | height: 30px; 231 | display: flex; 232 | align-self: flex-end; 233 | align-items: center; 234 | transition: all 0.2s ease-in; 235 | } 236 | 237 | .top button { 238 | background: transparent; 239 | border: 0; 240 | cursor: pointer; 241 | font-size: 1.25rem; 242 | line-height: 1.75rem; 243 | padding: 4px; 244 | } 245 | -------------------------------------------------------------------------------- /src/contexts/LiveAPIContext.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createContext, FC, ReactNode, useContext } from "react"; 18 | import { useLiveAPI, UseLiveAPIResults } from "../hooks/use-live-api"; 19 | import { LiveClientOptions } from "../types"; 20 | 21 | const LiveAPIContext = createContext(undefined); 22 | 23 | export type LiveAPIProviderProps = { 24 | children: ReactNode; 25 | options: LiveClientOptions; 26 | }; 27 | 28 | export const LiveAPIProvider: FC = ({ 29 | options, 30 | children, 31 | }) => { 32 | const liveAPI = useLiveAPI(options); 33 | 34 | return ( 35 | 36 | {children} 37 | 38 | ); 39 | }; 40 | 41 | export const useLiveAPIContext = () => { 42 | const context = useContext(LiveAPIContext); 43 | if (!context) { 44 | throw new Error("useLiveAPIContext must be used wihin a LiveAPIProvider"); 45 | } 46 | return context; 47 | }; 48 | -------------------------------------------------------------------------------- /src/hooks/use-live-api.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { useCallback, useEffect, useMemo, useRef, useState } from "react"; 18 | import { GenAILiveClient } from "../lib/genai-live-client"; 19 | import { LiveClientOptions } from "../types"; 20 | import { AudioStreamer } from "../lib/audio-streamer"; 21 | import { audioContext } from "../lib/utils"; 22 | import VolMeterWorket from "../lib/worklets/vol-meter"; 23 | import { LiveConnectConfig } from "@google/genai"; 24 | 25 | export type UseLiveAPIResults = { 26 | client: GenAILiveClient; 27 | setConfig: (config: LiveConnectConfig) => void; 28 | config: LiveConnectConfig; 29 | model: string; 30 | setModel: (model: string) => void; 31 | connected: boolean; 32 | connect: () => Promise; 33 | disconnect: () => Promise; 34 | volume: number; 35 | }; 36 | 37 | export function useLiveAPI(options: LiveClientOptions): UseLiveAPIResults { 38 | const client = useMemo(() => new GenAILiveClient(options), [options]); 39 | const audioStreamerRef = useRef(null); 40 | 41 | const [model, setModel] = useState("models/gemini-2.0-flash-exp"); 42 | const [config, setConfig] = useState({}); 43 | const [connected, setConnected] = useState(false); 44 | const [volume, setVolume] = useState(0); 45 | 46 | // register audio for streaming server -> speakers 47 | useEffect(() => { 48 | if (!audioStreamerRef.current) { 49 | audioContext({ id: "audio-out" }).then((audioCtx: AudioContext) => { 50 | audioStreamerRef.current = new AudioStreamer(audioCtx); 51 | audioStreamerRef.current 52 | .addWorklet("vumeter-out", VolMeterWorket, (ev: any) => { 53 | setVolume(ev.data.volume); 54 | }) 55 | .then(() => { 56 | // Successfully added worklet 57 | }); 58 | }); 59 | } 60 | }, [audioStreamerRef]); 61 | 62 | useEffect(() => { 63 | const onOpen = () => { 64 | setConnected(true); 65 | }; 66 | 67 | const onClose = () => { 68 | setConnected(false); 69 | }; 70 | 71 | const onError = (error: ErrorEvent) => { 72 | console.error("error", error); 73 | }; 74 | 75 | const stopAudioStreamer = () => audioStreamerRef.current?.stop(); 76 | 77 | const onAudio = (data: ArrayBuffer) => 78 | audioStreamerRef.current?.addPCM16(new Uint8Array(data)); 79 | 80 | client 81 | .on("error", onError) 82 | .on("open", onOpen) 83 | .on("close", onClose) 84 | .on("interrupted", stopAudioStreamer) 85 | .on("audio", onAudio); 86 | 87 | return () => { 88 | client 89 | .off("error", onError) 90 | .off("open", onOpen) 91 | .off("close", onClose) 92 | .off("interrupted", stopAudioStreamer) 93 | .off("audio", onAudio) 94 | .disconnect(); 95 | }; 96 | }, [client]); 97 | 98 | const connect = useCallback(async () => { 99 | if (!config) { 100 | throw new Error("config has not been set"); 101 | } 102 | client.disconnect(); 103 | await client.connect(model, config); 104 | }, [client, config, model]); 105 | 106 | const disconnect = useCallback(async () => { 107 | client.disconnect(); 108 | setConnected(false); 109 | }, [setConnected, client]); 110 | 111 | return { 112 | client, 113 | config, 114 | setConfig, 115 | model, 116 | setModel, 117 | connected, 118 | connect, 119 | disconnect, 120 | volume, 121 | }; 122 | } 123 | -------------------------------------------------------------------------------- /src/hooks/use-media-stream-mux.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export type UseMediaStreamResult = { 18 | type: "webcam" | "screen"; 19 | start: () => Promise; 20 | stop: () => void; 21 | isStreaming: boolean; 22 | stream: MediaStream | null; 23 | }; 24 | -------------------------------------------------------------------------------- /src/hooks/use-screen-capture.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { useState, useEffect } from "react"; 18 | import { UseMediaStreamResult } from "./use-media-stream-mux"; 19 | 20 | export function useScreenCapture(): UseMediaStreamResult { 21 | const [stream, setStream] = useState(null); 22 | const [isStreaming, setIsStreaming] = useState(false); 23 | 24 | useEffect(() => { 25 | const handleStreamEnded = () => { 26 | setIsStreaming(false); 27 | setStream(null); 28 | }; 29 | if (stream) { 30 | stream 31 | .getTracks() 32 | .forEach((track) => track.addEventListener("ended", handleStreamEnded)); 33 | return () => { 34 | stream 35 | .getTracks() 36 | .forEach((track) => 37 | track.removeEventListener("ended", handleStreamEnded), 38 | ); 39 | }; 40 | } 41 | }, [stream]); 42 | 43 | const start = async () => { 44 | // const controller = new CaptureController(); 45 | // controller.setFocusBehavior("no-focus-change"); 46 | const mediaStream = await navigator.mediaDevices.getDisplayMedia({ 47 | video: true, 48 | // controller 49 | }); 50 | setStream(mediaStream); 51 | setIsStreaming(true); 52 | return mediaStream; 53 | }; 54 | 55 | const stop = () => { 56 | if (stream) { 57 | stream.getTracks().forEach((track) => track.stop()); 58 | setStream(null); 59 | setIsStreaming(false); 60 | } 61 | }; 62 | 63 | const result: UseMediaStreamResult = { 64 | type: "screen", 65 | start, 66 | stop, 67 | isStreaming, 68 | stream, 69 | }; 70 | 71 | return result; 72 | } 73 | -------------------------------------------------------------------------------- /src/hooks/use-webcam.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { useState, useEffect } from "react"; 18 | import { UseMediaStreamResult } from "./use-media-stream-mux"; 19 | 20 | export function useWebcam(): UseMediaStreamResult { 21 | const [stream, setStream] = useState(null); 22 | const [isStreaming, setIsStreaming] = useState(false); 23 | 24 | useEffect(() => { 25 | const handleStreamEnded = () => { 26 | setIsStreaming(false); 27 | setStream(null); 28 | }; 29 | if (stream) { 30 | stream 31 | .getTracks() 32 | .forEach((track) => track.addEventListener("ended", handleStreamEnded)); 33 | return () => { 34 | stream 35 | .getTracks() 36 | .forEach((track) => 37 | track.removeEventListener("ended", handleStreamEnded), 38 | ); 39 | }; 40 | } 41 | }, [stream]); 42 | 43 | const start = async () => { 44 | const mediaStream = await navigator.mediaDevices.getUserMedia({ 45 | video: true, 46 | }); 47 | setStream(mediaStream); 48 | setIsStreaming(true); 49 | return mediaStream; 50 | }; 51 | 52 | const stop = () => { 53 | if (stream) { 54 | stream.getTracks().forEach((track) => track.stop()); 55 | setStream(null); 56 | setIsStreaming(false); 57 | } 58 | }; 59 | 60 | const result: UseMediaStreamResult = { 61 | type: "webcam", 62 | start, 63 | stop, 64 | isStreaming, 65 | stream, 66 | }; 67 | 68 | return result; 69 | } 70 | -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /src/index.tsx: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import React from 'react'; 18 | import ReactDOM from 'react-dom/client'; 19 | import './index.css'; 20 | import App from './App'; 21 | import reportWebVitals from './reportWebVitals'; 22 | 23 | const root = ReactDOM.createRoot( 24 | document.getElementById('root') as HTMLElement 25 | ); 26 | root.render( 27 | 28 | 29 | 30 | ); 31 | 32 | // If you want to start measuring performance in your app, pass a function 33 | // to log results (for example: reportWebVitals(console.log)) 34 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 35 | reportWebVitals(); 36 | -------------------------------------------------------------------------------- /src/lib/audio-recorder.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { audioContext } from "./utils"; 18 | import AudioRecordingWorklet from "./worklets/audio-processing"; 19 | import VolMeterWorket from "./worklets/vol-meter"; 20 | 21 | import { createWorketFromSrc } from "./audioworklet-registry"; 22 | import EventEmitter from "eventemitter3"; 23 | 24 | function arrayBufferToBase64(buffer: ArrayBuffer) { 25 | var binary = ""; 26 | var bytes = new Uint8Array(buffer); 27 | var len = bytes.byteLength; 28 | for (var i = 0; i < len; i++) { 29 | binary += String.fromCharCode(bytes[i]); 30 | } 31 | return window.btoa(binary); 32 | } 33 | 34 | export class AudioRecorder extends EventEmitter { 35 | stream: MediaStream | undefined; 36 | audioContext: AudioContext | undefined; 37 | source: MediaStreamAudioSourceNode | undefined; 38 | recording: boolean = false; 39 | recordingWorklet: AudioWorkletNode | undefined; 40 | vuWorklet: AudioWorkletNode | undefined; 41 | 42 | private starting: Promise | null = null; 43 | 44 | constructor(public sampleRate = 16000) { 45 | super(); 46 | } 47 | 48 | async start() { 49 | if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { 50 | throw new Error("Could not request user media"); 51 | } 52 | 53 | this.starting = new Promise(async (resolve, reject) => { 54 | this.stream = await navigator.mediaDevices.getUserMedia({ audio: true }); 55 | this.audioContext = await audioContext({ sampleRate: this.sampleRate }); 56 | this.source = this.audioContext.createMediaStreamSource(this.stream); 57 | 58 | const workletName = "audio-recorder-worklet"; 59 | const src = createWorketFromSrc(workletName, AudioRecordingWorklet); 60 | 61 | await this.audioContext.audioWorklet.addModule(src); 62 | this.recordingWorklet = new AudioWorkletNode( 63 | this.audioContext, 64 | workletName, 65 | ); 66 | 67 | this.recordingWorklet.port.onmessage = async (ev: MessageEvent) => { 68 | // worklet processes recording floats and messages converted buffer 69 | const arrayBuffer = ev.data.data.int16arrayBuffer; 70 | 71 | if (arrayBuffer) { 72 | const arrayBufferString = arrayBufferToBase64(arrayBuffer); 73 | this.emit("data", arrayBufferString); 74 | } 75 | }; 76 | this.source.connect(this.recordingWorklet); 77 | 78 | // vu meter worklet 79 | const vuWorkletName = "vu-meter"; 80 | await this.audioContext.audioWorklet.addModule( 81 | createWorketFromSrc(vuWorkletName, VolMeterWorket), 82 | ); 83 | this.vuWorklet = new AudioWorkletNode(this.audioContext, vuWorkletName); 84 | this.vuWorklet.port.onmessage = (ev: MessageEvent) => { 85 | this.emit("volume", ev.data.volume); 86 | }; 87 | 88 | this.source.connect(this.vuWorklet); 89 | this.recording = true; 90 | resolve(); 91 | this.starting = null; 92 | }); 93 | } 94 | 95 | stop() { 96 | // its plausible that stop would be called before start completes 97 | // such as if the websocket immediately hangs up 98 | const handleStop = () => { 99 | this.source?.disconnect(); 100 | this.stream?.getTracks().forEach((track) => track.stop()); 101 | this.stream = undefined; 102 | this.recordingWorklet = undefined; 103 | this.vuWorklet = undefined; 104 | }; 105 | if (this.starting) { 106 | this.starting.then(handleStop); 107 | return; 108 | } 109 | handleStop(); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/lib/audio-streamer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | createWorketFromSrc, 19 | registeredWorklets, 20 | } from "./audioworklet-registry"; 21 | 22 | export class AudioStreamer { 23 | private sampleRate: number = 24000; 24 | private bufferSize: number = 7680; 25 | // A queue of audio buffers to be played. Each buffer is a Float32Array. 26 | private audioQueue: Float32Array[] = []; 27 | private isPlaying: boolean = false; 28 | // Indicates if the stream has finished playing, e.g., interrupted. 29 | private isStreamComplete: boolean = false; 30 | private checkInterval: number | null = null; 31 | private scheduledTime: number = 0; 32 | private initialBufferTime: number = 0.1; //0.1 // 100ms initial buffer 33 | // Web Audio API nodes. source => gain => destination 34 | public gainNode: GainNode; 35 | public source: AudioBufferSourceNode; 36 | private endOfQueueAudioSource: AudioBufferSourceNode | null = null; 37 | 38 | public onComplete = () => {}; 39 | 40 | constructor(public context: AudioContext) { 41 | this.gainNode = this.context.createGain(); 42 | this.source = this.context.createBufferSource(); 43 | this.gainNode.connect(this.context.destination); 44 | this.addPCM16 = this.addPCM16.bind(this); 45 | } 46 | 47 | async addWorklet void>( 48 | workletName: string, 49 | workletSrc: string, 50 | handler: T 51 | ): Promise { 52 | let workletsRecord = registeredWorklets.get(this.context); 53 | if (workletsRecord && workletsRecord[workletName]) { 54 | // the worklet already exists on this context 55 | // add the new handler to it 56 | workletsRecord[workletName].handlers.push(handler); 57 | return Promise.resolve(this); 58 | //throw new Error(`Worklet ${workletName} already exists on context`); 59 | } 60 | 61 | if (!workletsRecord) { 62 | registeredWorklets.set(this.context, {}); 63 | workletsRecord = registeredWorklets.get(this.context)!; 64 | } 65 | 66 | // create new record to fill in as becomes available 67 | workletsRecord[workletName] = { handlers: [handler] }; 68 | 69 | const src = createWorketFromSrc(workletName, workletSrc); 70 | await this.context.audioWorklet.addModule(src); 71 | const worklet = new AudioWorkletNode(this.context, workletName); 72 | 73 | //add the node into the map 74 | workletsRecord[workletName].node = worklet; 75 | 76 | return this; 77 | } 78 | 79 | /** 80 | * Converts a Uint8Array of PCM16 audio data into a Float32Array. 81 | * PCM16 is a common raw audio format, but the Web Audio API generally 82 | * expects audio data as Float32Arrays with samples normalized between -1.0 and 1.0. 83 | * This function handles that conversion. 84 | * @param chunk The Uint8Array containing PCM16 audio data. 85 | * @returns A Float32Array representing the converted audio data. 86 | */ 87 | private _processPCM16Chunk(chunk: Uint8Array): Float32Array { 88 | const float32Array = new Float32Array(chunk.length / 2); 89 | const dataView = new DataView(chunk.buffer); 90 | 91 | for (let i = 0; i < chunk.length / 2; i++) { 92 | try { 93 | const int16 = dataView.getInt16(i * 2, true); 94 | float32Array[i] = int16 / 32768; 95 | } catch (e) { 96 | console.error(e); 97 | } 98 | } 99 | return float32Array; 100 | } 101 | 102 | addPCM16(chunk: Uint8Array) { 103 | // Reset the stream complete flag when a new chunk is added. 104 | this.isStreamComplete = false; 105 | // Process the chunk into a Float32Array 106 | let processingBuffer = this._processPCM16Chunk(chunk); 107 | // Add the processed buffer to the queue if it's larger than the buffer size. 108 | // This is to ensure that the buffer is not too large. 109 | while (processingBuffer.length >= this.bufferSize) { 110 | const buffer = processingBuffer.slice(0, this.bufferSize); 111 | this.audioQueue.push(buffer); 112 | processingBuffer = processingBuffer.slice(this.bufferSize); 113 | } 114 | // Add the remaining buffer to the queue if it's not empty. 115 | if (processingBuffer.length > 0) { 116 | this.audioQueue.push(processingBuffer); 117 | } 118 | // Start playing if not already playing. 119 | if (!this.isPlaying) { 120 | this.isPlaying = true; 121 | // Initialize scheduledTime only when we start playing 122 | this.scheduledTime = this.context.currentTime + this.initialBufferTime; 123 | this.scheduleNextBuffer(); 124 | } 125 | } 126 | 127 | private createAudioBuffer(audioData: Float32Array): AudioBuffer { 128 | const audioBuffer = this.context.createBuffer( 129 | 1, 130 | audioData.length, 131 | this.sampleRate 132 | ); 133 | audioBuffer.getChannelData(0).set(audioData); 134 | return audioBuffer; 135 | } 136 | 137 | private scheduleNextBuffer() { 138 | const SCHEDULE_AHEAD_TIME = 0.2; 139 | 140 | while ( 141 | this.audioQueue.length > 0 && 142 | this.scheduledTime < this.context.currentTime + SCHEDULE_AHEAD_TIME 143 | ) { 144 | const audioData = this.audioQueue.shift()!; 145 | const audioBuffer = this.createAudioBuffer(audioData); 146 | const source = this.context.createBufferSource(); 147 | 148 | if (this.audioQueue.length === 0) { 149 | if (this.endOfQueueAudioSource) { 150 | this.endOfQueueAudioSource.onended = null; 151 | } 152 | this.endOfQueueAudioSource = source; 153 | source.onended = () => { 154 | if ( 155 | !this.audioQueue.length && 156 | this.endOfQueueAudioSource === source 157 | ) { 158 | this.endOfQueueAudioSource = null; 159 | this.onComplete(); 160 | } 161 | }; 162 | } 163 | 164 | source.buffer = audioBuffer; 165 | source.connect(this.gainNode); 166 | 167 | const worklets = registeredWorklets.get(this.context); 168 | 169 | if (worklets) { 170 | Object.entries(worklets).forEach(([workletName, graph]) => { 171 | const { node, handlers } = graph; 172 | if (node) { 173 | source.connect(node); 174 | node.port.onmessage = function (ev: MessageEvent) { 175 | handlers.forEach((handler) => { 176 | handler.call(node.port, ev); 177 | }); 178 | }; 179 | node.connect(this.context.destination); 180 | } 181 | }); 182 | } 183 | // Ensure we never schedule in the past 184 | const startTime = Math.max(this.scheduledTime, this.context.currentTime); 185 | source.start(startTime); 186 | this.scheduledTime = startTime + audioBuffer.duration; 187 | } 188 | 189 | if (this.audioQueue.length === 0) { 190 | if (this.isStreamComplete) { 191 | this.isPlaying = false; 192 | if (this.checkInterval) { 193 | clearInterval(this.checkInterval); 194 | this.checkInterval = null; 195 | } 196 | } else { 197 | if (!this.checkInterval) { 198 | this.checkInterval = window.setInterval(() => { 199 | if (this.audioQueue.length > 0) { 200 | this.scheduleNextBuffer(); 201 | } 202 | }, 100) as unknown as number; 203 | } 204 | } 205 | } else { 206 | const nextCheckTime = 207 | (this.scheduledTime - this.context.currentTime) * 1000; 208 | setTimeout( 209 | () => this.scheduleNextBuffer(), 210 | Math.max(0, nextCheckTime - 50) 211 | ); 212 | } 213 | } 214 | 215 | stop() { 216 | this.isPlaying = false; 217 | this.isStreamComplete = true; 218 | this.audioQueue = []; 219 | this.scheduledTime = this.context.currentTime; 220 | 221 | if (this.checkInterval) { 222 | clearInterval(this.checkInterval); 223 | this.checkInterval = null; 224 | } 225 | 226 | this.gainNode.gain.linearRampToValueAtTime( 227 | 0, 228 | this.context.currentTime + 0.1 229 | ); 230 | 231 | setTimeout(() => { 232 | this.gainNode.disconnect(); 233 | this.gainNode = this.context.createGain(); 234 | this.gainNode.connect(this.context.destination); 235 | }, 200); 236 | } 237 | 238 | async resume() { 239 | if (this.context.state === "suspended") { 240 | await this.context.resume(); 241 | } 242 | this.isStreamComplete = false; 243 | this.scheduledTime = this.context.currentTime + this.initialBufferTime; 244 | this.gainNode.gain.setValueAtTime(1, this.context.currentTime); 245 | } 246 | 247 | complete() { 248 | this.isStreamComplete = true; 249 | this.onComplete(); 250 | } 251 | } 252 | 253 | // // Usage example: 254 | // const audioStreamer = new AudioStreamer(); 255 | // 256 | // // In your streaming code: 257 | // function handleChunk(chunk: Uint8Array) { 258 | // audioStreamer.handleChunk(chunk); 259 | // } 260 | // 261 | // // To start playing (call this in response to a user interaction) 262 | // await audioStreamer.resume(); 263 | // 264 | // // To stop playing 265 | // // audioStreamer.stop(); 266 | -------------------------------------------------------------------------------- /src/lib/audioworklet-registry.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * A registry to map attached worklets by their audio-context 19 | * any module using `audioContext.audioWorklet.addModule(` should register the worklet here 20 | */ 21 | export type WorkletGraph = { 22 | node?: AudioWorkletNode; 23 | handlers: Array<(this: MessagePort, ev: MessageEvent) => any>; 24 | }; 25 | 26 | export const registeredWorklets: Map< 27 | AudioContext, 28 | Record 29 | > = new Map(); 30 | 31 | export const createWorketFromSrc = ( 32 | workletName: string, 33 | workletSrc: string, 34 | ) => { 35 | const script = new Blob( 36 | [`registerProcessor("${workletName}", ${workletSrc})`], 37 | { 38 | type: "application/javascript", 39 | }, 40 | ); 41 | 42 | return URL.createObjectURL(script); 43 | }; 44 | -------------------------------------------------------------------------------- /src/lib/genai-live-client.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | Content, 19 | GoogleGenAI, 20 | LiveCallbacks, 21 | LiveClientToolResponse, 22 | LiveConnectConfig, 23 | LiveServerContent, 24 | LiveServerMessage, 25 | LiveServerToolCall, 26 | LiveServerToolCallCancellation, 27 | Part, 28 | Session, 29 | } from "@google/genai"; 30 | 31 | import { EventEmitter } from "eventemitter3"; 32 | import { difference } from "lodash"; 33 | import { LiveClientOptions, StreamingLog } from "../types"; 34 | import { base64ToArrayBuffer } from "./utils"; 35 | 36 | /** 37 | * Event types that can be emitted by the MultimodalLiveClient. 38 | * Each event corresponds to a specific message from GenAI or client state change. 39 | */ 40 | export interface LiveClientEventTypes { 41 | // Emitted when audio data is received 42 | audio: (data: ArrayBuffer) => void; 43 | // Emitted when the connection closes 44 | close: (event: CloseEvent) => void; 45 | // Emitted when content is received from the server 46 | content: (data: LiveServerContent) => void; 47 | // Emitted when an error occurs 48 | error: (error: ErrorEvent) => void; 49 | // Emitted when the server interrupts the current generation 50 | interrupted: () => void; 51 | // Emitted for logging events 52 | log: (log: StreamingLog) => void; 53 | // Emitted when the connection opens 54 | open: () => void; 55 | // Emitted when the initial setup is complete 56 | setupcomplete: () => void; 57 | // Emitted when a tool call is received 58 | toolcall: (toolCall: LiveServerToolCall) => void; 59 | // Emitted when a tool call is cancelled 60 | toolcallcancellation: ( 61 | toolcallCancellation: LiveServerToolCallCancellation 62 | ) => void; 63 | // Emitted when the current turn is complete 64 | turncomplete: () => void; 65 | } 66 | 67 | /** 68 | * A event-emitting class that manages the connection to the websocket and emits 69 | * events to the rest of the application. 70 | * If you dont want to use react you can still use this. 71 | */ 72 | export class GenAILiveClient extends EventEmitter { 73 | protected client: GoogleGenAI; 74 | 75 | private _status: "connected" | "disconnected" | "connecting" = "disconnected"; 76 | public get status() { 77 | return this._status; 78 | } 79 | 80 | private _session: Session | null = null; 81 | public get session() { 82 | return this._session; 83 | } 84 | 85 | private _model: string | null = null; 86 | public get model() { 87 | return this._model; 88 | } 89 | 90 | protected config: LiveConnectConfig | null = null; 91 | 92 | public getConfig() { 93 | return { ...this.config }; 94 | } 95 | 96 | constructor(options: LiveClientOptions) { 97 | super(); 98 | this.client = new GoogleGenAI(options); 99 | this.send = this.send.bind(this); 100 | this.onopen = this.onopen.bind(this); 101 | this.onerror = this.onerror.bind(this); 102 | this.onclose = this.onclose.bind(this); 103 | this.onmessage = this.onmessage.bind(this); 104 | } 105 | 106 | protected log(type: string, message: StreamingLog["message"]) { 107 | const log: StreamingLog = { 108 | date: new Date(), 109 | type, 110 | message, 111 | }; 112 | this.emit("log", log); 113 | } 114 | 115 | async connect(model: string, config: LiveConnectConfig): Promise { 116 | if (this._status === "connected" || this._status === "connecting") { 117 | return false; 118 | } 119 | 120 | this._status = "connecting"; 121 | this.config = config; 122 | this._model = model; 123 | 124 | const callbacks: LiveCallbacks = { 125 | onopen: this.onopen, 126 | onmessage: this.onmessage, 127 | onerror: this.onerror, 128 | onclose: this.onclose, 129 | }; 130 | 131 | try { 132 | this._session = await this.client.live.connect({ 133 | model, 134 | config, 135 | callbacks, 136 | }); 137 | } catch (e) { 138 | console.error("Error connecting to GenAI Live:", e); 139 | this._status = "disconnected"; 140 | return false; 141 | } 142 | 143 | this._status = "connected"; 144 | return true; 145 | } 146 | 147 | public disconnect() { 148 | if (!this.session) { 149 | return false; 150 | } 151 | this.session?.close(); 152 | this._session = null; 153 | this._status = "disconnected"; 154 | 155 | this.log("client.close", `Disconnected`); 156 | return true; 157 | } 158 | 159 | protected onopen() { 160 | this.log("client.open", "Connected"); 161 | this.emit("open"); 162 | } 163 | 164 | protected onerror(e: ErrorEvent) { 165 | this.log("server.error", e.message); 166 | this.emit("error", e); 167 | } 168 | 169 | protected onclose(e: CloseEvent) { 170 | this.log( 171 | `server.close`, 172 | `disconnected ${e.reason ? `with reason: ${e.reason}` : ``}` 173 | ); 174 | this.emit("close", e); 175 | } 176 | 177 | protected async onmessage(message: LiveServerMessage) { 178 | if (message.setupComplete) { 179 | this.log("server.send", "setupComplete"); 180 | this.emit("setupcomplete"); 181 | return; 182 | } 183 | if (message.toolCall) { 184 | this.log("server.toolCall", message); 185 | this.emit("toolcall", message.toolCall); 186 | return; 187 | } 188 | if (message.toolCallCancellation) { 189 | this.log("server.toolCallCancellation", message); 190 | this.emit("toolcallcancellation", message.toolCallCancellation); 191 | return; 192 | } 193 | 194 | // this json also might be `contentUpdate { interrupted: true }` 195 | // or contentUpdate { end_of_turn: true } 196 | if (message.serverContent) { 197 | const { serverContent } = message; 198 | if ("interrupted" in serverContent) { 199 | this.log("server.content", "interrupted"); 200 | this.emit("interrupted"); 201 | return; 202 | } 203 | if ("turnComplete" in serverContent) { 204 | this.log("server.content", "turnComplete"); 205 | this.emit("turncomplete"); 206 | } 207 | 208 | if ("modelTurn" in serverContent) { 209 | let parts: Part[] = serverContent.modelTurn?.parts || []; 210 | 211 | // when its audio that is returned for modelTurn 212 | const audioParts = parts.filter( 213 | (p) => p.inlineData && p.inlineData.mimeType?.startsWith("audio/pcm") 214 | ); 215 | const base64s = audioParts.map((p) => p.inlineData?.data); 216 | 217 | // strip the audio parts out of the modelTurn 218 | const otherParts = difference(parts, audioParts); 219 | // console.log("otherParts", otherParts); 220 | 221 | base64s.forEach((b64) => { 222 | if (b64) { 223 | const data = base64ToArrayBuffer(b64); 224 | this.emit("audio", data); 225 | this.log(`server.audio`, `buffer (${data.byteLength})`); 226 | } 227 | }); 228 | if (!otherParts.length) { 229 | return; 230 | } 231 | 232 | parts = otherParts; 233 | 234 | const content: { modelTurn: Content } = { modelTurn: { parts } }; 235 | this.emit("content", content); 236 | this.log(`server.content`, message); 237 | } 238 | } else { 239 | console.log("received unmatched message", message); 240 | } 241 | } 242 | 243 | /** 244 | * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg" 245 | */ 246 | sendRealtimeInput(chunks: Array<{ mimeType: string; data: string }>) { 247 | let hasAudio = false; 248 | let hasVideo = false; 249 | for (const ch of chunks) { 250 | this.session?.sendRealtimeInput({ media: ch }); 251 | if (ch.mimeType.includes("audio")) { 252 | hasAudio = true; 253 | } 254 | if (ch.mimeType.includes("image")) { 255 | hasVideo = true; 256 | } 257 | if (hasAudio && hasVideo) { 258 | break; 259 | } 260 | } 261 | const message = 262 | hasAudio && hasVideo 263 | ? "audio + video" 264 | : hasAudio 265 | ? "audio" 266 | : hasVideo 267 | ? "video" 268 | : "unknown"; 269 | this.log(`client.realtimeInput`, message); 270 | } 271 | 272 | /** 273 | * send a response to a function call and provide the id of the functions you are responding to 274 | */ 275 | sendToolResponse(toolResponse: LiveClientToolResponse) { 276 | if ( 277 | toolResponse.functionResponses && 278 | toolResponse.functionResponses.length 279 | ) { 280 | this.session?.sendToolResponse({ 281 | functionResponses: toolResponse.functionResponses, 282 | }); 283 | this.log(`client.toolResponse`, toolResponse); 284 | } 285 | } 286 | 287 | /** 288 | * send normal content parts such as { text } 289 | */ 290 | send(parts: Part | Part[], turnComplete: boolean = true) { 291 | this.session?.sendClientContent({ turns: parts, turnComplete }); 292 | this.log(`client.send`, { 293 | turns: Array.isArray(parts) ? parts : [parts], 294 | turnComplete, 295 | }); 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /src/lib/store-logger.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { create } from "zustand"; 18 | import { StreamingLog } from "../types"; 19 | import { mockLogs } from "../components/logger/mock-logs"; 20 | 21 | interface StoreLoggerState { 22 | maxLogs: number; 23 | logs: StreamingLog[]; 24 | log: (streamingLog: StreamingLog) => void; 25 | clearLogs: () => void; 26 | } 27 | 28 | export const useLoggerStore = create((set, get) => ({ 29 | maxLogs: 100, 30 | logs: [], //mockLogs, 31 | log: ({ date, type, message }: StreamingLog) => { 32 | set((state) => { 33 | const prevLog = state.logs.at(-1); 34 | if (prevLog && prevLog.type === type && prevLog.message === message) { 35 | return { 36 | logs: [ 37 | ...state.logs.slice(0, -1), 38 | { 39 | date, 40 | type, 41 | message, 42 | count: prevLog.count ? prevLog.count + 1 : 1, 43 | } as StreamingLog, 44 | ], 45 | }; 46 | } 47 | return { 48 | logs: [ 49 | ...state.logs.slice(-(get().maxLogs - 1)), 50 | { 51 | date, 52 | type, 53 | message, 54 | } as StreamingLog, 55 | ], 56 | }; 57 | }); 58 | }, 59 | 60 | clearLogs: () => { 61 | console.log("clear log"); 62 | set({ logs: [] }); 63 | }, 64 | setMaxLogs: (n: number) => set({ maxLogs: n }), 65 | })); 66 | -------------------------------------------------------------------------------- /src/lib/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export type GetAudioContextOptions = AudioContextOptions & { 18 | id?: string; 19 | }; 20 | 21 | const map: Map = new Map(); 22 | 23 | export const audioContext: ( 24 | options?: GetAudioContextOptions 25 | ) => Promise = (() => { 26 | const didInteract = new Promise((res) => { 27 | window.addEventListener("pointerdown", res, { once: true }); 28 | window.addEventListener("keydown", res, { once: true }); 29 | }); 30 | 31 | return async (options?: GetAudioContextOptions) => { 32 | try { 33 | const a = new Audio(); 34 | a.src = 35 | "data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"; 36 | await a.play(); 37 | if (options?.id && map.has(options.id)) { 38 | const ctx = map.get(options.id); 39 | if (ctx) { 40 | return ctx; 41 | } 42 | } 43 | const ctx = new AudioContext(options); 44 | if (options?.id) { 45 | map.set(options.id, ctx); 46 | } 47 | return ctx; 48 | } catch (e) { 49 | await didInteract; 50 | if (options?.id && map.has(options.id)) { 51 | const ctx = map.get(options.id); 52 | if (ctx) { 53 | return ctx; 54 | } 55 | } 56 | const ctx = new AudioContext(options); 57 | if (options?.id) { 58 | map.set(options.id, ctx); 59 | } 60 | return ctx; 61 | } 62 | }; 63 | })(); 64 | 65 | export function base64ToArrayBuffer(base64: string) { 66 | var binaryString = atob(base64); 67 | var bytes = new Uint8Array(binaryString.length); 68 | for (let i = 0; i < binaryString.length; i++) { 69 | bytes[i] = binaryString.charCodeAt(i); 70 | } 71 | return bytes.buffer; 72 | } 73 | -------------------------------------------------------------------------------- /src/lib/worklets/audio-processing.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const AudioRecordingWorklet = ` 18 | class AudioProcessingWorklet extends AudioWorkletProcessor { 19 | 20 | // send and clear buffer every 2048 samples, 21 | // which at 16khz is about 8 times a second 22 | buffer = new Int16Array(2048); 23 | 24 | // current write index 25 | bufferWriteIndex = 0; 26 | 27 | constructor() { 28 | super(); 29 | this.hasAudio = false; 30 | } 31 | 32 | /** 33 | * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0] 34 | * @param outputs Float32Array[][] 35 | */ 36 | process(inputs) { 37 | if (inputs[0].length) { 38 | const channel0 = inputs[0][0]; 39 | this.processChunk(channel0); 40 | } 41 | return true; 42 | } 43 | 44 | sendAndClearBuffer(){ 45 | this.port.postMessage({ 46 | event: "chunk", 47 | data: { 48 | int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer, 49 | }, 50 | }); 51 | this.bufferWriteIndex = 0; 52 | } 53 | 54 | processChunk(float32Array) { 55 | const l = float32Array.length; 56 | 57 | for (let i = 0; i < l; i++) { 58 | // convert float32 -1 to 1 to int16 -32768 to 32767 59 | const int16Value = float32Array[i] * 32768; 60 | this.buffer[this.bufferWriteIndex++] = int16Value; 61 | if(this.bufferWriteIndex >= this.buffer.length) { 62 | this.sendAndClearBuffer(); 63 | } 64 | } 65 | 66 | if(this.bufferWriteIndex >= this.buffer.length) { 67 | this.sendAndClearBuffer(); 68 | } 69 | } 70 | } 71 | `; 72 | 73 | export default AudioRecordingWorklet; 74 | -------------------------------------------------------------------------------- /src/lib/worklets/vol-meter.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const VolMeterWorket = ` 18 | class VolMeter extends AudioWorkletProcessor { 19 | volume 20 | updateIntervalInMS 21 | nextUpdateFrame 22 | 23 | constructor() { 24 | super() 25 | this.volume = 0 26 | this.updateIntervalInMS = 25 27 | this.nextUpdateFrame = this.updateIntervalInMS 28 | this.port.onmessage = event => { 29 | if (event.data.updateIntervalInMS) { 30 | this.updateIntervalInMS = event.data.updateIntervalInMS 31 | } 32 | } 33 | } 34 | 35 | get intervalInFrames() { 36 | return (this.updateIntervalInMS / 1000) * sampleRate 37 | } 38 | 39 | process(inputs) { 40 | const input = inputs[0] 41 | 42 | if (input.length > 0) { 43 | const samples = input[0] 44 | let sum = 0 45 | let rms = 0 46 | 47 | for (let i = 0; i < samples.length; ++i) { 48 | sum += samples[i] * samples[i] 49 | } 50 | 51 | rms = Math.sqrt(sum / samples.length) 52 | this.volume = Math.max(rms, this.volume * 0.7) 53 | 54 | this.nextUpdateFrame -= samples.length 55 | if (this.nextUpdateFrame < 0) { 56 | this.nextUpdateFrame += this.intervalInFrames 57 | this.port.postMessage({volume: this.volume}) 58 | } 59 | } 60 | 61 | return true 62 | } 63 | }`; 64 | 65 | export default VolMeterWorket; 66 | -------------------------------------------------------------------------------- /src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// 18 | -------------------------------------------------------------------------------- /src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { ReportHandler } from 'web-vitals'; 18 | 19 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 20 | if (onPerfEntry && onPerfEntry instanceof Function) { 21 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 22 | getCLS(onPerfEntry); 23 | getFID(onPerfEntry); 24 | getFCP(onPerfEntry); 25 | getLCP(onPerfEntry); 26 | getTTFB(onPerfEntry); 27 | }); 28 | } 29 | }; 30 | 31 | export default reportWebVitals; 32 | -------------------------------------------------------------------------------- /src/setupTests.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 18 | // allows you to do things like: 19 | // expect(element).toHaveTextContent(/react/i) 20 | // learn more: https://github.com/testing-library/jest-dom 21 | import '@testing-library/jest-dom'; 22 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { 18 | GoogleGenAIOptions, 19 | LiveClientToolResponse, 20 | LiveServerMessage, 21 | Part, 22 | } from "@google/genai"; 23 | 24 | /** 25 | * the options to initiate the client, ensure apiKey is required 26 | */ 27 | export type LiveClientOptions = GoogleGenAIOptions & { apiKey: string }; 28 | 29 | /** log types */ 30 | export type StreamingLog = { 31 | date: Date; 32 | type: string; 33 | count?: number; 34 | message: 35 | | string 36 | | ClientContentLog 37 | | Omit 38 | | LiveClientToolResponse; 39 | }; 40 | 41 | export type ClientContentLog = { 42 | turns: Part[]; 43 | turnComplete: boolean; 44 | }; 45 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "esModuleInterop": true, 8 | "allowSyntheticDefaultImports": true, 9 | "strict": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "module": "esnext", 13 | "moduleResolution": "node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react-jsx" 18 | }, 19 | "include": ["src", "src/**/*"], 20 | "ts-node": { 21 | "compilerOptions": { 22 | "module": "commonjs" 23 | } 24 | } 25 | } 26 | --------------------------------------------------------------------------------