├── .gitignore ├── README.md ├── demovideo └── AvatarAzureMediumDemo.mp4 ├── installCoturn.sh ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html ├── logo192.png ├── logo512.png ├── manifest.json └── robots.txt ├── src ├── App.css ├── App.js ├── App.test.js ├── components │ ├── Avatar.css │ ├── Avatar.jsx │ ├── Utility.js │ └── config.js ├── index.css ├── index.js ├── logo.svg ├── reportWebVitals.js └── setupTests.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Avatar Demo 2 | 3 | Welcome to the Azure Avatar Demo! This project showcases the integration of Azure AI's Text-to-Speech Avatar feature into a ReactJS application. With this application, you can bring lifelike synthetic talking avatars to your projects. 4 | 5 | [Watch the Demo Video](./demovideo/AvatarAzureMediumDemo.mp4) 6 | 7 | Click the link above to watch a demo of the Azure Avatar in action! 8 | 9 | 10 | ## NOTICE 11 | 12 | Microsoft is now retiring azure TURN services. Azure TTS avatar was using azure turn services for communication. 13 | I have added script to install coturn on ubuntu instance. Execute installCoturn.sh to setup your own TURN server. 14 | 15 | Refer this medium link -> 16 | 17 | https://raokarthik83.medium.com/azure-avatar-tts-update-migrating-from-azure-turn-to-coturn-14b6ac86d60c 18 | 19 | 20 | 21 | ## Getting Started 22 | 23 | Follow these steps to set up and run the application locally: 24 | 25 | 1. **Clone the Repository:** 26 | ```bash 27 | git clone https://github.com/hacktronaut/azure-avatar-demo.git 28 | cd azure-avatar-demo 29 | 30 | 2. **Install Dependencies:** 31 | ```bash 32 | npm install 33 | ``` 34 | 3. **Start the Application:** 35 | ```bash 36 | npm start 37 | ``` 38 | 39 | The application will be accessible at http://localhost:3000 in your web browser. 40 | 41 | ### Configuration 42 | 43 | Make sure to configure the necessary API keys and settings in the config.js file before running the application. 44 | 45 | ```javascript 46 | // config.js 47 | export const avatarAppConfig = { 48 | cogSvcRegion: 'your-region', 49 | cogSvcSubKey: 'your-subscription-key', 50 | // ... (other configuration options) 51 | }; 52 | ``` 53 | 54 | ### Feedback and Issues 55 | 56 | If you encounter any issues or have feedback, please open an issue. We welcome your contributions and suggestions! 57 | 58 | Happy coding! -------------------------------------------------------------------------------- /demovideo/AvatarAzureMediumDemo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacktronaut/azure-avatar-demo/4671b88c534c134d66e0355b11c1629950babfb4/demovideo/AvatarAzureMediumDemo.mp4 -------------------------------------------------------------------------------- /installCoturn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Update package repositories 4 | sudo apt-get -y update 5 | 6 | # Install Coturn server 7 | sudo apt-get install coturn -y 8 | 9 | # Stop Coturn server 10 | sudo service coturn stop 11 | 12 | # Enable TURN server by adding configuration in /etc/default/coturn 13 | echo 'TURNSERVER_ENABLED=1' | sudo tee -a /etc/default/coturn 14 | 15 | # Backup existing turnserver.conf and create a new one 16 | sudo mv /etc/turnserver.conf /etc/turnserver.conf.bak 17 | sudo touch /etc/turnserver.conf 18 | 19 | # Add configuration to the turnserver.conf file 20 | sudo tee /etc/turnserver.conf > /dev/null <0.2%", 29 | "not dead", 30 | "not op_mini all" 31 | ], 32 | "development": [ 33 | "last 1 chrome version", 34 | "last 1 firefox version", 35 | "last 1 safari version" 36 | ] 37 | }, 38 | "devDependencies": { 39 | "bootstrap": "^5.3.2", 40 | "microsoft-cognitiveservices-speech-sdk": "^1.33.1" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacktronaut/azure-avatar-demo/4671b88c534c134d66e0355b11c1629950babfb4/public/favicon.ico -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacktronaut/azure-avatar-demo/4671b88c534c134d66e0355b11c1629950babfb4/public/logo192.png -------------------------------------------------------------------------------- /public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacktronaut/azure-avatar-demo/4671b88c534c134d66e0355b11c1629950babfb4/public/logo512.png -------------------------------------------------------------------------------- /public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /src/App.css: -------------------------------------------------------------------------------- 1 | /* .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #282c34; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } */ 39 | -------------------------------------------------------------------------------- /src/App.js: -------------------------------------------------------------------------------- 1 | import logo from './logo.svg'; 2 | import './App.css'; 3 | import { Avatar } from './components/Avatar'; 4 | 5 | function App() { 6 | return ( 7 |
8 | 9 |
10 | ); 11 | } 12 | 13 | export default App; 14 | -------------------------------------------------------------------------------- /src/App.test.js: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import App from './App'; 3 | 4 | test('renders learn react link', () => { 5 | render(); 6 | const linkElement = screen.getByText(/learn react/i); 7 | expect(linkElement).toBeInTheDocument(); 8 | }); 9 | -------------------------------------------------------------------------------- /src/components/Avatar.css: -------------------------------------------------------------------------------- 1 | .myAvatarDemoText { 2 | font-size: larger; 3 | font-family: "Poppins"; 4 | font-weight: 600; 5 | } 6 | 7 | .myAvatarContainer { 8 | text-align: center; 9 | margin-top: 5rem; 10 | } 11 | 12 | .myAvatarVideoRootDiv { 13 | margin-top: 3rem; 14 | } 15 | 16 | .myTextArea { 17 | height: 11rem; 18 | width: 35rem; 19 | border-radius: 5px; 20 | border-color: grey; 21 | } 22 | 23 | .myAvatarVideo { 24 | /* background-color: grey; */ 25 | height: 20rem; 26 | width: 13rem; 27 | border-radius: 8px; 28 | } 29 | 30 | .myVideoDiv { 31 | height: 22rem; 32 | margin-bottom: 2rem; 33 | } 34 | 35 | video { 36 | margin: 0px 0px 20px 0px; 37 | padding-right: 5rem; 38 | width: 20rem; 39 | height: 22rem; 40 | border-radius: 8px; 41 | } -------------------------------------------------------------------------------- /src/components/Avatar.jsx: -------------------------------------------------------------------------------- 1 | import "./Avatar.css"; 2 | import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk"; 3 | import { createAvatarSynthesizer, createWebRTCConnection } from "./Utility"; 4 | import { avatarAppConfig } from "./config"; 5 | import { useState } from "react"; 6 | import { useRef } from "react"; 7 | 8 | export const Avatar = () => { 9 | 10 | const [avatarSynthesizer, setAvatarSynthesizer] = useState(null); 11 | const myAvatarVideoEleRef = useRef(); 12 | const myAvatarAudioEleRef = useRef(); 13 | const [mySpeechText, setMySpeechText] = useState(""); 14 | 15 | var iceUrl = avatarAppConfig.iceUrl 16 | var iceUsername = avatarAppConfig.iceUsername 17 | var iceCredential = avatarAppConfig.iceCredential 18 | 19 | const handleSpeechText = (event) => { 20 | setMySpeechText(event.target.value); 21 | } 22 | 23 | 24 | const handleOnTrack = (event) => { 25 | 26 | console.log("#### Printing handle onTrack ",event); 27 | 28 | // Update UI elements 29 | console.log("Printing event.track.kind ",event.track.kind); 30 | if (event.track.kind === 'video') { 31 | const mediaPlayer = myAvatarVideoEleRef.current; 32 | mediaPlayer.id = event.track.kind; 33 | mediaPlayer.srcObject = event.streams[0]; 34 | mediaPlayer.autoplay = true; 35 | mediaPlayer.playsInline = true; 36 | mediaPlayer.addEventListener('play', () => { 37 | window.requestAnimationFrame(()=>{}); 38 | }); 39 | } else { 40 | // Mute the audio player to make sure it can auto play, will unmute it when speaking 41 | // Refer to https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide 42 | //const mediaPlayer = myAvatarVideoEleRef.current; 43 | const audioPlayer = myAvatarAudioEleRef.current; 44 | audioPlayer.srcObject = event.streams[0]; 45 | audioPlayer.autoplay = true; 46 | audioPlayer.playsInline = true; 47 | audioPlayer.muted = true; 48 | } 49 | }; 50 | 51 | const stopSpeaking = () => { 52 | avatarSynthesizer.stopSpeakingAsync().then(() => { 53 | console.log("[" + (new Date()).toISOString() + "] Stop speaking request sent.") 54 | 55 | }).catch(); 56 | } 57 | 58 | const stopSession = () => { 59 | 60 | try{ 61 | //Stop speaking 62 | avatarSynthesizer.stopSpeakingAsync().then(() => { 63 | console.log("[" + (new Date()).toISOString() + "] Stop speaking request sent.") 64 | // Close the synthesizer 65 | avatarSynthesizer.close(); 66 | }).catch(); 67 | }catch(e) { 68 | } 69 | } 70 | 71 | const speakSelectedText = () => { 72 | 73 | //Start speaking the text 74 | const audioPlayer = myAvatarAudioEleRef.current; 75 | console.log("Audio muted status ",audioPlayer.muted); 76 | audioPlayer.muted = false; 77 | avatarSynthesizer.speakTextAsync(mySpeechText).then( 78 | (result) => { 79 | if (result.reason === SpeechSDK.ResultReason.SynthesizingAudioCompleted) { 80 | console.log("Speech and avatar synthesized to video stream.") 81 | } else { 82 | console.log("Unable to speak. Result ID: " + result.resultId) 83 | if (result.reason === SpeechSDK.ResultReason.Canceled) { 84 | let cancellationDetails = SpeechSDK.CancellationDetails.fromResult(result) 85 | console.log(cancellationDetails.reason) 86 | if (cancellationDetails.reason === SpeechSDK.CancellationReason.Error) { 87 | console.log(cancellationDetails.errorDetails) 88 | } 89 | } 90 | } 91 | }).catch((error) => { 92 | console.log(error) 93 | avatarSynthesizer.close() 94 | }); 95 | } 96 | 97 | const startSession = () => { 98 | 99 | let peerConnection = createWebRTCConnection(iceUrl,iceUsername, iceCredential); 100 | console.log("Peer connection ",peerConnection); 101 | peerConnection.ontrack = handleOnTrack; 102 | peerConnection.addTransceiver('video', { direction: 'sendrecv' }) 103 | peerConnection.addTransceiver('audio', { direction: 'sendrecv' }) 104 | 105 | let avatarSynthesizer = createAvatarSynthesizer(); 106 | setAvatarSynthesizer(avatarSynthesizer); 107 | peerConnection.oniceconnectionstatechange = e => { 108 | console.log("WebRTC status: " + peerConnection.iceConnectionState) 109 | 110 | if (peerConnection.iceConnectionState === 'connected') { 111 | console.log("Connected to Azure Avatar service"); 112 | } 113 | 114 | if (peerConnection.iceConnectionState === 'disconnected' || peerConnection.iceConnectionState === 'failed') { 115 | console.log("Azure Avatar service Disconnected"); 116 | } 117 | } 118 | 119 | avatarSynthesizer.startAvatarAsync(peerConnection).then((r) => { 120 | console.log("[" + (new Date()).toISOString() + "] Avatar started.") 121 | 122 | }).catch( 123 | (error) => { 124 | console.log("[" + (new Date()).toISOString() + "] Avatar failed to start. Error: " + error) 125 | } 126 | ); 127 | } 128 | 129 | 130 | 131 | return( 132 |
133 |

Azure Avatar Demo

134 |
135 |
136 |
137 | 138 | 141 | 142 | 145 |
146 |
147 | 151 | 155 |
156 |
157 |
158 | 159 | 162 |
163 | 166 | 169 |
170 |
171 |
172 |
173 | ) 174 | } -------------------------------------------------------------------------------- /src/components/Utility.js: -------------------------------------------------------------------------------- 1 | import * as SpeechSDK from "microsoft-cognitiveservices-speech-sdk"; 2 | import { avatarAppConfig } from "./config"; 3 | const cogSvcRegion = avatarAppConfig.cogSvcRegion 4 | const cogSvcSubKey = avatarAppConfig.cogSvcSubKey 5 | const voiceName = avatarAppConfig.voiceName 6 | const avatarCharacter = avatarAppConfig.avatarCharacter 7 | const avatarStyle = avatarAppConfig.avatarStyle 8 | const avatarBackgroundColor = "#FFFFFFFF"; 9 | 10 | 11 | export const createWebRTCConnection = (iceServerUrl, iceServerUsername, iceServerCredential) => { 12 | 13 | var peerConnection = new RTCPeerConnection({ 14 | iceServers: [{ 15 | urls: [ iceServerUrl ], 16 | username: iceServerUsername, 17 | credential: iceServerCredential 18 | }] 19 | }) 20 | 21 | return peerConnection; 22 | 23 | } 24 | 25 | export const createAvatarSynthesizer = () => { 26 | 27 | const speechSynthesisConfig = SpeechSDK.SpeechConfig.fromSubscription(cogSvcSubKey, cogSvcRegion) 28 | 29 | speechSynthesisConfig.speechSynthesisVoiceName = voiceName; 30 | 31 | const videoFormat = new SpeechSDK.AvatarVideoFormat() 32 | 33 | let videoCropTopLeftX = 600 34 | let videoCropBottomRightX = 1320 35 | videoFormat.setCropRange(new SpeechSDK.Coordinate(videoCropTopLeftX, 50), new SpeechSDK.Coordinate(videoCropBottomRightX, 1080)); 36 | 37 | 38 | const talkingAvatarCharacter = avatarCharacter 39 | const talkingAvatarStyle = avatarStyle 40 | 41 | const avatarConfig = new SpeechSDK.AvatarConfig(talkingAvatarCharacter, talkingAvatarStyle, videoFormat) 42 | avatarConfig.backgroundColor = avatarBackgroundColor; 43 | let avatarSynthesizer = new SpeechSDK.AvatarSynthesizer(speechSynthesisConfig, avatarConfig) 44 | 45 | avatarSynthesizer.avatarEventReceived = function (s, e) { 46 | var offsetMessage = ", offset from session start: " + e.offset / 10000 + "ms." 47 | if (e.offset === 0) { 48 | offsetMessage = "" 49 | } 50 | console.log("[" + (new Date()).toISOString() + "] Event received: " + e.description + offsetMessage) 51 | } 52 | 53 | return avatarSynthesizer; 54 | 55 | } -------------------------------------------------------------------------------- /src/components/config.js: -------------------------------------------------------------------------------- 1 | 2 | export const avatarAppConfig = { 3 | cogSvcRegion : "westus2", 4 | cogSvcSubKey : "YOUR_KEY", 5 | voiceName : "en-US-JennyNeural", 6 | avatarCharacter : "lisa", 7 | avatarStyle : "casual-sitting", 8 | avatarBackgroundColor : "#FFFFFFFF", 9 | iceUrl : "stun:relay.communication.microsoft.com:3478", 10 | iceUsername : "YOUR_ICE_USERNAME", 11 | iceCredential : "YOUR_ICE_CREDENTIAL" 12 | } -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | // Import Bootstrap CSS 7 | import 'bootstrap/dist/css/bootstrap.min.css'; 8 | // Import Bootstrap JS 9 | import 'bootstrap/dist/js/bootstrap.min.js'; 10 | 11 | const root = ReactDOM.createRoot(document.getElementById('root')); 12 | root.render( 13 | 14 | 15 | 16 | ); 17 | 18 | // If you want to start measuring performance in your app, pass a function 19 | // to log results (for example: reportWebVitals(console.log)) 20 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 21 | reportWebVitals(); 22 | -------------------------------------------------------------------------------- /src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/reportWebVitals.js: -------------------------------------------------------------------------------- 1 | const reportWebVitals = onPerfEntry => { 2 | if (onPerfEntry && onPerfEntry instanceof Function) { 3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 4 | getCLS(onPerfEntry); 5 | getFID(onPerfEntry); 6 | getFCP(onPerfEntry); 7 | getLCP(onPerfEntry); 8 | getTTFB(onPerfEntry); 9 | }); 10 | } 11 | }; 12 | 13 | export default reportWebVitals; 14 | -------------------------------------------------------------------------------- /src/setupTests.js: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | --------------------------------------------------------------------------------