├── .gitignore ├── architecture.png ├── mic.js ├── package.json ├── README.md ├── index.html ├── server.js ├── demo.js ├── main.js └── unminified-important.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /node_modules 3 | /temp 4 | 5 | cred.json 6 | -------------------------------------------------------------------------------- /architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hansonrb/webrtc-google-transcript/HEAD/architecture.png -------------------------------------------------------------------------------- /mic.js: -------------------------------------------------------------------------------- 1 | // var streams = require('memory-streams'); 2 | // var reader = new streams.ReadableStream(''); 3 | // reader.pipe(process.stdout); 4 | // 5 | // for (var i = 0; i < 1000; i++) { 6 | // reader.append('random string\n'); 7 | // } 8 | 9 | 10 | var streams = require('memory-streams'); 11 | var writer = new streams.WritableStream(); 12 | var fs = require('fs'); 13 | 14 | writer = fs.createWriteStream('./test.wav'); 15 | const record = require('node-record-lpcm16'); 16 | 17 | record 18 | .start({ 19 | sampleRateHertz: 16000, 20 | threshold: 0, 21 | // Other options, see https://www.npmjs.com/package/node-record-lpcm16#options 22 | verbose: false, 23 | recordProgram: 'rec', // Try also "arecord" or "sox" 24 | silence: '10.0', 25 | }) 26 | .on('error', console.error) 27 | .pipe(writer); 28 | // 29 | // console.log(writer.toString('hex')); 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "audio-google", 3 | "version": "1.0.0", 4 | "description": "WebRTC Audio transcription using Google Cloud to speech API", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "nodemon server.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/hansonrb/webrtc-google-transcript.git" 13 | }, 14 | "author": "Hanson Rynsburger", 15 | "license": "ISC", 16 | "bugs": { 17 | "url": "https://github.com/hansonrb/webrtc-google-transcript/issues" 18 | }, 19 | "homepage": "https://github.com/hansonrb/webrtc-google-transcript#readme", 20 | "dependencies": { 21 | "@google-cloud/speech": "1.5.0", 22 | "memory-streams": "^0.1.3", 23 | "ws": "^5.1.1" 24 | }, 25 | "devDependencies": { 26 | "node-record-lpcm16": "^0.3.0", 27 | "nodemon": "^1.17.4" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # webrtc-google-transcript 2 | 3 | WebRTC Audio transcription using Google Cloud to speech API 4 | 5 | 6 | ## Architecture 7 | ![Architecture](https://raw.githubusercontent.com/hansonrb/webrtc-google-transcript/master/architecture.png) 8 | 9 | 10 | ## Requirement 11 | 12 | You need to have nodejs and npm and you can use nvm for multiple node versions. 13 | 14 | More info about how to install nvm https://github.com/creationix/nvm 15 | 16 | ### For Backend ( NodeJS Middleware ) 17 | ``` 18 | nvm install 6.14.2 19 | nvm use 6.14.2 20 | 21 | npm install 22 | 23 | export GOOGLE_APPLICATION_CREDENTIALS=/Volumes/data/works/webrtc/audio-google/cred.json 24 | 25 | npm run start 26 | ``` 27 | 28 | environment variable GOOGLE_APPLICATION_CREDENTIALS should be set with absolute URL 29 | 30 | ### For frontend 31 | 32 | You can use any html server : apache or nginx. 33 | Or you can just use simple nodejs html server 34 | 35 | Open another terminal and run following for frontend 36 | ``` 37 | nvm use 6.14.2 38 | npm install -g serve 39 | serve . 40 | ``` 41 | 42 | And visit localhost:5000 43 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Audio Transcript 9 | 10 | 11 | 12 | 51 | 52 | 53 | 54 |
55 | 56 |
57 |
58 | 59 |
60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | const WebSocket = require('ws'); 2 | const Speech = require('@google-cloud/speech'); 3 | const fs = require('fs'); 4 | const streams = require('memory-streams'); 5 | const util = require('util'); 6 | const tempfile = 'temp/stream.wav'; 7 | 8 | let wsInstance = null; 9 | 10 | const request = { 11 | config: { 12 | encoding: 'LINEAR16', 13 | // sampleRateHertz: 16000, 14 | sampleRateHertz: 44100, 15 | languageCode: 'en-US', 16 | // enableWordTimeOffsets: true, 17 | enableAutomaticPunctuation: true, 18 | model: 'default', 19 | }, 20 | interimResults: true, // If you want interim results, set this to true 21 | verbose: true, 22 | }; 23 | 24 | const client = new Speech.SpeechClient(); 25 | 26 | const recognizeStream = client 27 | .streamingRecognize(request) 28 | .on('error', console.error) 29 | .on('data', data => { 30 | if (data.results && data.results[0]) { 31 | wsInstance && 32 | wsInstance.send( 33 | JSON.stringify({ 34 | isFinal: data.results[0].isFinal, 35 | text: data.results[0].alternatives[0].transcript, 36 | }) 37 | ); 38 | } 39 | }); 40 | 41 | console.log('Connected to google cloud speech api and ready to accept stream'); 42 | 43 | // var reader = fs.createReadStream('./test.wav'); 44 | // reader.pipe(recognizeStream); 45 | 46 | var reader = new streams.ReadableStream(''); 47 | // var writer = fs.createWriteStream('./test2.wav'); 48 | // reader.pipe(writer); 49 | 50 | // --- socket --- 51 | const wss = new WebSocket.Server({ port: 12345 }); 52 | wss.on('connection', function connection(ws) { 53 | ws.on('message', function incoming(message) { 54 | if (message === 'start') { 55 | reader.pipe(recognizeStream); 56 | // reader.pipe(process.stdout); 57 | console.log('streaming into google'); 58 | } else if (message === 'end') { 59 | console.log('\n\n\n======================\nstop'); 60 | } else { 61 | // const buf = Buffer.from(message); 62 | // console.log(util.inspect(message)); 63 | reader.append(message); 64 | } 65 | }); 66 | 67 | wsInstance = ws; // ws.send('something'); 68 | }); 69 | console.log('WebSocket is ready to accept audio streaming'); 70 | -------------------------------------------------------------------------------- /demo.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 | * 4 | * Use of this source code is governed by a BSD-style license 5 | * that can be found in the LICENSE file in the root of the source 6 | * tree. 7 | */ 8 | 9 | 'use strict'; 10 | 11 | // Put variables in global scope to make them available to the browser console. 12 | // var audio = document.querySelector('audio'); 13 | var button = document.querySelector('button'); 14 | 15 | // console.log(button); 16 | // button.addEventListener('click', function() { 17 | // button.value = 'Stop'; 18 | // }); 19 | 20 | try { 21 | window.AudioContext = window.AudioContext || window.webkitAudioContext; 22 | window.audioContext = new AudioContext(); 23 | } catch (e) { 24 | alert('Web Audio API not supported.'); 25 | } 26 | 27 | var constraints = window.constraints = { 28 | audio: true, 29 | video: false 30 | }; 31 | 32 | function handleSuccess(stream) { 33 | 34 | // var socket = new WebSocket("wss://cloudspeech.goog/ws"); 35 | 36 | socket.binaryType = "arraybuffer"; 37 | socket.onopen = function() { 38 | socket.send(JSON.stringify({ 39 | format: "LINEAR16", 40 | language: 'en-US', 41 | punctuation: true, 42 | rate: 44100 43 | })); 44 | } 45 | 46 | var audioTracks = stream.getAudioTracks(); 47 | 48 | stream.oninactive = function() { 49 | console.log('Stream ended'); 50 | }; 51 | 52 | window.stream = stream; // make variable available to browser console 53 | // audio.srcObject = stream; 54 | 55 | var script = window.audioContext.createScriptProcessor(4096, 1, 1); 56 | 57 | script.onaudioprocess = function(event) { 58 | var input = event.inputBuffer.getChannelData(0) || new Float32Array(4096); 59 | 60 | for (var idx = input.length, newData = new Int16Array(idx); idx--;) 61 | newData[idx] = 32767 * Math.min(1, input[idx]); 62 | 63 | if (socket.readyState) { 64 | socket.send(newData.buffer); 65 | } 66 | } 67 | 68 | socket.onmessage = function(b) { 69 | const data = JSON.parse(b.data); 70 | console.log(data) 71 | // b.isFinal ? a.transcripts[0].results ? (a.set("transcripts.0.tempResult", null), a.push("transcripts.0.results", b.text)) : a.setResults_([b.text]) : (a.resultsReady = !0, a.set("transcripts.0.isEnabled", !0), a.set("transcripts.0.tempResult", b.text)) 72 | }; 73 | 74 | var mic = window.audioContext.createMediaStreamSource(stream); 75 | mic.connect(script); 76 | script.connect(window.audioContext.destination); 77 | } 78 | 79 | function handleError(error) { 80 | console.log('navigator.getUserMedia error: ', error); 81 | } 82 | 83 | navigator.mediaDevices.getUserMedia(constraints). 84 | then(handleSuccess).catch(handleError); 85 | -------------------------------------------------------------------------------- /main.js: -------------------------------------------------------------------------------- 1 | try { 2 | window.AudioContext = window.AudioContext || window.webkitAudioContext; 3 | window.audioContext = new AudioContext(); 4 | } catch (e) { 5 | alert('Web Audio API not supported.'); 6 | } 7 | 8 | var constraints = window.constraints = { 9 | audio: true, 10 | video: false 11 | }; 12 | 13 | $(document).ready(() => { 14 | console.log('document.ready'); 15 | 16 | const button = $("button#control"); 17 | 18 | button.click(() => { 19 | if (button.text() === 'Start') { 20 | console.log('Start Recording, API calls'); 21 | 22 | navigator.mediaDevices.getUserMedia(constraints). 23 | then(handleSuccess).catch(handleError); 24 | 25 | button.text('Stop').addClass('stop').removeClass('start'); 26 | } else { 27 | if (window.script) { 28 | window.script.onaudioprocess = function() {}; 29 | } 30 | if (window.socket) { 31 | window.socket.readyState && window.socket.send('end'); 32 | window.socket.readyState && window.socket.close(); 33 | } 34 | 35 | button.text('Start').removeClass('stop').addClass('start'); 36 | } 37 | }) 38 | }); 39 | 40 | function handleSuccess(stream) { 41 | // var socket = new WebSocket("wss://cloudspeech.goog/ws"); 42 | var socket = new WebSocket("ws://localhost:12345/"); 43 | 44 | socket.binaryType = "arraybuffer"; 45 | socket.onopen = function() { 46 | // socket.send(JSON.stringify({ 47 | // format: "LINEAR16", 48 | // language: 'en-US', 49 | // punctuation: true, 50 | // rate: 44100 51 | // })); 52 | 53 | socket.send('start'); 54 | } 55 | 56 | var audioTracks = stream.getAudioTracks(); 57 | 58 | stream.oninactive = function() { 59 | console.log('Stream ended'); 60 | }; 61 | 62 | window.stream = stream; // make variable available to browser console 63 | // audio.srcObject = stream; 64 | 65 | var script = window.audioContext.createScriptProcessor(4096, 1, 1); 66 | 67 | script.onaudioprocess = function(event) { 68 | var input = event.inputBuffer.getChannelData(0) || new Float32Array(4096); 69 | 70 | for (var idx = input.length, newData = new Int16Array(idx); idx--;) 71 | newData[idx] = 32767 * Math.min(1, input[idx]); 72 | 73 | if (socket.readyState === 1) { 74 | // console.log(newData); 75 | // socket.send(newData.buffer); 76 | socket.send(newData); 77 | } 78 | } 79 | 80 | socket.onmessage = function(b) { 81 | const data = JSON.parse(b.data); 82 | // console.log(data); 83 | 84 | if (data.isFinal) { 85 | $(".output-final").append( 86 | "

" + data.text + "

" 87 | ); 88 | } else { 89 | $(".output").text(data.text); 90 | } 91 | }; 92 | 93 | var mic = window.audioContext.createMediaStreamSource(stream); 94 | mic.connect(script); 95 | script.connect(window.audioContext.destination); 96 | 97 | window.socket = socket; 98 | window.script = script; 99 | } 100 | 101 | function handleError(error) { 102 | console.log('navigator.getUserMedia error: ', error); 103 | } 104 | -------------------------------------------------------------------------------- /unminified-important.js: -------------------------------------------------------------------------------- 1 | 2 | customElements.define(X.SpResults.is, X.SpResults); 3 | r.Sp_results = {}; 4 | r.Sp_results.SpResults = {}; 5 | n.crypt = {}; 6 | n.crypt.stringToByteArray = function(a) { 7 | for (var b = [], d = 0, e = 0; e < a.length; e++) { 8 | var f = a.charCodeAt(e); 9 | 255 < f && (b[d++] = f & 255, f >>= 8); 10 | b[d++] = f 11 | } 12 | return b 13 | }; 14 | n.crypt.byteArrayToString = function(a) { 15 | if (8192 >= a.length) return String.fromCharCode.apply(null, a); 16 | for (var b = "", d = 0; d < a.length; d += 8192) { 17 | var e = n.array.slice(a, d, d + 8192); 18 | b += String.fromCharCode.apply(null, e) 19 | } 20 | return b 21 | }; 22 | n.crypt.byteArrayToHex = function(a, b) { 23 | return n.array.map(a, function(a) { 24 | a = a.toString(16); 25 | return 1 < a.length ? a : "0" + a 26 | }).join(b || "") 27 | }; 28 | n.crypt.hexToByteArray = function(a) { 29 | n.asserts.assert(0 == a.length % 2, "Key string length must be multiple of 2"); 30 | for (var b = [], d = 0; d < a.length; d += 2) b.push(parseInt(a.substring(d, d + 2), 16)); 31 | return b 32 | }; 33 | n.crypt.stringToUtf8ByteArray = function(a) { 34 | for (var b = [], d = 0, e = 0; e < a.length; e++) { 35 | var f = a.charCodeAt(e); 36 | 128 > f ? b[d++] = f : (2048 > f ? b[d++] = f >> 6 | 192 : (55296 == (f & 64512) && e + 1 < a.length && 56320 == (a.charCodeAt(e + 1) & 64512) ? (f = 65536 + ((f & 1023) << 10) + (a.charCodeAt(++e) & 1023), b[d++] = f >> 18 | 240, b[d++] = f >> 12 & 63 | 128) : b[d++] = f >> 12 | 224, b[d++] = f >> 6 & 63 | 128), b[d++] = f & 63 | 128) 37 | } 38 | return b 39 | }; 40 | n.crypt.utf8ByteArrayToString = function(a) { 41 | for (var b = [], d = 0, e = 0; d < a.length;) { 42 | var f = a[d++]; 43 | if (128 > f) b[e++] = String.fromCharCode(f); 44 | else if (191 < f && 224 > f) { 45 | var g = a[d++]; 46 | b[e++] = String.fromCharCode((f & 31) << 6 | g & 63) 47 | } else if (239 < f && 365 > f) { 48 | g = a[d++]; 49 | var h = a[d++], 50 | l = a[d++]; 51 | f = ((f & 7) << 18 | (g & 63) << 12 | (h & 63) << 6 | l & 63) - 65536; 52 | b[e++] = String.fromCharCode(55296 + (f >> 10)); 53 | b[e++] = String.fromCharCode(56320 + (f & 1023)) 54 | } else g = a[d++], h = a[d++], b[e++] = String.fromCharCode((f & 15) << 12 | (g & 63) << 6 | h & 63) 55 | } 56 | return b.join("") 57 | }; 58 | n.crypt.xorByteArray = function(a, b) { 59 | n.asserts.assert(a.length == b.length, "XOR array lengths must match"); 60 | for (var d = [], e = 0; e < a.length; e++) d.push(a[e] ^ b[e]); 61 | return d 62 | }; 63 | n.crypt.base64 = {}; 64 | n.crypt.base64.byteToCharMap_ = null; 65 | n.crypt.base64.charToByteMap_ = null; 66 | n.crypt.base64.byteToCharMapWebSafe_ = null; 67 | n.crypt.base64.ENCODED_VALS_BASE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; 68 | n.crypt.base64.ENCODED_VALS = n.crypt.base64.ENCODED_VALS_BASE + "+/="; 69 | n.crypt.base64.ENCODED_VALS_WEBSAFE = n.crypt.base64.ENCODED_VALS_BASE + "-_."; 70 | n.crypt.base64.ASSUME_NATIVE_SUPPORT_ = n.userAgent.GECKO || n.userAgent.WEBKIT && !n.userAgent.product.SAFARI || n.userAgent.OPERA; 71 | n.crypt.base64.HAS_NATIVE_ENCODE_ = n.crypt.base64.ASSUME_NATIVE_SUPPORT_ || "function" == typeof n.global.btoa; 72 | n.crypt.base64.HAS_NATIVE_DECODE_ = n.crypt.base64.ASSUME_NATIVE_SUPPORT_ || !n.userAgent.product.SAFARI && !n.userAgent.IE && "function" == typeof n.global.atob; 73 | n.crypt.base64.encodeByteArray = function(a, b) { 74 | n.asserts.assert(n.isArrayLike(a), "encodeByteArray takes an array as a parameter"); 75 | n.crypt.base64.init_(); 76 | b = b ? n.crypt.base64.byteToCharMapWebSafe_ : n.crypt.base64.byteToCharMap_; 77 | for (var d = [], e = 0; e < a.length; e += 3) { 78 | var f = a[e], 79 | g = e + 1 < a.length, 80 | h = g ? a[e + 1] : 0, 81 | l = e + 2 < a.length, 82 | p = l ? a[e + 2] : 0, 83 | m = f >> 2; 84 | f = (f & 3) << 4 | h >> 4; 85 | h = (h & 15) << 2 | p >> 6; 86 | p &= 63; 87 | l || (p = 64, g || (h = 64)); 88 | d.push(b[m], b[f], b[h], b[p]) 89 | } 90 | return d.join("") 91 | }; 92 | n.crypt.base64.encodeString = function(a, b) { 93 | return n.crypt.base64.HAS_NATIVE_ENCODE_ && !b ? n.global.btoa(a) : n.crypt.base64.encodeByteArray(n.crypt.stringToByteArray(a), b) 94 | }; 95 | n.crypt.base64.decodeString = function(a, b) { 96 | function d(a) { 97 | e += String.fromCharCode(a) 98 | } 99 | if (n.crypt.base64.HAS_NATIVE_DECODE_ && !b) return n.global.atob(a); 100 | var e = ""; 101 | n.crypt.base64.decodeStringInternal_(a, d); 102 | return e 103 | }; 104 | n.crypt.base64.decodeStringToByteArray = function(a) { 105 | function b(a) { 106 | d.push(a) 107 | } 108 | var d = []; 109 | n.crypt.base64.decodeStringInternal_(a, b); 110 | return d 111 | }; 112 | n.crypt.base64.decodeStringToUint8Array = function(a) { 113 | function b(a) { 114 | f[g++] = a 115 | } 116 | n.asserts.assert(!n.userAgent.IE || n.userAgent.isVersionOrHigher("10"), "Browser does not support typed arrays"); 117 | var d = a.length, 118 | e = 0; 119 | "=" === a[d - 2] ? e = 2 : "=" === a[d - 1] && (e = 1); 120 | var f = new Uint8Array(Math.ceil(3 * d / 4) - e), 121 | g = 0; 122 | n.crypt.base64.decodeStringInternal_(a, b); 123 | return f.subarray(0, g) 124 | }; 125 | n.crypt.base64.decodeStringInternal_ = function(a, b) { 126 | function d(b) { 127 | for (; e < a.length;) { 128 | var d = a.charAt(e++), 129 | f = n.crypt.base64.charToByteMap_[d]; 130 | if (null != f) return f; 131 | if (!n.string.isEmptyOrWhitespace(d)) throw Error("Unknown base64 encoding at char: " + d); 132 | } 133 | return b 134 | } 135 | n.crypt.base64.init_(); 136 | for (var e = 0;;) { 137 | var f = d(-1), 138 | g = d(0), 139 | h = d(64), 140 | l = d(64); 141 | if (64 === l && -1 === f) break; 142 | f = f << 2 | g >> 4; 143 | b(f); 144 | 64 != h && (g = g << 4 & 240 | h >> 2, b(g), 64 != l && (h = h << 6 & 192 | l, b(h))) 145 | } 146 | }; 147 | n.crypt.base64.init_ = function() { 148 | if (!n.crypt.base64.byteToCharMap_) { 149 | n.crypt.base64.byteToCharMap_ = {}; 150 | n.crypt.base64.charToByteMap_ = {}; 151 | n.crypt.base64.byteToCharMapWebSafe_ = {}; 152 | for (var a = 0; a < n.crypt.base64.ENCODED_VALS.length; a++) n.crypt.base64.byteToCharMap_[a] = n.crypt.base64.ENCODED_VALS.charAt(a), n.crypt.base64.charToByteMap_[n.crypt.base64.byteToCharMap_[a]] = a, n.crypt.base64.byteToCharMapWebSafe_[a] = n.crypt.base64.ENCODED_VALS_WEBSAFE.charAt(a), a >= n.crypt.base64.ENCODED_VALS_BASE.length && (n.crypt.base64.charToByteMap_[n.crypt.base64.ENCODED_VALS_WEBSAFE.charAt(a)] = 153 | a) 154 | } 155 | }; 156 | 157 | xc = { 158 | DEFAULT: "Default", 159 | COMMAND_AND_SEARCH: "Command / Search", 160 | PHONE_CALL: "Phone call", 161 | VIDEO: "Video" 162 | }; 163 | 164 | Y.SpApp = function() { 165 | var a = Polymer.Element.call(this) || this; 166 | try { 167 | window.AudioContext = window.AudioContext || window.webkitAudioContext, navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia 168 | } catch (b) { 169 | return a.errorClient = "BROWSER_UNSUPPORTED", a.controlsDisabled = !1, a 170 | } 171 | if (!navigator.getUserMedia || !window.AudioContext) return a.errorClient = "BROWSER_UNSUPPORTED", a.controlsDisabled = !0, a; 172 | a.audioContext = new AudioContext; 173 | a.checkServer_(); 174 | a.transcripts = 175 | a.recognitionModels.map(function(a) { 176 | a.isEnabled = !1; 177 | return a 178 | }); 179 | return a 180 | }; 181 | k.inherits(Y.SpApp, Polymer.Element); 182 | c = Y.SpApp.prototype; 183 | c.checkServer_ = function() { 184 | var a = this; 185 | this.initFallback_(); 186 | var b = new XMLHttpRequest; 187 | b.onload = function() { 188 | 200 <= b.status && 400 > b.status && (a.streamingAvailable = !0, a.controlsDisabled = !1) 189 | }; 190 | b.onerror = function() { 191 | a.initFallback_() 192 | }; 193 | b.open("GET", "//" + this.urlServer + "/status", !0); 194 | b.send() 195 | }; 196 | c.convertFileToArrayBuffer_ = function(a) { 197 | return new Promise(function(b, d) { 198 | var e = new FileReader; 199 | e.onload = function(a) { 200 | b(a.target.result) 201 | }; 202 | e.onerror = function(a) { 203 | d(a.target.error) 204 | }; 205 | e.readAsArrayBuffer(a) 206 | }) 207 | }; 208 | c.decodeArrayBuffer_ = function(a) { 209 | var b = this, 210 | d = a.slice(0); 211 | return new Promise(function(e, f) { 212 | b.audioContext.decodeAudioData(d, function(d) { 213 | d.duration > b.audioDurationLimit && (b.errorClient = "AUDIO_LENGTH", f()); 214 | b.audioDuration = d.duration; 215 | if (1 === d.numberOfChannels) e(a); 216 | else { 217 | b.fileUploadStatus = "CONVERTING"; 218 | window.OfflineAudioContext || (f(), b.errorClient = "MULTI_CHANNEL"); 219 | b.sampleRate = d.sampleRate; 220 | var g = new OfflineAudioContext(1, d.length, d.sampleRate), 221 | l = g.createBufferSource(), 222 | p = g.createChannelMerger(1); 223 | l.buffer = 224 | d; 225 | l.connect(p); 226 | p.connect(g.destination); 227 | l.start(); 228 | g.startRendering().then(function(a) { 229 | var b = a, 230 | d = void 0; 231 | d = d || {}; 232 | a = b.numberOfChannels; 233 | var f = b.sampleRate, 234 | g = d.float32 ? 3 : 1; 235 | d = 3 === g ? 32 : 16; 236 | if (2 === a) { 237 | var h = b.getChannelData(0); 238 | b = b.getChannelData(1); 239 | for (var l = h.length + b.length, p = new Float32Array(l), m = 0, y = 0; m < l;) p[m++] = h[y], p[m++] = b[y], y++; 240 | h = p 241 | } else h = b.getChannelData(0); 242 | b = a; 243 | l = d / 8; 244 | p = b * l; 245 | a = new ArrayBuffer(44 + h.length * l); 246 | m = new DataView(a); 247 | tc(m, 0, "RIFF"); 248 | m.setUint32(4, 36 + h.length * l, !0); 249 | tc(m, 8, "WAVE"); 250 | tc(m, 12, "fmt "); 251 | m.setUint32(16, 16, !0); 252 | m.setUint16(20, g, !0); 253 | m.setUint16(22, b, !0); 254 | m.setUint32(24, f, !0); 255 | m.setUint32(28, f * p, !0); 256 | m.setUint16(32, p, !0); 257 | m.setUint16(34, d, !0); 258 | tc(m, 36, "data"); 259 | m.setUint32(40, h.length * l, !0); 260 | if (1 === g) 261 | for (d = m, f = 44, g = 0; g < h.length; g++, f += 2) b = Math.max(-1, Math.min(1, h[g])), d.setInt16(f, 0 > b ? 32768 * b : 32767 * b, !0); 262 | else 263 | for (d = m, f = 44, g = 0; g < h.length; g++, f += 4) d.setFloat32(f, h[g], !0); 264 | e(a) 265 | }).catch(function(a) { 266 | f(a) 267 | }) 268 | } 269 | }, function(a) { 270 | f(a) 271 | }) 272 | }) 273 | }; 274 | c.fileUploadValueChanged_ = function(a) { 275 | var b = this; 276 | a ? (this.resetResults_(), this.controlsDisabled = !0, this.fileUploadProgress = 20, this.convertFileToArrayBuffer_(a).then(function(a) { 277 | b.decodeArrayBuffer_(a).then(function(a) { 278 | a = new Uint8Array(a); 279 | b.getTranscripts_(n.crypt.base64.encodeByteArray(a)) 280 | }, function(a) { 281 | a && (console.log(a), b.errorClient = "DECODE_FAILED"); 282 | b.controlsDisabled = !1; 283 | b.fileUploadProgress = 0 284 | }) 285 | }), mc({ 286 | type: "speech-api", 287 | name: "fileUploaded" 288 | })) : this.fileUploadProgress = 0 289 | }; 290 | c.getErrorMessage_ = function(a) { 291 | return this.messages.error[a] || a 292 | }; 293 | c.getTranscripts_ = function(a) { 294 | var b = this; 295 | if (!this.fileUploadCanceled) { 296 | this.fileUploadProgress = 30; 297 | this.fileUploadStatus = "TRANSCRIBING"; 298 | var d = { 299 | config: { 300 | encoding: "LINEAR16", 301 | languageCode: this.languageSelected, 302 | sampleRateHertz: this.sampleRate, 303 | enableAutomaticPunctuation: this.punctuationEnabled 304 | }, 305 | audio: { 306 | content: a 307 | } 308 | }; 309 | var e = setInterval(function() { 310 | return b.updateProgressMeter_() 311 | }, 50); 312 | this.recognitionModels.forEach(function(a) { 313 | var f = b.transcripts.map(function(a) { 314 | return a.id 315 | }).indexOf(a.id); 316 | if (a.enabledLanguages && 317 | !a.enabledLanguages.includes(b.languageSelected)) { 318 | b.set("transcripts." + f + ".isEnabled", !1); 319 | var h = b.getErrorMessage_("MODEL_UNAVAILABLE"); 320 | b.set("transcripts." + f + ".error", h) 321 | } else h = JSON.parse(JSON.stringify(d)), h.config.model = a.id, b.set("transcripts." + f + ".isEnabled", !0), b.postAudioRequest_(JSON.stringify(h)).then(JSON.parse).then(function(d) { 322 | b.fileUploadProgress = 0; 323 | clearInterval(e); 324 | b.controlsDisabled = !1; 325 | b.errorServer = null; 326 | d.results && d.results.length ? b.setResults_(d.results, a.id) : (d = b.getErrorMessage_("MODEL_ERROR"), 327 | b.set("transcripts." + f + ".error", d)) 328 | }, function(a) { 329 | console.log(a); 330 | try { 331 | var d = a && "" !== a && JSON.parse(a); 332 | var f = d.error && d.error.message 333 | } catch (u) { 334 | f = a && "" !== a ? a : "API_POST_ERROR" 335 | } 336 | b.fileUploadProgress = 0; 337 | clearInterval(e); 338 | b.controlsDisabled = !1; 339 | b.errorServer = f 340 | }), a.showEnhanced && (h.config.useEnhanced = !0, b.postAudioRequest_(JSON.stringify(h)).then(JSON.parse).then(function(a) { 341 | b.fileUploadCanceled || (a = a.results.map(function(a) { 342 | return a.alternatives[0].transcript 343 | }), b.set("transcripts." + f + ".enhancedResults", a)) 344 | }, function(a) { 345 | console.log(a) 346 | })) 347 | }) 348 | } 349 | }; 350 | c.initFallback_ = function() { 351 | var a = this; 352 | if (!this.checkRecorder && !window.Recorder) { 353 | var b = document.createElement("script"); 354 | b.src = this.recorderLib; 355 | b.type = "text/javascript"; 356 | b.async = !0; 357 | document.head.appendChild(b); 358 | this.checkRecorder = setInterval(function() { 359 | window.Recorder && (a.controlsDisabled = !1, a.maxRecordTime = 3E4, clearInterval(a.checkRecorder)) 360 | }, 200); 361 | this.streamingAvailable = !1 362 | } 363 | }; 364 | c.onInputTypeSelectedChanged_ = function() { 365 | this.resetResults_() 366 | }; 367 | c.onIsRecordingChanged_ = function(a, b) { 368 | "undefined" !== typeof b && (this.isRecording ? this.startRecording_() : this.stopRecording_()) 369 | }; 370 | c.postAudioRequest_ = function(a, b) { 371 | var d = this; 372 | b = void 0 === b ? "recognize" : b; 373 | return new Promise(function(e, f) { 374 | var g = d.urlApi + b; 375 | g = d.urlProxy + "?url=" + encodeURIComponent(g); 376 | var h = new XMLHttpRequest; 377 | h.onload = function() { 378 | 200 <= h.status && 400 > h.status ? e(h.response) : f(h.response) 379 | }; 380 | h.onerror = function() { 381 | f(h.response) 382 | }; 383 | h.open("POST", g, !0); 384 | h.send(a) 385 | }) 386 | }; 387 | 388 | c.processAudio_ = function(a) { 389 | a = a.inputBuffer.getChannelData(0) || new Float32Array(this.bufferSize); 390 | for (var b = a.length, d = new Int16Array(b); b--;) d[b] = 32767 * Math.min(1, a[b]); 391 | 1 !== this.socket.readyState ? this.errorServer = "SERVICE_UNAVAILABLE" : this.socket.send(d.buffer) 392 | }; 393 | 394 | c.resetResults_ = function() { 395 | var a = this; 396 | this.resultsReady = !1; 397 | this.transcripts && this.transcripts.length && this.transcripts.forEach(function(b, d) { 398 | a.set("transcripts." + d + ".tempResult", null); 399 | a.set("transcripts." + d + ".results", null); 400 | a.set("transcripts." + d + ".enhancedResults", null); 401 | a.set("transcripts." + d + ".isEnabled", !1) 402 | }) 403 | }; 404 | c.setResults_ = function(a, b) { 405 | b = void 0 === b ? "default" : b; 406 | this.fileUploadCanceled || (b = this.transcripts.map(function(a) { 407 | return a.id 408 | }).indexOf(b), a = a.map(function(a) { 409 | if ("string" === typeof a) var b = a; 410 | else a.text ? b = a.text : a.alternatives && a.alternatives[0].transcript && (b = a.alternatives[0].transcript); 411 | return b 412 | }), this.resultsReady = !0, this.set("transcripts." + b + ".error", !1), this.set("transcripts." + b + ".isEnabled", !0), this.set("transcripts." + b + ".results", a), this.set("transcripts." + b + ".tempResult", null)) 413 | }; 414 | c.setTimeDisplay_ = function() { 415 | var a = Date.now() - this.startTime; 416 | a >= this.maxRecordTime ? this.stopRecording_() : 1E3 <= a && (a = 1E4 > a ? "0" + a.toString().slice(0, -3) : a.toString().slice(0, -3), this.timeDisplay = "00:" + a + " / 00:" + this.maxRecordTime / 1E3) 417 | }; 418 | 419 | c.startRecording_ = function() { 420 | var a = this; 421 | this.controlsDisabled = !0; 422 | this.fileUploadValue = null; 423 | this.resetResults_(); 424 | this.tabSelected = 0; 425 | this.timeDisplay = "00:00 / 00:" + this.maxRecordTime / 1E3; 426 | this.startTime = Date.now(); 427 | this.streamingAvailable ? this.startStreaming_() : (this.startFallback_(), this.updateInterval = setInterval(function() { 428 | a.setTimeDisplay_() 429 | }, 500)); 430 | mc({ 431 | type: "speech-api", 432 | name: "recordStarted" 433 | }) 434 | }; 435 | 436 | c.startFallback_ = function() { 437 | var a = this; 438 | navigator.getUserMedia({ 439 | audio: { 440 | mandatory: { 441 | googEchoCancellation: "false", 442 | googAutoGainControl: "false", 443 | googNoiseSuppression: "false", 444 | googHighpassFilter: "false" 445 | }, 446 | optional: [] 447 | } 448 | }, function(b) { 449 | var d = a.audioContext.createMediaStreamSource(b); 450 | a.mediaTrack = b.getTracks()[0]; 451 | a.recorder = new window.Recorder(d, { 452 | numChannels: 1, 453 | workerPath: a.recorderWorkerLib 454 | }); 455 | a.recorder.clear(); 456 | a.recorder.record() 457 | }, function() { 458 | a.errorClient = "MICROPHONE_INACCESSIBLE"; 459 | a.stopFallback_() 460 | }) 461 | }; 462 | 463 | c.startStreaming_ = function() { 464 | var a = this; 465 | this.socket = new WebSocket("wss://" + this.urlServer + "/ws"); 466 | this.socket.binaryType = "arraybuffer"; 467 | this.socket.onopen = function() { 468 | a.socket.send(JSON.stringify({ 469 | format: "LINEAR16", 470 | language: a.languageSelected, 471 | punctuation: a.punctuationEnabled, 472 | rate: a.audioContext.sampleRate 473 | })); 474 | navigator.getUserMedia({ 475 | audio: !0 476 | }, function(b) { 477 | a.tabSelected = 0; 478 | var d = a.audioContext.createMediaStreamSource(b); 479 | a.processor = a.audioContext.createScriptProcessor(a.bufferSize, 1, 1); 480 | a.processor.onaudioprocess = 481 | function(b) { 482 | a.processAudio_(b) 483 | }; 484 | a.processor.connect(a.audioContext.destination); 485 | d.connect(a.processor); 486 | a.mediaTrack = b.getTracks()[0]; 487 | a.updateInterval = setInterval(function() { 488 | a.setTimeDisplay_() 489 | }, 500) 490 | }, function() { 491 | a.errorClient = "MICROPHONE_INACCESSIBLE"; 492 | a.stopRecording_() 493 | }) 494 | }; 495 | this.socket.onmessage = function(b) { 496 | b = JSON.parse(b.data); 497 | b.isFinal ? a.transcripts[0].results ? (a.set("transcripts.0.tempResult", null), a.push("transcripts.0.results", b.text)) : a.setResults_([b.text]) : (a.resultsReady = !0, a.set("transcripts.0.isEnabled", !0), a.set("transcripts.0.tempResult", b.text)) 498 | }; 499 | this.socket.onclose = function(b) { 500 | 1006 === b.code ? a.errorServer = "SERVICE_UNAVAILABLE" : a.controlsDisabled = !1 501 | }; 502 | this.socket.onerror = function() { 503 | a.errorServer = "SERVICE_UNAVAILABLE" 504 | } 505 | }; 506 | 507 | c.stopFallback_ = function() { 508 | var a = this; 509 | this.controlsDisabled = !1; 510 | this.recorder && setTimeout(function() { 511 | a.recorder.stop(); 512 | a.mediaTrack && (a.mediaTrack.stop(), a.mediaTrack = {}); 513 | a.recorder.exportWAV(function(b) { 514 | a.fileUploadValue = b 515 | }) 516 | }, 500) 517 | }; 518 | 519 | c.stopRecording_ = function() { 520 | var a = Date.now() - this.startTime; 521 | this.isRecording = !1; 522 | clearInterval(this.updateInterval); 523 | this.timeDisplay = "00:00 / 00:" + this.maxRecordTime / 1E3; 524 | this.streamingAvailable ? this.stopStreaming_() : this.stopFallback_(); 525 | mc({ 526 | type: "speech-api", 527 | name: "recordStopped", 528 | metadata: { 529 | recordingTime: a 530 | } 531 | }) 532 | }; 533 | 534 | c.stopStreaming_ = function() { 535 | var a = this; 536 | setTimeout(function() { 537 | a.audioContext && "running" == a.audioContext.state && (a.processor && (a.processor.onaudioprocess = function() {}), a.processor = {}); 538 | a.socket && 1 == a.socket.readyState && (a.socket && a.socket.close(), a.socket = {}); 539 | a.set("transcripts.0.tempResult", null); 540 | a.mediaTrack && a.mediaTrack.stop && (a.mediaTrack.stop(), a.mediaTrack = {}) 541 | }, 1500); 542 | clearInterval(this.updateInterval); 543 | this.timeDisplay = "00:00 / 00:" + this.maxRecordTime / 1E3 544 | }; 545 | 546 | 547 | k.global.Object.defineProperties(Y.SpApp, { 548 | is: { 549 | configurable: !0, 550 | enumerable: !0, 551 | get: function() { 552 | return "sp-app" 553 | } 554 | }, 555 | properties: { 556 | configurable: !0, 557 | enumerable: !0, 558 | get: function() { 559 | return { 560 | audioContext: { 561 | type: Object 562 | }, 563 | audioDuration: { 564 | type: Number 565 | }, 566 | audioDurationLimit: { 567 | readOnly: !0, 568 | type: Number, 569 | value: 59 570 | }, 571 | bufferSize: { 572 | readOnly: !0, 573 | type: Number, 574 | value: 4096 575 | }, 576 | checkRecorder: { 577 | type: Number, 578 | value: 0 579 | }, 580 | controlsDisabled: { 581 | type: Boolean, 582 | value: !1 583 | }, 584 | errorClient: { 585 | notify: !0, 586 | type: String 587 | }, 588 | errorServer: { 589 | type: String 590 | }, 591 | fileUploadCanceled: { 592 | type: Boolean, 593 | value: !1 594 | }, 595 | fileUploadProgress: { 596 | notify: !0, 597 | type: Number, 598 | value: null 599 | }, 600 | fileUploadStatus: { 601 | notify: !0, 602 | type: String 603 | }, 604 | fileUploadValue: { 605 | observer: "fileUploadValueChanged_", 606 | type: Object, 607 | value: null 608 | }, 609 | inputTypes: { 610 | type: Array, 611 | value: function() { 612 | return [{ 613 | id: "microphone", 614 | name: uc.MICROPHONE 615 | }, { 616 | id: "file", 617 | name: uc.FILE 618 | }] 619 | } 620 | }, 621 | inputTypeSelected: { 622 | observer: "onInputTypeSelectedChanged_", 623 | type: String, 624 | value: "microphone" 625 | }, 626 | isRecording: { 627 | observer: "onIsRecordingChanged_", 628 | type: Boolean, 629 | value: !1 630 | }, 631 | languages: { 632 | type: Array, 633 | value: wc 634 | }, 635 | languageSelected: { 636 | type: String, 637 | value: "en-US" 638 | }, 639 | maxRecordTime: { 640 | type: Number, 641 | value: 5E4 642 | }, 643 | mediaTrack: { 644 | type: Object 645 | }, 646 | processor: { 647 | type: Object 648 | }, 649 | punctuationEnabled: { 650 | notify: !0, 651 | type: Boolean, 652 | value: !0 653 | }, 654 | recognitionModels: { 655 | notify: !0, 656 | type: Array, 657 | value: function() { 658 | return [{ 659 | id: "default", 660 | name: xc.DEFAULT 661 | }, { 662 | id: "command_and_search", 663 | name: xc.COMMAND_AND_SEARCH 664 | }, { 665 | id: "phone_call", 666 | name: xc.PHONE_CALL, 667 | showEnhanced: !0, 668 | enabledLanguages: ["en-US"] 669 | }, { 670 | id: "video", 671 | name: xc.VIDEO, 672 | enabledLanguages: ["en-US"] 673 | }] 674 | } 675 | }, 676 | recorder: { 677 | type: Object 678 | }, 679 | recorderLib: { 680 | type: String, 681 | value: "https://cloud.google.com/_static/js/recorder-bundle.js" 682 | }, 683 | recorderWorkerLib: { 684 | type: String, 685 | value: "https://cloud.google.com/_static/js/recorderWorker-bundle.js" 686 | }, 687 | resultsReady: { 688 | type: Boolean, 689 | value: !1 690 | }, 691 | sampleRate: { 692 | type: Number, 693 | value: null 694 | }, 695 | startTime: { 696 | type: Number 697 | }, 698 | messages: { 699 | type: Object, 700 | value: function() { 701 | return { 702 | error: vc 703 | } 704 | } 705 | }, 706 | socket: { 707 | type: Object 708 | }, 709 | streamingAvailable: { 710 | type: Boolean 711 | }, 712 | tabSelected: { 713 | notify: !0, 714 | type: Number, 715 | value: 0 716 | }, 717 | timeDisplay: { 718 | type: String, 719 | value: "00:00" 720 | }, 721 | transcripts: { 722 | notify: !0, 723 | type: Array, 724 | value: function() { 725 | return [] 726 | } 727 | }, 728 | updateInterval: { 729 | type: Number 730 | }, 731 | urlApi: { 732 | readOnly: !0, 733 | type: String, 734 | value: "https://speech.googleapis.com/v1p1beta1/speech:" 735 | }, 736 | urlProxy: { 737 | readOnly: !0, 738 | type: String, 739 | value: "https://cxl-services.appspot.com/proxy" 740 | }, 741 | urlServer: { 742 | type: String, 743 | value: "cloudspeech.goog" 744 | } 745 | } 746 | } 747 | } 748 | }); 749 | Y.SpApp.prototype._setAudioDurationLimit = function() {}; 750 | Y.SpApp.prototype._setBufferSize = function() {}; 751 | Y.SpApp.prototype._setUrlApi = function() {}; 752 | Y.SpApp.prototype._setUrlProxy = function() {}; 753 | customElements.define(Y.SpApp.is, Y.SpApp); 754 | r.SpApp = {}; 755 | r.SpApp.SpApp = {}; 756 | --------------------------------------------------------------------------------