├── _config.yml ├── samples └── browser │ ├── sample_app.js │ ├── package.json │ ├── README.md │ ├── sample-server.js │ └── Sample.html ├── src ├── common │ ├── IDetachable.ts │ ├── IDictionary.ts │ ├── IKeyValueStorage.ts │ ├── IWebsocketMessageFormatter.ts │ ├── ITimer.ts │ ├── ConnectionOpenResponse.ts │ ├── Guid.ts │ ├── IDisposable.ts │ ├── IEventSource.ts │ ├── IAudioSource.ts │ ├── Events.ts │ ├── IConnection.ts │ ├── Exports.ts │ ├── Storage.ts │ ├── PlatformEvent.ts │ ├── InMemoryStorage.ts │ ├── RawWebsocketMessage.ts │ ├── Error.ts │ ├── ConnectionMessage.ts │ ├── EventSource.ts │ ├── AudioSourceEvents.ts │ ├── ConnectionEvents.ts │ ├── RiffPcmEncoder.ts │ ├── Stream.ts │ ├── Queue.ts │ ├── List.ts │ └── Promise.ts ├── sdk │ ├── speech.browser │ │ ├── Exports.ts │ │ ├── Recognizer.ts │ │ └── SpeechConnectionFactory.ts │ └── speech │ │ ├── IConnectionFactory.ts │ │ ├── Exports.ts │ │ ├── IAuthentication.ts │ │ ├── CognitiveSubscriptionKeyAuthentication.ts │ │ ├── SpeechResults.ts │ │ ├── CognitiveTokenAuthentication.ts │ │ ├── SpeechConnectionMessage.Internal.ts │ │ ├── RecognizerConfig.ts │ │ ├── WebsocketMessageFormatter.ts │ │ ├── ServiceTelemetryListener.Internal.ts │ │ ├── RecognitionEvents.ts │ │ └── Recognizer.ts └── common.browser │ ├── IRecorder.ts │ ├── Exports.ts │ ├── Timer.ts │ ├── LocalStorage.ts │ ├── SessionStorage.ts │ ├── OpusRecorder.ts │ ├── ConsoleLoggingListener.ts │ ├── PCMRecorder.ts │ ├── WebsocketConnection.ts │ ├── FileAudioSource.ts │ ├── MicAudioSource.ts │ └── WebsocketMessageAdapter.ts ├── .npmignore ├── .gitignore ├── Speech.Browser.Sdk.ts ├── tslint.json ├── LICENSE ├── package.json ├── gulpfile.js └── README.md /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /samples/browser/sample_app.js: -------------------------------------------------------------------------------- 1 | window.SDK = require('../../distrib/Speech.Browser.Sdk.js') -------------------------------------------------------------------------------- /src/common/IDetachable.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface IDetachable { 3 | Detach(): void; 4 | } 5 | -------------------------------------------------------------------------------- /src/sdk/speech.browser/Exports.ts: -------------------------------------------------------------------------------- 1 | 2 | export * from "./Recognizer"; 3 | export * from "./SpeechConnectionFactory"; 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /.vscode/* 2 | /samples/* 3 | /node_modules/* 4 | /.gitattributes 5 | /.gitignore 6 | /.npmignore 7 | /.npmrc 8 | /.vscode 9 | /web.config 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore the npm packages 2 | /**/node_modules/ 3 | # ignore auto-generated (transpiled) js files 4 | src/**/*.js 5 | distrib/* 6 | 7 | /**/speech.key 8 | /samples/**/package-lock.json 9 | .idea/ 10 | -------------------------------------------------------------------------------- /src/common/IDictionary.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface IStringDictionary { 3 | [propName: string]: TValue; 4 | } 5 | 6 | export interface INumberDictionary extends Object { 7 | [propName: number]: TValue; 8 | } 9 | -------------------------------------------------------------------------------- /src/common/IKeyValueStorage.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface IKeyValueStorage { 3 | Get(key: string): string; 4 | GetOrAdd(key: string, valueToAdd: string): string; 5 | Set(key: string, value: string): void; 6 | Remove(key: string): void; 7 | } 8 | -------------------------------------------------------------------------------- /src/common.browser/IRecorder.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from "../common/Exports"; 2 | 3 | export interface IRecorder { 4 | Record(context: AudioContext, mediaStream: MediaStream, outputStream: Stream): void; 5 | ReleaseMediaResources(context: AudioContext): void; 6 | } 7 | -------------------------------------------------------------------------------- /samples/browser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sample_speech_server", 3 | "description": "A simple http server to facilitate SDK testing on mobile devices as well as provide a token authentication example", 4 | "main": "sample-server.js", 5 | "dependencies": { 6 | "localtunnel": "^1.8.3", 7 | "qrcode-terminal": "^0.11.0" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/browser/README.md: -------------------------------------------------------------------------------- 1 | To launch sample-server.js 2 | 1. `echo YOUR_BING_SPEECH_API_KEY > speech.key` 3 | 4 | 2. `npm install` 5 | 6 | 3. * The server and the client are on the same subnet: 7 | `node sample-server.js`. 8 | * The server and the client are on different subnets, server is not accessible from the outside: 9 | `node sample-server.js enableTunnel` 10 | -------------------------------------------------------------------------------- /src/sdk/speech/IConnectionFactory.ts: -------------------------------------------------------------------------------- 1 | import { IConnection } from "../../common/Exports"; 2 | import { AuthInfo } from "./IAuthentication"; 3 | import { RecognizerConfig } from "./RecognizerConfig"; 4 | 5 | export interface IConnectionFactory { 6 | Create( 7 | config: RecognizerConfig, 8 | authInfo: AuthInfo, 9 | connectionId?: string): IConnection; 10 | } 11 | -------------------------------------------------------------------------------- /src/common/IWebsocketMessageFormatter.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionMessage } from "./ConnectionMessage"; 2 | import { Promise } from "./Promise"; 3 | import { RawWebsocketMessage } from "./RawWebsocketMessage"; 4 | 5 | export interface IWebsocketMessageFormatter { 6 | ToConnectionMessage(message: RawWebsocketMessage): Promise; 7 | FromConnectionMessage(message: ConnectionMessage): Promise; 8 | } 9 | -------------------------------------------------------------------------------- /src/common.browser/Exports.ts: -------------------------------------------------------------------------------- 1 | 2 | export * from "./ConsoleLoggingListener"; 3 | export * from "./IRecorder"; 4 | export * from "./LocalStorage"; 5 | export * from "./MicAudioSource"; 6 | export * from "./FileAudioSource"; 7 | export * from "./OpusRecorder"; 8 | export * from "./PCMRecorder"; 9 | export * from "./SessionStorage"; 10 | export * from "./Timer"; 11 | export * from "./WebsocketConnection"; 12 | export * from "./WebsocketMessageAdapter"; 13 | -------------------------------------------------------------------------------- /src/common/ITimer.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface ITimer { 3 | /** 4 | * start timer 5 | * 6 | * @param {number} delay 7 | * @param {(...args: any[]) => any} successCallback 8 | * @returns {*} 9 | * 10 | * @memberOf ITimer 11 | */ 12 | start(): void; 13 | 14 | /** 15 | * stops timer 16 | * 17 | * @param {*} timerId 18 | * 19 | * @memberOf ITimer 20 | */ 21 | stop(): void; 22 | } 23 | -------------------------------------------------------------------------------- /src/sdk/speech/Exports.ts: -------------------------------------------------------------------------------- 1 | 2 | // IMPORTANT - Dont publish internal modules. 3 | 4 | export * from "./CognitiveSubscriptionKeyAuthentication"; 5 | export * from "./CognitiveTokenAuthentication"; 6 | export * from "./IAuthentication"; 7 | export * from "./IConnectionFactory"; 8 | export * from "./RecognitionEvents"; 9 | export * from "./Recognizer"; 10 | export * from "./RecognizerConfig"; 11 | export * from "./SpeechResults"; 12 | export * from "./WebsocketMessageFormatter"; 13 | -------------------------------------------------------------------------------- /src/common/ConnectionOpenResponse.ts: -------------------------------------------------------------------------------- 1 | 2 | export class ConnectionOpenResponse { 3 | private statusCode: number; 4 | private reason: string; 5 | 6 | constructor(statusCode: number, reason: string) { 7 | this.statusCode = statusCode; 8 | this.reason = reason; 9 | } 10 | 11 | public get StatusCode(): number { 12 | return this.statusCode; 13 | } 14 | 15 | public get Reason(): string { 16 | return this.reason; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Speech.Browser.Sdk.ts: -------------------------------------------------------------------------------- 1 | import { ConsoleLoggingListener, LocalStorage, SessionStorage } from "./src/common.browser/Exports"; 2 | import { Events, Storage } from "./src/common/Exports"; 3 | 4 | // Common.Storage.SetLocalStorage(new Common.Browser.LocalStorage()); 5 | // Common.Storage.SetSessionStorage(new Common.Browser.SessionStorage()); 6 | Events.Instance.AttachListener(new ConsoleLoggingListener()); 7 | 8 | export * from "./src/common/Exports"; 9 | export * from "./src/common.browser/Exports"; 10 | export * from "./src/sdk/speech/Exports"; 11 | export * from "./src/sdk/speech.browser/Exports"; 12 | -------------------------------------------------------------------------------- /src/common/Guid.ts: -------------------------------------------------------------------------------- 1 | 2 | const CreateGuid: () => string = (): string => { 3 | let d = new Date().getTime(); 4 | const guid = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, (c: string) => { 5 | const r = (d + Math.random() * 16) % 16 | 0; 6 | d = Math.floor(d / 16); 7 | return (c === "x" ? r : (r & 0x3 | 0x8)).toString(16); 8 | }); 9 | 10 | return guid; 11 | }; 12 | 13 | const CreateNoDashGuid: () => string = (): string => { 14 | return CreateGuid().replace(new RegExp("-", "g"), "").toUpperCase(); 15 | }; 16 | 17 | export { CreateGuid, CreateNoDashGuid }; 18 | -------------------------------------------------------------------------------- /src/common/IDisposable.ts: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * 4 | * 5 | * @export 6 | * @interface IDisposable 7 | */ 8 | export interface IDisposable { 9 | 10 | /** 11 | * 12 | * 13 | * @returns {boolean} 14 | * 15 | * @memberOf IDisposable 16 | */ 17 | IsDisposed(): boolean; 18 | 19 | /** 20 | * Performs cleanup operations on this instance 21 | * 22 | * @param {string} [reason] optional reason for disposing the instance. 23 | * This will be used to throw errors when a operations are performed on the disposed object. 24 | * 25 | * @memberOf IDisposable 26 | */ 27 | Dispose(reason?: string): void; 28 | } 29 | -------------------------------------------------------------------------------- /src/common/IEventSource.ts: -------------------------------------------------------------------------------- 1 | import { IDetachable } from "./IDetachable"; 2 | import { IStringDictionary } from "./IDictionary"; 3 | import { IDisposable } from "./IDisposable"; 4 | import { PlatformEvent } from "./PlatformEvent"; 5 | 6 | export interface IEventListener { 7 | OnEvent(e: TEvent): void; 8 | } 9 | 10 | export interface IEventSource extends IDisposable { 11 | Metadata: IStringDictionary; 12 | 13 | OnEvent(e: TEvent): void; 14 | 15 | Attach(onEventCallback: (event: TEvent) => void): IDetachable; 16 | 17 | AttachListener(listener: IEventListener): IDetachable; 18 | } 19 | -------------------------------------------------------------------------------- /src/sdk/speech/IAuthentication.ts: -------------------------------------------------------------------------------- 1 | import { Promise } from "../../common/Exports"; 2 | 3 | export interface IAuthentication { 4 | Fetch(authFetchEventId: string): Promise; 5 | FetchOnExpiry(authFetchEventId: string): Promise; 6 | } 7 | 8 | export class AuthInfo { 9 | private headerName: string; 10 | private token: string; 11 | 12 | public constructor(headerName: string, token: string) { 13 | this.headerName = headerName; 14 | this.token = token; 15 | } 16 | 17 | public get HeaderName(): string { 18 | return this.headerName; 19 | } 20 | 21 | public get Token(): string { 22 | return this.token; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/common/IAudioSource.ts: -------------------------------------------------------------------------------- 1 | import { AudioSourceEvent } from "./AudioSourceEvents"; 2 | import { EventSource } from "./EventSource"; 3 | import { IDetachable } from "./IDetachable"; 4 | import { Promise } from "./Promise"; 5 | import { IStreamChunk } from "./Stream"; 6 | 7 | export interface IAudioSource { 8 | Id(): string; 9 | TurnOn(): Promise; 10 | Attach(audioNodeId: string): Promise; 11 | Detach(audioNodeId: string): void; 12 | TurnOff(): Promise; 13 | Events: EventSource; 14 | } 15 | 16 | export interface IAudioStreamNode extends IDetachable { 17 | Id(): string; 18 | Read(): Promise>; 19 | } 20 | -------------------------------------------------------------------------------- /src/common/Events.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError } from "./Error"; 2 | import { EventSource } from "./EventSource"; 3 | import { IEventSource } from "./IEventSource"; 4 | import { PlatformEvent } from "./PlatformEvent"; 5 | 6 | export class Events { 7 | private static instance: IEventSource = new EventSource(); 8 | 9 | public static SetEventSource = (eventSource: IEventSource): void => { 10 | if (!eventSource) { 11 | throw new ArgumentNullError("eventSource"); 12 | } 13 | 14 | Events.instance = eventSource; 15 | } 16 | 17 | public static get Instance(): IEventSource { 18 | return Events.instance; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/common.browser/Timer.ts: -------------------------------------------------------------------------------- 1 | import { ITimer } from "../common/Exports"; 2 | 3 | export class Timer implements ITimer { 4 | private delayInMillisec: number; 5 | private timerId: number; 6 | private successCallback: any; 7 | constructor(delayInMillisec: number, successCallback: any) { 8 | this.delayInMillisec = delayInMillisec; 9 | this.successCallback = successCallback; 10 | } 11 | public start = (...params: any[]): void => { 12 | if (this.timerId) { 13 | this.stop(); 14 | } 15 | this.timerId = setTimeout(this.successCallback, this.delayInMillisec, params); 16 | } 17 | 18 | public stop = (): void => { 19 | clearTimeout(this.timerId); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/common/IConnection.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionEvent } from "./ConnectionEvents"; 2 | import { ConnectionMessage } from "./ConnectionMessage"; 3 | import { ConnectionOpenResponse } from "./ConnectionOpenResponse"; 4 | import { EventSource } from "./EventSource"; 5 | import { IDisposable } from "./IDisposable"; 6 | import { Promise } from "./Promise"; 7 | 8 | export enum ConnectionState { 9 | None, 10 | Connected, 11 | Connecting, 12 | Disconnected, 13 | } 14 | 15 | export interface IConnection extends IDisposable { 16 | Id: string; 17 | State(): ConnectionState; 18 | Open(): Promise; 19 | Send(message: ConnectionMessage): Promise; 20 | Read(): Promise; 21 | Events: EventSource; 22 | } 23 | -------------------------------------------------------------------------------- /src/common/Exports.ts: -------------------------------------------------------------------------------- 1 | 2 | export * from "./AudioSourceEvents"; 3 | export * from "./ConnectionEvents"; 4 | export * from "./ConnectionMessage"; 5 | export * from "./ConnectionOpenResponse"; 6 | export * from "./Error"; 7 | export * from "./Events"; 8 | export * from "./EventSource"; 9 | export * from "./Guid"; 10 | export * from "./IAudioSource"; 11 | export * from "./IConnection"; 12 | export * from "./IDetachable"; 13 | export * from "./IDictionary"; 14 | export * from "./IDisposable"; 15 | export * from "./IEventSource"; 16 | export * from "./IKeyValueStorage"; 17 | export * from "./InMemoryStorage"; 18 | export * from "./ITimer"; 19 | export * from "./IWebsocketMessageFormatter"; 20 | export * from "./List"; 21 | export * from "./PlatformEvent"; 22 | export * from "./Promise"; 23 | export * from "./Queue"; 24 | export * from "./RawWebsocketMessage"; 25 | export * from "./RiffPcmEncoder"; 26 | export * from "./Storage"; 27 | export * from "./Stream"; 28 | -------------------------------------------------------------------------------- /src/sdk/speech/CognitiveSubscriptionKeyAuthentication.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError, Promise, PromiseHelper } from "../../common/Exports"; 2 | import { AuthInfo, IAuthentication } from "./IAuthentication"; 3 | 4 | const AuthHeader: string = "Ocp-Apim-Subscription-Key"; 5 | 6 | export class CognitiveSubscriptionKeyAuthentication implements IAuthentication { 7 | private authInfo: AuthInfo; 8 | 9 | constructor(subscriptionKey: string) { 10 | if (!subscriptionKey) { 11 | throw new ArgumentNullError("subscriptionKey"); 12 | } 13 | 14 | this.authInfo = new AuthInfo(AuthHeader, subscriptionKey); 15 | } 16 | 17 | public Fetch = (authFetchEventId: string): Promise => { 18 | return PromiseHelper.FromResult(this.authInfo); 19 | } 20 | 21 | public FetchOnExpiry = (authFetchEventId: string): Promise => { 22 | return PromiseHelper.FromResult(this.authInfo); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/sdk/speech/SpeechResults.ts: -------------------------------------------------------------------------------- 1 | 2 | export enum RecognitionStatus { 3 | Success, 4 | NoMatch, 5 | InitialSilenceTimeout, 6 | BabbleTimeout, 7 | Error, 8 | EndOfDictation, 9 | } 10 | 11 | export interface ISpeechStartDetectedResult { 12 | Offset?: number; 13 | } 14 | 15 | export interface ISpeechFragment { 16 | Text: string; 17 | Offset?: number; 18 | Duration?: number; 19 | } 20 | 21 | export interface ISpeechEndDetectedResult { 22 | Offset?: number; 23 | } 24 | 25 | export interface ISimpleSpeechPhrase { 26 | RecognitionStatus: RecognitionStatus; 27 | DisplayText: string; 28 | Duration?: number; 29 | Offset?: number; 30 | } 31 | 32 | export interface IDetailedSpeechPhrase { 33 | RecognitionStatus: RecognitionStatus; 34 | NBest: IPhrase[]; 35 | Duration?: number; 36 | Offset?: number; 37 | } 38 | 39 | export interface IPhrase { 40 | Confidence?: number; 41 | Lexical: string; 42 | ITN: string; 43 | MaskedITN: string; 44 | Display: string; 45 | } 46 | -------------------------------------------------------------------------------- /src/common/Storage.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError } from "./Error"; 2 | import { IKeyValueStorage } from "./IKeyValueStorage"; 3 | import { InMemoryStorage } from "./InMemoryStorage"; 4 | 5 | export class Storage { 6 | private static sessionStorage: IKeyValueStorage = new InMemoryStorage(); 7 | private static localStorage: IKeyValueStorage = new InMemoryStorage(); 8 | 9 | public static SetSessionStorage = (sessionStorage: IKeyValueStorage): void => { 10 | if (!sessionStorage) { 11 | throw new ArgumentNullError("sessionStorage"); 12 | } 13 | 14 | Storage.sessionStorage = sessionStorage; 15 | } 16 | 17 | public static SetLocalStorage = (localStorage: IKeyValueStorage): void => { 18 | if (!localStorage) { 19 | throw new ArgumentNullError("localStorage"); 20 | } 21 | 22 | Storage.localStorage = localStorage; 23 | } 24 | 25 | public static get Session(): IKeyValueStorage { 26 | return Storage.sessionStorage; 27 | } 28 | 29 | public static get Local(): IKeyValueStorage { 30 | return Storage.localStorage; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | /* 3 | Using recommended community rules 4 | https://github.com/palantir/tslint/blob/master/src/configs/recommended.ts 5 | */ 6 | "extends": "tslint:recommended", 7 | "rules": { 8 | 9 | "max-line-length": [false], 10 | 11 | "member-ordering":[ 12 | true, { 13 | "order": [ 14 | "private-static-field", 15 | "private-instance-field", 16 | "constructor", 17 | "public-static-method", 18 | "public-instance-method", 19 | "protected-static-method", 20 | "protected-instance-method", 21 | "private-static-method", 22 | "private-instance-method" 23 | ] 24 | } 25 | ], 26 | 27 | "no-reference": true, 28 | "no-namespace": true, 29 | "no-bitwise": false, 30 | "no-shadowed-variable": false, 31 | 32 | "only-arrow-functions": [ 33 | true 34 | ], 35 | 36 | /* Enabling strictest possible type checking on every thing we write */ 37 | "typedef": [ 38 | true, 39 | "arrow-parameter", 40 | "call-signature", 41 | "member-variable-declaration", 42 | "parameter", 43 | "property-declaration" 44 | ] 45 | } 46 | } -------------------------------------------------------------------------------- /src/common/PlatformEvent.ts: -------------------------------------------------------------------------------- 1 | import { CreateNoDashGuid } from "./Guid"; 2 | import { IStringDictionary } from "./IDictionary"; 3 | 4 | export enum EventType { 5 | Debug, 6 | Info, 7 | Warning, 8 | Error, 9 | } 10 | 11 | export class PlatformEvent { 12 | private name: string; 13 | private eventId: string; 14 | private eventTime: string; 15 | private eventType: EventType; 16 | private metadata: IStringDictionary; 17 | 18 | constructor(eventName: string, eventType: EventType) { 19 | this.name = eventName; 20 | this.eventId = CreateNoDashGuid(); 21 | this.eventTime = new Date().toISOString(); 22 | this.eventType = eventType; 23 | this.metadata = { }; 24 | } 25 | 26 | public get Name(): string { 27 | return this.name; 28 | } 29 | 30 | public get EventId(): string { 31 | return this.eventId; 32 | } 33 | 34 | public get EventTime(): string { 35 | return this.eventTime; 36 | } 37 | 38 | public get EventType(): EventType { 39 | return this.eventType; 40 | } 41 | 42 | public get Metadata(): IStringDictionary { 43 | return this.metadata; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/common.browser/LocalStorage.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError, IKeyValueStorage } from "../common/Exports"; 2 | 3 | export class LocalStorage implements IKeyValueStorage { 4 | 5 | public Get = (key: string): string => { 6 | if (!key) { 7 | throw new ArgumentNullError("key"); 8 | } 9 | 10 | return localStorage.getItem(key); 11 | } 12 | 13 | public GetOrAdd = (key: string, valueToAdd: string): string => { 14 | if (!key) { 15 | throw new ArgumentNullError("key"); 16 | } 17 | 18 | const value = localStorage.getItem(key); 19 | if (value === null || value === undefined) { 20 | localStorage.setItem(key, valueToAdd); 21 | } 22 | 23 | return localStorage.getItem(key); 24 | } 25 | 26 | public Set = (key: string, value: string): void => { 27 | if (!key) { 28 | throw new ArgumentNullError("key"); 29 | } 30 | 31 | localStorage.setItem(key, value); 32 | } 33 | 34 | public Remove = (key: string): void => { 35 | if (!key) { 36 | throw new ArgumentNullError("key"); 37 | } 38 | 39 | localStorage.removeItem(key); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/common.browser/SessionStorage.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError, IKeyValueStorage } from "../common/Exports"; 2 | 3 | export class SessionStorage implements IKeyValueStorage { 4 | 5 | public Get = (key: string): string => { 6 | if (!key) { 7 | throw new ArgumentNullError("key"); 8 | } 9 | 10 | return sessionStorage.getItem(key); 11 | } 12 | 13 | public GetOrAdd = (key: string, valueToAdd: string): string => { 14 | if (!key) { 15 | throw new ArgumentNullError("key"); 16 | } 17 | 18 | const value = sessionStorage.getItem(key); 19 | if (value === null || value === undefined) { 20 | sessionStorage.setItem(key, valueToAdd); 21 | } 22 | 23 | return sessionStorage.getItem(key); 24 | } 25 | 26 | public Set = (key: string, value: string): void => { 27 | if (!key) { 28 | throw new ArgumentNullError("key"); 29 | } 30 | 31 | sessionStorage.setItem(key, value); 32 | } 33 | 34 | public Remove = (key: string): void => { 35 | if (!key) { 36 | throw new ArgumentNullError("key"); 37 | } 38 | 39 | sessionStorage.removeItem(key); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /src/common/InMemoryStorage.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError } from "./Error"; 2 | import { IStringDictionary } from "./IDictionary"; 3 | import { IKeyValueStorage } from "./IKeyValueStorage"; 4 | 5 | export class InMemoryStorage implements IKeyValueStorage { 6 | 7 | private store: IStringDictionary = {}; 8 | 9 | public Get = (key: string): string => { 10 | if (!key) { 11 | throw new ArgumentNullError("key"); 12 | } 13 | 14 | return this.store[key]; 15 | } 16 | 17 | public GetOrAdd = (key: string, valueToAdd: string): string => { 18 | if (!key) { 19 | throw new ArgumentNullError("key"); 20 | } 21 | 22 | if (this.store[key] === undefined) { 23 | this.store[key] = valueToAdd; 24 | } 25 | 26 | return this.store[key]; 27 | } 28 | 29 | public Set = (key: string, value: string): void => { 30 | if (!key) { 31 | throw new ArgumentNullError("key"); 32 | } 33 | 34 | this.store[key] = value; 35 | } 36 | 37 | public Remove = (key: string): void => { 38 | if (!key) { 39 | throw new ArgumentNullError("key"); 40 | } 41 | 42 | if (this.store[key] !== undefined) { 43 | delete this.store[key]; 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/sdk/speech/CognitiveTokenAuthentication.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError, Promise, PromiseHelper } from "../../common/Exports"; 2 | import { AuthInfo, IAuthentication } from "./IAuthentication"; 3 | 4 | const AuthHeader: string = "Authorization"; 5 | 6 | export class CognitiveTokenAuthentication implements IAuthentication { 7 | private fetchCallback: (authFetchEventId: string) => Promise; 8 | private fetchOnExpiryCallback: (authFetchEventId: string) => Promise; 9 | 10 | constructor(fetchCallback: (authFetchEventId: string) => Promise, fetchOnExpiryCallback: (authFetchEventId: string) => Promise) { 11 | if (!fetchCallback) { 12 | throw new ArgumentNullError("fetchCallback"); 13 | } 14 | 15 | if (!fetchOnExpiryCallback) { 16 | throw new ArgumentNullError("fetchOnExpiryCallback"); 17 | } 18 | 19 | this.fetchCallback = fetchCallback; 20 | this.fetchOnExpiryCallback = fetchOnExpiryCallback; 21 | } 22 | 23 | public Fetch = (authFetchEventId: string): Promise => { 24 | return this.fetchCallback(authFetchEventId).OnSuccessContinueWith((token: string) => new AuthInfo(AuthHeader, token)); 25 | } 26 | 27 | public FetchOnExpiry = (authFetchEventId: string): Promise => { 28 | return this.fetchOnExpiryCallback(authFetchEventId).OnSuccessContinueWith((token: string) => new AuthInfo(AuthHeader, token)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "microsoft-speech-browser-sdk", 3 | "version": "0.0.12", 4 | "description": "Microsoft Speech SDK for browsers", 5 | "main": "distrib/Speech.Browser.Sdk.js", 6 | "types": "distrib/speech.browser.sdk.d.ts", 7 | "keywords": [ 8 | "microsoft", 9 | "speech", 10 | "sdk", 11 | "javascript", 12 | "typescript", 13 | "ts", 14 | "js", 15 | "browser", 16 | "websocket", 17 | "speechtotext" 18 | ], 19 | "bugs": { 20 | "url": "https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript/issues" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript.git" 25 | }, 26 | "author": "Microsoft", 27 | "license": "MIT", 28 | "devDependencies": { 29 | "gulp": "^3.9.1", 30 | "gulp-bump": "^2.8.0", 31 | "gulp-git": "^2.2.0", 32 | "gulp-minify": "0.0.15", 33 | "gulp-sourcemaps": "^2.6.0", 34 | "gulp-tag-version": "^1.3.0", 35 | "gulp-tslint": "^8.0.0", 36 | "gulp-typescript": "^3.2.3", 37 | "source-map-loader": "^0.2.3", 38 | "tslint": "^5.8.0", 39 | "typescript": "^2.6.2", 40 | "webpack-stream": "^4.0.0" 41 | }, 42 | "scripts": { 43 | "prepublishOnly": "npm install & gulp build", 44 | "build": "npm install & gulp build", 45 | "bundle": "npm install & gulp bundle", 46 | "patchRelease": "npm install & gulp patchRelease", 47 | "featureRelease": "npm install & gulp featureRelease", 48 | "majorRelease": "npm install & gulp majorRelease", 49 | "preRelease": "npm install & gulp preRelease" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/sdk/speech.browser/Recognizer.ts: -------------------------------------------------------------------------------- 1 | import { FileAudioSource, MicAudioSource, PcmRecorder } from "../../common.browser/Exports"; 2 | import { IAudioSource, Promise, Storage } from "../../common/Exports"; 3 | import { IAuthentication, Recognizer, RecognizerConfig } from "../speech/Exports"; 4 | import { SpeechConnectionFactory } from "./SpeechConnectionFactory"; 5 | 6 | const CreateRecognizer = (recognizerConfig: RecognizerConfig, authentication: IAuthentication): Recognizer => { 7 | return CreateRecognizerWithPcmRecorder( 8 | recognizerConfig, 9 | authentication); 10 | }; 11 | 12 | const CreateRecognizerWithPcmRecorder = (recognizerConfig: RecognizerConfig, authentication: IAuthentication): Recognizer => { 13 | return CreateRecognizerWithCustomAudioSource( 14 | recognizerConfig, 15 | authentication, 16 | new MicAudioSource(new PcmRecorder())); 17 | }; 18 | 19 | const CreateRecognizerWithFileAudioSource = (recognizerConfig: RecognizerConfig, authentication: IAuthentication, file: File): Recognizer => { 20 | return CreateRecognizerWithCustomAudioSource( 21 | recognizerConfig, 22 | authentication, 23 | new FileAudioSource(file)); 24 | }; 25 | 26 | const CreateRecognizerWithCustomAudioSource = (recognizerConfig: RecognizerConfig, authentication: IAuthentication, audioSource: IAudioSource): Recognizer => { 27 | return new Recognizer ( 28 | authentication, 29 | new SpeechConnectionFactory(), 30 | audioSource, 31 | recognizerConfig); 32 | }; 33 | 34 | export { CreateRecognizer, CreateRecognizerWithCustomAudioSource, CreateRecognizerWithFileAudioSource, CreateRecognizerWithPcmRecorder }; 35 | -------------------------------------------------------------------------------- /src/common/RawWebsocketMessage.ts: -------------------------------------------------------------------------------- 1 | import { MessageType } from "./ConnectionMessage"; 2 | import { ArgumentNullError, InvalidOperationError } from "./Error"; 3 | import { CreateNoDashGuid } from "./Guid"; 4 | 5 | export class RawWebsocketMessage { 6 | 7 | private messageType: MessageType; 8 | private payload: any = null; 9 | private id: string; 10 | 11 | public constructor(messageType: MessageType, payload: any, id?: string) { 12 | if (!payload) { 13 | throw new ArgumentNullError("payload"); 14 | } 15 | 16 | if (messageType === MessageType.Binary && !(payload instanceof ArrayBuffer)) { 17 | throw new InvalidOperationError("Payload must be ArrayBuffer"); 18 | } 19 | 20 | if (messageType === MessageType.Text && !(typeof (payload) === "string")) { 21 | throw new InvalidOperationError("Payload must be a string"); 22 | } 23 | 24 | this.messageType = messageType; 25 | this.payload = payload; 26 | this.id = id ? id : CreateNoDashGuid(); 27 | } 28 | 29 | public get MessageType(): MessageType { 30 | return this.messageType; 31 | } 32 | 33 | public get Payload(): any { 34 | return this.payload; 35 | } 36 | 37 | public get TextContent(): string { 38 | if (this.messageType === MessageType.Binary) { 39 | throw new InvalidOperationError("Not supported for binary message"); 40 | } 41 | 42 | return this.payload as string; 43 | } 44 | 45 | public get BinaryContent(): ArrayBuffer { 46 | if (this.messageType === MessageType.Text) { 47 | throw new InvalidOperationError("Not supported for text message"); 48 | } 49 | 50 | return this.payload; 51 | } 52 | 53 | public get Id(): string { 54 | return this.id; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/common/Error.ts: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * The error that is thrown when an argument passed in is null. 4 | * 5 | * @export 6 | * @class ArgumentNullError 7 | * @extends {Error} 8 | */ 9 | export class ArgumentNullError extends Error { 10 | 11 | /** 12 | * Creates an instance of ArgumentNullError. 13 | * 14 | * @param {string} argumentName Name of the argument that is null 15 | * 16 | * @memberOf ArgumentNullError 17 | */ 18 | public constructor(argumentName: string) { 19 | super(argumentName); 20 | this.name = "ArgumentNull"; 21 | this.message = argumentName; 22 | } 23 | } 24 | 25 | /** 26 | * The error that is thrown when an invalid operation is performed in the code. 27 | * 28 | * @export 29 | * @class InvalidOperationError 30 | * @extends {Error} 31 | */ 32 | // tslint:disable-next-line:max-classes-per-file 33 | export class InvalidOperationError extends Error { 34 | 35 | /** 36 | * Creates an instance of InvalidOperationError. 37 | * 38 | * @param {string} error The error 39 | * 40 | * @memberOf InvalidOperationError 41 | */ 42 | public constructor(error: string) { 43 | super(error); 44 | this.name = "InvalidOperation"; 45 | this.message = error; 46 | } 47 | } 48 | 49 | /** 50 | * The error that is thrown when an object is disposed. 51 | * 52 | * @export 53 | * @class ObjectDisposedError 54 | * @extends {Error} 55 | */ 56 | // tslint:disable-next-line:max-classes-per-file 57 | export class ObjectDisposedError extends Error { 58 | 59 | /** 60 | * Creates an instance of ObjectDisposedError. 61 | * 62 | * @param {string} objectName The object that is disposed 63 | * @param {string} error The error 64 | * 65 | * @memberOf ObjectDisposedError 66 | */ 67 | public constructor(objectName: string, error?: string) { 68 | super(error); 69 | this.name = objectName + "ObjectDisposed"; 70 | this.message = error; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/common/ConnectionMessage.ts: -------------------------------------------------------------------------------- 1 | import { InvalidOperationError } from "./Error"; 2 | import { CreateNoDashGuid } from "./Guid"; 3 | import { IStringDictionary } from "./IDictionary"; 4 | 5 | export enum MessageType { 6 | Text, 7 | Binary, 8 | } 9 | 10 | export class ConnectionMessage { 11 | 12 | private messageType: MessageType; 13 | private headers: IStringDictionary; 14 | private body: any = null; 15 | 16 | private id: string; 17 | 18 | public constructor( 19 | messageType: MessageType, 20 | body: any, 21 | headers?: IStringDictionary, 22 | id?: string) { 23 | 24 | if (messageType === MessageType.Text && body && !(typeof (body) === "string")) { 25 | throw new InvalidOperationError("Payload must be a string"); 26 | } 27 | 28 | if (messageType === MessageType.Binary && body && !(body instanceof ArrayBuffer)) { 29 | throw new InvalidOperationError("Payload must be ArrayBuffer"); 30 | } 31 | 32 | this.messageType = messageType; 33 | this.body = body; 34 | this.headers = headers ? headers : {}; 35 | this.id = id ? id : CreateNoDashGuid(); 36 | } 37 | 38 | public get MessageType(): MessageType { 39 | return this.messageType; 40 | } 41 | 42 | public get Headers(): any { 43 | return this.headers; 44 | } 45 | 46 | public get Body(): any { 47 | return this.body; 48 | } 49 | 50 | public get TextBody(): string { 51 | if (this.messageType === MessageType.Binary) { 52 | throw new InvalidOperationError("Not supported for binary message"); 53 | } 54 | 55 | return this.body as string; 56 | } 57 | 58 | public get BinaryBody(): ArrayBuffer { 59 | if (this.messageType === MessageType.Text) { 60 | throw new InvalidOperationError("Not supported for text message"); 61 | } 62 | 63 | return this.body; 64 | } 65 | 66 | public get Id(): string { 67 | return this.id; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/common.browser/OpusRecorder.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from "../common/Exports"; 2 | import { IRecorder } from "./IRecorder"; 3 | 4 | // getting around the build error for MediaRecorder as Typescript does not have a definition for this one. 5 | declare var MediaRecorder: any; 6 | 7 | export class OpusRecorder implements IRecorder { 8 | private mediaResources: IMediaResources; 9 | private mediaRecorderOptions: any; 10 | 11 | constructor(options?: { mimeType: string, bitsPerSecond: number }) { 12 | this.mediaRecorderOptions = options; 13 | } 14 | 15 | public Record = (context: AudioContext, mediaStream: MediaStream, outputStream: Stream): void => { 16 | const mediaRecorder: any = new MediaRecorder(mediaStream, this.mediaRecorderOptions); 17 | const timeslice = 100; // this is in ms - 100 ensures that the chunk doesn't exceed the max size of chunk allowed in WS connection 18 | mediaRecorder.ondataavailable = (dataAvailableEvent: any) => { 19 | if (outputStream) { 20 | const reader = new FileReader(); 21 | reader.readAsArrayBuffer(dataAvailableEvent.data); 22 | reader.onloadend = (event: ProgressEvent) => { 23 | outputStream.Write(reader.result); 24 | }; 25 | } 26 | }; 27 | 28 | this.mediaResources = { 29 | recorder: mediaRecorder, 30 | stream: mediaStream, 31 | }; 32 | mediaRecorder.start(timeslice); 33 | } 34 | 35 | public ReleaseMediaResources = (context: AudioContext): void => { 36 | if (this.mediaResources.recorder.state !== "inactive") { 37 | this.mediaResources.recorder.stop(); 38 | } 39 | this.mediaResources.stream.getTracks().forEach((track: any) => track.stop()); 40 | } 41 | } 42 | 43 | interface IMediaResources { 44 | stream: MediaStream; 45 | recorder: any; 46 | } 47 | 48 | /* Declaring this inline to avoid compiler warnings 49 | declare class MediaRecorder { 50 | constructor(mediaStream: MediaStream, options: any); 51 | 52 | public state: string; 53 | 54 | public ondataavailable(dataAvailableEvent: any): void; 55 | public stop(): void; 56 | }*/ 57 | -------------------------------------------------------------------------------- /src/common/EventSource.ts: -------------------------------------------------------------------------------- 1 | import { ObjectDisposedError } from "./Error"; 2 | import { CreateNoDashGuid } from "./Guid"; 3 | import { IDetachable } from "./IDetachable"; 4 | import { IStringDictionary } from "./IDictionary"; 5 | import { IEventListener, IEventSource } from "./IEventSource"; 6 | import { PlatformEvent } from "./PlatformEvent"; 7 | 8 | export class EventSource implements IEventSource { 9 | private eventListeners: IStringDictionary<(event: TEvent) => void> = {}; 10 | private metadata: IStringDictionary; 11 | private isDisposed: boolean = false; 12 | 13 | constructor(metadata?: IStringDictionary) { 14 | this.metadata = metadata; 15 | } 16 | 17 | public OnEvent = (event: TEvent): void => { 18 | if (this.IsDisposed()) { 19 | throw (new ObjectDisposedError("EventSource")); 20 | } 21 | 22 | if (this.Metadata) { 23 | for (const paramName in this.Metadata) { 24 | if (paramName) { 25 | if (event.Metadata) { 26 | if (!event.Metadata[paramName]) { 27 | event.Metadata[paramName] = this.Metadata[paramName]; 28 | } 29 | } 30 | } 31 | } 32 | } 33 | 34 | for (const eventId in this.eventListeners) { 35 | if (eventId && this.eventListeners[eventId]) { 36 | this.eventListeners[eventId](event); 37 | } 38 | } 39 | } 40 | 41 | public Attach = (onEventCallback: (event: TEvent) => void): IDetachable => { 42 | const id = CreateNoDashGuid(); 43 | this.eventListeners[id] = onEventCallback; 44 | return { 45 | Detach: () => { 46 | delete this.eventListeners[id]; 47 | }, 48 | }; 49 | } 50 | 51 | public AttachListener = (listener: IEventListener): IDetachable => { 52 | return this.Attach(listener.OnEvent); 53 | } 54 | 55 | public IsDisposed = (): boolean => { 56 | return this.isDisposed; 57 | } 58 | 59 | public Dispose = (): void => { 60 | this.eventListeners = null; 61 | this.isDisposed = true; 62 | } 63 | 64 | public get Metadata(): IStringDictionary { 65 | return this.metadata; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/common.browser/ConsoleLoggingListener.ts: -------------------------------------------------------------------------------- 1 | import { EventType, IEventListener, PlatformEvent } from "../common/Exports"; 2 | 3 | export class ConsoleLoggingListener implements IEventListener { 4 | private logLevelFilter: EventType; 5 | 6 | public constructor(logLevelFilter: EventType = EventType.Warning) { 7 | this.logLevelFilter = logLevelFilter; 8 | } 9 | 10 | public OnEvent = (event: PlatformEvent): void => { 11 | if (event.EventType >= this.logLevelFilter) { 12 | const log = this.ToString(event); 13 | 14 | switch (event.EventType) { 15 | case EventType.Debug: 16 | // tslint:disable-next-line:no-console 17 | console.debug(log); 18 | break; 19 | case EventType.Info: 20 | // tslint:disable-next-line:no-console 21 | console.info(log); 22 | break; 23 | case EventType.Warning: 24 | // tslint:disable-next-line:no-console 25 | console.warn(log); 26 | break; 27 | case EventType.Error: 28 | // tslint:disable-next-line:no-console 29 | console.error(log); 30 | break; 31 | default: 32 | // tslint:disable-next-line:no-console 33 | console.log(log); 34 | break; 35 | } 36 | } 37 | } 38 | 39 | private ToString = (event: any): string => { 40 | const logFragments = [ 41 | `${event.EventTime}`, 42 | `${event.Name}`, 43 | ]; 44 | 45 | for (const prop in event) { 46 | if (prop && event.hasOwnProperty(prop) && prop !== "eventTime" && prop !== "eventType" && prop !== "eventId" && prop !== "name" && prop !== "constructor") { 47 | const value = event[prop]; 48 | let valueToLog = ""; 49 | if (value !== undefined && value !== null) { 50 | if (typeof (value) === "number" || typeof (value) === "string") { 51 | valueToLog = value.toString(); 52 | } else { 53 | valueToLog = JSON.stringify(value); 54 | } 55 | } 56 | 57 | logFragments.push(`${prop}: ${valueToLog}`); 58 | } 59 | 60 | } 61 | 62 | return logFragments.join(" | "); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | var gulp = require("gulp"); 2 | var ts = require('gulp-typescript'); 3 | var sourcemaps = require('gulp-sourcemaps'); 4 | var tslint = require("gulp-tslint"); 5 | var minify = require('gulp-minify'); 6 | var git = require('gulp-git'); 7 | var versionBump = require('gulp-bump') 8 | var tagVersion = require('gulp-tag-version'); 9 | var webpack = require('webpack-stream'); 10 | 11 | gulp.task("build_ES5", function() { 12 | return gulp.src([ 13 | "src/**/*.ts", 14 | "Speech.Browser.Sdk.ts"], 15 | {base: '.'}) 16 | .pipe(tslint({ 17 | formatter: "prose", 18 | configuration: "tslint.json" 19 | })) 20 | .pipe(tslint.report({ 21 | summarizeFailureOutput: true 22 | })) 23 | .pipe(sourcemaps.init()) 24 | .pipe(ts({ 25 | target: "ES5", 26 | declaration: true, 27 | noImplicitAny: true, 28 | removeComments: false, 29 | outDir: 'distrib' 30 | })) 31 | .pipe(sourcemaps.write('.')) 32 | .pipe(gulp.dest('distrib')); 33 | }); 34 | 35 | 36 | gulp.task("build", ["build_ES5"]); 37 | 38 | gulp.task("bundle", ["build_ES5"], function () { 39 | return gulp.src('samples/browser/sample_app.js') 40 | .pipe(webpack({ 41 | output: {filename: 'speech.sdk.bundle.js'}, 42 | devtool: 'source-map', 43 | module: { 44 | rules: [{ 45 | enforce: 'pre', 46 | test: /\.js$/, 47 | loader: "source-map-loader" 48 | }] 49 | } 50 | })) 51 | .pipe(gulp.dest('distrib')); 52 | }); 53 | 54 | 55 | // We don't want to release anything without a successful build. So build task is dependency for these tasks. 56 | gulp.task('patchRelease', ['build'], function() { return BumpVersionTagAndCommit('patch'); }) 57 | gulp.task('featureRelease', ['build'], function() { return BumpVersionTagAndCommit('minor'); }) 58 | gulp.task('majorRelease', ['build'], function() { return BumpVersionTagAndCommit('major'); }) 59 | gulp.task('preRelease', ['build'], function() { return BumpVersionTagAndCommit('prerelease'); }) 60 | 61 | function BumpVersionTagAndCommit(versionType) { 62 | return gulp.src(['./package.json']) 63 | // bump the version number 64 | .pipe(versionBump({type:versionType})) 65 | // save it back to filesystem 66 | .pipe(gulp.dest('./')) 67 | // commit the changed version number 68 | .pipe(git.commit('Bumping package version')) 69 | // tag it in the repository 70 | .pipe(tagVersion()); 71 | } 72 | -------------------------------------------------------------------------------- /src/sdk/speech.browser/SpeechConnectionFactory.ts: -------------------------------------------------------------------------------- 1 | import { WebsocketConnection } from "../../common.browser/Exports"; 2 | import { 3 | IConnection, 4 | IStringDictionary, 5 | Promise, 6 | Storage, 7 | } from "../../common/Exports"; 8 | import { 9 | AuthInfo, 10 | IAuthentication, 11 | IConnectionFactory, 12 | RecognitionMode, 13 | RecognizerConfig, 14 | SpeechResultFormat, 15 | WebsocketMessageFormatter, 16 | } from "../speech/Exports"; 17 | 18 | const TestHooksParamName: string = "testhooks"; 19 | const ConnectionIdHeader: string = "X-ConnectionId"; 20 | 21 | export class SpeechConnectionFactory implements IConnectionFactory { 22 | 23 | public Create = ( 24 | config: RecognizerConfig, 25 | authInfo: AuthInfo, 26 | connectionId?: string): IConnection => { 27 | 28 | let endpoint = ""; 29 | switch (config.RecognitionMode) { 30 | case RecognitionMode.Conversation: 31 | endpoint = this.Host + this.ConversationRelativeUri; 32 | break; 33 | case RecognitionMode.Dictation: 34 | endpoint = this.Host + this.DictationRelativeUri; 35 | break; 36 | default: 37 | endpoint = this.Host + this.InteractiveRelativeUri; // default is interactive 38 | break; 39 | } 40 | 41 | const queryParams: IStringDictionary = { 42 | format: SpeechResultFormat[config.Format].toString().toLowerCase(), 43 | language: config.Language, 44 | }; 45 | 46 | if (this.IsDebugModeEnabled) { 47 | queryParams[TestHooksParamName] = "1"; 48 | } 49 | 50 | const headers: IStringDictionary = {}; 51 | headers[authInfo.HeaderName] = authInfo.Token; 52 | headers[ConnectionIdHeader] = connectionId; 53 | 54 | return new WebsocketConnection(endpoint, queryParams, headers, new WebsocketMessageFormatter(), connectionId); 55 | } 56 | 57 | private get Host(): string { 58 | return Storage.Local.GetOrAdd("Host", "wss://speech.platform.bing.com"); 59 | } 60 | 61 | private get InteractiveRelativeUri(): string { 62 | return Storage.Local.GetOrAdd("InteractiveRelativeUri", "/speech/recognition/interactive/cognitiveservices/v1"); 63 | } 64 | 65 | private get ConversationRelativeUri(): string { 66 | return Storage.Local.GetOrAdd("ConversationRelativeUri", "/speech/recognition/conversation/cognitiveservices/v1"); 67 | } 68 | 69 | private get DictationRelativeUri(): string { 70 | return Storage.Local.GetOrAdd("DictationRelativeUri", "/speech/recognition/dictation/cognitiveservices/v1"); 71 | } 72 | 73 | private get IsDebugModeEnabled(): boolean { 74 | const value = Storage.Local.GetOrAdd("IsDebugModeEnabled", "false"); 75 | return value.toLowerCase() === "true"; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/common.browser/PCMRecorder.ts: -------------------------------------------------------------------------------- 1 | import { RiffPcmEncoder, Stream } from "../common/Exports"; 2 | import { IRecorder } from "./IRecorder"; 3 | 4 | export class PcmRecorder implements IRecorder { 5 | private mediaResources: IMediaResources; 6 | 7 | public Record = (context: AudioContext, mediaStream: MediaStream, outputStream: Stream): void => { 8 | const desiredSampleRate = 16000; 9 | 10 | // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/createScriptProcessor 11 | const scriptNode = (() => { 12 | let bufferSize = 0; 13 | try { 14 | return context.createScriptProcessor(bufferSize, 1, 1); 15 | } catch (error) { 16 | // Webkit (<= version 31) requires a valid bufferSize. 17 | bufferSize = 2048; 18 | let audioSampleRate = context.sampleRate; 19 | while (bufferSize < 16384 && audioSampleRate >= (2 * desiredSampleRate)) { 20 | bufferSize <<= 1 ; 21 | audioSampleRate >>= 1; 22 | } 23 | return context.createScriptProcessor(bufferSize, 1, 1); 24 | } 25 | })(); 26 | 27 | const waveStreamEncoder = new RiffPcmEncoder(context.sampleRate, desiredSampleRate); 28 | let needHeader: boolean = true; 29 | const that = this; 30 | scriptNode.onaudioprocess = (event: AudioProcessingEvent) => { 31 | const inputFrame = event.inputBuffer.getChannelData(0); 32 | 33 | if (outputStream && !outputStream.IsClosed) { 34 | const waveFrame = waveStreamEncoder.Encode(needHeader, inputFrame); 35 | if (!!waveFrame) { 36 | outputStream.Write(waveFrame); 37 | needHeader = false; 38 | } 39 | } 40 | }; 41 | 42 | // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/createMediaStreamSource 43 | const micInput = context.createMediaStreamSource(mediaStream); 44 | 45 | this.mediaResources = { 46 | scriptProcessorNode: scriptNode, 47 | source: micInput, 48 | stream: mediaStream, 49 | }; 50 | 51 | micInput.connect(scriptNode); 52 | scriptNode.connect(context.destination); 53 | } 54 | 55 | public ReleaseMediaResources = (context: AudioContext): void => { 56 | if (this.mediaResources) { 57 | if (this.mediaResources.scriptProcessorNode) { 58 | this.mediaResources.scriptProcessorNode.disconnect(context.destination); 59 | this.mediaResources.scriptProcessorNode = null; 60 | } 61 | if (this.mediaResources.source) { 62 | this.mediaResources.source.disconnect(); 63 | this.mediaResources.stream.getTracks().forEach((track: any) => track.stop()); 64 | this.mediaResources.source = null; 65 | } 66 | } 67 | } 68 | } 69 | 70 | interface IMediaResources { 71 | source: MediaStreamAudioSourceNode; 72 | scriptProcessorNode: ScriptProcessorNode; 73 | stream: MediaStream; 74 | } 75 | -------------------------------------------------------------------------------- /src/common/AudioSourceEvents.ts: -------------------------------------------------------------------------------- 1 | import { EventType, PlatformEvent } from "./PlatformEvent"; 2 | 3 | export class AudioSourceEvent extends PlatformEvent { 4 | private audioSourceId: string; 5 | 6 | constructor(eventName: string, audioSourceId: string, eventType: EventType = EventType.Info) { 7 | super(eventName, eventType); 8 | this.audioSourceId = audioSourceId; 9 | } 10 | 11 | public get AudioSourceId(): string { 12 | return this.audioSourceId; 13 | } 14 | } 15 | 16 | // tslint:disable-next-line:max-classes-per-file 17 | export class AudioSourceInitializingEvent extends AudioSourceEvent { 18 | constructor(audioSourceId: string) { 19 | super("AudioSourceInitializingEvent", audioSourceId); 20 | } 21 | } 22 | 23 | // tslint:disable-next-line:max-classes-per-file 24 | export class AudioSourceReadyEvent extends AudioSourceEvent { 25 | constructor(audioSourceId: string) { 26 | super("AudioSourceReadyEvent", audioSourceId); 27 | } 28 | } 29 | 30 | // tslint:disable-next-line:max-classes-per-file 31 | export class AudioSourceOffEvent extends AudioSourceEvent { 32 | constructor(audioSourceId: string) { 33 | super("AudioSourceOffEvent", audioSourceId); 34 | } 35 | } 36 | 37 | // tslint:disable-next-line:max-classes-per-file 38 | export class AudioSourceErrorEvent extends AudioSourceEvent { 39 | private error: string; 40 | constructor(audioSourceId: string, error: string) { 41 | super("AudioSourceErrorEvent", audioSourceId, EventType.Error); 42 | this.error = error; 43 | } 44 | 45 | public get Error(): string { 46 | return this.error; 47 | } 48 | } 49 | 50 | // tslint:disable-next-line:max-classes-per-file 51 | export class AudioStreamNodeEvent extends AudioSourceEvent { 52 | private audioNodeId: string; 53 | 54 | constructor(eventName: string, audioSourceId: string, audioNodeId: string) { 55 | super(eventName, audioSourceId); 56 | this.audioNodeId = audioNodeId; 57 | } 58 | 59 | public get AudioNodeId(): string { 60 | return this.audioNodeId; 61 | } 62 | } 63 | 64 | // tslint:disable-next-line:max-classes-per-file 65 | export class AudioStreamNodeAttachingEvent extends AudioStreamNodeEvent { 66 | constructor(audioSourceId: string, audioNodeId: string) { 67 | super("AudioStreamNodeAttachingEvent", audioSourceId, audioNodeId); 68 | } 69 | } 70 | 71 | // tslint:disable-next-line:max-classes-per-file 72 | export class AudioStreamNodeAttachedEvent extends AudioStreamNodeEvent { 73 | constructor(audioSourceId: string, audioNodeId: string) { 74 | super("AudioStreamNodeAttachedEvent", audioSourceId, audioNodeId); 75 | } 76 | } 77 | 78 | // tslint:disable-next-line:max-classes-per-file 79 | export class AudioStreamNodeDetachedEvent extends AudioStreamNodeEvent { 80 | constructor(audioSourceId: string, audioNodeId: string) { 81 | super("AudioStreamNodeDetachedEvent", audioSourceId, audioNodeId); 82 | } 83 | } 84 | 85 | // tslint:disable-next-line:max-classes-per-file 86 | export class AudioStreamNodeErrorEvent extends AudioStreamNodeEvent { 87 | private error: string; 88 | 89 | constructor(audioSourceId: string, audioNodeId: string, error: string) { 90 | super("AudioStreamNodeErrorEvent", audioSourceId, audioNodeId); 91 | this.error = error; 92 | } 93 | 94 | public get Error(): string { 95 | return this.error; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/common.browser/WebsocketConnection.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArgumentNullError, 3 | ConnectionEvent, 4 | ConnectionMessage, 5 | ConnectionOpenResponse, 6 | ConnectionState, 7 | CreateNoDashGuid, 8 | EventSource, 9 | IConnection, 10 | IStringDictionary, 11 | IWebsocketMessageFormatter, 12 | PlatformEvent, 13 | Promise, 14 | } from "../common/Exports"; 15 | import { WebsocketMessageAdapter } from "./WebsocketMessageAdapter"; 16 | 17 | export class WebsocketConnection implements IConnection { 18 | 19 | private uri: string; 20 | private messageFormatter: IWebsocketMessageFormatter; 21 | private connectionMessageAdapter: WebsocketMessageAdapter; 22 | private id: string; 23 | private isDisposed: boolean = false; 24 | 25 | public constructor( 26 | uri: string, 27 | queryParameters: IStringDictionary, 28 | headers: IStringDictionary, 29 | messageFormatter: IWebsocketMessageFormatter, 30 | connectionId?: string) { 31 | 32 | if (!uri) { 33 | throw new ArgumentNullError("uri"); 34 | } 35 | 36 | if (!messageFormatter) { 37 | throw new ArgumentNullError("messageFormatter"); 38 | } 39 | 40 | this.messageFormatter = messageFormatter; 41 | 42 | let queryParams = ""; 43 | let i = 0; 44 | 45 | if (queryParameters) { 46 | for (const paramName in queryParameters) { 47 | if (paramName) { 48 | queryParams += i === 0 ? "?" : "&"; 49 | const val = encodeURIComponent(queryParameters[paramName]); 50 | queryParams += `${paramName}=${val}`; 51 | i++; 52 | } 53 | } 54 | } 55 | 56 | if (headers) { 57 | for (const headerName in headers) { 58 | if (headerName) { 59 | queryParams += i === 0 ? "?" : "&"; 60 | const val = encodeURIComponent(headers[headerName]); 61 | queryParams += `${headerName}=${val}`; 62 | i++; 63 | } 64 | } 65 | } 66 | 67 | this.uri = uri + queryParams; 68 | this.id = connectionId ? connectionId : CreateNoDashGuid(); 69 | 70 | this.connectionMessageAdapter = new WebsocketMessageAdapter( 71 | this.uri, 72 | this.Id, 73 | this.messageFormatter); 74 | } 75 | 76 | public Dispose = (): void => { 77 | this.isDisposed = true; 78 | 79 | if (this.connectionMessageAdapter) { 80 | this.connectionMessageAdapter.Close(); 81 | } 82 | } 83 | 84 | public IsDisposed = (): boolean => { 85 | return this.isDisposed; 86 | } 87 | 88 | public get Id(): string { 89 | return this.id; 90 | } 91 | 92 | public State = (): ConnectionState => { 93 | return this.connectionMessageAdapter.State; 94 | } 95 | 96 | public Open = (): Promise => { 97 | return this.connectionMessageAdapter.Open(); 98 | } 99 | 100 | public Send = (message: ConnectionMessage): Promise => { 101 | return this.connectionMessageAdapter.Send(message); 102 | } 103 | 104 | public Read = (): Promise => { 105 | return this.connectionMessageAdapter.Read(); 106 | } 107 | 108 | public get Events(): EventSource { 109 | return this.connectionMessageAdapter.Events; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /samples/browser/sample-server.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var https = require('https'); 3 | var url = require('url'); 4 | var os = require("os"); 5 | var fs = require('fs'); 6 | var qrcode = require('qrcode-terminal'); // "qrcode-terminal": "^0.11.0", 7 | 8 | var sample = fs.readFileSync(__dirname + '/Sample.html', 'utf8'); 9 | 10 | var enableTunnel = false; 11 | for (let j = 0; j < process.argv.length; j++) { 12 | enableTunnel |= process.argv[j] == 'enableTunnel'; 13 | } 14 | 15 | var keyFile = __dirname+'/speech.key'; 16 | if (fs.existsSync(keyFile)) { 17 | var key = fs.readFileSync(keyFile, 'utf8'); 18 | key = key.replace(/\s/g, ""); 19 | if (!!key) { 20 | var before = "value=\"YOUR_BING_SPEECH_API_KEY\""; 21 | var after = " disabled value=\"Using token-based auth mechanism.\""; 22 | sample = sample.replace(before, after); 23 | sample = sample.replace('var useTokenAuth = false;', 'var useTokenAuth = true;'); 24 | } 25 | } 26 | 27 | 28 | var port = 8765; 29 | var server = http.createServer(function(request, response){ 30 | var respond = function(status, data, contentType) { 31 | if (typeof(contentType) === 'undefined') { 32 | contentType = 'text/plain'; 33 | } 34 | response.writeHead(status, {'Content-Type': contentType}); 35 | !!data && response.write(data); 36 | response.end(); 37 | } 38 | 39 | path= url.parse(request.url).pathname; 40 | console.log("Incoming request:" + request.url); 41 | 42 | if (path == '/token') { 43 | getToken(key, function(token){ 44 | respond(200, token); 45 | }) 46 | } else if (path == '/') { 47 | respond(200, sample, 'text/html'); 48 | } else { 49 | var pathExists = fs.existsSync(__dirname + '/../../'+path); 50 | 51 | if (!pathExists || 52 | !path.endsWith('speech.sdk.bundle.js') && 53 | !path.endsWith('speech.sdk.bundle.js.map')) { 54 | respond(404); 55 | } else { 56 | var type = 'application/javascript'; 57 | respond(200, fs.readFileSync(__dirname+'/../../'+path, 'utf8'), type); 58 | } 59 | } 60 | }); 61 | 62 | 63 | var quit = false; 64 | if (enableTunnel) { 65 | var localtunnel = require('localtunnel'); 66 | var tunnel = localtunnel(port, function(err, tunnel) { 67 | if (err) { 68 | quit = true; 69 | server.close(); 70 | console.log('Something went south...' + err.message) 71 | } else { 72 | printServerInfo(tunnel.url) 73 | } 74 | }); 75 | 76 | tunnel.on('close', function() { 77 | quit = true; 78 | server.close(); 79 | }); 80 | } else { 81 | printServerInfo('http://'+os.hostname() + ':' + port); 82 | } 83 | 84 | 85 | 86 | if (!quit) { 87 | server.listen(port); 88 | } 89 | 90 | function printServerInfo(url) { 91 | console.log('Up and running @ ' + url); 92 | qrcode.setErrorLevel('H'); 93 | qrcode.generate(url, {small: true}); 94 | } 95 | 96 | function getToken(apiKey, result) { 97 | var options = { 98 | host: 'api.cognitive.microsoft.com', 99 | path: '/sts/v1.0/issueToken', 100 | method: 'POST', 101 | headers: { 102 | 'Content-type': 'application/x-www-form-urlencoded', 103 | 'Content-Length': '0', 104 | 'Ocp-Apim-Subscription-Key': apiKey 105 | } 106 | }; 107 | 108 | var callback = function(response) { 109 | var token = '' 110 | response.on('data', function (chunk) { 111 | token += chunk; 112 | }); 113 | 114 | response.on('end', function () { 115 | result(token); 116 | }); 117 | } 118 | 119 | var issueTokenRequest = https.request(options, callback); 120 | issueTokenRequest.end(); 121 | } -------------------------------------------------------------------------------- /src/sdk/speech/SpeechConnectionMessage.Internal.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArgumentNullError, 3 | ConnectionMessage, 4 | IStringDictionary, 5 | MessageType, 6 | } from "../../common/Exports"; 7 | 8 | const PathHeaderName: string = "path"; 9 | const ContentTypeHeaderName: string = "content-type"; 10 | const RequestIdHeaderName: string = "x-requestid"; 11 | const RequestTimestampHeaderName: string = "x-timestamp"; 12 | 13 | export class SpeechConnectionMessage extends ConnectionMessage { 14 | 15 | private path: string; 16 | private requestId: string; 17 | private contentType: string; 18 | private additionalHeaders: IStringDictionary; 19 | 20 | public constructor( 21 | messageType: MessageType, 22 | path: string, 23 | requestId: string, 24 | contentType: string, 25 | body: any, 26 | additionalHeaders?: IStringDictionary, 27 | id?: string) { 28 | 29 | if (!path) { 30 | throw new ArgumentNullError("path"); 31 | } 32 | 33 | if (!requestId) { 34 | throw new ArgumentNullError("requestId"); 35 | } 36 | 37 | const headers: IStringDictionary = {}; 38 | headers[PathHeaderName] = path; 39 | headers[RequestIdHeaderName] = requestId; 40 | headers[RequestTimestampHeaderName] = new Date().toISOString(); 41 | if (contentType) { 42 | headers[ContentTypeHeaderName] = contentType; 43 | } 44 | 45 | if (additionalHeaders) { 46 | for (const headerName in additionalHeaders) { 47 | if (headerName) { 48 | headers[headerName] = additionalHeaders[headerName]; 49 | } 50 | 51 | } 52 | } 53 | 54 | if (id) { 55 | super(messageType, body, headers, id); 56 | } else { 57 | super(messageType, body, headers); 58 | } 59 | 60 | this.path = path; 61 | this.requestId = requestId; 62 | this.contentType = contentType; 63 | this.additionalHeaders = additionalHeaders; 64 | } 65 | 66 | public get Path(): string { 67 | return this.path; 68 | } 69 | 70 | public get RequestId(): string { 71 | return this.requestId; 72 | } 73 | 74 | public get ContentType(): string { 75 | return this.contentType; 76 | } 77 | 78 | public get AdditionalHeaders(): IStringDictionary { 79 | return this.additionalHeaders; 80 | } 81 | 82 | public static FromConnectionMessage = (message: ConnectionMessage): SpeechConnectionMessage => { 83 | let path = null; 84 | let requestId = null; 85 | let contentType = null; 86 | let requestTimestamp = null; 87 | const additionalHeaders: IStringDictionary = {}; 88 | 89 | if (message.Headers) { 90 | for (const headerName in message.Headers) { 91 | if (headerName) { 92 | if (headerName.toLowerCase() === PathHeaderName.toLowerCase()) { 93 | path = message.Headers[headerName]; 94 | } else if (headerName.toLowerCase() === RequestIdHeaderName.toLowerCase()) { 95 | requestId = message.Headers[headerName]; 96 | } else if (headerName.toLowerCase() === RequestTimestampHeaderName.toLowerCase()) { 97 | requestTimestamp = message.Headers[headerName]; 98 | } else if (headerName.toLowerCase() === ContentTypeHeaderName.toLowerCase()) { 99 | contentType = message.Headers[headerName]; 100 | } else { 101 | additionalHeaders[headerName] = message.Headers[headerName]; 102 | } 103 | } 104 | } 105 | } 106 | 107 | return new SpeechConnectionMessage( 108 | message.MessageType, 109 | path, 110 | requestId, 111 | contentType, 112 | message.Body, 113 | additionalHeaders, 114 | message.Id); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/common/ConnectionEvents.ts: -------------------------------------------------------------------------------- 1 | import { ConnectionMessage } from "./ConnectionMessage"; 2 | import { IStringDictionary } from "./IDictionary"; 3 | import { EventType, PlatformEvent } from "./PlatformEvent"; 4 | 5 | export class ConnectionEvent extends PlatformEvent { 6 | private connectionId: string; 7 | 8 | constructor(eventName: string, connectionId: string, eventType: EventType = EventType.Info) { 9 | super(eventName, eventType); 10 | this.connectionId = connectionId; 11 | } 12 | 13 | public get ConnectionId(): string { 14 | return this.connectionId; 15 | } 16 | } 17 | 18 | // tslint:disable-next-line:max-classes-per-file 19 | export class ConnectionStartEvent extends ConnectionEvent { 20 | private uri: string; 21 | private headers: IStringDictionary; 22 | 23 | constructor(connectionId: string, uri: string, headers?: IStringDictionary) { 24 | super("ConnectionStartEvent", connectionId); 25 | this.uri = uri; 26 | this.headers = headers; 27 | } 28 | 29 | public get Uri(): string { 30 | return this.uri; 31 | } 32 | 33 | public get Headers(): IStringDictionary { 34 | return this.headers; 35 | } 36 | } 37 | 38 | // tslint:disable-next-line:max-classes-per-file 39 | export class ConnectionEstablishedEvent extends ConnectionEvent { 40 | constructor(connectionId: string, metadata?: IStringDictionary) { 41 | super("ConnectionEstablishedEvent", connectionId); 42 | } 43 | } 44 | 45 | // tslint:disable-next-line:max-classes-per-file 46 | export class ConnectionClosedEvent extends ConnectionEvent { 47 | private reason: string; 48 | private statusCode: number; 49 | 50 | constructor(connectionId: string, statusCode: number, reason: string) { 51 | super("ConnectionClosedEvent", connectionId, EventType.Warning); 52 | this.reason = reason; 53 | this.statusCode = statusCode; 54 | } 55 | 56 | public get Reason(): string { 57 | return this.reason; 58 | } 59 | 60 | public get StatusCode(): number { 61 | return this.statusCode; 62 | } 63 | } 64 | 65 | // tslint:disable-next-line:max-classes-per-file 66 | export class ConnectionEstablishErrorEvent extends ConnectionEvent { 67 | private statusCode: number; 68 | private reason: string; 69 | 70 | constructor(connectionId: string, statuscode: number, reason: string) { 71 | super("ConnectionEstablishErrorEvent", connectionId, EventType.Error); 72 | this.statusCode = statuscode; 73 | this.reason = reason; 74 | } 75 | 76 | public get Reason(): string { 77 | return this.reason; 78 | } 79 | 80 | public get StatusCode(): number { 81 | return this.statusCode; 82 | } 83 | } 84 | 85 | // tslint:disable-next-line:max-classes-per-file 86 | export class ConnectionMessageReceivedEvent extends ConnectionEvent { 87 | private networkReceivedTime: string; 88 | private message: ConnectionMessage; 89 | 90 | constructor(connectionId: string, networkReceivedTimeISO: string, message: ConnectionMessage) { 91 | super("ConnectionMessageReceivedEvent", connectionId); 92 | this.networkReceivedTime = networkReceivedTimeISO; 93 | this.message = message; 94 | } 95 | 96 | public get NetworkReceivedTime(): string { 97 | return this.networkReceivedTime; 98 | } 99 | 100 | public get Message(): ConnectionMessage { 101 | return this.message; 102 | } 103 | } 104 | 105 | // tslint:disable-next-line:max-classes-per-file 106 | export class ConnectionMessageSentEvent extends ConnectionEvent { 107 | private networkSentTime: string; 108 | private message: ConnectionMessage; 109 | 110 | constructor(connectionId: string, networkSentTimeISO: string, message: ConnectionMessage) { 111 | super("ConnectionMessageSentEvent", connectionId); 112 | this.networkSentTime = networkSentTimeISO; 113 | this.message = message; 114 | } 115 | 116 | public get NetworkSentTime(): string { 117 | return this.networkSentTime; 118 | } 119 | 120 | public get Message(): ConnectionMessage { 121 | return this.message; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/common/RiffPcmEncoder.ts: -------------------------------------------------------------------------------- 1 | 2 | export class RiffPcmEncoder { 3 | 4 | private actualSampleRate: number; 5 | private desiredSampleRate: number; 6 | private channelCount: number = 1; 7 | 8 | public constructor(actualSampleRate: number, desiredSampleRate: number) { 9 | this.actualSampleRate = actualSampleRate; 10 | this.desiredSampleRate = desiredSampleRate; 11 | } 12 | 13 | public Encode = ( 14 | needHeader: boolean, 15 | actualAudioFrame: Float32Array): ArrayBuffer => { 16 | 17 | const audioFrame = this.DownSampleAudioFrame(actualAudioFrame, this.actualSampleRate, this.desiredSampleRate); 18 | 19 | if (!audioFrame) { 20 | return null; 21 | } 22 | 23 | const audioLength = audioFrame.length * 2; 24 | 25 | if (!needHeader) { 26 | const buffer = new ArrayBuffer(audioLength); 27 | const view = new DataView(buffer); 28 | this.FloatTo16BitPCM(view, 0, audioFrame); 29 | 30 | return buffer; 31 | } 32 | 33 | const buffer = new ArrayBuffer(44 + audioLength); 34 | 35 | const bitsPerSample = 16; 36 | const bytesPerSample = bitsPerSample / 8; 37 | // We dont know ahead of time about the length of audio to stream. So set to 0. 38 | const fileLength = 0; 39 | 40 | // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DataView 41 | const view = new DataView(buffer); 42 | 43 | /* RIFF identifier */ 44 | this.SetString(view, 0, "RIFF"); 45 | /* file length */ 46 | view.setUint32(4, fileLength, true); 47 | /* RIFF type & Format */ 48 | this.SetString(view, 8, "WAVEfmt "); 49 | /* format chunk length */ 50 | view.setUint32(16, 16, true); 51 | /* sample format (raw) */ 52 | view.setUint16(20, 1, true); 53 | /* channel count */ 54 | view.setUint16(22, this.channelCount, true); 55 | /* sample rate */ 56 | view.setUint32(24, this.desiredSampleRate, true); 57 | /* byte rate (sample rate * block align) */ 58 | view.setUint32(28, this.desiredSampleRate * this.channelCount * bytesPerSample, true); 59 | /* block align (channel count * bytes per sample) */ 60 | view.setUint16(32, this.channelCount * bytesPerSample, true); 61 | /* bits per sample */ 62 | view.setUint16(34, bitsPerSample, true); 63 | /* data chunk identifier */ 64 | this.SetString(view, 36, "data"); 65 | /* data chunk length */ 66 | view.setUint32(40, fileLength, true); 67 | 68 | this.FloatTo16BitPCM(view, 44, audioFrame); 69 | 70 | return buffer; 71 | } 72 | 73 | private SetString = (view: DataView, offset: number, str: string): void => { 74 | for (let i = 0; i < str.length; i++) { 75 | view.setUint8(offset + i, str.charCodeAt(i)); 76 | } 77 | } 78 | 79 | private FloatTo16BitPCM = (view: DataView, offset: number, input: Float32Array): void => { 80 | for (let i = 0; i < input.length; i++ , offset += 2) { 81 | const s = Math.max(-1, Math.min(1, input[i])); 82 | view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); 83 | } 84 | } 85 | 86 | private DownSampleAudioFrame = ( 87 | srcFrame: Float32Array, 88 | srcRate: number, 89 | dstRate: number): Float32Array => { 90 | 91 | if (dstRate === srcRate || dstRate > srcRate) { 92 | return srcFrame; 93 | } 94 | 95 | const ratio = srcRate / dstRate; 96 | const dstLength = Math.round(srcFrame.length / ratio); 97 | const dstFrame = new Float32Array(dstLength); 98 | let srcOffset = 0; 99 | let dstOffset = 0; 100 | while (dstOffset < dstLength) { 101 | const nextSrcOffset = Math.round((dstOffset + 1) * ratio); 102 | let accum = 0; 103 | let count = 0; 104 | while (srcOffset < nextSrcOffset && srcOffset < srcFrame.length) { 105 | accum += srcFrame[srcOffset++]; 106 | count++; 107 | } 108 | dstFrame[dstOffset++] = accum / count; 109 | } 110 | 111 | return dstFrame; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/common/Stream.ts: -------------------------------------------------------------------------------- 1 | import { InvalidOperationError } from "./Error"; 2 | import { CreateNoDashGuid } from "./Guid"; 3 | import { IStringDictionary } from "./IDictionary"; 4 | import { Promise } from "./Promise"; 5 | import { Queue } from "./Queue"; 6 | import { IStreamChunk } from "./Stream"; 7 | 8 | export interface IStreamChunk { 9 | IsEnd: boolean; 10 | Buffer: TBuffer; 11 | } 12 | 13 | export class Stream { 14 | private id: string; 15 | private readerIdCounter: number = 1; 16 | private streambuffer: Array>; 17 | private isEnded: boolean = false; 18 | private readerQueues: IStringDictionary>>; 19 | 20 | public constructor(streamId?: string) { 21 | this.id = streamId ? streamId : CreateNoDashGuid(); 22 | this.streambuffer = []; 23 | this.readerQueues = {}; 24 | } 25 | 26 | public get IsClosed(): boolean { 27 | return this.isEnded; 28 | } 29 | 30 | public get Id(): string { 31 | return this.id; 32 | } 33 | 34 | public Write = (buffer: TBuffer): void => { 35 | this.ThrowIfClosed(); 36 | this.WriteStreamChunk({ 37 | Buffer: buffer, 38 | IsEnd: false, 39 | }); 40 | } 41 | 42 | public GetReader = (): StreamReader => { 43 | const readerId = this.readerIdCounter; 44 | this.readerIdCounter++; 45 | const readerQueue = new Queue>(); 46 | const currentLength = this.streambuffer.length; 47 | this.readerQueues[readerId] = readerQueue; 48 | for (let i = 0; i < currentLength; i++) { 49 | readerQueue.Enqueue(this.streambuffer[i]); 50 | } 51 | return new StreamReader( 52 | this.id, 53 | readerQueue, 54 | () => { 55 | delete this.readerQueues[readerId]; 56 | }); 57 | } 58 | 59 | public Close = (): void => { 60 | if (!this.isEnded) { 61 | this.WriteStreamChunk({ 62 | Buffer: null, 63 | IsEnd: true, 64 | }); 65 | this.isEnded = true; 66 | } 67 | } 68 | 69 | private WriteStreamChunk = (streamChunk: IStreamChunk): void => { 70 | this.ThrowIfClosed(); 71 | this.streambuffer.push(streamChunk); 72 | for (const readerId in this.readerQueues) { 73 | if (!this.readerQueues[readerId].IsDisposed()) { 74 | try { 75 | this.readerQueues[readerId].Enqueue(streamChunk); 76 | } catch (e) { 77 | // Do nothing 78 | } 79 | } 80 | } 81 | } 82 | 83 | private ThrowIfClosed = (): void => { 84 | if (this.isEnded) { 85 | throw new InvalidOperationError("Stream closed"); 86 | } 87 | } 88 | } 89 | 90 | // tslint:disable-next-line:max-classes-per-file 91 | export class StreamReader { 92 | private readerQueue: Queue>; 93 | private onClose: () => void; 94 | private isClosed: boolean = false; 95 | private streamId: string; 96 | 97 | public constructor(streamId: string, readerQueue: Queue>, onClose: () => void) { 98 | this.readerQueue = readerQueue; 99 | this.onClose = onClose; 100 | this.streamId = streamId; 101 | } 102 | 103 | public get IsClosed(): boolean { 104 | return this.isClosed; 105 | } 106 | 107 | public get StreamId(): string { 108 | return this.streamId; 109 | } 110 | 111 | public Read = (): Promise> => { 112 | if (this.IsClosed) { 113 | throw new InvalidOperationError("StreamReader closed"); 114 | } 115 | 116 | return this.readerQueue 117 | .Dequeue() 118 | .OnSuccessContinueWith((streamChunk: IStreamChunk) => { 119 | if (streamChunk.IsEnd) { 120 | this.readerQueue.Dispose("End of stream reached"); 121 | } 122 | 123 | return streamChunk; 124 | }); 125 | } 126 | 127 | public Close = (): void => { 128 | if (!this.isClosed) { 129 | this.isClosed = true; 130 | this.readerQueue.Dispose("StreamReader closed"); 131 | this.onClose(); 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/sdk/speech/RecognizerConfig.ts: -------------------------------------------------------------------------------- 1 | 2 | export enum RecognitionMode { 3 | Interactive, 4 | Conversation, 5 | Dictation, 6 | } 7 | 8 | export enum SpeechResultFormat { 9 | Simple, 10 | Detailed, 11 | } 12 | 13 | export class RecognizerConfig { 14 | private recognitionMode: RecognitionMode = RecognitionMode.Interactive; 15 | private language: string; 16 | private format: SpeechResultFormat; 17 | private speechConfig: SpeechConfig; 18 | private recognitionActivityTimeout: number; 19 | 20 | constructor( 21 | platformConfig: SpeechConfig, 22 | recognitionMode: RecognitionMode = RecognitionMode.Interactive, 23 | language: string = "en-us", 24 | format: SpeechResultFormat = SpeechResultFormat.Simple) { 25 | this.speechConfig = platformConfig ? platformConfig : new SpeechConfig(new Context(null, null)); 26 | this.recognitionMode = recognitionMode; 27 | this.language = language; 28 | this.format = format; 29 | this.recognitionActivityTimeout = recognitionMode === RecognitionMode.Interactive ? 8000 : 25000; 30 | } 31 | 32 | public get RecognitionMode(): RecognitionMode { 33 | return this.recognitionMode; 34 | } 35 | 36 | public get Language(): string { 37 | return this.language; 38 | } 39 | 40 | public get Format(): SpeechResultFormat { 41 | return this.format; 42 | } 43 | 44 | public get SpeechConfig(): SpeechConfig { 45 | return this.speechConfig; 46 | } 47 | 48 | public get RecognitionActivityTimeout(): number { 49 | return this.recognitionActivityTimeout; 50 | } 51 | 52 | public get IsContinuousRecognition(): boolean { 53 | return this.recognitionMode !== RecognitionMode.Interactive; 54 | } 55 | } 56 | 57 | // tslint:disable-next-line:max-classes-per-file 58 | export class SpeechConfig { 59 | private context: Context; 60 | 61 | constructor(context: Context) { 62 | this.context = context; 63 | } 64 | 65 | public Serialize = (): string => { 66 | return JSON.stringify(this, (key: any, value: any): any => { 67 | if (value && typeof value === "object") { 68 | const replacement: any = {}; 69 | for (const k in value) { 70 | if (Object.hasOwnProperty.call(value, k)) { 71 | replacement[k && k.charAt(0).toLowerCase() + k.substring(1)] = value[k]; 72 | } 73 | } 74 | return replacement; 75 | } 76 | return value; 77 | }); 78 | } 79 | 80 | public get Context(): Context { 81 | return this.context; 82 | } 83 | 84 | } 85 | 86 | // tslint:disable-next-line:max-classes-per-file 87 | export class Context { 88 | private system: System; 89 | private os: OS; 90 | private device: Device; 91 | 92 | constructor(os: OS, device: Device) { 93 | this.system = new System(); 94 | this.os = os; 95 | this.device = device; 96 | } 97 | 98 | public get System(): System { 99 | return this.system; 100 | } 101 | 102 | public get OS(): OS { 103 | return this.os; 104 | } 105 | 106 | public get Device(): Device { 107 | return this.device; 108 | } 109 | } 110 | 111 | // tslint:disable-next-line:max-classes-per-file 112 | export class System { 113 | private version: string; 114 | constructor() { 115 | // TODO: Tie this with the sdk Version somehow 116 | this.version = "1.0.00000"; 117 | } 118 | public get Version(): string { 119 | // Controlled by sdk 120 | return this.version; 121 | } 122 | } 123 | 124 | // tslint:disable-next-line:max-classes-per-file 125 | export class OS { 126 | 127 | private platform: string; 128 | private name: string; 129 | private version: string; 130 | 131 | constructor(platform: string, name: string, version: string) { 132 | this.platform = platform; 133 | this.name = name; 134 | this.version = version; 135 | } 136 | 137 | public get Platform(): string { 138 | return this.platform; 139 | } 140 | 141 | public get Name(): string { 142 | return this.name; 143 | } 144 | 145 | public get Version(): string { 146 | return this.version; 147 | } 148 | } 149 | 150 | // tslint:disable-next-line:max-classes-per-file 151 | export class Device { 152 | 153 | private manufacturer: string; 154 | private model: string; 155 | private version: string; 156 | 157 | constructor(manufacturer: string, model: string, version: string) { 158 | this.manufacturer = manufacturer; 159 | this.model = model; 160 | this.version = version; 161 | } 162 | 163 | public get Manufacturer(): string { 164 | return this.manufacturer; 165 | } 166 | 167 | public get Model(): string { 168 | return this.model; 169 | } 170 | 171 | public get Version(): string { 172 | return this.version; 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/common/Queue.ts: -------------------------------------------------------------------------------- 1 | import { InvalidOperationError, ObjectDisposedError } from "./Error"; 2 | import { IDetachable } from "./IDetachable"; 3 | import { IDisposable } from "./IDisposable"; 4 | import { List } from "./List"; 5 | import { Deferred, Promise, PromiseHelper } from "./Promise"; 6 | 7 | export interface IQueue extends IDisposable { 8 | Enqueue(item: TItem): void; 9 | EnqueueFromPromise(promise: Promise): void; 10 | Dequeue(): Promise; 11 | Peek(): Promise; 12 | Length(): number; 13 | } 14 | 15 | enum SubscriberType { 16 | Dequeue, 17 | Peek, 18 | } 19 | 20 | export class Queue implements IQueue { 21 | private promiseStore: List> = new List>(); 22 | private list: List; 23 | private detachables: IDetachable[]; 24 | private subscribers: List<{ type: SubscriberType, deferral: Deferred }>; 25 | private isDrainInProgress: boolean = false; 26 | private isDisposing: boolean = false; 27 | private disposeReason: string = null; 28 | 29 | public constructor(list?: List) { 30 | this.list = list ? list : new List(); 31 | this.detachables = []; 32 | this.subscribers = new List<{ type: SubscriberType, deferral: Deferred }>(); 33 | this.detachables.push(this.list.OnAdded(this.Drain)); 34 | } 35 | 36 | public Enqueue = (item: TItem): void => { 37 | this.ThrowIfDispose(); 38 | this.EnqueueFromPromise(PromiseHelper.FromResult(item)); 39 | } 40 | 41 | public EnqueueFromPromise = (promise: Promise): void => { 42 | this.ThrowIfDispose(); 43 | this.promiseStore.Add(promise); 44 | promise.Finally(() => { 45 | while (this.promiseStore.Length() > 0) { 46 | if (!this.promiseStore.First().Result().IsCompleted) { 47 | break; 48 | } else { 49 | const p = this.promiseStore.RemoveFirst(); 50 | if (!p.Result().IsError) { 51 | this.list.Add(p.Result().Result); 52 | } else { 53 | // TODO: Log as warning. 54 | } 55 | } 56 | } 57 | }); 58 | } 59 | 60 | public Dequeue = (): Promise => { 61 | this.ThrowIfDispose(); 62 | const deferredSubscriber = new Deferred(); 63 | this.subscribers.Add({ deferral: deferredSubscriber, type: SubscriberType.Dequeue }); 64 | this.Drain(); 65 | return deferredSubscriber.Promise(); 66 | } 67 | 68 | public Peek = (): Promise => { 69 | this.ThrowIfDispose(); 70 | const deferredSubscriber = new Deferred(); 71 | this.subscribers.Add({ deferral: deferredSubscriber, type: SubscriberType.Peek }); 72 | this.Drain(); 73 | return deferredSubscriber.Promise(); 74 | } 75 | 76 | public Length = (): number => { 77 | this.ThrowIfDispose(); 78 | return this.list.Length(); 79 | } 80 | 81 | public IsDisposed = (): boolean => { 82 | return this.subscribers == null; 83 | } 84 | 85 | public DrainAndDispose = (pendingItemProcessor: (pendingItemInQueue: TItem) => void, reason?: string): Promise => { 86 | if (!this.IsDisposed() && !this.isDisposing) { 87 | this.disposeReason = reason; 88 | this.isDisposing = true; 89 | while (this.subscribers.Length() > 0) { 90 | const subscriber = this.subscribers.RemoveFirst(); 91 | // TODO: this needs work (Resolve(null) instead?). 92 | subscriber.deferral.Reject("Disposed"); 93 | } 94 | 95 | for (const detachable of this.detachables) { 96 | detachable.Detach(); 97 | } 98 | 99 | if (this.promiseStore.Length() > 0 && pendingItemProcessor) { 100 | return PromiseHelper 101 | .WhenAll(this.promiseStore.ToArray()) 102 | .ContinueWith(() => { 103 | this.subscribers = null; 104 | this.list.ForEach((item: TItem, index: number): void => { 105 | pendingItemProcessor(item); 106 | }); 107 | this.list = null; 108 | return true; 109 | }); 110 | } else { 111 | this.subscribers = null; 112 | this.list = null; 113 | } 114 | } 115 | 116 | return PromiseHelper.FromResult(true); 117 | } 118 | 119 | public Dispose = (reason?: string): void => { 120 | this.DrainAndDispose(null, reason); 121 | } 122 | 123 | private Drain = (): void => { 124 | if (!this.isDrainInProgress && !this.isDisposing) { 125 | this.isDrainInProgress = true; 126 | 127 | while (this.list.Length() > 0 && this.subscribers.Length() > 0 && !this.isDisposing) { 128 | const subscriber = this.subscribers.RemoveFirst(); 129 | if (subscriber.type === SubscriberType.Peek) { 130 | subscriber.deferral.Resolve(this.list.First()); 131 | } else { 132 | const dequeuedItem = this.list.RemoveFirst(); 133 | subscriber.deferral.Resolve(dequeuedItem); 134 | } 135 | } 136 | 137 | this.isDrainInProgress = false; 138 | } 139 | } 140 | 141 | private ThrowIfDispose = (): void => { 142 | if (this.IsDisposed()) { 143 | if (this.disposeReason) { 144 | throw new InvalidOperationError(this.disposeReason); 145 | } 146 | 147 | throw new ObjectDisposedError("Queue"); 148 | } else if (this.isDisposing) { 149 | throw new InvalidOperationError("Queue disposing"); 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/sdk/speech/WebsocketMessageFormatter.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ConnectionMessage, 3 | Deferred, 4 | IStringDictionary, 5 | IWebsocketMessageFormatter, 6 | MessageType, 7 | Promise, 8 | RawWebsocketMessage, 9 | } from "../../common/Exports"; 10 | 11 | const CRLF: string = "\r\n"; 12 | 13 | export class WebsocketMessageFormatter implements IWebsocketMessageFormatter { 14 | 15 | public ToConnectionMessage = (message: RawWebsocketMessage): Promise => { 16 | const deferral = new Deferred(); 17 | 18 | try { 19 | if (message.MessageType === MessageType.Text) { 20 | const textMessage: string = message.TextContent; 21 | let headers: IStringDictionary = {}; 22 | let body: string = null; 23 | 24 | if (textMessage) { 25 | const headerBodySplit = textMessage.split("\r\n\r\n"); 26 | if (headerBodySplit && headerBodySplit.length > 0) { 27 | headers = this.ParseHeaders(headerBodySplit[0]); 28 | if (headerBodySplit.length > 1) { 29 | body = headerBodySplit[1]; 30 | } 31 | } 32 | } 33 | 34 | deferral.Resolve(new ConnectionMessage(message.MessageType, body, headers, message.Id)); 35 | } else if (message.MessageType === MessageType.Binary) { 36 | const binaryMessage: ArrayBuffer = message.BinaryContent; 37 | let headers: IStringDictionary = {}; 38 | let body: ArrayBuffer = null; 39 | 40 | if (!binaryMessage || binaryMessage.byteLength < 2) { 41 | throw new Error("Invalid binary message format. Header length missing."); 42 | } 43 | 44 | const dataView = new DataView(binaryMessage); 45 | const headerLength = dataView.getInt16(0); 46 | 47 | if (binaryMessage.byteLength < headerLength + 2) { 48 | throw new Error("Invalid binary message format. Header content missing."); 49 | } 50 | 51 | let headersString = ""; 52 | for (let i = 0; i < headerLength; i++) { 53 | headersString += String.fromCharCode((dataView).getInt8(i + 2)); 54 | } 55 | 56 | headers = this.ParseHeaders(headersString); 57 | 58 | if (binaryMessage.byteLength > headerLength + 2) { 59 | body = binaryMessage.slice(2 + headerLength); 60 | } 61 | 62 | deferral.Resolve(new ConnectionMessage(message.MessageType, body, headers, message.Id)); 63 | } 64 | } catch (e) { 65 | deferral.Reject(`Error formatting the message. Error: ${e}`); 66 | } 67 | 68 | return deferral.Promise(); 69 | } 70 | 71 | public FromConnectionMessage = (message: ConnectionMessage): Promise => { 72 | const deferral = new Deferred(); 73 | 74 | try { 75 | if (message.MessageType === MessageType.Text) { 76 | const payload = `${this.MakeHeaders(message)}${CRLF}${message.TextBody ? message.TextBody : ""}`; 77 | 78 | deferral.Resolve(new RawWebsocketMessage(MessageType.Text, payload, message.Id)); 79 | 80 | } else if (message.MessageType === MessageType.Binary) { 81 | const headersString = this.MakeHeaders(message); 82 | const content = message.BinaryBody; 83 | 84 | const headerInt8Array = new Int8Array(this.StringToArrayBuffer(headersString)); 85 | 86 | const payload = new ArrayBuffer(2 + headerInt8Array.byteLength + (content ? content.byteLength : 0)); 87 | const dataView = new DataView(payload); 88 | 89 | dataView.setInt16(0, headerInt8Array.length); 90 | 91 | for (let i = 0; i < headerInt8Array.byteLength; i++) { 92 | dataView.setInt8(2 + i, headerInt8Array[i]); 93 | } 94 | 95 | if (content) { 96 | const bodyInt8Array = new Int8Array(content); 97 | for (let i = 0; i < bodyInt8Array.byteLength; i++) { 98 | dataView.setInt8(2 + headerInt8Array.byteLength + i, bodyInt8Array[i]); 99 | } 100 | } 101 | 102 | deferral.Resolve(new RawWebsocketMessage(MessageType.Binary, payload, message.Id)); 103 | } 104 | } catch (e) { 105 | deferral.Reject(`Error formatting the message. ${e}`); 106 | } 107 | 108 | return deferral.Promise(); 109 | } 110 | 111 | private MakeHeaders = (message: ConnectionMessage): string => { 112 | let headersString: string = ""; 113 | 114 | if (message.Headers) { 115 | for (const header in message.Headers) { 116 | if (header) { 117 | headersString += `${header}: ${message.Headers[header]}${CRLF}`; 118 | } 119 | } 120 | } 121 | 122 | return headersString; 123 | } 124 | 125 | private ParseHeaders = (headersString: string): IStringDictionary => { 126 | const headers: IStringDictionary = {}; 127 | 128 | if (headersString) { 129 | const headerMatches = headersString.match(/[^\r\n]+/g); 130 | if (headers) { 131 | for (const header of headerMatches) { 132 | if (header) { 133 | const separatorIndex = header.indexOf(":"); 134 | const headerName = separatorIndex > 0 ? header.substr(0, separatorIndex).trim().toLowerCase() : header; 135 | const headerValue = 136 | separatorIndex > 0 && header.length > (separatorIndex + 1) ? 137 | header.substr(separatorIndex + 1).trim() : 138 | ""; 139 | 140 | headers[headerName] = headerValue; 141 | } 142 | } 143 | } 144 | } 145 | 146 | return headers; 147 | } 148 | 149 | private StringToArrayBuffer = (str: string): ArrayBuffer => { 150 | const buffer = new ArrayBuffer(str.length); 151 | const view = new DataView(buffer); 152 | for (let i = 0; i < str.length; i++) { 153 | view.setUint8(i, str.charCodeAt(i)); 154 | } 155 | return buffer; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/sdk/speech/ServiceTelemetryListener.Internal.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AudioSourceErrorEvent, 3 | AudioStreamNodeAttachedEvent, 4 | AudioStreamNodeAttachingEvent, 5 | AudioStreamNodeDetachedEvent, 6 | AudioStreamNodeErrorEvent, 7 | ConnectionClosedEvent, 8 | ConnectionEstablishedEvent, 9 | ConnectionEstablishErrorEvent, 10 | ConnectionMessageReceivedEvent, 11 | ConnectionStartEvent, 12 | IEventListener, 13 | IStringDictionary, 14 | PlatformEvent, 15 | } from "../../common/Exports"; 16 | import { 17 | ConnectingToServiceEvent, 18 | RecognitionTriggeredEvent, 19 | } from "./RecognitionEvents"; 20 | 21 | interface ITelemetry { 22 | Metrics: IMetric[]; 23 | ReceivedMessages: IStringDictionary; 24 | } 25 | 26 | // tslint:disable-next-line:max-classes-per-file 27 | interface IMetric { 28 | End: string; 29 | Error?: string; 30 | Id?: string; 31 | Name: string; 32 | Start: string; 33 | } 34 | 35 | // tslint:disable-next-line:max-classes-per-file 36 | export class ServiceTelemetryListener implements IEventListener { 37 | private isDisposed: boolean = false; 38 | 39 | private requestId: string; 40 | private audioSourceId: string; 41 | private audioNodeId: string; 42 | 43 | private listeningTriggerMetric: IMetric = null; 44 | private micMetric: IMetric = null; 45 | private connectionEstablishMetric: IMetric = null; 46 | 47 | private micStartTime: string; 48 | 49 | private connectionId: string; 50 | private connectionStartTime: string; 51 | 52 | private receivedMessages: IStringDictionary; 53 | 54 | constructor(requestId: string, audioSourceId: string, audioNodeId: string) { 55 | this.requestId = requestId; 56 | this.audioSourceId = audioSourceId; 57 | this.audioNodeId = audioNodeId; 58 | 59 | this.receivedMessages = {}; 60 | } 61 | 62 | public OnEvent = (e: PlatformEvent): void => { 63 | if (this.isDisposed) { 64 | return; 65 | } 66 | 67 | if (e instanceof RecognitionTriggeredEvent && e.RequestId === this.requestId) { 68 | this.listeningTriggerMetric = { 69 | End: e.EventTime, 70 | Name: "ListeningTrigger", 71 | Start: e.EventTime, 72 | }; 73 | } 74 | 75 | if (e instanceof AudioStreamNodeAttachingEvent && e.AudioSourceId === this.audioSourceId && e.AudioNodeId === this.audioNodeId) { 76 | this.micStartTime = e.EventTime; 77 | } 78 | 79 | if (e instanceof AudioStreamNodeAttachedEvent && e.AudioSourceId === this.audioSourceId && e.AudioNodeId === this.audioNodeId) { 80 | this.micStartTime = e.EventTime; 81 | } 82 | 83 | if (e instanceof AudioSourceErrorEvent && e.AudioSourceId === this.audioSourceId) { 84 | if (!this.micMetric) { 85 | this.micMetric = { 86 | End: e.EventTime, 87 | Error: e.Error, 88 | Name: "Microphone", 89 | Start: this.micStartTime, 90 | }; 91 | } 92 | } 93 | 94 | if (e instanceof AudioStreamNodeErrorEvent && e.AudioSourceId === this.audioSourceId && e.AudioNodeId === this.audioNodeId) { 95 | if (!this.micMetric) { 96 | this.micMetric = { 97 | End: e.EventTime, 98 | Error: e.Error, 99 | Name: "Microphone", 100 | Start: this.micStartTime, 101 | }; 102 | } 103 | } 104 | 105 | if (e instanceof AudioStreamNodeDetachedEvent && e.AudioSourceId === this.audioSourceId && e.AudioNodeId === this.audioNodeId) { 106 | if (!this.micMetric) { 107 | this.micMetric = { 108 | End: e.EventTime, 109 | Name: "Microphone", 110 | Start: this.micStartTime, 111 | }; 112 | } 113 | } 114 | 115 | if (e instanceof ConnectingToServiceEvent && e.RequestId === this.requestId) { 116 | this.connectionId = e.ConnectionId; 117 | } 118 | 119 | if (e instanceof ConnectionStartEvent && e.ConnectionId === this.connectionId) { 120 | this.connectionStartTime = e.EventTime; 121 | } 122 | 123 | if (e instanceof ConnectionEstablishedEvent && e.ConnectionId === this.connectionId) { 124 | if (!this.connectionEstablishMetric) { 125 | this.connectionEstablishMetric = { 126 | End: e.EventTime, 127 | Id: this.connectionId, 128 | Name: "Connection", 129 | Start: this.connectionStartTime, 130 | }; 131 | } 132 | } 133 | 134 | if (e instanceof ConnectionEstablishErrorEvent && e.ConnectionId === this.connectionId) { 135 | if (!this.connectionEstablishMetric) { 136 | this.connectionEstablishMetric = { 137 | End: e.EventTime, 138 | Error: this.GetConnectionError(e.StatusCode), 139 | Id: this.connectionId, 140 | Name: "Connection", 141 | Start: this.connectionStartTime, 142 | }; 143 | } 144 | } 145 | 146 | if (e instanceof ConnectionMessageReceivedEvent && e.ConnectionId === this.connectionId) { 147 | if (e.Message && e.Message.Headers && e.Message.Headers.path) { 148 | if (!this.receivedMessages[e.Message.Headers.path]) { 149 | this.receivedMessages[e.Message.Headers.path] = new Array(); 150 | } 151 | 152 | this.receivedMessages[e.Message.Headers.path].push(e.NetworkReceivedTime); 153 | } 154 | } 155 | } 156 | 157 | public GetTelemetry = (): string => { 158 | const metrics = new Array(); 159 | 160 | if (this.listeningTriggerMetric) { 161 | metrics.push(this.listeningTriggerMetric); 162 | } 163 | 164 | if (this.micMetric) { 165 | metrics.push(this.micMetric); 166 | } 167 | 168 | if (this.connectionEstablishMetric) { 169 | metrics.push(this.connectionEstablishMetric); 170 | } 171 | 172 | const telemetry: ITelemetry = { 173 | Metrics: metrics, 174 | ReceivedMessages: this.receivedMessages, 175 | }; 176 | 177 | const json = JSON.stringify(telemetry); 178 | 179 | // We dont want to send the same telemetry again. So clean those out. 180 | this.receivedMessages = {}; 181 | this.listeningTriggerMetric = null; 182 | this.micMetric = null; 183 | this.connectionEstablishMetric = null; 184 | 185 | return json; 186 | } 187 | 188 | public Dispose = (): void => { 189 | this.isDisposed = true; 190 | } 191 | 192 | private GetConnectionError = (statusCode: number): string => { 193 | /* 194 | -- Websocket status codes -- 195 | NormalClosure = 1000, 196 | EndpointUnavailable = 1001, 197 | ProtocolError = 1002, 198 | InvalidMessageType = 1003, 199 | Empty = 1005, 200 | InvalidPayloadData = 1007, 201 | PolicyViolation = 1008, 202 | MessageTooBig = 1009, 203 | MandatoryExtension = 1010, 204 | InternalServerError = 1011 205 | */ 206 | 207 | switch (statusCode) { 208 | case 400: 209 | case 1002: 210 | case 1003: 211 | case 1005: 212 | case 1007: 213 | case 1008: 214 | case 1009: return "BadRequest"; 215 | case 401: return "Unauthorized"; 216 | case 403: return "Forbidden"; 217 | case 503: 218 | case 1001: return "ServerUnavailable"; 219 | case 500: 220 | case 1011: return "ServerError"; 221 | case 408: 222 | case 504: return "Timeout"; 223 | default: return "statuscode:" + statusCode.toString(); 224 | } 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /src/common.browser/FileAudioSource.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AudioSourceErrorEvent, 3 | AudioSourceEvent, 4 | AudioSourceInitializingEvent, 5 | AudioSourceOffEvent, 6 | AudioSourceReadyEvent, 7 | AudioStreamNodeAttachedEvent, 8 | AudioStreamNodeAttachingEvent, 9 | AudioStreamNodeDetachedEvent, 10 | AudioStreamNodeErrorEvent, 11 | CreateNoDashGuid, 12 | Events, 13 | EventSource, 14 | IAudioSource, 15 | IAudioStreamNode, 16 | IStringDictionary, 17 | PlatformEvent, 18 | Promise, 19 | PromiseHelper, 20 | Stream, 21 | StreamReader, 22 | } from "../common/Exports"; 23 | 24 | import { Timer } from "../common.browser/Exports"; 25 | 26 | export class FileAudioSource implements IAudioSource { 27 | 28 | // Recommended sample rate (bytes/second). 29 | private static readonly SAMPLE_RATE: number = 16000 * 2; // 16 kHz * 16 bits 30 | 31 | // We should stream audio at no faster than 2x real-time (i.e., send five chunks 32 | // per second, with the chunk size == sample rate in bytes per second * 2 / 5). 33 | private static readonly CHUNK_SIZE: number = FileAudioSource.SAMPLE_RATE * 2 / 5; 34 | 35 | private static readonly UPLOAD_INTERVAL: number = 200; // milliseconds 36 | 37 | // 10 seconds of audio in bytes = 38 | // sample rate (bytes/second) * 600 (seconds) + 44 (size of the wave header). 39 | private static readonly MAX_SIZE: number = FileAudioSource.SAMPLE_RATE * 600 + 44; 40 | 41 | private streams: IStringDictionary> = {}; 42 | 43 | private id: string; 44 | 45 | private events: EventSource; 46 | 47 | private file: File; 48 | 49 | public constructor(file: File, audioSourceId?: string) { 50 | this.id = audioSourceId ? audioSourceId : CreateNoDashGuid(); 51 | this.events = new EventSource(); 52 | this.file = file; 53 | } 54 | 55 | public TurnOn = (): Promise => { 56 | if (typeof FileReader === "undefined") { 57 | const errorMsg = "Browser does not support FileReader."; 58 | this.OnEvent(new AudioSourceErrorEvent(errorMsg, "")); // initialization error - no streamid at this point 59 | return PromiseHelper.FromError(errorMsg); 60 | } else if (this.file.name.lastIndexOf(".wav") !== this.file.name.length - 4) { 61 | const errorMsg = this.file.name + " is not supported. Only WAVE files are allowed at the moment."; 62 | this.OnEvent(new AudioSourceErrorEvent(errorMsg, "")); 63 | return PromiseHelper.FromError(errorMsg); 64 | } else if (this.file.size > FileAudioSource.MAX_SIZE) { 65 | const errorMsg = this.file.name + " exceeds the maximum allowed file size (" + FileAudioSource.MAX_SIZE + ")."; 66 | this.OnEvent(new AudioSourceErrorEvent(errorMsg, "")); 67 | return PromiseHelper.FromError(errorMsg); 68 | } 69 | 70 | this.OnEvent(new AudioSourceInitializingEvent(this.id)); // no stream id 71 | this.OnEvent(new AudioSourceReadyEvent(this.id)); 72 | return PromiseHelper.FromResult(true); 73 | } 74 | 75 | public Id = (): string => { 76 | return this.id; 77 | } 78 | 79 | public Attach = (audioNodeId: string): Promise => { 80 | this.OnEvent(new AudioStreamNodeAttachingEvent(this.id, audioNodeId)); 81 | 82 | return this.Upload(audioNodeId).OnSuccessContinueWith( 83 | (streamReader: StreamReader) => { 84 | this.OnEvent(new AudioStreamNodeAttachedEvent(this.id, audioNodeId)); 85 | return { 86 | Detach: () => { 87 | streamReader.Close(); 88 | delete this.streams[audioNodeId]; 89 | this.OnEvent(new AudioStreamNodeDetachedEvent(this.id, audioNodeId)); 90 | this.TurnOff(); 91 | }, 92 | Id: () => { 93 | return audioNodeId; 94 | }, 95 | Read: () => { 96 | return streamReader.Read(); 97 | }, 98 | }; 99 | }); 100 | } 101 | 102 | public Detach = (audioNodeId: string): void => { 103 | if (audioNodeId && this.streams[audioNodeId]) { 104 | this.streams[audioNodeId].Close(); 105 | delete this.streams[audioNodeId]; 106 | this.OnEvent(new AudioStreamNodeDetachedEvent(this.id, audioNodeId)); 107 | } 108 | } 109 | 110 | public TurnOff = (): Promise => { 111 | for (const streamId in this.streams) { 112 | if (streamId) { 113 | const stream = this.streams[streamId]; 114 | if (stream && !stream.IsClosed) { 115 | stream.Close(); 116 | } 117 | } 118 | } 119 | 120 | this.OnEvent(new AudioSourceOffEvent(this.id)); // no stream now 121 | return PromiseHelper.FromResult(true); 122 | } 123 | 124 | public get Events(): EventSource { 125 | return this.events; 126 | } 127 | 128 | private Upload = (audioNodeId: string): Promise> => { 129 | return this.TurnOn() 130 | .OnSuccessContinueWith>((_: boolean) => { 131 | const stream = new Stream(audioNodeId); 132 | 133 | this.streams[audioNodeId] = stream; 134 | 135 | const reader: FileReader = new FileReader(); 136 | 137 | let startOffset = 0; 138 | let endOffset = FileAudioSource.CHUNK_SIZE; 139 | let lastWriteTimestamp = 0; 140 | 141 | const processNextChunk = (event: Event): void => { 142 | if (stream.IsClosed) { 143 | return; // output stream was closed (somebody called TurnOff). We're done here. 144 | } 145 | 146 | if (lastWriteTimestamp !== 0) { 147 | const delay = Date.now() - lastWriteTimestamp; 148 | if (delay < FileAudioSource.UPLOAD_INTERVAL) { 149 | // It's been less than the "upload interval" since we've uploaded the 150 | // last chunk. Schedule the next upload to make sure that we're sending 151 | // upstream roughly one chunk per upload interval. 152 | new Timer(FileAudioSource.UPLOAD_INTERVAL - delay, processNextChunk).start(); 153 | return; 154 | } 155 | } 156 | 157 | stream.Write(reader.result); 158 | lastWriteTimestamp = Date.now(); 159 | 160 | if (endOffset < this.file.size) { 161 | startOffset = endOffset; 162 | endOffset = Math.min(endOffset + FileAudioSource.CHUNK_SIZE, this.file.size); 163 | const chunk = this.file.slice(startOffset, endOffset); 164 | reader.readAsArrayBuffer(chunk); 165 | } else { 166 | // we've written the entire file to the output stream, can close it now. 167 | stream.Close(); 168 | } 169 | }; 170 | 171 | reader.onload = processNextChunk; 172 | 173 | reader.onerror = (event: ErrorEvent) => { 174 | const errorMsg = `Error occurred while processing '${this.file.name}'. ${event.error}`; 175 | this.OnEvent(new AudioStreamNodeErrorEvent(this.id, audioNodeId, event.error)); 176 | throw new Error(errorMsg); 177 | }; 178 | 179 | const chunk = this.file.slice(startOffset, endOffset); 180 | reader.readAsArrayBuffer(chunk); 181 | 182 | return stream.GetReader(); 183 | }); 184 | } 185 | 186 | private OnEvent = (event: AudioSourceEvent): void => { 187 | this.events.OnEvent(event); 188 | Events.Instance.OnEvent(event); 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/sdk/speech/RecognitionEvents.ts: -------------------------------------------------------------------------------- 1 | import { EventType, PlatformEvent } from "../../common/Exports"; 2 | import { 3 | IDetailedSpeechPhrase, 4 | ISimpleSpeechPhrase, 5 | ISpeechEndDetectedResult, 6 | ISpeechFragment, 7 | ISpeechStartDetectedResult, 8 | } from "./SpeechResults"; 9 | 10 | export class SpeechRecognitionEvent extends PlatformEvent { 11 | private requestId: string; 12 | 13 | constructor(eventName: string, requestId: string, eventType: EventType = EventType.Info) { 14 | super(eventName, eventType); 15 | 16 | this.requestId = requestId; 17 | } 18 | 19 | public get RequestId(): string { 20 | return this.requestId; 21 | } 22 | } 23 | 24 | // tslint:disable-next-line:max-classes-per-file 25 | export class SpeechRecognitionResultEvent extends SpeechRecognitionEvent { 26 | private result: TResult; 27 | 28 | constructor(eventName: string, requestId: string, result: TResult) { 29 | super(eventName, requestId); 30 | this.result = result; 31 | } 32 | 33 | public get Result(): TResult { 34 | return this.result; 35 | } 36 | } 37 | 38 | // tslint:disable-next-line:max-classes-per-file 39 | export class RecognitionTriggeredEvent extends SpeechRecognitionEvent { 40 | private audioSourceId: string; 41 | private audioNodeId: string; 42 | 43 | constructor(requestId: string, audioSourceId: string, audioNodeId: string) { 44 | super("RecognitionTriggeredEvent", requestId); 45 | 46 | this.audioSourceId = audioSourceId; 47 | this.audioNodeId = audioNodeId; 48 | } 49 | 50 | public get AudioSourceId(): string { 51 | return this.audioSourceId; 52 | } 53 | 54 | public get AudioNodeId(): string { 55 | return this.audioNodeId; 56 | } 57 | } 58 | 59 | // tslint:disable-next-line:max-classes-per-file 60 | export class ListeningStartedEvent extends SpeechRecognitionEvent { 61 | private audioSourceId: string; 62 | private audioNodeId: string; 63 | 64 | constructor(requestId: string, audioSourceId: string, audioNodeId: string) { 65 | super("ListeningStartedEvent", requestId); 66 | this.audioSourceId = audioSourceId; 67 | this.audioNodeId = audioNodeId; 68 | } 69 | 70 | public get AudioSourceId(): string { 71 | return this.audioSourceId; 72 | } 73 | 74 | public get AudioNodeId(): string { 75 | return this.audioNodeId; 76 | } 77 | } 78 | 79 | // tslint:disable-next-line:max-classes-per-file 80 | export class ConnectingToServiceEvent extends SpeechRecognitionEvent { 81 | private authFetchEventid: string; 82 | private connectionId: string; 83 | 84 | constructor(requestId: string, authFetchEventid: string, connectionId: string) { 85 | super("ConnectingToServiceEvent", requestId); 86 | this.authFetchEventid = authFetchEventid; 87 | this.connectionId = connectionId; 88 | } 89 | 90 | public get AuthFetchEventid(): string { 91 | return this.authFetchEventid; 92 | } 93 | 94 | public get ConnectionId(): string { 95 | return this.connectionId; 96 | } 97 | } 98 | 99 | // tslint:disable-next-line:max-classes-per-file 100 | export class RecognitionStartedEvent extends SpeechRecognitionEvent { 101 | private audioSourceId: string; 102 | private audioNodeId: string; 103 | private authFetchEventId: string; 104 | private connectionId: string; 105 | 106 | constructor(requestId: string, audioSourceId: string, audioNodeId: string, authFetchEventId: string, connectionId: string) { 107 | super("RecognitionStartedEvent", requestId); 108 | 109 | this.audioSourceId = audioSourceId; 110 | this.audioNodeId = audioNodeId; 111 | this.authFetchEventId = authFetchEventId; 112 | this.connectionId = connectionId; 113 | } 114 | 115 | public get AudioSourceId(): string { 116 | return this.audioSourceId; 117 | } 118 | 119 | public get AudioNodeId(): string { 120 | return this.audioNodeId; 121 | } 122 | 123 | public get AuthFetchEventId(): string { 124 | return this.authFetchEventId; 125 | } 126 | 127 | public get ConnectionId(): string { 128 | return this.connectionId; 129 | } 130 | } 131 | 132 | // tslint:disable-next-line:max-classes-per-file 133 | export class SpeechStartDetectedEvent extends SpeechRecognitionResultEvent { 134 | constructor(requestId: string, result: ISpeechStartDetectedResult) { 135 | super("SpeechStartDetectedEvent", requestId, result); 136 | } 137 | } 138 | 139 | // tslint:disable-next-line:max-classes-per-file 140 | export class SpeechHypothesisEvent extends SpeechRecognitionResultEvent { 141 | constructor(requestId: string, result: ISpeechFragment) { 142 | super("SpeechHypothesisEvent", requestId, result); 143 | } 144 | } 145 | 146 | // tslint:disable-next-line:max-classes-per-file 147 | export class SpeechFragmentEvent extends SpeechRecognitionResultEvent { 148 | constructor(requestId: string, result: ISpeechFragment) { 149 | super("SpeechFragmentEvent", requestId, result); 150 | } 151 | } 152 | 153 | // tslint:disable-next-line:max-classes-per-file 154 | export class SpeechEndDetectedEvent extends SpeechRecognitionResultEvent { 155 | constructor(requestId: string, result: ISpeechEndDetectedResult) { 156 | super("SpeechEndDetectedEvent", requestId, result); 157 | } 158 | } 159 | 160 | // tslint:disable-next-line:max-classes-per-file 161 | export class SpeechSimplePhraseEvent extends SpeechRecognitionResultEvent { 162 | constructor(requestId: string, result: ISimpleSpeechPhrase) { 163 | super("SpeechSimplePhraseEvent", requestId, result); 164 | } 165 | } 166 | 167 | // tslint:disable-next-line:max-classes-per-file 168 | export class SpeechDetailedPhraseEvent extends SpeechRecognitionResultEvent { 169 | constructor(requestId: string, result: IDetailedSpeechPhrase) { 170 | super("SpeechDetailedPhraseEvent", requestId, result); 171 | } 172 | } 173 | 174 | export enum RecognitionCompletionStatus { 175 | Success, 176 | AudioSourceError, 177 | AudioSourceTimeout, 178 | AuthTokenFetchError, 179 | AuthTokenFetchTimeout, 180 | UnAuthorized, 181 | ConnectTimeout, 182 | ConnectError, 183 | ClientRecognitionActivityTimeout, 184 | UnknownError, 185 | } 186 | 187 | // tslint:disable-next-line:max-classes-per-file 188 | export class RecognitionEndedEvent extends SpeechRecognitionEvent { 189 | private audioSourceId: string; 190 | private audioNodeId: string; 191 | private authFetchEventId: string; 192 | private connectionId: string; 193 | private serviceTag: string; 194 | private status: RecognitionCompletionStatus; 195 | private error: string; 196 | 197 | constructor( 198 | requestId: string, 199 | audioSourceId: string, 200 | audioNodeId: string, 201 | authFetchEventId: string, 202 | connectionId: string, 203 | serviceTag: string, 204 | status: RecognitionCompletionStatus, 205 | error: string) { 206 | 207 | super("RecognitionEndedEvent", requestId, status === RecognitionCompletionStatus.Success ? EventType.Info : EventType.Error); 208 | 209 | this.audioSourceId = audioSourceId; 210 | this.audioNodeId = audioNodeId; 211 | this.connectionId = connectionId; 212 | this.authFetchEventId = authFetchEventId; 213 | this.status = status; 214 | this.error = error; 215 | this.serviceTag = serviceTag; 216 | } 217 | 218 | public get AudioSourceId(): string { 219 | return this.audioSourceId; 220 | } 221 | 222 | public get AudioNodeId(): string { 223 | return this.audioNodeId; 224 | } 225 | 226 | public get AuthFetchEventId(): string { 227 | return this.authFetchEventId; 228 | } 229 | 230 | public get ConnectionId(): string { 231 | return this.connectionId; 232 | } 233 | 234 | public get ServiceTag(): string { 235 | return this.serviceTag; 236 | } 237 | 238 | public get Status(): RecognitionCompletionStatus { 239 | return this.status; 240 | } 241 | 242 | public get Error(): string { 243 | return this.error; 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/common/List.ts: -------------------------------------------------------------------------------- 1 | import { ObjectDisposedError } from "./Error"; 2 | import { IDetachable } from "./IDetachable"; 3 | import { IStringDictionary } from "./IDictionary"; 4 | import { IDisposable } from "./IDisposable"; 5 | 6 | export interface IList extends IDisposable { 7 | Get(itemIndex: number): TItem; 8 | First(): TItem; 9 | Last(): TItem; 10 | 11 | Add(item: TItem): void; 12 | InsertAt(index: number, item: TItem): void; 13 | 14 | RemoveFirst(): TItem; 15 | RemoveLast(): TItem; 16 | RemoveAt(index: number): TItem; 17 | Remove(index: number, count: number): TItem[]; 18 | Clear(): void; 19 | 20 | Length(): number; 21 | 22 | OnAdded(addedCallback: () => void): IDetachable; 23 | OnRemoved(removedCallback: () => void): IDetachable; 24 | OnDisposed(disposedCallback: () => void): IDetachable; 25 | 26 | Join(seperator?: string): string; 27 | 28 | ToArray(): TItem[]; 29 | 30 | Any(callback?: (item: TItem, index: number) => boolean): boolean; 31 | All(callback: (item: TItem) => boolean): boolean; 32 | ForEach(callback: (item: TItem, index: number) => void): void; 33 | Select(callback: (item: TItem, index: number) => T2): List; 34 | Where(callback: (item: TItem, index: number) => boolean): List; 35 | OrderBy(compareFn: (a: TItem, b: TItem) => number): List; 36 | OrderByDesc(compareFn: (a: TItem, b: TItem) => number): List; 37 | Clone(): List; 38 | Concat(list: List): List; 39 | ConcatArray(array: TItem[]): List; 40 | } 41 | 42 | export class List implements IList { 43 | private list: TItem[]; 44 | private subscriptionIdCounter: number = 0; 45 | private addSubscriptions: IStringDictionary<() => void> = {}; 46 | private removeSubscriptions: IStringDictionary<() => void> = {}; 47 | private disposedSubscriptions: IStringDictionary<() => void> = {}; 48 | private disposeReason: string = null; 49 | 50 | public constructor(list?: TItem[]) { 51 | this.list = []; 52 | // copy the list rather than taking as is. 53 | if (list) { 54 | for (const item of list) { 55 | this.list.push(item); 56 | } 57 | } 58 | } 59 | 60 | public Get = (itemIndex: number): TItem => { 61 | this.ThrowIfDisposed(); 62 | return this.list[itemIndex]; 63 | } 64 | 65 | public First = (): TItem => { 66 | return this.Get(0); 67 | } 68 | 69 | public Last = (): TItem => { 70 | return this.Get(this.Length() - 1); 71 | } 72 | 73 | public Add = (item: TItem): void => { 74 | this.ThrowIfDisposed(); 75 | this.InsertAt(this.list.length, item); 76 | } 77 | 78 | public InsertAt = (index: number, item: TItem): void => { 79 | this.ThrowIfDisposed(); 80 | if (index === 0) { 81 | this.list.unshift(item); 82 | } else if (index === this.list.length) { 83 | this.list.push(item); 84 | } else { 85 | this.list.splice(index, 0, item); 86 | } 87 | this.TriggerSubscriptions(this.addSubscriptions); 88 | } 89 | 90 | public RemoveFirst = (): TItem => { 91 | this.ThrowIfDisposed(); 92 | return this.RemoveAt(0); 93 | } 94 | 95 | public RemoveLast = (): TItem => { 96 | this.ThrowIfDisposed(); 97 | return this.RemoveAt(this.Length() - 1); 98 | } 99 | 100 | public RemoveAt = (index: number): TItem => { 101 | this.ThrowIfDisposed(); 102 | return this.Remove(index, 1)[0]; 103 | } 104 | 105 | public Remove = (index: number, count: number): TItem[] => { 106 | this.ThrowIfDisposed(); 107 | const removedElements = this.list.splice(index, count); 108 | this.TriggerSubscriptions(this.removeSubscriptions); 109 | return removedElements; 110 | } 111 | 112 | public Clear = (): void => { 113 | this.ThrowIfDisposed(); 114 | this.Remove(0, this.Length()); 115 | } 116 | 117 | public Length = (): number => { 118 | this.ThrowIfDisposed(); 119 | return this.list.length; 120 | } 121 | 122 | public OnAdded = (addedCallback: () => void): IDetachable => { 123 | this.ThrowIfDisposed(); 124 | const subscriptionId = this.subscriptionIdCounter++; 125 | 126 | this.addSubscriptions[subscriptionId] = addedCallback; 127 | 128 | return { 129 | Detach: () => { 130 | delete this.addSubscriptions[subscriptionId]; 131 | }, 132 | }; 133 | } 134 | 135 | public OnRemoved = (removedCallback: () => void): IDetachable => { 136 | this.ThrowIfDisposed(); 137 | const subscriptionId = this.subscriptionIdCounter++; 138 | 139 | this.removeSubscriptions[subscriptionId] = removedCallback; 140 | 141 | return { 142 | Detach: () => { 143 | delete this.removeSubscriptions[subscriptionId]; 144 | }, 145 | }; 146 | } 147 | 148 | public OnDisposed = (disposedCallback: () => void): IDetachable => { 149 | this.ThrowIfDisposed(); 150 | const subscriptionId = this.subscriptionIdCounter++; 151 | 152 | this.disposedSubscriptions[subscriptionId] = disposedCallback; 153 | 154 | return { 155 | Detach: () => { 156 | delete this.disposedSubscriptions[subscriptionId]; 157 | }, 158 | }; 159 | } 160 | 161 | public Join = (seperator?: string): string => { 162 | this.ThrowIfDisposed(); 163 | return this.list.join(seperator); 164 | } 165 | 166 | public ToArray = (): TItem[] => { 167 | const cloneCopy = Array(); 168 | this.list.forEach((val: TItem) => { 169 | cloneCopy.push(val); 170 | }); 171 | return cloneCopy; 172 | } 173 | 174 | public Any = (callback?: (item: TItem, index: number) => boolean): boolean => { 175 | this.ThrowIfDisposed(); 176 | if (callback) { 177 | return this.Where(callback).Length() > 0; 178 | } else { 179 | return this.Length() > 0; 180 | } 181 | } 182 | 183 | public All = (callback: (item: TItem) => boolean): boolean => { 184 | this.ThrowIfDisposed(); 185 | return this.Where(callback).Length() === this.Length(); 186 | } 187 | 188 | public ForEach = (callback: (item: TItem, index: number) => void): void => { 189 | this.ThrowIfDisposed(); 190 | for (let i = 0; i < this.Length(); i++) { 191 | callback(this.list[i], i); 192 | } 193 | } 194 | 195 | public Select = (callback: (item: TItem, index: number) => T2): List => { 196 | this.ThrowIfDisposed(); 197 | const selectList: T2[] = []; 198 | for (let i = 0; i < this.list.length; i++) { 199 | selectList.push(callback(this.list[i], i)); 200 | } 201 | 202 | return new List(selectList); 203 | } 204 | 205 | public Where = (callback: (item: TItem, index: number) => boolean): List => { 206 | this.ThrowIfDisposed(); 207 | const filteredList = new List(); 208 | for (let i = 0; i < this.list.length; i++) { 209 | if (callback(this.list[i], i)) { 210 | filteredList.Add(this.list[i]); 211 | } 212 | } 213 | return filteredList; 214 | } 215 | 216 | public OrderBy = (compareFn: (a: TItem, b: TItem) => number): List => { 217 | this.ThrowIfDisposed(); 218 | const clonedArray = this.ToArray(); 219 | const orderedArray = clonedArray.sort(compareFn); 220 | return new List(orderedArray); 221 | } 222 | 223 | public OrderByDesc = (compareFn: (a: TItem, b: TItem) => number): List => { 224 | this.ThrowIfDisposed(); 225 | return this.OrderBy((a: TItem, b: TItem) => compareFn(b, a)); 226 | } 227 | 228 | public Clone = (): List => { 229 | this.ThrowIfDisposed(); 230 | return new List(this.ToArray()); 231 | } 232 | 233 | public Concat = (list: List): List => { 234 | this.ThrowIfDisposed(); 235 | return new List(this.list.concat(list.ToArray())); 236 | } 237 | 238 | public ConcatArray = (array: TItem[]): List => { 239 | this.ThrowIfDisposed(); 240 | return new List(this.list.concat(array)); 241 | } 242 | 243 | public IsDisposed = (): boolean => { 244 | return this.list == null; 245 | } 246 | 247 | public Dispose = (reason?: string): void => { 248 | if (!this.IsDisposed()) { 249 | this.disposeReason = reason; 250 | this.list = null; 251 | this.addSubscriptions = null; 252 | this.removeSubscriptions = null; 253 | this.TriggerSubscriptions(this.disposedSubscriptions); 254 | } 255 | } 256 | 257 | private ThrowIfDisposed = (): void => { 258 | if (this.IsDisposed()) { 259 | throw new ObjectDisposedError("List", this.disposeReason); 260 | } 261 | } 262 | 263 | private TriggerSubscriptions = (subscriptions: IStringDictionary<() => void>): void => { 264 | if (subscriptions) { 265 | for (const subscriptionId in subscriptions) { 266 | if (subscriptionId) { 267 | subscriptions[subscriptionId](); 268 | } 269 | } 270 | } 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![npm version](https://badge.fury.io/js/microsoft-speech-browser-sdk.svg)](https://www.npmjs.com/package/microsoft-speech-browser-sdk) 2 | 3 | ## Prerequisites 4 | 5 | ### Subscribe to the Speech Recognition API, and get a free trial subscription key 6 | 7 | The Speech API is part of Cognitive Services. You can get free trial subscription keys from the [Cognitive Services subscription](https://azure.microsoft.com/try/cognitive-services/) page. After you select the Speech API, select **Get API Key** to get the key. It returns a primary and secondary key. Both keys are tied to the same quota, so you can use either key. 8 | 9 | **Note:** Before you can use Speech client libraries, you must have a [subscription key](https://azure.microsoft.com/try/cognitive-services/). 10 | 11 | ## Get started 12 | 13 | In this section we will walk you through the necessary steps to load a sample HTML page. The sample is located in our [github repository](https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript). You can **open the sample directly** from the repository, or **open the sample from a local copy** of the repository. 14 | 15 | **Note:** Some browsers block microphone access on un-secure origin. So, it is recommended to host the 'sample'/'your app' on https to get it working on all supported browsers. 16 | 17 | ### Open the sample directly 18 | 19 | Acquire a subscription key as described above. Then open the [link to the sample](https://htmlpreview.github.io/?https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript/blob/preview/samples/browser/Sample.html). This will load the page into your default browser (Rendered using [htmlPreview](https://github.com/htmlpreview/htmlpreview.github.com)). 20 | 21 | ### Open the sample from a local copy 22 | 23 | To try the sample locally, clone this repository: 24 | 25 | ``` 26 | git clone https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript 27 | ``` 28 | 29 | compile the TypeScript sources and bundle/browserfy them into a single JavaScript file ([npm](https://www.npmjs.com/) needs to be installed on your machine). Change into the root of the cloned repository and run the commands: 30 | 31 | ``` 32 | cd SpeechToText-WebSockets-Javascript && npm run bundle 33 | ``` 34 | 35 | Open `samples\browser\Sample.html` in your favorite browser. 36 | 37 | ## Next steps 38 | 39 | ### Installation of npm package 40 | 41 | An npm package of the Microsoft Speech Javascript Websocket SDK is available. To install the [npm package](https://www.npmjs.com/package/microsoft-speech-browser-sdk) run 42 | ``` 43 | npm install microsoft-speech-browser-sdk 44 | ``` 45 | 46 | ### As a Node module 47 | 48 | If you're building a node app and want to use the Speech SDK, all you need to do is add the following import statement: 49 | 50 | ```javascript 51 | import * as SDK from 'microsoft-speech-browser-sdk'; 52 | ``` 53 | 54 | and setup the recognizer: 55 | 56 | ```javascript 57 | function RecognizerSetup(SDK, recognitionMode, language, format, subscriptionKey) { 58 | let recognizerConfig = new SDK.RecognizerConfig( 59 | new SDK.SpeechConfig( 60 | new SDK.Context( 61 | new SDK.OS(navigator.userAgent, "Browser", null), 62 | new SDK.Device("SpeechSample", "SpeechSample", "1.0.00000"))), 63 | recognitionMode, // SDK.RecognitionMode.Interactive (Options - Interactive/Conversation/Dictation) 64 | language, // Supported languages are specific to each recognition mode Refer to docs. 65 | format); // SDK.SpeechResultFormat.Simple (Options - Simple/Detailed) 66 | 67 | // Alternatively use SDK.CognitiveTokenAuthentication(fetchCallback, fetchOnExpiryCallback) for token auth 68 | let authentication = new SDK.CognitiveSubscriptionKeyAuthentication(subscriptionKey); 69 | 70 | return SDK.Recognizer.Create(recognizerConfig, authentication); 71 | } 72 | 73 | function RecognizerStart(SDK, recognizer) { 74 | recognizer.Recognize((event) => { 75 | /* 76 | Alternative syntax for typescript devs. 77 | if (event instanceof SDK.RecognitionTriggeredEvent) 78 | */ 79 | switch (event.Name) { 80 | case "RecognitionTriggeredEvent" : 81 | UpdateStatus("Initializing"); 82 | break; 83 | case "ListeningStartedEvent" : 84 | UpdateStatus("Listening"); 85 | break; 86 | case "RecognitionStartedEvent" : 87 | UpdateStatus("Listening_Recognizing"); 88 | break; 89 | case "SpeechStartDetectedEvent" : 90 | UpdateStatus("Listening_DetectedSpeech_Recognizing"); 91 | console.log(JSON.stringify(event.Result)); // check console for other information in result 92 | break; 93 | case "SpeechHypothesisEvent" : 94 | UpdateRecognizedHypothesis(event.Result.Text); 95 | console.log(JSON.stringify(event.Result)); // check console for other information in result 96 | break; 97 | case "SpeechFragmentEvent" : 98 | UpdateRecognizedHypothesis(event.Result.Text); 99 | console.log(JSON.stringify(event.Result)); // check console for other information in result 100 | break; 101 | case "SpeechEndDetectedEvent" : 102 | OnSpeechEndDetected(); 103 | UpdateStatus("Processing_Adding_Final_Touches"); 104 | console.log(JSON.stringify(event.Result)); // check console for other information in result 105 | break; 106 | case "SpeechSimplePhraseEvent" : 107 | UpdateRecognizedPhrase(JSON.stringify(event.Result, null, 3)); 108 | break; 109 | case "SpeechDetailedPhraseEvent" : 110 | UpdateRecognizedPhrase(JSON.stringify(event.Result, null, 3)); 111 | break; 112 | case "RecognitionEndedEvent" : 113 | OnComplete(); 114 | UpdateStatus("Idle"); 115 | console.log(JSON.stringify(event)); // Debug information 116 | break; 117 | } 118 | }) 119 | .On(() => { 120 | // The request succeeded. Nothing to do here. 121 | }, 122 | (error) => { 123 | console.error(error); 124 | }); 125 | } 126 | 127 | function RecognizerStop(SDK, recognizer) { 128 | // recognizer.AudioSource.Detach(audioNodeId) can be also used here. (audioNodeId is part of ListeningStartedEvent) 129 | recognizer.AudioSource.TurnOff(); 130 | } 131 | ``` 132 | 133 | ### In a Browser, using Webpack 134 | 135 | Currently, the TypeScript code in this SDK is compiled using the default module system (CommonJS), which means that the compilation produces a number of distinct JS source files. To make the SDK usable in a browser, it first needs to be "browserified" (all the javascript sources need to be glued together). Towards this end, this is what you need to do: 136 | 137 | 1. Add `require` statement to you web app source file, for instance (take a look at [sample_app.js](samples/browser/sample_app.js)): 138 | 139 | ```javascript 140 | var SDK = require('/Speech.Browser.Sdk.js'); 141 | ``` 142 | 143 | 2. Setup the recognizer, same as [above](#reco_setup). 144 | 145 | 3. Run your web-app through the webpack (see "bundle" task in [gulpfile.js](gulpfile.js), to execute it, run `npm run bundle`). 146 | 147 | 4. Add the generated bundle to your html page: 148 | 149 | ``` 150 | 151 | ``` 152 | 153 | ### In a Browser, as a native ES6 module 154 | 155 | ...in progress, will be available soon 156 | 157 | ### Token-based authentication 158 | 159 | To use token-based authentication, please launch a local node server, as described [here](https://github.com/Azure-Samples/SpeechToText-WebSockets-Javascript/blob/master/samples/browser/README.md) 160 | 161 | ## Docs 162 | The SDK is a reference implementation for the speech websocket protocol. Check the [API reference](https://docs.microsoft.com/en-us/azure/cognitive-services/speech/API-reference-rest/bingvoicerecognition#websocket) and [Websocket protocol reference](https://docs.microsoft.com/en-us/azure/cognitive-services/speech/API-reference-rest/websocketprotocol) for more details. 163 | 164 | ## Browser support 165 | The SDK depends on WebRTC APIs to get access to the microphone and read the audio stream. Most of todays browsers(Edge/Chrome/Firefox) support this. For more details about supported browsers refer to [navigator.getUserMedia#BrowserCompatibility](https://developer.mozilla.org/en-US/docs/Web/API/Navigator/getUserMedia#Browser_compatibility) 166 | 167 | **Note:** The SDK currently depends on [navigator.getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/Navigator/getUserMedia#Browser_compatibility) API. However this API is in process of being dropped as browsers are moving towards newer [MediaDevices.getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) instead. The SDK will add support to the newer API soon. 168 | 169 | ## Contributing 170 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 171 | -------------------------------------------------------------------------------- /src/common.browser/MicAudioSource.ts: -------------------------------------------------------------------------------- 1 | import { 2 | AudioSourceErrorEvent, 3 | AudioSourceEvent, 4 | AudioSourceInitializingEvent, 5 | AudioSourceOffEvent, 6 | AudioSourceReadyEvent, 7 | AudioStreamNodeAttachedEvent, 8 | AudioStreamNodeAttachingEvent, 9 | AudioStreamNodeDetachedEvent, 10 | AudioStreamNodeErrorEvent, 11 | CreateNoDashGuid, 12 | Deferred, 13 | Events, 14 | EventSource, 15 | IAudioSource, 16 | IAudioStreamNode, 17 | IStringDictionary, 18 | PlatformEvent, 19 | Promise, 20 | PromiseHelper, 21 | Stream, 22 | StreamReader, 23 | } from "../common/Exports"; 24 | import { IRecorder } from "./IRecorder"; 25 | 26 | // Extending the default definition with browser specific definitions for backward compatibility 27 | interface INavigatorUserMedia extends NavigatorUserMedia { 28 | webkitGetUserMedia?: (constraints: MediaStreamConstraints, successCallback: NavigatorUserMediaSuccessCallback, errorCallback: NavigatorUserMediaErrorCallback) => void; 29 | mozGetUserMedia?: (constraints: MediaStreamConstraints, successCallback: NavigatorUserMediaSuccessCallback, errorCallback: NavigatorUserMediaErrorCallback) => void; 30 | msGetUserMedia?: (constraints: MediaStreamConstraints, successCallback: NavigatorUserMediaSuccessCallback, errorCallback: NavigatorUserMediaErrorCallback) => void; 31 | } 32 | 33 | export class MicAudioSource implements IAudioSource { 34 | 35 | private streams: IStringDictionary> = {}; 36 | 37 | private id: string; 38 | 39 | private events: EventSource; 40 | 41 | private initializeDeferral: Deferred; 42 | 43 | private recorder: IRecorder; 44 | 45 | private mediaStream: MediaStream; 46 | 47 | private context: AudioContext; 48 | 49 | public constructor(recorder: IRecorder, audioSourceId?: string) { 50 | this.id = audioSourceId ? audioSourceId : CreateNoDashGuid(); 51 | this.events = new EventSource(); 52 | this.recorder = recorder; 53 | } 54 | 55 | public TurnOn = (): Promise => { 56 | if (this.initializeDeferral) { 57 | return this.initializeDeferral.Promise(); 58 | } 59 | 60 | this.initializeDeferral = new Deferred(); 61 | 62 | this.CreateAudioContext(); 63 | 64 | const nav = window.navigator as INavigatorUserMedia; 65 | 66 | let getUserMedia = ( 67 | nav.getUserMedia || 68 | nav.webkitGetUserMedia || 69 | nav.mozGetUserMedia || 70 | nav.msGetUserMedia 71 | ); 72 | 73 | if (!!nav.mediaDevices) { 74 | getUserMedia = (constraints: MediaStreamConstraints, successCallback: NavigatorUserMediaSuccessCallback, errorCallback: NavigatorUserMediaErrorCallback): void => { 75 | nav.mediaDevices 76 | .getUserMedia(constraints) 77 | .then(successCallback) 78 | .catch(errorCallback); 79 | }; 80 | } 81 | 82 | if (!getUserMedia) { 83 | const errorMsg = "Browser does not support getUserMedia."; 84 | this.initializeDeferral.Reject(errorMsg); 85 | this.OnEvent(new AudioSourceErrorEvent(errorMsg, "")); // mic initialized error - no streamid at this point 86 | } else { 87 | const next = () => { 88 | this.OnEvent(new AudioSourceInitializingEvent(this.id)); // no stream id 89 | getUserMedia( 90 | { audio: true, video: false }, 91 | (mediaStream: MediaStream) => { 92 | this.mediaStream = mediaStream; 93 | this.OnEvent(new AudioSourceReadyEvent(this.id)); 94 | this.initializeDeferral.Resolve(true); 95 | }, (error: MediaStreamError) => { 96 | const errorMsg = `Error occurred during microphone initialization: ${error}`; 97 | const tmp = this.initializeDeferral; 98 | // HACK: this should be handled through onError callbacks of all promises up the stack. 99 | // Unfortunately, the current implementation does not provide an easy way to reject promises 100 | // without a lot of code replication. 101 | // TODO: fix promise implementation, allow for a graceful reject chaining. 102 | this.initializeDeferral = null; 103 | tmp.Reject(errorMsg); // this will bubble up through the whole chain of promises, 104 | // with each new level adding extra "Unhandled callback error" prefix to the error message. 105 | // The following line is not guaranteed to be executed. 106 | this.OnEvent(new AudioSourceErrorEvent(this.id, errorMsg)); 107 | }); 108 | }; 109 | 110 | if (this.context.state === "suspended") { 111 | // NOTE: On iOS, the Web Audio API requires sounds to be triggered from an explicit user action. 112 | // https://github.com/WebAudio/web-audio-api/issues/790 113 | this.context.resume().then(next, (reason: any) => { 114 | this.initializeDeferral.Reject(`Failed to initialize audio context: ${reason}`); 115 | }); 116 | } else { 117 | next(); 118 | } 119 | } 120 | 121 | return this.initializeDeferral.Promise(); 122 | } 123 | 124 | public Id = (): string => { 125 | return this.id; 126 | } 127 | 128 | public Attach = (audioNodeId: string): Promise => { 129 | this.OnEvent(new AudioStreamNodeAttachingEvent(this.id, audioNodeId)); 130 | 131 | return this.Listen(audioNodeId).OnSuccessContinueWith( 132 | (streamReader: StreamReader) => { 133 | this.OnEvent(new AudioStreamNodeAttachedEvent(this.id, audioNodeId)); 134 | return { 135 | Detach: () => { 136 | streamReader.Close(); 137 | delete this.streams[audioNodeId]; 138 | this.OnEvent(new AudioStreamNodeDetachedEvent(this.id, audioNodeId)); 139 | this.TurnOff(); 140 | }, 141 | Id: () => { 142 | return audioNodeId; 143 | }, 144 | Read: () => { 145 | return streamReader.Read(); 146 | }, 147 | }; 148 | }); 149 | } 150 | 151 | public Detach = (audioNodeId: string): void => { 152 | if (audioNodeId && this.streams[audioNodeId]) { 153 | this.streams[audioNodeId].Close(); 154 | delete this.streams[audioNodeId]; 155 | this.OnEvent(new AudioStreamNodeDetachedEvent(this.id, audioNodeId)); 156 | } 157 | } 158 | 159 | public TurnOff = (): Promise => { 160 | for (const streamId in this.streams) { 161 | if (streamId) { 162 | const stream = this.streams[streamId]; 163 | if (stream) { 164 | stream.Close(); 165 | } 166 | } 167 | } 168 | 169 | this.OnEvent(new AudioSourceOffEvent(this.id)); // no stream now 170 | this.initializeDeferral = null; 171 | 172 | this.DestroyAudioContext(); 173 | 174 | return PromiseHelper.FromResult(true); 175 | } 176 | 177 | public get Events(): EventSource { 178 | return this.events; 179 | } 180 | 181 | private Listen = (audioNodeId: string): Promise> => { 182 | return this.TurnOn() 183 | .OnSuccessContinueWith>((_: boolean) => { 184 | const stream = new Stream(audioNodeId); 185 | this.streams[audioNodeId] = stream; 186 | 187 | try { 188 | this.recorder.Record(this.context, this.mediaStream, stream); 189 | } catch (error) { 190 | this.OnEvent(new AudioStreamNodeErrorEvent(this.id, audioNodeId, error)); 191 | throw error; 192 | } 193 | 194 | return stream.GetReader(); 195 | }); 196 | } 197 | 198 | private OnEvent = (event: AudioSourceEvent): void => { 199 | this.events.OnEvent(event); 200 | Events.Instance.OnEvent(event); 201 | } 202 | 203 | private CreateAudioContext = (): void => { 204 | if (!!this.context) { 205 | return; 206 | } 207 | 208 | // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext 209 | const AudioContext = ((window as any).AudioContext) 210 | || ((window as any).webkitAudioContext) 211 | || false; 212 | 213 | if (!AudioContext) { 214 | throw new Error("Browser does not support Web Audio API (AudioContext is not available)."); 215 | } 216 | 217 | this.context = new AudioContext(); 218 | } 219 | 220 | private DestroyAudioContext = (): void => { 221 | if (!this.context) { 222 | return; 223 | } 224 | 225 | this.recorder.ReleaseMediaResources(this.context); 226 | 227 | if ("close" in this.context) { 228 | this.context.close(); 229 | this.context = null; 230 | } else if (this.context.state === "running") { 231 | // Suspend actually takes a callback, but analogous to the 232 | // resume method, it'll be only fired if suspend is called 233 | // in a direct response to a user action. The later is not always 234 | // the case, as TurnOff is also called, when we receive an 235 | // end-of-speech message from the service. So, doing a best effort 236 | // fire-and-forget here. 237 | this.context.suspend(); 238 | } 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/common.browser/WebsocketMessageAdapter.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArgumentNullError, 3 | ConnectionClosedEvent, 4 | ConnectionEstablishedEvent, 5 | ConnectionEstablishErrorEvent, 6 | ConnectionEvent, 7 | ConnectionMessage, 8 | ConnectionMessageReceivedEvent, 9 | ConnectionMessageSentEvent, 10 | ConnectionOpenResponse, 11 | ConnectionStartEvent, 12 | ConnectionState, 13 | Deferred, 14 | Events, 15 | EventSource, 16 | IWebsocketMessageFormatter, 17 | MessageType, 18 | PlatformEvent, 19 | Promise, 20 | PromiseHelper, 21 | Queue, 22 | RawWebsocketMessage, 23 | } from "../common/Exports"; 24 | 25 | interface ISendItem { 26 | Message: ConnectionMessage; 27 | RawWebsocketMessage: RawWebsocketMessage; 28 | SendStatusDeferral: Deferred; 29 | } 30 | 31 | export class WebsocketMessageAdapter { 32 | 33 | private connectionState: ConnectionState; 34 | private messageFormatter: IWebsocketMessageFormatter; 35 | private websocketClient: WebSocket; 36 | 37 | private sendMessageQueue: Queue; 38 | private receivingMessageQueue: Queue; 39 | private connectionEstablishDeferral: Deferred; 40 | private disconnectDeferral: Deferred; 41 | private connectionEvents: EventSource; 42 | private connectionId: string; 43 | private uri: string; 44 | 45 | public constructor( 46 | uri: string, 47 | connectionId: string, 48 | messageFormatter: IWebsocketMessageFormatter) { 49 | 50 | if (!uri) { 51 | throw new ArgumentNullError("uri"); 52 | } 53 | 54 | if (!messageFormatter) { 55 | throw new ArgumentNullError("messageFormatter"); 56 | } 57 | 58 | this.connectionEvents = new EventSource(); 59 | this.connectionId = connectionId; 60 | this.messageFormatter = messageFormatter; 61 | this.connectionState = ConnectionState.None; 62 | this.uri = uri; 63 | } 64 | 65 | public get State(): ConnectionState { 66 | return this.connectionState; 67 | } 68 | 69 | public Open = (): Promise => { 70 | if (this.connectionState === ConnectionState.Disconnected) { 71 | return PromiseHelper.FromError(`Cannot open a connection that is in ${this.connectionState} state`); 72 | } 73 | 74 | if (this.connectionEstablishDeferral) { 75 | return this.connectionEstablishDeferral.Promise(); 76 | } 77 | 78 | this.connectionEstablishDeferral = new Deferred(); 79 | this.connectionState = ConnectionState.Connecting; 80 | 81 | this.websocketClient = new WebSocket(this.uri); 82 | this.websocketClient.binaryType = "arraybuffer"; 83 | this.receivingMessageQueue = new Queue(); 84 | this.disconnectDeferral = new Deferred(); 85 | this.sendMessageQueue = new Queue(); 86 | this.ProcessSendQueue(); 87 | 88 | this.OnEvent(new ConnectionStartEvent(this.connectionId, this.uri)); 89 | 90 | this.websocketClient.onopen = (e: Event) => { 91 | this.connectionState = ConnectionState.Connected; 92 | this.OnEvent(new ConnectionEstablishedEvent(this.connectionId)); 93 | this.connectionEstablishDeferral.Resolve(new ConnectionOpenResponse(200, "")); 94 | }; 95 | 96 | this.websocketClient.onerror = (e: Event) => { 97 | // TODO: Understand what this is error is. Will we still get onClose ? 98 | if (this.connectionState !== ConnectionState.Connecting) { 99 | // TODO: Is this required ? 100 | // this.OnEvent(new ConnectionErrorEvent(errorMsg, connectionId)); 101 | } 102 | }; 103 | 104 | this.websocketClient.onclose = (e: CloseEvent) => { 105 | if (this.connectionState === ConnectionState.Connecting) { 106 | this.connectionState = ConnectionState.Disconnected; 107 | this.OnEvent(new ConnectionEstablishErrorEvent(this.connectionId, e.code, e.reason)); 108 | this.connectionEstablishDeferral.Resolve(new ConnectionOpenResponse(e.code, e.reason)); 109 | } else { 110 | this.OnEvent(new ConnectionClosedEvent(this.connectionId, e.code, e.reason)); 111 | } 112 | 113 | this.OnClose(e.code, e.reason); 114 | }; 115 | 116 | this.websocketClient.onmessage = (e: MessageEvent) => { 117 | const networkReceivedTime = new Date().toISOString(); 118 | if (this.connectionState === ConnectionState.Connected) { 119 | const deferred = new Deferred(); 120 | // let id = ++this.idCounter; 121 | this.receivingMessageQueue.EnqueueFromPromise(deferred.Promise()); 122 | if (e.data instanceof ArrayBuffer) { 123 | const rawMessage = new RawWebsocketMessage(MessageType.Binary, e.data); 124 | this.messageFormatter 125 | .ToConnectionMessage(rawMessage) 126 | .On((connectionMessage: ConnectionMessage) => { 127 | this.OnEvent(new ConnectionMessageReceivedEvent(this.connectionId, networkReceivedTime, connectionMessage)); 128 | deferred.Resolve(connectionMessage); 129 | }, (error: string) => { 130 | // TODO: Events for these ? 131 | deferred.Reject(`Invalid binary message format. Error: ${error}`); 132 | }); 133 | } else { 134 | const rawMessage = new RawWebsocketMessage(MessageType.Text, e.data); 135 | this.messageFormatter 136 | .ToConnectionMessage(rawMessage) 137 | .On((connectionMessage: ConnectionMessage) => { 138 | this.OnEvent(new ConnectionMessageReceivedEvent(this.connectionId, networkReceivedTime, connectionMessage)); 139 | deferred.Resolve(connectionMessage); 140 | }, (error: string) => { 141 | // TODO: Events for these ? 142 | deferred.Reject(`Invalid text message format. Error: ${error}`); 143 | }); 144 | } 145 | } 146 | }; 147 | 148 | return this.connectionEstablishDeferral.Promise(); 149 | } 150 | 151 | public Send = (message: ConnectionMessage): Promise => { 152 | if (this.connectionState !== ConnectionState.Connected) { 153 | return PromiseHelper.FromError(`Cannot send on connection that is in ${this.connectionState} state`); 154 | } 155 | 156 | const messageSendStatusDeferral = new Deferred(); 157 | const messageSendDeferral = new Deferred(); 158 | 159 | this.sendMessageQueue.EnqueueFromPromise(messageSendDeferral.Promise()); 160 | 161 | this.messageFormatter 162 | .FromConnectionMessage(message) 163 | .On((rawMessage: RawWebsocketMessage) => { 164 | messageSendDeferral.Resolve({ 165 | Message: message, 166 | RawWebsocketMessage: rawMessage, 167 | SendStatusDeferral: messageSendStatusDeferral, 168 | }); 169 | }, (error: string) => { 170 | messageSendDeferral.Reject(`Error formatting the message. ${error}`); 171 | }); 172 | 173 | return messageSendStatusDeferral.Promise(); 174 | } 175 | 176 | public Read = (): Promise => { 177 | if (this.connectionState !== ConnectionState.Connected) { 178 | return PromiseHelper.FromError(`Cannot read on connection that is in ${this.connectionState} state`); 179 | } 180 | 181 | return this.receivingMessageQueue.Dequeue(); 182 | } 183 | 184 | public Close = (reason?: string): Promise => { 185 | if (this.websocketClient) { 186 | if (this.connectionState !== ConnectionState.Connected) { 187 | this.websocketClient.close(1000, reason ? reason : "Normal closure by client"); 188 | } 189 | } else { 190 | const deferral = new Deferred(); 191 | deferral.Resolve(true); 192 | return deferral.Promise(); 193 | } 194 | 195 | return this.disconnectDeferral.Promise(); 196 | } 197 | 198 | public get Events(): EventSource { 199 | return this.connectionEvents; 200 | } 201 | 202 | private SendRawMessage = (sendItem: ISendItem): Promise => { 203 | try { 204 | this.OnEvent(new ConnectionMessageSentEvent(this.connectionId, new Date().toISOString(), sendItem.Message)); 205 | this.websocketClient.send(sendItem.RawWebsocketMessage.Payload); 206 | return PromiseHelper.FromResult(true); 207 | } catch (e) { 208 | return PromiseHelper.FromError(`websocket send error: ${e}`); 209 | } 210 | } 211 | 212 | private OnClose = (code: number, reason: string): void => { 213 | const closeReason = `Connection closed. ${code}: ${reason}`; 214 | this.connectionState = ConnectionState.Disconnected; 215 | this.disconnectDeferral.Resolve(true); 216 | this.receivingMessageQueue.Dispose(reason); 217 | this.receivingMessageQueue.DrainAndDispose((pendingReceiveItem: ConnectionMessage) => { 218 | // TODO: Events for these ? 219 | // Logger.Instance.OnEvent(new LoggingEvent(LogType.Warning, null, `Failed to process received message. Reason: ${closeReason}, Message: ${JSON.stringify(pendingReceiveItem)}`)); 220 | }, closeReason); 221 | 222 | this.sendMessageQueue.DrainAndDispose((pendingSendItem: ISendItem) => { 223 | pendingSendItem.SendStatusDeferral.Reject(closeReason); 224 | }, closeReason); 225 | } 226 | 227 | private ProcessSendQueue = (): void => { 228 | this.sendMessageQueue 229 | .Dequeue() 230 | .On((sendItem: ISendItem) => { 231 | this.SendRawMessage(sendItem) 232 | .On((result: boolean) => { 233 | sendItem.SendStatusDeferral.Resolve(result); 234 | this.ProcessSendQueue(); 235 | }, (sendError: string) => { 236 | sendItem.SendStatusDeferral.Reject(sendError); 237 | this.ProcessSendQueue(); 238 | }); 239 | }, (error: string) => { 240 | // do nothing 241 | }); 242 | } 243 | 244 | private OnEvent = (event: ConnectionEvent): void => { 245 | this.connectionEvents.OnEvent(event); 246 | Events.Instance.OnEvent(event); 247 | } 248 | } 249 | -------------------------------------------------------------------------------- /src/common/Promise.ts: -------------------------------------------------------------------------------- 1 | import { ArgumentNullError } from "./Error"; 2 | 3 | export enum PromiseState { 4 | None, 5 | Resolved, 6 | Rejected, 7 | } 8 | 9 | export interface IPromise { 10 | Result(): PromiseResult; 11 | 12 | ContinueWith( 13 | continuationCallback: (promiseResult: PromiseResult) => TContinuationResult): IPromise; 14 | 15 | ContinueWithPromise( 16 | continuationCallback: (promiseResult: PromiseResult) => IPromise): IPromise; 17 | 18 | OnSuccessContinueWith( 19 | continuationCallback: (result: T) => TContinuationResult): IPromise; 20 | 21 | OnSuccessContinueWithPromise( 22 | continuationCallback: (result: T) => IPromise): IPromise; 23 | 24 | On(successCallback: (result: T) => void, errorCallback: (error: string) => void): IPromise; 25 | 26 | Finally(callback: () => void): IPromise; 27 | } 28 | 29 | export interface IDeferred { 30 | State(): PromiseState; 31 | 32 | Promise(): IPromise; 33 | 34 | Resolve(result: T): IDeferred; 35 | 36 | Reject(error: string): IDeferred; 37 | } 38 | 39 | export class PromiseResult { 40 | protected isCompleted: boolean; 41 | protected isError: boolean; 42 | protected error: string; 43 | protected result: T; 44 | 45 | public constructor(promiseResultEventSource: PromiseResultEventSource) { 46 | promiseResultEventSource.On((result: T) => { 47 | if (!this.isCompleted) { 48 | this.isCompleted = true; 49 | this.isError = false; 50 | this.result = result; 51 | } 52 | }, (error: string) => { 53 | if (!this.isCompleted) { 54 | this.isCompleted = true; 55 | this.isError = true; 56 | this.error = error; 57 | } 58 | }); 59 | } 60 | 61 | public get IsCompleted(): boolean { 62 | return this.isCompleted; 63 | } 64 | 65 | public get IsError(): boolean { 66 | return this.isError; 67 | } 68 | 69 | public get Error(): string { 70 | return this.error; 71 | } 72 | 73 | public get Result(): T { 74 | return this.result; 75 | } 76 | 77 | public ThrowIfError = (): void => { 78 | if (this.IsError) { 79 | throw this.Error; 80 | } 81 | } 82 | } 83 | 84 | // tslint:disable-next-line:max-classes-per-file 85 | export class PromiseResultEventSource { 86 | 87 | private onSetResult: (result: T) => void; 88 | private onSetError: (error: string) => void; 89 | 90 | public SetResult = (result: T): void => { 91 | this.onSetResult(result); 92 | } 93 | 94 | public SetError = (error: string): void => { 95 | this.onSetError(error); 96 | } 97 | 98 | public On = (onSetResult: (result: T) => void, onSetError: (error: string) => void): void => { 99 | this.onSetResult = onSetResult; 100 | this.onSetError = onSetError; 101 | } 102 | } 103 | 104 | // tslint:disable-next-line:max-classes-per-file 105 | export class PromiseHelper { 106 | public static WhenAll = (promises: Array>): Promise => { 107 | if (!promises || promises.length === 0) { 108 | throw new ArgumentNullError("promises"); 109 | } 110 | 111 | const deferred = new Deferred(); 112 | const errors: string[] = []; 113 | let completedPromises: number = 0; 114 | 115 | const checkForCompletion = () => { 116 | completedPromises++; 117 | if (completedPromises === promises.length) { 118 | if (errors.length === 0) { 119 | deferred.Resolve(true); 120 | } else { 121 | deferred.Reject(errors.join(", ")); 122 | } 123 | } 124 | }; 125 | 126 | for (const promise of promises) { 127 | promise.On((r: any) => { 128 | checkForCompletion(); 129 | }, (e: string) => { 130 | errors.push(e); 131 | checkForCompletion(); 132 | }); 133 | } 134 | 135 | return deferred.Promise(); 136 | } 137 | 138 | public static FromResult = (result: TResult): Promise => { 139 | const deferred = new Deferred(); 140 | deferred.Resolve(result); 141 | return deferred.Promise(); 142 | } 143 | 144 | public static FromError = (error: string): Promise => { 145 | const deferred = new Deferred(); 146 | deferred.Reject(error); 147 | return deferred.Promise(); 148 | } 149 | } 150 | 151 | // TODO: replace with ES6 promises 152 | // tslint:disable-next-line:max-classes-per-file 153 | export class Promise implements IPromise { 154 | 155 | private sink: Sink; 156 | 157 | public constructor(sink: Sink) { 158 | this.sink = sink; 159 | } 160 | 161 | public Result = (): PromiseResult => { 162 | return this.sink.Result; 163 | } 164 | 165 | public ContinueWith = ( 166 | continuationCallback: (promiseResult: PromiseResult) => TContinuationResult): Promise => { 167 | 168 | if (!continuationCallback) { 169 | throw new ArgumentNullError("continuationCallback"); 170 | } 171 | 172 | const continuationDeferral = new Deferred(); 173 | 174 | this.sink.on( 175 | (r: T) => { 176 | try { 177 | const continuationResult: TContinuationResult = continuationCallback(this.sink.Result); 178 | continuationDeferral.Resolve(continuationResult); 179 | } catch (e) { 180 | continuationDeferral.Reject(`'Unhandled callback error: ${e}'`); 181 | } 182 | }, 183 | (error: string) => { 184 | try { 185 | const continuationResult: TContinuationResult = continuationCallback(this.sink.Result); 186 | continuationDeferral.Resolve(continuationResult); 187 | } catch (e) { 188 | continuationDeferral.Reject(`'Unhandled callback error: ${e}. InnerError: ${error}'`); 189 | } 190 | }, 191 | ); 192 | 193 | return continuationDeferral.Promise(); 194 | } 195 | 196 | public OnSuccessContinueWith = ( 197 | continuationCallback: (result: T) => TContinuationResult): Promise => { 198 | 199 | if (!continuationCallback) { 200 | throw new ArgumentNullError("continuationCallback"); 201 | } 202 | 203 | const continuationDeferral = new Deferred(); 204 | 205 | this.sink.on( 206 | (r: T) => { 207 | try { 208 | const continuationResult: TContinuationResult = continuationCallback(r); 209 | continuationDeferral.Resolve(continuationResult); 210 | } catch (e) { 211 | continuationDeferral.Reject(`'Unhandled callback error: ${e}'`); 212 | } 213 | }, 214 | (error: string) => { 215 | continuationDeferral.Reject(`'Unhandled callback error: ${error}'`); 216 | }, 217 | ); 218 | 219 | return continuationDeferral.Promise(); 220 | } 221 | 222 | public ContinueWithPromise = ( 223 | continuationCallback: (promiseResult: PromiseResult) => Promise): Promise => { 224 | 225 | if (!continuationCallback) { 226 | throw new ArgumentNullError("continuationCallback"); 227 | } 228 | 229 | const continuationDeferral = new Deferred(); 230 | 231 | this.sink.on( 232 | (r: T) => { 233 | try { 234 | const continuationPromise: Promise = continuationCallback(this.sink.Result); 235 | if (!continuationPromise) { 236 | throw new Error("'Continuation callback did not return promise'"); 237 | } 238 | continuationPromise.On((continuationResult: TContinuationResult) => { 239 | continuationDeferral.Resolve(continuationResult); 240 | }, (e: string) => { 241 | continuationDeferral.Reject(e); 242 | }); 243 | } catch (e) { 244 | continuationDeferral.Reject(`'Unhandled callback error: ${e}'`); 245 | } 246 | }, 247 | (error: string) => { 248 | try { 249 | const continuationPromise: Promise = continuationCallback(this.sink.Result); 250 | if (!continuationPromise) { 251 | throw new Error("Continuation callback did not return promise"); 252 | } 253 | continuationPromise.On((continuationResult: TContinuationResult) => { 254 | continuationDeferral.Resolve(continuationResult); 255 | }, (e: string) => { 256 | continuationDeferral.Reject(e); 257 | }); 258 | } catch (e) { 259 | continuationDeferral.Reject(`'Unhandled callback error: ${e}. InnerError: ${error}'`); 260 | } 261 | }, 262 | ); 263 | 264 | return continuationDeferral.Promise(); 265 | } 266 | 267 | public OnSuccessContinueWithPromise = ( 268 | continuationCallback: (result: T) => Promise): Promise => { 269 | 270 | if (!continuationCallback) { 271 | throw new ArgumentNullError("continuationCallback"); 272 | } 273 | 274 | const continuationDeferral = new Deferred(); 275 | 276 | this.sink.on( 277 | (r: T) => { 278 | try { 279 | const continuationPromise: Promise = continuationCallback(r); 280 | if (!continuationPromise) { 281 | throw new Error("Continuation callback did not return promise"); 282 | } 283 | continuationPromise.On((continuationResult: TContinuationResult) => { 284 | continuationDeferral.Resolve(continuationResult); 285 | }, (e: string) => { 286 | continuationDeferral.Reject(e); 287 | }); 288 | } catch (e) { 289 | continuationDeferral.Reject(`'Unhandled callback error: ${e}'`); 290 | } 291 | }, 292 | (error: string) => { 293 | continuationDeferral.Reject(`'Unhandled callback error: ${error}.'`); 294 | }, 295 | ); 296 | 297 | return continuationDeferral.Promise(); 298 | } 299 | 300 | public On = ( 301 | successCallback: (result: T) => void, 302 | errorCallback: (error: string) => void): Promise => { 303 | if (!successCallback) { 304 | throw new ArgumentNullError("successCallback"); 305 | } 306 | 307 | if (!errorCallback) { 308 | throw new ArgumentNullError("errorCallback"); 309 | } 310 | 311 | this.sink.on(successCallback, errorCallback); 312 | return this; 313 | } 314 | 315 | public Finally = (callback: () => void): Promise => { 316 | if (!callback) { 317 | throw new ArgumentNullError("callback"); 318 | } 319 | 320 | const callbackWrapper = (_: any) => { 321 | callback(); 322 | }; 323 | 324 | return this.On(callbackWrapper, callbackWrapper); 325 | } 326 | } 327 | 328 | // tslint:disable-next-line:max-classes-per-file 329 | export class Deferred implements IDeferred { 330 | 331 | private promise: Promise; 332 | private sink: Sink; 333 | 334 | public constructor() { 335 | this.sink = new Sink(); 336 | this.promise = new Promise(this.sink); 337 | } 338 | 339 | public State = (): PromiseState => { 340 | return this.sink.State; 341 | } 342 | 343 | public Promise = (): Promise => { 344 | return this.promise; 345 | } 346 | 347 | public Resolve = (result: T): Deferred => { 348 | this.sink.Resolve(result); 349 | return this; 350 | } 351 | 352 | public Reject = (error: string): Deferred => { 353 | this.sink.Reject(error); 354 | return this; 355 | } 356 | } 357 | 358 | // tslint:disable-next-line:max-classes-per-file 359 | export class Sink { 360 | 361 | private state: PromiseState = PromiseState.None; 362 | private promiseResult: PromiseResult = null; 363 | private promiseResultEvents: PromiseResultEventSource = null; 364 | 365 | private successHandlers: Array<((result: T) => void)> = []; 366 | private errorHandlers: Array<(e: string) => void> = []; 367 | 368 | public constructor() { 369 | this.promiseResultEvents = new PromiseResultEventSource(); 370 | this.promiseResult = new PromiseResult(this.promiseResultEvents); 371 | } 372 | 373 | public get State(): PromiseState { 374 | return this.state; 375 | } 376 | 377 | public get Result(): PromiseResult { 378 | return this.promiseResult; 379 | } 380 | 381 | public Resolve = (result: T): void => { 382 | if (this.state !== PromiseState.None) { 383 | throw new Error("'Cannot resolve a completed promise'"); 384 | } 385 | 386 | this.state = PromiseState.Resolved; 387 | this.promiseResultEvents.SetResult(result); 388 | 389 | for (let i = 0; i < this.successHandlers.length; i++) { 390 | this.ExecuteSuccessCallback(result, this.successHandlers[i], this.errorHandlers[i]); 391 | } 392 | 393 | this.DetachHandlers(); 394 | } 395 | 396 | public Reject = (error: string): void => { 397 | if (this.state !== PromiseState.None) { 398 | throw new Error("'Cannot reject a completed promise'"); 399 | } 400 | 401 | this.state = PromiseState.Rejected; 402 | this.promiseResultEvents.SetError(error); 403 | 404 | for (const errorHandler of this.errorHandlers) { 405 | this.ExecuteErrorCallback(error, errorHandler); 406 | } 407 | 408 | this.DetachHandlers(); 409 | } 410 | 411 | public on = ( 412 | successCallback: (result: T) => void, 413 | errorCallback: (error: string) => void): void => { 414 | 415 | if (successCallback == null) { 416 | successCallback = (r: T) => { return; }; 417 | } 418 | 419 | if (this.state === PromiseState.None) { 420 | this.successHandlers.push(successCallback); 421 | this.errorHandlers.push(errorCallback); 422 | } else { 423 | if (this.state === PromiseState.Resolved) { 424 | this.ExecuteSuccessCallback(this.promiseResult.Result, successCallback, errorCallback); 425 | } else if (this.state === PromiseState.Rejected) { 426 | this.ExecuteErrorCallback(this.promiseResult.Error, errorCallback); 427 | } 428 | 429 | this.DetachHandlers(); 430 | } 431 | } 432 | 433 | private ExecuteSuccessCallback = (result: T, successCallback: (result: T) => void, errorCallback: (error: string) => void): void => { 434 | try { 435 | successCallback(result); 436 | } catch (e) { 437 | this.ExecuteErrorCallback(`'Unhandled callback error: ${e}'`, errorCallback); 438 | } 439 | } 440 | 441 | private ExecuteErrorCallback = (error: string, errorCallback: (error: string) => void): void => { 442 | if (errorCallback) { 443 | try { 444 | errorCallback(error); 445 | } catch (e) { 446 | throw new Error(`'Unhandled callback error: ${e}. InnerError: ${error}'`); 447 | } 448 | } else { 449 | throw new Error(`'Unhandled error: ${error}'`); 450 | } 451 | } 452 | 453 | private DetachHandlers = (): void => { 454 | this.errorHandlers = []; 455 | this.successHandlers = []; 456 | } 457 | } 458 | -------------------------------------------------------------------------------- /samples/browser/Sample.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Speech Sample 5 | 6 | 7 | 8 |
9 |

Speech Recognition SDK not found.

10 |

Please execute npm run bundle and reload.

11 |
12 | 104 | 105 | 106 | 107 | 108 | 109 | 246 | 247 | 248 | 365 | 366 | 367 | -------------------------------------------------------------------------------- /src/sdk/speech/Recognizer.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ArgumentNullError, 3 | ConnectionMessage, 4 | ConnectionOpenResponse, 5 | ConnectionState, 6 | CreateNoDashGuid, 7 | Deferred, 8 | Events, 9 | IAudioSource, 10 | IAudioStreamNode, 11 | IConnection, 12 | IDetachable, 13 | IEventSource, 14 | IStreamChunk, 15 | MessageType, 16 | PlatformEvent, 17 | Promise, 18 | PromiseHelper, 19 | PromiseResult, 20 | } from "../../common/Exports"; 21 | import { AuthInfo, IAuthentication } from "./IAuthentication"; 22 | import { IConnectionFactory } from "./IConnectionFactory"; 23 | import { 24 | ConnectingToServiceEvent, 25 | ListeningStartedEvent, 26 | RecognitionCompletionStatus, 27 | RecognitionEndedEvent, 28 | RecognitionStartedEvent, 29 | RecognitionTriggeredEvent, 30 | SpeechDetailedPhraseEvent, 31 | SpeechEndDetectedEvent, 32 | SpeechFragmentEvent, 33 | SpeechHypothesisEvent, 34 | SpeechRecognitionEvent, 35 | SpeechRecognitionResultEvent, 36 | SpeechSimplePhraseEvent, 37 | SpeechStartDetectedEvent, 38 | } from "./RecognitionEvents"; 39 | import { RecognitionMode, RecognizerConfig, SpeechResultFormat } from "./RecognizerConfig"; 40 | import { ServiceTelemetryListener } from "./ServiceTelemetryListener.Internal"; 41 | import { SpeechConnectionMessage } from "./SpeechConnectionMessage.Internal"; 42 | import { 43 | IDetailedSpeechPhrase, 44 | ISimpleSpeechPhrase, 45 | ISpeechEndDetectedResult, 46 | ISpeechFragment, 47 | ISpeechStartDetectedResult, 48 | } from "./SpeechResults"; 49 | 50 | export class Recognizer { 51 | private authentication: IAuthentication; 52 | private connectionFactory: IConnectionFactory; 53 | private audioSource: IAudioSource; 54 | private recognizerConfig: RecognizerConfig; 55 | private speechConfigConnectionId: string; 56 | private connectionFetchPromise: Promise; 57 | private connectionId: string; 58 | private authFetchEventId: string; 59 | 60 | public constructor( 61 | authentication: IAuthentication, 62 | connectionFactory: IConnectionFactory, 63 | audioSource: IAudioSource, 64 | recognizerConfig: RecognizerConfig) { 65 | 66 | if (!authentication) { 67 | throw new ArgumentNullError("authentication"); 68 | } 69 | 70 | if (!connectionFactory) { 71 | throw new ArgumentNullError("connectionFactory"); 72 | } 73 | 74 | if (!audioSource) { 75 | throw new ArgumentNullError("audioSource"); 76 | } 77 | 78 | if (!recognizerConfig) { 79 | throw new ArgumentNullError("recognizerConfig"); 80 | } 81 | 82 | this.authentication = authentication; 83 | this.connectionFactory = connectionFactory; 84 | this.audioSource = audioSource; 85 | this.recognizerConfig = recognizerConfig; 86 | } 87 | 88 | public get AudioSource(): IAudioSource { 89 | return this.audioSource; 90 | } 91 | 92 | public Recognize = (onEventCallback: (event: SpeechRecognitionEvent) => void, speechContextJson?: string): Promise => { 93 | const requestSession = new RequestSession(this.audioSource.Id(), onEventCallback); 94 | requestSession.ListenForServiceTelemetry(this.audioSource.Events); 95 | 96 | return this.audioSource 97 | .Attach(requestSession.AudioNodeId) 98 | .ContinueWithPromise((result: PromiseResult) => { 99 | if (result.IsError) { 100 | requestSession.OnAudioSourceAttachCompleted(null, true, result.Error); 101 | throw new Error(result.Error); 102 | } else { 103 | requestSession.OnAudioSourceAttachCompleted(result.Result, false); 104 | } 105 | 106 | const audioNode = result.Result; 107 | 108 | this.FetchConnection(requestSession) 109 | .OnSuccessContinueWith((connection: IConnection) => { 110 | const messageRetrievalPromise = this.ReceiveMessage(connection, requestSession); 111 | const messageSendPromise = this.SendSpeechConfig(requestSession.RequestId, connection, this.recognizerConfig.SpeechConfig.Serialize()) 112 | .OnSuccessContinueWithPromise((_: boolean) => { 113 | return this.SendSpeechContext(requestSession.RequestId, connection, speechContextJson) 114 | .OnSuccessContinueWithPromise((_: boolean) => { 115 | return this.SendAudio(requestSession.RequestId, connection, audioNode, requestSession); 116 | }); 117 | }); 118 | 119 | const completionPromise = PromiseHelper.WhenAll([messageRetrievalPromise, messageSendPromise]); 120 | 121 | completionPromise.On((r: boolean) => { 122 | requestSession.Dispose(); 123 | this.SendTelemetryData(requestSession.RequestId, connection, requestSession.GetTelemetry()); 124 | }, (error: string) => { 125 | requestSession.Dispose(error); 126 | this.SendTelemetryData(requestSession.RequestId, connection, requestSession.GetTelemetry()); 127 | }); 128 | 129 | return completionPromise; 130 | }); 131 | 132 | return requestSession.CompletionPromise; 133 | }); 134 | } 135 | 136 | private FetchConnection = (requestSession: RequestSession, isUnAuthorized: boolean = false): Promise => { 137 | if (this.connectionFetchPromise) { 138 | if (this.connectionFetchPromise.Result().IsError 139 | || this.connectionFetchPromise.Result().Result.State() === ConnectionState.Disconnected) { 140 | this.connectionId = null; 141 | this.connectionFetchPromise = null; 142 | return this.FetchConnection(requestSession); 143 | } else { 144 | requestSession.OnPreConnectionStart(this.authFetchEventId, this.connectionId); 145 | requestSession.OnConnectionEstablishCompleted(200); 146 | requestSession.ListenForServiceTelemetry(this.connectionFetchPromise.Result().Result.Events); 147 | return this.connectionFetchPromise; 148 | } 149 | } 150 | 151 | this.authFetchEventId = CreateNoDashGuid(); 152 | this.connectionId = CreateNoDashGuid(); 153 | 154 | requestSession.OnPreConnectionStart(this.authFetchEventId, this.connectionId); 155 | 156 | const authPromise = isUnAuthorized ? this.authentication.FetchOnExpiry(this.authFetchEventId) : this.authentication.Fetch(this.authFetchEventId); 157 | 158 | this.connectionFetchPromise = authPromise 159 | .ContinueWithPromise((result: PromiseResult) => { 160 | if (result.IsError) { 161 | requestSession.OnAuthCompleted(true, result.Error); 162 | throw new Error(result.Error); 163 | } else { 164 | requestSession.OnAuthCompleted(false); 165 | } 166 | 167 | const connection = this.connectionFactory.Create(this.recognizerConfig, result.Result, this.connectionId); 168 | requestSession.ListenForServiceTelemetry(connection.Events); 169 | 170 | return connection.Open().OnSuccessContinueWithPromise((response: ConnectionOpenResponse) => { 171 | if (response.StatusCode === 200) { 172 | requestSession.OnConnectionEstablishCompleted(response.StatusCode); 173 | return PromiseHelper.FromResult(connection); 174 | } else if (response.StatusCode === 403 && !isUnAuthorized) { 175 | return this.FetchConnection(requestSession, true); 176 | } else { 177 | requestSession.OnConnectionEstablishCompleted(response.StatusCode, response.Reason); 178 | return PromiseHelper.FromError(`Unable to contact server. StatusCode: ${response.StatusCode}, Reason: ${response.Reason}`); 179 | } 180 | }); 181 | }); 182 | 183 | return this.connectionFetchPromise; 184 | } 185 | 186 | private ReceiveMessage = (connection: IConnection, requestSession: RequestSession): Promise => { 187 | return connection 188 | .Read() 189 | .OnSuccessContinueWithPromise((message: ConnectionMessage) => { 190 | const connectionMessage = SpeechConnectionMessage.FromConnectionMessage(message); 191 | if (connectionMessage.RequestId.toLowerCase() === requestSession.RequestId.toLowerCase()) { 192 | switch (connectionMessage.Path.toLowerCase()) { 193 | case "turn.start": 194 | requestSession.OnServiceTurnStartResponse(JSON.parse(connectionMessage.TextBody)); 195 | break; 196 | case "speech.startDetected": 197 | requestSession.OnServiceSpeechStartDetectedResponse(JSON.parse(connectionMessage.TextBody)); 198 | break; 199 | case "speech.hypothesis": 200 | requestSession.OnServiceSpeechHypothesisResponse(JSON.parse(connectionMessage.TextBody)); 201 | break; 202 | case "speech.fragment": 203 | requestSession.OnServiceSpeechFragmentResponse(JSON.parse(connectionMessage.TextBody)); 204 | break; 205 | case "speech.enddetected": 206 | requestSession.OnServiceSpeechEndDetectedResponse(JSON.parse(connectionMessage.TextBody)); 207 | break; 208 | case "speech.phrase": 209 | if (this.recognizerConfig.IsContinuousRecognition) { 210 | // For continuous recognition telemetry has to be sent for every phrase as per spec. 211 | this.SendTelemetryData(requestSession.RequestId, connection, requestSession.GetTelemetry()); 212 | } 213 | if (this.recognizerConfig.Format === SpeechResultFormat.Simple) { 214 | requestSession.OnServiceSimpleSpeechPhraseResponse(JSON.parse(connectionMessage.TextBody)); 215 | } else { 216 | requestSession.OnServiceDetailedSpeechPhraseResponse(JSON.parse(connectionMessage.TextBody)); 217 | } 218 | break; 219 | case "turn.end": 220 | requestSession.OnServiceTurnEndResponse(); 221 | return PromiseHelper.FromResult(true); 222 | default: 223 | break; 224 | } 225 | } 226 | 227 | return this.ReceiveMessage(connection, requestSession); 228 | }); 229 | } 230 | 231 | private SendSpeechConfig = (requestId: string, connection: IConnection, speechConfigJson: string) => { 232 | if (speechConfigJson && this.connectionId !== this.speechConfigConnectionId) { 233 | this.speechConfigConnectionId = this.connectionId; 234 | return connection 235 | .Send(new SpeechConnectionMessage( 236 | MessageType.Text, 237 | "speech.config", 238 | requestId, 239 | "application/json", 240 | speechConfigJson)); 241 | } 242 | 243 | return PromiseHelper.FromResult(true); 244 | } 245 | 246 | private SendSpeechContext = (requestId: string, connection: IConnection, speechContextJson: string) => { 247 | if (speechContextJson) { 248 | return connection 249 | .Send(new SpeechConnectionMessage( 250 | MessageType.Text, 251 | "speech.context", 252 | requestId, 253 | "application/json", 254 | speechContextJson)); 255 | } 256 | return PromiseHelper.FromResult(true); 257 | } 258 | 259 | private SendTelemetryData = (requestId: string, connection: IConnection, telemetryData: string) => { 260 | return connection 261 | .Send(new SpeechConnectionMessage( 262 | MessageType.Text, 263 | "telemetry", 264 | requestId, 265 | "application/json", 266 | telemetryData)); 267 | } 268 | 269 | private SendAudio = ( 270 | requestId: string, 271 | connection: IConnection, 272 | audioStreamNode: IAudioStreamNode, 273 | requestSession: RequestSession): Promise => { 274 | // NOTE: Home-baked promises crash ios safari during the invocation 275 | // of the error callback chain (looks like the recursion is way too deep, and 276 | // it blows up the stack). The following construct is a stop-gap that does not 277 | // bubble the error up the callback chain and hence circumvents this problem. 278 | // TODO: rewrite with ES6 promises. 279 | const deferred = new Deferred(); 280 | 281 | const readAndUploadCycle = (_: boolean) => { 282 | audioStreamNode.Read().On( 283 | (audioStreamChunk: IStreamChunk) => { 284 | // we have a new audio chunk to upload. 285 | if (requestSession.IsSpeechEnded) { 286 | // If service already recognized audio end then dont send any more audio 287 | deferred.Resolve(true); 288 | return; 289 | } 290 | 291 | const payload = (audioStreamChunk.IsEnd) ? null : audioStreamChunk.Buffer; 292 | const uploaded = connection.Send( 293 | new SpeechConnectionMessage( 294 | MessageType.Binary, "audio", requestId, null, payload)); 295 | 296 | if (!audioStreamChunk.IsEnd) { 297 | uploaded.OnSuccessContinueWith(readAndUploadCycle); 298 | } else { 299 | // the audio stream has been closed, no need to schedule next 300 | // read-upload cycle. 301 | deferred.Resolve(true); 302 | } 303 | }, 304 | (error: string) => { 305 | if (requestSession.IsSpeechEnded) { 306 | // For whatever reason, Reject is used to remove queue subscribers inside 307 | // the Queue.DrainAndDispose invoked from DetachAudioNode down blow, which 308 | // means that sometimes things can be rejected in normal circumstances, without 309 | // any errors. 310 | deferred.Resolve(true); // TODO: remove the argument, it's is completely meaningless. 311 | } else { 312 | // Only reject, if there was a proper error. 313 | deferred.Reject(error); 314 | } 315 | }); 316 | }; 317 | 318 | readAndUploadCycle(true); 319 | 320 | return deferred.Promise(); 321 | } 322 | } 323 | 324 | // tslint:disable-next-line:max-classes-per-file 325 | class RequestSession { 326 | private isDisposed: boolean = false; 327 | private serviceTelemetryListener: ServiceTelemetryListener; 328 | private detachables: IDetachable[] = new Array(); 329 | private requestId: string; 330 | private audioSourceId: string; 331 | private audioNodeId: string; 332 | private audioNode: IAudioStreamNode; 333 | private authFetchEventId: string; 334 | private connectionId: string; 335 | private serviceTag: string; 336 | private isAudioNodeDetached: boolean = false; 337 | private isCompleted: boolean = false; 338 | private onEventCallback: (event: SpeechRecognitionEvent) => void; 339 | 340 | private requestCompletionDeferral: Deferred; 341 | 342 | constructor(audioSourceId: string, onEventCallback: (event: SpeechRecognitionEvent) => void) { 343 | this.audioSourceId = audioSourceId; 344 | this.onEventCallback = onEventCallback; 345 | this.requestId = CreateNoDashGuid(); 346 | this.audioNodeId = CreateNoDashGuid(); 347 | this.requestCompletionDeferral = new Deferred(); 348 | 349 | this.serviceTelemetryListener = new ServiceTelemetryListener(this.requestId, this.audioSourceId, this.audioNodeId); 350 | 351 | this.OnEvent(new RecognitionTriggeredEvent(this.RequestId, this.audioSourceId, this.audioNodeId)); 352 | } 353 | 354 | public get RequestId(): string { 355 | return this.requestId; 356 | } 357 | 358 | public get AudioNodeId(): string { 359 | return this.audioNodeId; 360 | } 361 | 362 | public get CompletionPromise(): Promise { 363 | return this.requestCompletionDeferral.Promise(); 364 | } 365 | 366 | public get IsSpeechEnded(): boolean { 367 | return this.isAudioNodeDetached; 368 | } 369 | 370 | public get IsCompleted(): boolean { 371 | return this.isCompleted; 372 | } 373 | 374 | public ListenForServiceTelemetry(eventSource: IEventSource): void { 375 | this.detachables.push(eventSource.AttachListener(this.serviceTelemetryListener)); 376 | } 377 | 378 | public OnAudioSourceAttachCompleted = (audioNode: IAudioStreamNode, isError: boolean, error?: string): void => { 379 | this.audioNode = audioNode; 380 | if (isError) { 381 | this.OnComplete(RecognitionCompletionStatus.AudioSourceError, error); 382 | } else { 383 | this.OnEvent(new ListeningStartedEvent(this.requestId, this.audioSourceId, this.audioNodeId)); 384 | } 385 | } 386 | 387 | public OnPreConnectionStart = (authFetchEventId: string, connectionId: string): void => { 388 | this.authFetchEventId = authFetchEventId; 389 | this.connectionId = connectionId; 390 | this.OnEvent(new ConnectingToServiceEvent(this.requestId, this.authFetchEventId, this.connectionId)); 391 | } 392 | 393 | public OnAuthCompleted = (isError: boolean, error?: string): void => { 394 | if (isError) { 395 | this.OnComplete(RecognitionCompletionStatus.AuthTokenFetchError, error); 396 | } 397 | } 398 | 399 | public OnConnectionEstablishCompleted = (statusCode: number, reason?: string): void => { 400 | if (statusCode === 200) { 401 | this.OnEvent(new RecognitionStartedEvent(this.RequestId, this.audioSourceId, this.audioNodeId, this.authFetchEventId, this.connectionId)); 402 | return; 403 | } else if (statusCode === 403) { 404 | this.OnComplete(RecognitionCompletionStatus.UnAuthorized, reason); 405 | } else { 406 | this.OnComplete(RecognitionCompletionStatus.ConnectError, reason); 407 | } 408 | } 409 | 410 | public OnServiceTurnStartResponse = (response: ITurnStartResponse): void => { 411 | if (response && response.context && response.context.serviceTag) { 412 | this.serviceTag = response.context.serviceTag; 413 | } 414 | } 415 | 416 | public OnServiceSpeechStartDetectedResponse = (result: ISpeechStartDetectedResult): void => { 417 | this.OnEvent(new SpeechStartDetectedEvent(this.RequestId, result)); 418 | } 419 | 420 | public OnServiceSpeechHypothesisResponse = (result: ISpeechFragment): void => { 421 | this.OnEvent(new SpeechHypothesisEvent(this.RequestId, result)); 422 | } 423 | 424 | public OnServiceSpeechFragmentResponse = (result: ISpeechFragment): void => { 425 | this.OnEvent(new SpeechFragmentEvent(this.RequestId, result)); 426 | } 427 | 428 | public OnServiceSpeechEndDetectedResponse = (result: ISpeechEndDetectedResult): void => { 429 | this.DetachAudioNode(); 430 | this.OnEvent(new SpeechEndDetectedEvent(this.RequestId, result)); 431 | } 432 | 433 | public OnServiceSimpleSpeechPhraseResponse = (result: ISimpleSpeechPhrase): void => { 434 | this.OnEvent(new SpeechSimplePhraseEvent(this.RequestId, result)); 435 | } 436 | 437 | public OnServiceDetailedSpeechPhraseResponse = (result: IDetailedSpeechPhrase): void => { 438 | this.OnEvent(new SpeechDetailedPhraseEvent(this.RequestId, result)); 439 | } 440 | 441 | public OnServiceTurnEndResponse = (): void => { 442 | this.OnComplete(RecognitionCompletionStatus.Success); 443 | } 444 | 445 | public OnConnectionError = (error: string): void => { 446 | this.OnComplete(RecognitionCompletionStatus.UnknownError, error); 447 | } 448 | 449 | public Dispose = (error?: string): void => { 450 | if (!this.isDisposed) { 451 | // we should have completed by now. If we did not its an unknown error. 452 | this.OnComplete(RecognitionCompletionStatus.UnknownError, error); 453 | this.isDisposed = true; 454 | for (const detachable of this.detachables) { 455 | detachable.Detach(); 456 | } 457 | 458 | this.serviceTelemetryListener.Dispose(); 459 | } 460 | } 461 | 462 | public GetTelemetry = (): string => { 463 | return this.serviceTelemetryListener.GetTelemetry(); 464 | } 465 | 466 | private OnComplete = (status: RecognitionCompletionStatus, error?: string): void => { 467 | if (!this.isCompleted) { 468 | this.isCompleted = true; 469 | this.DetachAudioNode(); 470 | this.OnEvent(new RecognitionEndedEvent(this.RequestId, this.audioSourceId, this.audioNodeId, this.authFetchEventId, this.connectionId, this.serviceTag, status, error ? error : "")); 471 | } 472 | } 473 | 474 | private DetachAudioNode = (): void => { 475 | if (!this.isAudioNodeDetached) { 476 | this.isAudioNodeDetached = true; 477 | if (this.audioNode) { 478 | this.audioNode.Detach(); 479 | } 480 | } 481 | } 482 | 483 | private OnEvent = (event: SpeechRecognitionEvent): void => { 484 | this.serviceTelemetryListener.OnEvent(event); 485 | Events.Instance.OnEvent(event); 486 | if (this.onEventCallback) { 487 | this.onEventCallback(event); 488 | } 489 | } 490 | } 491 | 492 | interface ITurnStartResponse { 493 | context: ITurnStartContext; 494 | } 495 | 496 | interface ITurnStartContext { 497 | serviceTag: string; 498 | } 499 | --------------------------------------------------------------------------------