├── cypress ├── fixtures │ └── example.json ├── tsconfig.json ├── plugins │ └── index.js ├── support │ └── helpers │ │ ├── clientMethods.helpers.ts │ │ └── socket.helpers.ts └── integration │ ├── createAssistantWeb.spec.ts │ ├── initializeAssistantSDK.spec.ts │ ├── disableDubbing.spec.ts │ ├── greetings.spec.ts │ ├── sendAction.spec.ts │ └── refreshToken.spec.ts ├── .eslintignore ├── cypress.json ├── examples ├── todo-canvas-app │ ├── .env.sample │ ├── cypress.json │ ├── src │ │ ├── react-app-env.d.ts │ │ ├── global.d.ts │ │ ├── index.css │ │ ├── setupTests.ts │ │ ├── reportWebVitals.ts │ │ ├── index.tsx │ │ ├── App.css │ │ ├── store.ts │ │ └── App.tsx │ ├── public │ │ ├── robots.txt │ │ ├── favicon.ico │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── index.html │ ├── cypress │ │ ├── tsconfig.json │ │ ├── plugins │ │ │ └── index.js │ │ └── integration │ │ │ └── sample.spec.ts │ ├── tsconfig.json │ ├── README.md │ └── package.json └── umd-example │ ├── README.md │ └── index.html ├── .prettierignore ├── .npmrc ├── .lintstagedrc ├── .huskyrc.json ├── .editorconfig ├── .commitlintrc.json ├── .prettierrc ├── src ├── declarations.d.ts ├── index.ts ├── createRecoveryStateRepository.ts ├── record │ ├── record-downloader.ts │ ├── console-logger.ts │ ├── recorder.ts │ ├── online-player.ts │ ├── createMockWsCreator.ts │ ├── callback-recorder.ts │ ├── index.tsx │ ├── offline-player.ts │ └── mock-recorder.ts ├── nanoobservable.ts ├── NativePanel │ ├── fonts.ts │ ├── components │ │ ├── Bubble.tsx │ │ ├── VoiceTouch.tsx │ │ ├── TextInput.tsx │ │ ├── Suggests.tsx │ │ └── KeyboardTouch.tsx │ ├── styles.tsx │ ├── NativePanelStyles.ts │ └── NativePanel.tsx ├── assistantSdk │ ├── voice │ │ ├── recognizers │ │ │ ├── mtt │ │ │ │ └── index.proto │ │ │ ├── musicRecognizer.ts │ │ │ ├── speechRecognizer.ts │ │ │ └── asr │ │ │ │ └── index.proto │ │ ├── player │ │ │ ├── trackCollection.ts │ │ │ ├── chunkQueue.ts │ │ │ ├── voicePlayer.ts │ │ │ └── trackStream.ts │ │ ├── listener │ │ │ ├── voiceListener.ts │ │ │ └── navigatorAudioProvider.ts │ │ └── audioContext.ts │ ├── README.md │ ├── client │ │ ├── transport.ts │ │ ├── client.ts │ │ └── methods.ts │ └── meta.ts ├── nanoevents.ts ├── appInitialData.ts ├── createAssistantDev.ts ├── proto │ └── index.proto ├── mock.ts └── debug.ts ├── global.d.ts ├── .gitignore ├── tsconfig.json ├── .github └── workflows │ └── release.yml ├── rollup.config.js ├── .eslintrc.js └── package.json /cypress/fixtures/example.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | *.js 4 | *.d.ts 5 | examples 6 | -------------------------------------------------------------------------------- /cypress.json: -------------------------------------------------------------------------------- 1 | { 2 | "projectId": "ac6epj", 3 | "video": false 4 | } 5 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/.env.sample: -------------------------------------------------------------------------------- 1 | REACT_APP_TOKEN= 2 | REACT_APP_SMARTAPP= 3 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/cypress.json: -------------------------------------------------------------------------------- 1 | { 2 | "baseUrl": "http://localhost:3000" 3 | } 4 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /package.json 3 | /package-lock.json 4 | /dist 5 | README.md 6 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | @sberdevices:registry=https://registry.npmjs.org/ 2 | //registry.npmjs.org/:_authToken=${NPM_REGISTRY_TOKEN} 3 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /.lintstagedrc: -------------------------------------------------------------------------------- 1 | { 2 | "src/*.{js,jsx,ts,tsx}": [ 3 | "eslint --fix", 4 | "pretty-quick --staged" 5 | ], 6 | } -------------------------------------------------------------------------------- /examples/todo-canvas-app/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sberdevices/assistant-client/HEAD/examples/todo-canvas-app/public/favicon.ico -------------------------------------------------------------------------------- /examples/todo-canvas-app/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sberdevices/assistant-client/HEAD/examples/todo-canvas-app/public/logo192.png -------------------------------------------------------------------------------- /examples/todo-canvas-app/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sberdevices/assistant-client/HEAD/examples/todo-canvas-app/public/logo512.png -------------------------------------------------------------------------------- /.huskyrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "hooks": { 3 | "pre-commit": "lint-staged", 4 | "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/global.d.ts: -------------------------------------------------------------------------------- 1 | export declare global { 2 | interface Window { 3 | Cypress?: Record; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/index.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | font-family: sans-serif; 4 | } 5 | 6 | html, 7 | body { 8 | margin: 0; 9 | padding: 0; 10 | } 11 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | max_line_length = 120 10 | trim_trailing_whitespace = true 11 | 12 | [*.{json,md,yaml,yml}] 13 | indent_size = 2 14 | -------------------------------------------------------------------------------- /.commitlintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@commitlint/config-conventional"], 3 | "rules": { 4 | "subject-case": [0], 5 | "type-enum": [2, "always", ["build", "ci", "chore", "feat", "fix", "docs", "style", "perf", "refactor", "test", "revert", "breaking"]] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /cypress/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "noEmit": false, 5 | "types": [ 6 | "cypress" 7 | ] 8 | }, 9 | "include": [ 10 | "../node_modules/cypress", 11 | "../*/*.ts" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "typescript", 3 | "arrowParens": "always", 4 | "printWidth": 120, 5 | "jsxBracketSameLine": false, 6 | "jsxSingleQuote": false, 7 | "endOfLine": "auto", 8 | "semi": true, 9 | "singleQuote": true, 10 | "tabWidth": 4, 11 | "trailingComma": "all" 12 | } 13 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/cypress/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "noEmit": false, 5 | "types": [ 6 | "cypress" 7 | ] 8 | }, 9 | "include": [ 10 | "../node_modules/cypress", 11 | "../*/*.ts" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /src/declarations.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | declare module '*.svg' { 4 | export const ReactComponent: React.FunctionComponent>; 5 | export default ReactComponent; 6 | } 7 | 8 | declare module '*.png' { 9 | const content: string; 10 | export default content; 11 | } 12 | -------------------------------------------------------------------------------- /global.d.ts: -------------------------------------------------------------------------------- 1 | import { AssistantWindow } from './src/typings'; 2 | 3 | export declare global { 4 | // eslint-disable-next-line @typescript-eslint/no-empty-interface 5 | interface Window extends AssistantWindow {} 6 | 7 | interface Window { 8 | __ASSISTANT_CLIENT__: { version: string }; 9 | webkitAudioContext?: new () => AudioContext; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ##### OS FILES 2 | .DS_Store 3 | .history 4 | 5 | ##### Editors 6 | .*.swp 7 | .idea 8 | .vscode 9 | .vs 10 | *.iml 11 | *.sublime-project 12 | *.sublime-workspace 13 | 14 | ##### BUILD 15 | node_modules/ 16 | cypress/videos/ 17 | npm-debug.log 18 | dist/ 19 | esm/ 20 | umd/ 21 | *.tsbuildinfo 22 | build/ 23 | 24 | # cypress 25 | cypress/screenshots/ 26 | cypress/videos/ 27 | 28 | .cache 29 | .env 30 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './createAssistant'; 2 | export * from './createAssistantDev'; 3 | export { 4 | AssistantEvent, 5 | AppEvent, 6 | VpsEvent, 7 | ActionCommandEvent, 8 | AssistantEvents as AssistantClientEvents, 9 | createAssistant as createAssistantClient, 10 | } from './assistantSdk/assistant'; 11 | export { createNavigatorAudioProvider } from './assistantSdk/voice/listener/navigatorAudioProvider'; 12 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | } 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import "./index.css"; 4 | import { App } from "./App"; 5 | import reportWebVitals from "./reportWebVitals"; 6 | 7 | ReactDOM.render( 8 | 9 | 10 | , 11 | document.getElementById("root") 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /src/createRecoveryStateRepository.ts: -------------------------------------------------------------------------------- 1 | const prefix = 'recovery_'; 2 | 3 | export const createRecoveryStateRepository = () => { 4 | const get = (key: string): unknown => { 5 | const value = localStorage.getItem(`${prefix}${key}`); 6 | 7 | return value ? JSON.parse(value) : null; 8 | }; 9 | const set = (key: string, state: unknown) => { 10 | state && localStorage.setItem(`${prefix}${key}`, JSON.stringify(state)); 11 | }; 12 | const remove = (key: string) => { 13 | localStorage.removeItem(`${prefix}${key}`); 14 | }; 15 | 16 | return { get, set, remove }; 17 | }; 18 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /src/record/record-downloader.ts: -------------------------------------------------------------------------------- 1 | import { RecordSaver } from '../typings'; 2 | 3 | export const createRecordDownloader = (): RecordSaver => { 4 | return { 5 | save: (record: object) => { 6 | const dataStr = `data:text/json;charset=utf-8,${encodeURIComponent(JSON.stringify(record))}`; 7 | const anchor = document.createElement('a'); 8 | anchor.setAttribute('href', dataStr); 9 | anchor.setAttribute('download', 'assistant-log.json'); 10 | document.body.appendChild(anchor); 11 | anchor.click(); 12 | anchor.remove(); 13 | }, 14 | }; 15 | }; 16 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /src/record/console-logger.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import { ClientLogger } from '../typings'; 3 | 4 | export type ConsoleLoggerCreator = (level?: 'debug' | 'log') => ClientLogger; 5 | 6 | export const createConsoleLogger: ConsoleLoggerCreator = (level = 'debug') => (entry) => { 7 | switch (entry.type) { 8 | case 'init': { 9 | console[level]('Initialize', entry.params); 10 | 11 | break; 12 | } 13 | case 'incoming': { 14 | console[level]('Received message', entry.message); 15 | 16 | break; 17 | } 18 | case 'outcoming': { 19 | console[level]('Sended message', entry.message); 20 | 21 | break; 22 | } 23 | default: { 24 | break; 25 | } 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": true, 3 | "compilerOptions": { 4 | "jsx": "react", 5 | "target": "ES5", 6 | "lib": ["ES2020.Promise", "esnext.asynciterable", "DOM"], 7 | "declaration": true, 8 | "declarationMap": true, 9 | "composite": false, 10 | "strict": true, 11 | "strictNullChecks": true, 12 | "noUnusedLocals": true, 13 | "noUnusedParameters": true, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "moduleResolution": "node", 17 | "esModuleInterop": true, 18 | "resolveJsonModule": true, 19 | "skipLibCheck": true, 20 | "skipDefaultLibCheck": true, 21 | "rootDir": "src", 22 | "allowJs": true 23 | }, 24 | "include": ["src", "global.d.ts"], 25 | "exclude": ["**/tests/**/*", "**/dist/**/*"] 26 | } 27 | -------------------------------------------------------------------------------- /src/nanoobservable.ts: -------------------------------------------------------------------------------- 1 | import { createNanoEvents } from './nanoevents'; 2 | 3 | export type ObserverFunc = (data: T) => void; 4 | 5 | interface Events { 6 | next: ObserverFunc; 7 | } 8 | 9 | export interface Observer { 10 | next: ObserverFunc; 11 | } 12 | 13 | export interface Observable { 14 | subscribe: (observer: Observer) => { unsubscribe: () => void }; 15 | } 16 | 17 | export const createNanoObservable = (observerFunc: (observer: Observer) => void): Observable => { 18 | const { on, emit } = createNanoEvents>(); 19 | 20 | const subscribe = ({ next }: Observer) => { 21 | const unsubscribe = on('next', next); 22 | return { unsubscribe }; 23 | }; 24 | 25 | observerFunc({ 26 | next: (data: T) => { 27 | emit('next', data); 28 | }, 29 | }); 30 | 31 | return { 32 | subscribe, 33 | }; 34 | }; 35 | -------------------------------------------------------------------------------- /src/NativePanel/fonts.ts: -------------------------------------------------------------------------------- 1 | const FONTS_CDN = 'https://cdn-app.sberdevices.ru/shared-static/0.0.0/fonts/SBSansText/'; 2 | 3 | export const fontFace = ` 4 | @font-face { 5 | font-family: 'SB Sans Text'; 6 | src: local('SB Sans Text Medium'), local('SBSansText-Medium'), url('${FONTS_CDN}SBSansText-Medium.woff2') format('woff2'), url('${FONTS_CDN}SBSansText-Medium.woff') format('woff'); 7 | font-weight: 500; 8 | font-style: normal; 9 | } 10 | 11 | @font-face { 12 | font-family: 'SB Sans Text'; 13 | src: local('SB Sans Text Regular'), local('SBSansText-Regular'), url('${FONTS_CDN}SBSansText-Regular.woff2') format('woff2'), url('${FONTS_CDN}SBSansText-Regular.woff') format('woff'); 14 | font-weight: 400; 15 | font-style: normal; 16 | } 17 | `; 18 | 19 | export const fontFamily500 = ` 20 | font-family: 'SB Sans Text'; 21 | font-weight: 500; 22 | `; 23 | 24 | export const fontFamily400 = ` 25 | font-family: 'SB Sans Text'; 26 | font-weight: 400; 27 | `; 28 | -------------------------------------------------------------------------------- /examples/umd-example/README.md: -------------------------------------------------------------------------------- 1 | # umd-example 2 | 3 | Это небольшое Todo приложение (добавление, выполнение и удаление задач. [См. видео](https://youtu.be/P-o2rwHhARo)) демонстрирует пример взаимодействия с [Assistant Client](https://github.com/sberdevices/assistant-client).Голосовой помощник интегрирован с помощью подключения umd-сборки в теге 23 | 24 | 25 | 26 | 27 |

Мой список дел:

28 |
    29 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /src/assistantSdk/client/transport.ts: -------------------------------------------------------------------------------- 1 | import { createNanoEvents } from '../../nanoevents'; 2 | import { WSCreator } from '../../typings'; 3 | 4 | export interface TransportEvents { 5 | connecting: () => void; 6 | ready: () => void; 7 | close: () => void; 8 | error: (error?: Event) => void; 9 | message: (data: Uint8Array) => void; 10 | } 11 | 12 | const defaultWSCreator: WSCreator = (url: string) => new WebSocket(url); 13 | 14 | export const createTransport = (createWS: WSCreator = defaultWSCreator) => { 15 | const { on, emit } = createNanoEvents(); 16 | 17 | let status: 'connecting' | 'ready' | 'closed' = 'closed'; 18 | let stopped = false; 19 | let ws: WebSocket; 20 | let timeOut: number | undefined; // ид таймера автореконнекта 21 | let retries = 0; // количество попыток коннекта при ошибке 22 | 23 | const close = () => { 24 | stopped = true; 25 | ws && ws.close(); // статус изменится по подписке 26 | clearTimeout(timeOut); 27 | timeOut = undefined; 28 | }; 29 | 30 | const send = (data: Uint8Array) => { 31 | if (!navigator.onLine) { 32 | close(); 33 | emit('error'); 34 | return; 35 | } 36 | 37 | ws.send(data); 38 | }; 39 | 40 | const open = (url: string) => { 41 | if (status !== 'closed') { 42 | return; 43 | } 44 | 45 | status = 'connecting'; 46 | emit('connecting'); 47 | // TODO: нужен таймаут для подключения 48 | ws = createWS(url); 49 | 50 | ws.binaryType = 'arraybuffer'; 51 | ws.addEventListener('open', () => { 52 | if (ws.readyState === 1) { 53 | retries = 0; // сбрасываем количество попыток реконнекта 54 | status = 'ready'; 55 | emit('ready'); 56 | } 57 | }); 58 | 59 | ws.addEventListener('close', () => { 60 | status = 'closed'; 61 | emit('close'); 62 | }); 63 | 64 | ws.addEventListener('error', (e) => { 65 | if (status !== 'connecting') { 66 | throw e; 67 | } 68 | 69 | // пробуем переподключаться, если возникла ошибка при коннекте 70 | if (!ws || (ws.readyState === 3 && !stopped)) { 71 | if (timeOut) { 72 | clearTimeout(timeOut); 73 | } 74 | if (retries < 3) { 75 | timeOut = window.setTimeout(() => { 76 | open(url); 77 | retries++; 78 | }, 300 * retries); 79 | } else { 80 | retries = 0; 81 | emit('error', e); 82 | } 83 | } 84 | }); 85 | 86 | ws.addEventListener('message', (e) => { 87 | emit('message', e.data); 88 | }); 89 | }; 90 | 91 | const reconnect = (url: string) => { 92 | if (status === 'closed') { 93 | open(url); 94 | return; 95 | } 96 | 97 | close(); 98 | setTimeout(() => reconnect(url)); 99 | }; 100 | 101 | return { 102 | send, 103 | open, 104 | close, 105 | reconnect, 106 | on, 107 | }; 108 | }; 109 | -------------------------------------------------------------------------------- /src/NativePanel/styles.tsx: -------------------------------------------------------------------------------- 1 | import { fontFace } from './fonts'; 2 | import { BubbleStyles, BubbleMDStyles, BubbleSMStyles } from './components/Bubble'; 3 | import { SphereButtonStyles, SphereButtonMDStyles, SphereButtonSMStyles } from './components/SphereButton'; 4 | import { CarouselTouchStyles } from './components/CarouselTouch'; 5 | import { KeyboardTouchStyles } from './components/KeyboardTouch'; 6 | import { VoiceTouchStyles } from './components/VoiceTouch'; 7 | import { 8 | SuggestsStyles, 9 | SuggestLGStyles, 10 | SuggestMDStyles, 11 | SuggestSMStyles, 12 | SuggestFilledStyles, 13 | SuggestOutlinedStyles, 14 | } from './components/Suggests'; 15 | import { TextInputStyles, TextInputPureStyles, TextInputFilledStyles } from './components/TextInput'; 16 | import { 17 | NativePanelStyles, 18 | NativePanelTouchStyles, 19 | NativePanelDesktopStyles, 20 | NativePanelDesktopBubblePositionMD, 21 | NativePanelDesktopBubblePositionLG, 22 | NativePanelInputOffsetLGStyles, 23 | NativePanelInputOffsetMDStyles, 24 | NativePanelPaddingsLGStyles, 25 | NativePanelPaddingsMDStyles, 26 | NativePanelPaddingsSMStyles, 27 | } from './NativePanelStyles'; 28 | 29 | export const styles = ` 30 | ${fontFace} 31 | ${NativePanelStyles} 32 | ${BubbleStyles} 33 | ${CarouselTouchStyles} 34 | ${KeyboardTouchStyles} 35 | ${SphereButtonStyles} 36 | ${SuggestsStyles} 37 | ${TextInputStyles} 38 | ${VoiceTouchStyles} 39 | 40 | @keyframes rotation { 41 | 0% { 42 | transform: rotate(0deg); 43 | } 44 | 100% { 45 | transform: rotate(360deg); 46 | } 47 | } 48 | 49 | /** small */ 50 | @media screen and (max-width: 639px) { 51 | ${NativePanelTouchStyles} 52 | ${NativePanelPaddingsSMStyles} 53 | 54 | .Bubble { 55 | ${BubbleMDStyles} 56 | } 57 | 58 | .SphereButton { 59 | ${SphereButtonMDStyles} 60 | } 61 | 62 | .TextInput { 63 | ${TextInputFilledStyles} 64 | } 65 | 66 | .SuggestsSuggest { 67 | ${SuggestMDStyles} 68 | ${SuggestOutlinedStyles} 69 | } 70 | } 71 | 72 | /** medium */ 73 | @media screen and (min-width: 640px) and (max-width: 959px) { 74 | ${NativePanelDesktopStyles} 75 | ${NativePanelDesktopBubblePositionMD} 76 | ${NativePanelInputOffsetMDStyles} 77 | ${NativePanelPaddingsMDStyles} 78 | 79 | .Bubble { 80 | ${BubbleSMStyles} 81 | } 82 | 83 | .SphereButton { 84 | ${SphereButtonSMStyles} 85 | } 86 | 87 | .TextInput { 88 | ${TextInputPureStyles} 89 | } 90 | 91 | .SuggestsSuggest { 92 | ${SuggestSMStyles} 93 | ${SuggestFilledStyles} 94 | } 95 | } 96 | 97 | /** large */ 98 | @media screen and (min-width: 960px) { 99 | ${NativePanelDesktopStyles} 100 | ${NativePanelDesktopBubblePositionLG} 101 | ${NativePanelInputOffsetLGStyles} 102 | ${NativePanelPaddingsLGStyles} 103 | 104 | .bubble { 105 | font-size: calc(16px * 1.5); 106 | } 107 | 108 | .Bubble { 109 | ${BubbleMDStyles} 110 | } 111 | 112 | .SphereButton { 113 | ${SphereButtonMDStyles} 114 | } 115 | 116 | .TextInput { 117 | ${TextInputPureStyles} 118 | } 119 | 120 | .SuggestsSuggest { 121 | ${SuggestLGStyles} 122 | /* ${SuggestMDStyles} */ 123 | ${SuggestFilledStyles} 124 | } 125 | } 126 | `; 127 | -------------------------------------------------------------------------------- /src/assistantSdk/voice/player/voicePlayer.ts: -------------------------------------------------------------------------------- 1 | import { createNanoEvents } from '../../../nanoevents'; 2 | 3 | import { createTrackCollection } from './trackCollection'; 4 | import { createTrackStream } from './trackStream'; 5 | 6 | export interface VoicePlayerSettings { 7 | startVoiceDelay?: number; 8 | sampleRate?: number; 9 | numberOfChannels?: number; 10 | } 11 | 12 | export type EventsType = { 13 | play: (trackId: string) => void; 14 | end: (trackId: string) => void; 15 | }; 16 | 17 | export const createVoicePlayer = ( 18 | actx: AudioContext, 19 | { startVoiceDelay = 0.2, sampleRate, numberOfChannels }: VoicePlayerSettings = {}, 20 | ) => { 21 | const { on, emit } = createNanoEvents(); 22 | const tracks = createTrackCollection>(); 23 | // true - воспроизводим все треки в очереди (новые в том числе), false - скипаем всю очередь (новые в т.ч.) 24 | let active = true; 25 | // индекс текущего трека в tracks 26 | let cursor = 0; 27 | 28 | const play = () => { 29 | if (cursor >= tracks.length) { 30 | if (tracks.some((track) => !track.loaded)) { 31 | return; 32 | } 33 | 34 | // очищаем коллекцию, если все треки были воспроизведены 35 | cursor = 0; 36 | tracks.clear(); 37 | return; 38 | } 39 | 40 | // рекурсивно последовательно включаем треки из очереди 41 | const current = tracks.getByIndex(cursor); 42 | if (current.status === 'end') { 43 | if (cursor < tracks.length) { 44 | cursor++; 45 | play(); 46 | } 47 | } else { 48 | current.play(); 49 | } 50 | }; 51 | 52 | const append = (data: Uint8Array, trackId: string, last = false) => { 53 | let current = tracks.has(trackId) ? tracks.get(trackId) : undefined; 54 | if (current == null) { 55 | /// если trackId нет в коллекции - создаем трек 56 | /// по окончании проигрывания - запускаем следующий трек, вызывая play 57 | current = createTrackStream(actx, { 58 | sampleRate, 59 | numberOfChannels, 60 | delay: startVoiceDelay, 61 | onPlay: () => emit('play', trackId), 62 | onEnd: () => { 63 | emit('end', trackId); 64 | play(); 65 | }, 66 | trackStatus: active ? 'stop' : 'end', 67 | }); 68 | tracks.push(trackId, current); 69 | } 70 | 71 | if (current.status !== 'end' && data.length) { 72 | current.write(data); 73 | } 74 | 75 | if (last) { 76 | // все чанки трека загружены 77 | current.setLoaded(); 78 | } 79 | 80 | play(); 81 | }; 82 | 83 | const stop = () => { 84 | while (cursor < tracks.length) { 85 | const cur = cursor; 86 | 87 | cursor++; 88 | tracks.getByIndex(cur).stop(); 89 | } 90 | }; 91 | 92 | return { 93 | append, 94 | setActive(value: boolean) { 95 | active = value; 96 | if (value) { 97 | play(); 98 | } else { 99 | stop(); 100 | } 101 | }, 102 | on, 103 | stop, 104 | }; 105 | }; 106 | -------------------------------------------------------------------------------- /cypress/support/helpers/socket.helpers.ts: -------------------------------------------------------------------------------- 1 | import { WebSocket } from 'mock-socket'; 2 | 3 | import { appendHeader } from '../../../src/assistantSdk/client/protocol'; 4 | import { IStatus, Message } from '../../../src/proto'; 5 | import { 6 | AssistantNavigationCommand, 7 | AssistantSmartAppCommand, 8 | SystemMessageDataType, 9 | MessageNames, 10 | AppInfo, 11 | Character, 12 | } from '../../../src/typings'; 13 | 14 | export const APP_INFO: AppInfo = { 15 | applicationId: 'test_app', 16 | appversionId: '0.0.0', 17 | frontendType: 'WEB_APP', 18 | projectId: 'test_project', 19 | frontendStateId: 'test_app', 20 | }; 21 | 22 | export const sendMessage = ( 23 | socket: WebSocket, 24 | messageId: number | Long, 25 | { 26 | systemMessageData, 27 | textData, 28 | statusData, 29 | }: { systemMessageData?: SystemMessageDataType; textData?: string; statusData?: IStatus }, 30 | { messageName = MessageNames.ANSWER_TO_USER }: { messageName?: string } = {}, 31 | ) => { 32 | const message = Message.create({ 33 | messageName, 34 | messageId, 35 | text: textData ? { data: textData } : undefined, 36 | systemMessage: systemMessageData != null ? { data: JSON.stringify(systemMessageData) } : undefined, 37 | status: statusData, 38 | last: 1, 39 | version: 5, 40 | }); 41 | 42 | const buffer = Message.encode(message).finish(); 43 | const bufferWithHeader = appendHeader(buffer); 44 | 45 | socket.dispatchEvent({ 46 | type: 'message', 47 | // eslint-disable-next-line @typescript-eslint/ban-ts-ignore 48 | // @ts-ignore 49 | data: bufferWithHeader, 50 | }); 51 | }; 52 | 53 | export const initProtocol = ( 54 | socket: WebSocket, 55 | { 56 | initPhrase, 57 | character = { id: 'sber', name: 'Сбер', gender: 'male', appeal: 'official' }, 58 | items = [], 59 | }: { 60 | initPhrase?: string; 61 | character?: Character; 62 | items?: Array<{ command: AssistantSmartAppCommand | AssistantNavigationCommand }>; 63 | } = {}, 64 | ) => { 65 | socket.on('message', (data) => { 66 | const message = Message.decode((data as Uint8Array).slice(4)); 67 | if (message.messageName === 'OPEN_ASSISTANT') { 68 | sendMessage(socket, message.messageId, { 69 | systemMessageData: { 70 | // eslint-disable-next-line @typescript-eslint/camelcase 71 | auto_listening: false, 72 | // eslint-disable-next-line @typescript-eslint/camelcase 73 | app_info: { 74 | applicationId: '', 75 | appversionId: '', 76 | frontendEndpoint: '', 77 | frontendType: 'WEB_APP', 78 | projectId: '', 79 | }, 80 | items: [], 81 | }, 82 | }); 83 | } 84 | if (initPhrase && message.text?.data === initPhrase) { 85 | sendMessage(socket, message.messageId, { 86 | systemMessageData: { 87 | // eslint-disable-next-line @typescript-eslint/camelcase 88 | auto_listening: false, 89 | // eslint-disable-next-line @typescript-eslint/camelcase 90 | app_info: APP_INFO, 91 | character, 92 | items, 93 | }, 94 | }); 95 | } 96 | }); 97 | }; 98 | -------------------------------------------------------------------------------- /src/assistantSdk/voice/audioContext.ts: -------------------------------------------------------------------------------- 1 | import { createNanoEvents } from '../../nanoevents'; 2 | 3 | export const isAudioSupported = typeof window !== 'undefined' && (window.AudioContext || window.webkitAudioContext); 4 | 5 | /** 6 | * Возвращает новый инстанс AudioContext или ошибку 7 | * @param options AudioContextOptions 8 | * @returns AudioContext 9 | */ 10 | export const createAudioContext = (options?: AudioContextOptions): AudioContext => { 11 | if (window.AudioContext) { 12 | return new AudioContext(options); 13 | } 14 | 15 | if (window.webkitAudioContext) { 16 | // eslint-disable-next-line new-cap 17 | return new window.webkitAudioContext(); 18 | } 19 | 20 | throw new Error('Audio not supported'); 21 | }; 22 | 23 | interface ContextEvents { 24 | ready: () => void; 25 | } 26 | 27 | const { on, emit } = createNanoEvents(); 28 | 29 | let audioContext: { context: AudioContext; ready: boolean; on: typeof on }; 30 | 31 | /** 32 | * При помощи вызова функции из аргумента, возвращает, готовый к воспроизведению звука, AudioContext. 33 | * Всегда возвращает один и тот же AudioContext 34 | * @param onReady Функция, в аргумент которой будет возвращен AudioContext 35 | */ 36 | export const resolveAudioContext = (onReady: (context: AudioContext) => void) => { 37 | if (!audioContext) { 38 | const isSafari = navigator.vendor.search('Apple') >= 0; 39 | const context = createAudioContext(); 40 | 41 | audioContext = { 42 | context, 43 | ready: !isSafari && context.state === 'running', 44 | on, 45 | }; 46 | 47 | /// Контекст может быть не готов для использования сразу после создания 48 | /// Если попробовать что-то воспроизвести в этом контексте - звука не будет 49 | if (!audioContext.ready) { 50 | const handleClick = () => { 51 | document.removeEventListener('click', handleClick); 52 | document.removeEventListener('touchstart', handleClick); 53 | 54 | if (isSafari) { 55 | /// проигрываем тишину, т.к нужно что-то проиграть, 56 | /// чтобы сафари разрешил воспроизводить звуки в любой момент в этом контексте 57 | const oscillator = audioContext.context.createOscillator(); 58 | oscillator.frequency.value = 0; 59 | oscillator.connect(audioContext.context.destination); 60 | oscillator.start(0); 61 | oscillator.stop(0.5); 62 | } 63 | 64 | if (audioContext.context.state === 'suspended') { 65 | /// Developers who write games, WebRTC applications, or other websites that use the Web Audio API 66 | /// should call context.resume() after the first user gesture (e.g. a click, or tap) 67 | /// https://sites.google.com/a/chromium.org/dev/audio-video/autoplay 68 | audioContext.context.resume(); 69 | } 70 | 71 | audioContext.ready = true; 72 | emit('ready'); 73 | }; 74 | 75 | /// чтобы сделать контекст готовым к использованию (воспроизведению звука), 76 | /// необходимо событие от пользователя 77 | 78 | // для пк 79 | document.addEventListener('click', handleClick); 80 | // для мобильных устройств 81 | document.addEventListener('touchstart', handleClick); 82 | } 83 | } 84 | 85 | if (audioContext.ready) { 86 | onReady && onReady(audioContext.context); 87 | } else { 88 | const unsubscribe = on('ready', () => { 89 | onReady(audioContext.context); 90 | unsubscribe(); 91 | }); 92 | } 93 | }; 94 | -------------------------------------------------------------------------------- /src/assistantSdk/voice/recognizers/asr/index.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | // --- добавлено Бондарь Р.О 4 | // --- документация ASR : https://sbtatlas.sigma.sbrf.ru/wiki/pages/viewpage.action?pageId=1880294872 5 | 6 | option optimize_for = LITE_RUNTIME; 7 | 8 | message Variables { 9 | map variables = 1; 10 | } 11 | 12 | message UndecodedSeconds { 13 | float undecoded_seconds = 1; 14 | } 15 | 16 | message FullyFinalized { // Пакет об окончании обработки от сервера к клиента. Должен быть упакован в PacketWrapperFromServer.fully_finalized_field 17 | 18 | } 19 | 20 | message EmotionResult { // Результаты определения эмоций. Эмоции определяются только для всего аудио целиком, 21 | // высылаются только когда DecoderResult.is_final = true, т.е. в конце распознавания. 22 | string name = 1; // Имя эмоции: сейчас может быть "positive", "negative" и "neutral" 23 | float confidence = 2; // Уверенность в эмоции от 0.0 до 1.0. 24 | } 25 | 26 | message Hypothesis { // Гипотеза распознавания 27 | string words = 1; // Ненормализованный результат, например "раз два три москва" 28 | float acoustic_cost = 2; 29 | float linguistic_cost = 3; 30 | float final_cost = 4; 31 | float phrase_start = 5; 32 | float phrase_end = 6; 33 | string normalized_text = 7; // Нормализованный результат, например "1 2 3 Москва" 34 | } 35 | 36 | message DecoderResult { // Результат работы ASR. Должен быть упакован в PacketWrapperFromServer.decoder_result_field 37 | message ContextAnswer { 38 | message ContextRef { // Результат поиска в контекстах 39 | string id = 1; // В какой контекст попали. 40 | int32 index = 2; // В какой элемент попали (индекс в исходном списке, считается с 0) 41 | string original_value = 3; // Оригинальное значение элемента контекста, например "Иван Петров". 42 | string predicted_value = 4; // Распознанное значение контекста, например "Ивану Петрову" 43 | float score = 5; // Мера уверенности от 0.0 до 1.0 44 | } 45 | repeated ContextRef context_result = 1; 46 | } 47 | repeated Hypothesis hypothesis = 1; 48 | float chunk_start = 2; 49 | float chunk_end = 3; 50 | float time_endpoint_detection_ms = 4; 51 | float time_decoding_ms = 5; 52 | Variables variables = 6; 53 | bool is_final = 7; // Признак того, что это финальный результат 54 | repeated EmotionResult emotion_result = 8; // Результат оценки эмоций, заполняется только если DecoderResult.is_final = true 55 | repeated ContextAnswer context_answer = 9; // Результат попадания в контексты, заполняется только если DecoderResult.is_final = true 56 | } 57 | 58 | message ErrorResponse { 59 | string error_message = 1; 60 | } 61 | 62 | message PacketWrapperFromServer { // Все сообщения от сервера должны быть упакованы в этот тип сообщений 63 | oneof MessageType { 64 | UndecodedSeconds undecoded_seconds_field = 1; 65 | FullyFinalized fully_finalized_field = 2; 66 | // DecoderRestarted decoder_restarted_field = 3; 67 | DecoderResult decoder_result_field = 4; 68 | // EmotionList emotion_list = 5; 69 | // RequestCapabilitiesResponse request_capabilities_response = 6; 70 | // GetCapabilitiesResponse get_capabilities_response = 7; 71 | ErrorResponse error_response = 8; 72 | // GetEmotionParametersResponse get_emotion_parameters_response = 9; 73 | } 74 | } -------------------------------------------------------------------------------- /src/record/offline-player.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/camelcase */ 2 | import { 3 | AssistantRecord, 4 | SystemMessageDataType, 5 | AssistantClientCommand, 6 | OutcomingMessage, 7 | MessageNames, 8 | AssistantSmartAppCommand, 9 | } from '../typings'; 10 | 11 | import { CURRENT_VERSION } from './index'; 12 | 13 | export const createRecordOfflinePlayer = ( 14 | record?: AssistantRecord, 15 | { context = window }: { context?: Window } = {}, 16 | ) => { 17 | let currentRecord = record; 18 | let entryCursor = 0; 19 | 20 | const playMessage = (message: SystemMessageDataType, onPlay?: (command: AssistantClientCommand) => void) => { 21 | for (const item of message.items || []) { 22 | if (item.command) { 23 | const command: AssistantClientCommand = { ...(item.command as AssistantSmartAppCommand) }; 24 | 25 | onPlay ? onPlay(command) : context.AssistantClient?.onData && context.AssistantClient.onData(command); 26 | } 27 | } 28 | }; 29 | 30 | const playNext = (onPlay?: (command: AssistantClientCommand) => void) => { 31 | if (!currentRecord || entryCursor + 1 >= currentRecord.entries.length) { 32 | return false; 33 | } 34 | 35 | let entry = currentRecord.entries[entryCursor++]; 36 | while ( 37 | (entry.type !== 'incoming' || 38 | entry.message?.data == null || 39 | entry.message.name !== MessageNames.ANSWER_TO_USER || 40 | !(entry.message.data.items || []).some(({ command }) => command != null)) && 41 | entryCursor < currentRecord.entries.length 42 | ) { 43 | entry = currentRecord.entries[entryCursor++]; 44 | } 45 | if (entry.type === 'incoming' && entryCursor <= currentRecord.entries.length) { 46 | entry.message && playMessage(entry.message.data, onPlay); 47 | } 48 | 49 | return currentRecord.entries.some( 50 | (e, i) => 51 | i >= entryCursor && 52 | e.type === 'incoming' && 53 | e.message?.data != null && 54 | e.message.name === MessageNames.ANSWER_TO_USER && 55 | (e.message.data.items || []).some(({ command }) => command != null), 56 | ); 57 | }; 58 | 59 | const play = (onPlay?: (command: AssistantClientCommand) => void) => { 60 | context.AssistantClient?.onStart && context.AssistantClient.onStart(); 61 | if (!currentRecord) { 62 | return; 63 | } 64 | 65 | let end = false; 66 | while (!end) { 67 | end = !playNext(onPlay); 68 | } 69 | }; 70 | 71 | const getNextAction = () => { 72 | if (!currentRecord || entryCursor + 1 >= currentRecord.entries.length) { 73 | return undefined; 74 | } 75 | 76 | let cursor = entryCursor; 77 | let entry = currentRecord.entries[cursor++]; 78 | while ( 79 | entry.type === 'outcoming' && 80 | entry.message?.data?.systemMessage?.data == null && 81 | cursor < currentRecord.entries.length 82 | ) { 83 | entry = currentRecord.entries[cursor++]; 84 | } 85 | 86 | if (cursor >= currentRecord.entries.length) { 87 | return undefined; 88 | } 89 | 90 | return { 91 | action: (entry as OutcomingMessage).message?.data.server_action, 92 | name: entry.message?.name, 93 | requestId: entry.message?.data.sdk_meta?.requestId, 94 | }; 95 | }; 96 | 97 | const setRecord = (rec: AssistantRecord) => { 98 | if (rec.version !== CURRENT_VERSION) { 99 | throw new Error('Unsupported log version'); 100 | } 101 | currentRecord = rec; 102 | entryCursor = 0; 103 | }; 104 | 105 | return { 106 | continue: playNext, 107 | play, 108 | getNextAction, 109 | setRecord, 110 | }; 111 | }; 112 | -------------------------------------------------------------------------------- /src/appInitialData.ts: -------------------------------------------------------------------------------- 1 | import { AssistantClientCommand, AssistantClientCustomizedCommand, AssistantSmartAppCommand } from './typings'; 2 | 3 | const findCommandIndex = (arr: AssistantClientCommand[], command: AssistantClientCommand) => { 4 | let index = -1; 5 | 6 | if (command.type === 'character') { 7 | index = arr.findIndex((c) => c.type === 'character' && c.character.id === command.character.id); 8 | } else if (command.type === 'insets') { 9 | index = arr.findIndex((c) => c.type === 'insets'); 10 | } else if (command.type === 'app_context') { 11 | index = arr.findIndex((c) => c.type === 'app_context'); 12 | } else if (command.sdk_meta && command.sdk_meta?.mid && command.sdk_meta?.mid !== '-1') { 13 | index = arr.findIndex((c) => c.sdk_meta?.mid === command.sdk_meta?.mid); 14 | } 15 | 16 | return index; 17 | }; 18 | 19 | export const appInitialData = (() => { 20 | let isPulled = false; 21 | let pulled: Array = []; 22 | let committed: Array = []; 23 | let diff: Array = []; 24 | 25 | const isCommandWasPulled = (command: AssistantClientCommand) => findCommandIndex(pulled, command) >= 0; 26 | 27 | return { 28 | /** 29 | * Прочитать appInitialData 30 | * @returns Массив комманд 31 | */ 32 | pull: () => { 33 | isPulled = true; 34 | pulled = [...(window.appInitialData || [])]; 35 | return [...pulled]; 36 | }, 37 | /** 38 | * Зафиксировать текущее состояние appInitialData 39 | */ 40 | commit: () => { 41 | committed = [...(window.appInitialData || [])]; 42 | diff = 43 | isPulled === true 44 | ? (window.appInitialData || []).filter((c) => !isCommandWasPulled(c)) 45 | : [...(window.appInitialData || [])]; 46 | }, 47 | /** 48 | * Возвращает диф appInitialData между pull и commit 49 | * @returns Массив комманд 50 | */ 51 | diff: () => { 52 | return [...diff]; 53 | }, 54 | /** 55 | * Возвращает флаг наличия command в appInitialData на момент commit 56 | * @param command Команда, которую нужно проверить на наличие в appInitialData 57 | * @returns true - если команда была в appInitialData 58 | */ 59 | isCommitted: (command: AssistantClientCommand) => { 60 | const commandIndex = findCommandIndex(committed, command); 61 | const isCommitted = commandIndex >= 0; 62 | if (isCommitted) { 63 | committed.splice(commandIndex, 1); 64 | } 65 | 66 | return isCommitted; 67 | }, 68 | /** 69 | * Возвращает первое сообщение из appInitialData, подходящее под фильтры param 70 | * @param param Параметры: тип сообщения (например, smart_app_data) 71 | * и тип команды (значение поля smart_app_data.type) 72 | * @returns Первое сообщение, соответствующее параметрам или undefined 73 | */ 74 | find: ({ type, command }: { type?: string; command?: string }): T | undefined => { 75 | const data = [...(window.appInitialData || [])]; 76 | const result = data.find((data) => { 77 | if (!command && type && type === data.type) { 78 | return true; 79 | } 80 | const isCommandInSmartAppData = command && 'smart_app_data' in data; 81 | if (!isCommandInSmartAppData) { 82 | return; 83 | } 84 | if ( 85 | command === ((data.smart_app_data as unknown) as { command: string }).command || 86 | command === (data.smart_app_data as AssistantSmartAppCommand['smart_app_data']).type 87 | ) { 88 | return true; 89 | } 90 | return false; 91 | }) as AssistantClientCustomizedCommand; 92 | return ((result && 'smart_app_data' in result ? result.smart_app_data : result) as unknown) as T; 93 | }, 94 | }; 95 | })(); 96 | -------------------------------------------------------------------------------- /src/assistantSdk/meta.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/camelcase */ 2 | import { AppInfo, Meta, PermissionStatus, PermissionType, SystemMessageDataType } from '../typings'; 3 | 4 | interface CommandResponse extends SystemMessageDataType { 5 | app_info: AppInfo; 6 | meta: Meta; 7 | server_action: { 8 | action_id: 'command_response'; 9 | request_message_id: number | Long; 10 | command_response: { 11 | request_permissions?: { 12 | permissions: Array<{ 13 | type: PermissionType; 14 | status: PermissionStatus; 15 | }>; 16 | }; 17 | }; 18 | }; 19 | } 20 | 21 | type Permission = Record; 22 | 23 | const getMetaPermissons = (permission: Permission): Meta['permissions'] => 24 | Object.keys(permission).map((key: string) => ({ 25 | type: key as PermissionType, 26 | status: permission[key as PermissionType], 27 | })); 28 | 29 | export const getCurrentLocation = async (): Promise => 30 | new Promise((resolve, reject) => { 31 | navigator.geolocation.getCurrentPosition( 32 | ({ coords, timestamp }) => { 33 | resolve({ 34 | lat: coords.latitude.toString(), 35 | lon: coords.longitude.toString(), 36 | accuracy: coords.accuracy, 37 | timestamp, 38 | }); 39 | }, 40 | reject, 41 | { timeout: 5000 }, 42 | ); 43 | }); 44 | 45 | export const getTime = (): Meta['time'] => ({ 46 | // Здесь нужен полифилл, т.к. `Intl.DateTimeFormat().resolvedOptions().timeZone` - возвращает пустую строку 47 | 48 | timezone_id: Intl.DateTimeFormat().resolvedOptions().timeZone, 49 | timezone_offset_sec: -new Date().getTimezoneOffset() * 60, 50 | timestamp: Date.now(), 51 | }); 52 | 53 | export const getAnswerForRequestPermissions = async ( 54 | requestMessageId: number | Long, 55 | appInfo: AppInfo, 56 | items: PermissionType[], 57 | ): Promise => { 58 | const permissions: Permission = { 59 | record_audio: 'denied_once', 60 | geo: 'denied_once', 61 | read_contacts: 'denied_permanently', 62 | push: 'denied_once', 63 | }; 64 | 65 | const response: CommandResponse = { 66 | auto_listening: false, 67 | app_info: appInfo, 68 | meta: { 69 | time: getTime(), 70 | permissions: [], 71 | }, 72 | server_action: { 73 | action_id: 'command_response', 74 | request_message_id: requestMessageId, 75 | command_response: { 76 | request_permissions: { 77 | permissions: [], 78 | }, 79 | }, 80 | }, 81 | }; 82 | 83 | return Promise.all( 84 | items.map(async (permission: PermissionType) => { 85 | switch (permission) { 86 | case 'geo': 87 | try { 88 | const location = await getCurrentLocation(); 89 | permissions.geo = 'granted'; 90 | response.meta.location = location; 91 | response.server_action.command_response.request_permissions?.permissions.push({ 92 | type: 'geo', 93 | status: 'granted', 94 | }); 95 | } catch { 96 | permissions.geo = 'denied_permanently'; 97 | response.server_action.command_response.request_permissions?.permissions.push({ 98 | type: 'geo', 99 | status: 'denied_permanently', 100 | }); 101 | } 102 | 103 | break; 104 | default: 105 | // eslint-disable-next-line no-console 106 | console.warn('Unsupported permission request:', permission); 107 | } 108 | }), 109 | ).then(() => { 110 | response.meta.permissions = getMetaPermissons(permissions); 111 | return response; 112 | }); 113 | }; 114 | -------------------------------------------------------------------------------- /src/createAssistantDev.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/camelcase */ 2 | 3 | import { AssistantSettings, AssistantSmartAppData } from './typings'; 4 | import { initializeAssistantSDK, InitializeAssistantSDKParams } from './dev'; 5 | import { createAssistant, CreateAssistantParams } from './createAssistant'; 6 | 7 | export type Surface = 'SBERBOX' | 'STARGATE' | 'SATELLITE' | 'COMPANION' | 'SBOL' | 'TV' | 'TV_HUAWEI' | 'TIME'; 8 | export type Channel = 'B2C' | 'COMPANION_B2C' | 'SBOL'; 9 | 10 | const channelForSurface: Record = { 11 | COMPANION: 'COMPANION_B2C', 12 | SBOL: 'SBOL', 13 | }; 14 | 15 | export type CreateAssistantDevParams = CreateAssistantParams & { 16 | surface?: Surface | string; 17 | userChannel?: Channel | string; 18 | } & Pick< 19 | InitializeAssistantSDKParams, 20 | | 'initPhrase' 21 | | 'url' 22 | | 'userId' 23 | | 'token' 24 | | 'surfaceVersion' 25 | | 'nativePanel' 26 | | 'sdkVersion' 27 | | 'enableRecord' 28 | | 'recordParams' 29 | | 'fakeVps' 30 | | 'settings' 31 | | 'getMeta' 32 | | 'features' 33 | >; 34 | 35 | export const createAssistantDev = ({ 36 | getState, 37 | getRecoveryState, 38 | ready, 39 | surface = 'SBERBOX', 40 | userChannel, 41 | ...sdkParams 42 | }: CreateAssistantDevParams) => { 43 | const { nativePanel } = initializeAssistantSDK({ 44 | ...sdkParams, 45 | surface, 46 | userChannel: userChannel || channelForSurface[surface] || 'B2C', 47 | }); 48 | 49 | return { 50 | ...createAssistant({ getState, getRecoveryState, ready }), 51 | nativePanel, 52 | }; 53 | }; 54 | 55 | const parseJwt = (token: string) => { 56 | const base64Url = token.split('.')[1]; 57 | const base64 = base64Url.replace(/-/g, '+').replace(/_/g, '/'); 58 | const jsonPayload = decodeURIComponent( 59 | atob(base64) 60 | .split('') 61 | .map((c) => `%${`00${c.charCodeAt(0).toString(16)}`.slice(-2)}`) 62 | .join(''), 63 | ); 64 | 65 | return JSON.parse(jsonPayload); 66 | }; 67 | 68 | // Публичный метод, использующий токен из SmartApp Studio 69 | export const createSmartappDebugger = ({ 70 | token, 71 | getState, 72 | getRecoveryState, 73 | ready, 74 | settings = {}, 75 | nativePanel, 76 | ...sdkParams 77 | }: { 78 | token: string; 79 | settings?: Pick; 80 | } & CreateAssistantParams & 81 | Pick< 82 | CreateAssistantDevParams, 83 | 'surface' | 'userChannel' | 'nativePanel' | 'initPhrase' | 'enableRecord' | 'recordParams' | 'getMeta' 84 | >) => { 85 | try { 86 | const { exp } = parseJwt(token); 87 | if (exp * 1000 <= Date.now()) { 88 | // eslint-disable-next-line no-alert 89 | alert('Срок действия токена истек!'); 90 | throw new Error('Token expired'); 91 | } 92 | } catch (exc) { 93 | if (exc.message !== 'Token expired') { 94 | // eslint-disable-next-line no-alert 95 | alert('Указан невалидный токен!'); 96 | throw new Error('Wrong token'); 97 | } 98 | throw exc; 99 | } 100 | 101 | return createAssistantDev({ 102 | ...sdkParams, 103 | token, 104 | settings: { 105 | ...settings, 106 | authConnector: 'developer_portal_jwt', 107 | }, 108 | nativePanel, 109 | getState, 110 | getRecoveryState, 111 | ready, 112 | url: 'wss://nlp2vps.online.sberbank.ru:443/vps/', 113 | }); 114 | }; 115 | 116 | export { createRecordOfflinePlayer as createRecordPlayer } from './record/offline-player'; 117 | export { createOnlineRecordPlayer } from './record/online-player'; 118 | export { NativePanelParams } from './NativePanel/NativePanel'; 119 | // export * from './typings'; 120 | export * from './dev'; 121 | export { initializeDebugging } from './debug'; 122 | export * from './record/mock-recorder'; 123 | export * from './record/createMockWsCreator'; 124 | export { 125 | createAssistantHostMock, 126 | createAssistantHostMockWithRecord, 127 | AssistantActionResult, 128 | CommandParams, 129 | } from './mock'; 130 | -------------------------------------------------------------------------------- /examples/todo-canvas-app/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC, memo, useReducer, useState, useRef, useEffect } from 'react'; 2 | import { createSmartappDebugger, createAssistant, AssistantAppState } from '@sberdevices/assistant-client'; 3 | import './App.css'; 4 | 5 | import { reducer } from './store'; 6 | 7 | const initializeAssistant = (getState: any) => { 8 | if (process.env.NODE_ENV === 'development' && window.Cypress == null) { 9 | return createSmartappDebugger({ 10 | token: process.env.REACT_APP_TOKEN ?? '', 11 | initPhrase: `Запусти ${process.env.REACT_APP_SMARTAPP}`, 12 | getState, 13 | nativePanel: { 14 | defaultText: 'Покажи что-нибудь', 15 | screenshotMode: false, 16 | tabIndex: -1, 17 | }, 18 | }); 19 | } 20 | 21 | return createAssistant({ getState }); 22 | }; 23 | 24 | export const App: FC = memo(() => { 25 | const [appState, dispatch] = useReducer(reducer, { 26 | notes: [{ id: 'uinmh', title: 'купить хлеб', completed: false }], 27 | }); 28 | 29 | const [note, setNote] = useState(''); 30 | 31 | const assistantStateRef = useRef(); 32 | const assistantRef = useRef>(); 33 | 34 | useEffect(() => { 35 | assistantRef.current = initializeAssistant(() => assistantStateRef.current); 36 | 37 | assistantRef.current.on('data', ({ navigation, action }: any) => { 38 | if (navigation) { 39 | switch (navigation.command) { 40 | case 'UP': 41 | window.scrollTo(0, window.scrollY - 500); 42 | break; 43 | case 'DOWN': 44 | window.scrollTo(0, window.scrollY + 500); 45 | break; 46 | } 47 | } 48 | 49 | if (action) { 50 | dispatch(action); 51 | } 52 | }); 53 | }, []); 54 | 55 | useEffect(() => { 56 | assistantStateRef.current = { 57 | item_selector: { 58 | items: appState.notes.map(({ id, title }, index) => ({ 59 | number: index + 1, 60 | id, 61 | title, 62 | })), 63 | }, 64 | }; 65 | }, [appState]); 66 | 67 | const doneNote = (title: string) => { 68 | assistantRef.current?.sendData({ action: { action_id: 'done', parameters: { title } } }); 69 | }; 70 | 71 | return ( 72 |
    73 |
    { 75 | event.preventDefault(); 76 | dispatch({ type: 'add_note', note }); 77 | setNote(''); 78 | }} 79 | > 80 | setNote(value)} 86 | required 87 | autoFocus 88 | /> 89 |
    90 |
      91 | {appState.notes.map((note, index) => ( 92 |
    • 93 | 94 | {index + 1}. 95 | 100 | {note.title} 101 | 102 | 103 | doneNote(note.title)} 109 | disabled={note.completed} 110 | /> 111 |
      • 112 | ))} 113 |
    114 |
    115 | ); 116 | }); 117 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'airbnb', 4 | 'airbnb/hooks', 5 | 'plugin:@typescript-eslint/eslint-recommended', 6 | 'plugin:@typescript-eslint/recommended', 7 | 'plugin:cypress/recommended', 8 | 'prettier/react', 9 | ], 10 | parser: '@typescript-eslint/parser', 11 | plugins: ['react', 'import', 'prettier', 'cypress'], 12 | rules: { 13 | '@typescript-eslint/no-empty-function': 'off', 14 | 'no-restricted-syntax': 'off', // В for...of циклах ничего плохого нет 15 | 'spaced-comment': ['error', 'always', { markers: ['/'] }], /// разрешаем ts-require directive 16 | 'comma-dangle': ['error', 'always-multiline'], 17 | 'arrow-parens': ['error', 'always'], 18 | 19 | 'space-before-function-paren': [ 20 | 'error', 21 | { 22 | anonymous: 'never', 23 | named: 'never', 24 | asyncArrow: 'always', 25 | }, 26 | ], 27 | indent: 'off', 28 | 'max-len': [ 29 | 'error', 30 | 120, 31 | 2, 32 | { 33 | ignoreUrls: true, 34 | ignoreComments: false, 35 | ignoreRegExpLiterals: true, 36 | ignoreStrings: true, 37 | ignoreTemplateLiterals: true, 38 | }, 39 | ], 40 | 'padding-line-between-statements': [ 41 | 'error', 42 | { blankLine: 'always', prev: '*', next: 'return' }, 43 | { blankLine: 'always', prev: '*', next: 'if' }, 44 | ], 45 | 'implicit-arrow-linebreak': 'off', 46 | 'no-plusplus': 'off', 47 | 'max-classes-per-file': 'off', 48 | 'operator-linebreak': 'off', 49 | 'object-curly-newline': 'off', 50 | 'class-methods-use-this': 'off', 51 | 'no-confusing-arrow': 'off', 52 | 'function-paren-newline': 'off', 53 | 'no-param-reassign': 'off', 54 | 'no-shadow': 'warn', 55 | 'space-before-function-paren': 'off', 56 | 'consistent-return': 'off', 57 | 'prettier/prettier': 'error', 58 | 59 | '@typescript-eslint/explicit-function-return-type': 'off', 60 | 61 | 'react/prop-types': 'off', 62 | 'react/static-property-placement': 'off', 63 | 'react/state-in-constructor': 'off', 64 | 'react/jsx-filename-extension': ['error', { extensions: ['.tsx'] }], 65 | 'react/jsx-one-expression-per-line': 'off', 66 | 'react/jsx-indent': ['error', 4], 67 | 'react/jsx-indent-props': ['error', 4], 68 | 'react/jsx-props-no-spreading': 'off', 69 | 'react/destructuring-assignment': 'off', 70 | 'react/sort-comp': 'off', 71 | 'react/no-array-index-key': 'off', 72 | 73 | 'jsx-a11y/no-static-element-interactions': 'off', 74 | 'jsx-a11y/click-events-have-key-events': 'off', 75 | 'jsx-a11y/no-noninteractive-tabindex': 'off', 76 | 77 | 'import/prefer-default-export': 'off', // @grape: https://humanwhocodes.com/blog/2019/01/stop-using-default-exports-javascript-module/ 78 | 'import/order': [ 79 | 'error', 80 | { 81 | groups: [['builtin', 'external'], 'internal', 'parent', 'sibling', 'index'], 82 | 'newlines-between': 'always', 83 | }, 84 | ], 85 | 'import/no-unresolved': 'off', 86 | 'import/extensions': 'off', 87 | 'import/no-extraneous-dependencies': ['off'], //можно включить тока нужно резолвы разрулить 88 | 'arrow-body-style': 'off', 89 | 'padding-line-between-statements': 'off', 90 | 'no-unused-expressions': 'off', 91 | '@typescript-eslint/no-empty-function': 'off', 92 | }, 93 | overrides: [ 94 | { 95 | files: ['*.tsx?'], 96 | env: { 97 | browser: true, 98 | }, 99 | globals: { 100 | window: true, 101 | document: true, 102 | }, 103 | }, 104 | { 105 | files: ['*.test.tsx?', '*.test.js'], 106 | plugins: ['jest'], 107 | env: { 108 | browser: true, 109 | mocha: true, 110 | 'jest/globals': true, 111 | }, 112 | }, 113 | ], 114 | settings: { 115 | react: { 116 | version: '16.13.1', 117 | }, 118 | }, 119 | }; 120 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@sberdevices/assistant-client", 3 | "version": "4.20.0", 4 | "description": "Модуль взаимодействия с виртуальным ассистентом", 5 | "main": "dist/index.js", 6 | "module": "esm/index.js", 7 | "unpkgdev": "umd/assistant.development.min.js", 8 | "unpkg": "umd/assistant.production.min.js", 9 | "scripts": { 10 | "prebuild": "rm -rf ./dist ./esm ./umd", 11 | "build": "rollup -c", 12 | "prepublishOnly": "npm run build", 13 | "release": "auto shipit", 14 | "proto": "pbjs -t static-module src/proto/index.proto > src/proto/index.js && pbts src/proto/index.js -o src/proto/index.d.ts", 15 | "asr": "pbjs -t static-module src/assistantSdk/voice/recognizers/asr/index.proto > src/assistantSdk/voice/recognizers/asr/index.js && pbts src/assistantSdk/voice/recognizers/asr/index.js -o src/assistantSdk/voice/recognizers/asr/index.d.ts", 16 | "mtt": "pbjs -t static-module src/assistantSdk/voice/recognizers/mtt/index.proto > src/assistantSdk/voice/recognizers/mtt/index.js && pbts src/assistantSdk/voice/recognizers/mtt/index.js -o src/assistantSdk/voice/recognizers/mtt/index.d.ts", 17 | "cy:open": "cypress open", 18 | "cy:run": "cypress run", 19 | "test:cy": "cypress run -b chromium --headless", 20 | "lint": "eslint --ext .js,.ts,.tsx src/." 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "git+ssh://git@github.com/sberdevices/assistant-client" 25 | }, 26 | "keywords": [ 27 | "sber", 28 | "assistant", 29 | "smartapp" 30 | ], 31 | "author": "SberDevices Frontend Team ", 32 | "license": "Sber Public License at-nc-sa v.2", 33 | "dependencies": { 34 | "@salutejs/scenario": "0.20.0", 35 | "axios": "0.21.1", 36 | "lodash.clonedeep": "^4.5.0", 37 | "protobufjs": "6.10.2", 38 | "uuid": "8.0.0" 39 | }, 40 | "browserslist": [ 41 | "last 1 Chrome versions" 42 | ], 43 | "devDependencies": { 44 | "@auto-it/conventional-commits": "^10.25.0", 45 | "@auto-it/npm": "^10.25.0", 46 | "@auto-it/slack": "^10.25.0", 47 | "@commitlint/cli": "11.0.0", 48 | "@commitlint/config-conventional": "11.0.0", 49 | "@cypress/webpack-preprocessor": "5.9.1", 50 | "@rollup/plugin-commonjs": "^16.0.0", 51 | "@rollup/plugin-json": "^4.1.0", 52 | "@rollup/plugin-node-resolve": "^10.0.0", 53 | "@rollup/plugin-typescript": "^6.1.0", 54 | "@types/jest": "26.0.14", 55 | "@types/lodash.clonedeep": "^4.5.6", 56 | "@types/mocha": "8.0.3", 57 | "@types/react": "16.9.35", 58 | "@types/react-dom": "16.9.8", 59 | "@types/uuid": "7.0.3", 60 | "auto": "^10.25.0", 61 | "cypress": "7.6.0", 62 | "eslint": "6.8.0", 63 | "eslint-config-airbnb": "18.1.0", 64 | "eslint-config-prettier": "6.11.0", 65 | "eslint-plugin-cypress": "2.11.3", 66 | "eslint-plugin-flowtype": "4.7.0", 67 | "eslint-plugin-import": "2.20.2", 68 | "eslint-plugin-jest": "23.8.2", 69 | "eslint-plugin-jsx-a11y": "6.2.3", 70 | "eslint-plugin-prettier": "3.1.3", 71 | "eslint-plugin-react": "7.19.0", 72 | "eslint-plugin-react-hooks": "3.0.0", 73 | "eslint-plugin-testing-library": "^3.10.1", 74 | "file-loader": "6.1.0", 75 | "husky": "4.3.0", 76 | "lint-staged": "^10.5.4", 77 | "mock-socket": "9.0.3", 78 | "prettier": "2.1.2", 79 | "pretty-quick": "3.1.0", 80 | "react": "^17.0.1", 81 | "react-dom": "^17.0.1", 82 | "react-scripts": "3.4.3", 83 | "rollup": "^2.33.2", 84 | "rollup-plugin-copy": "^3.4.0", 85 | "rollup-plugin-replace": "^2.2.0", 86 | "rollup-plugin-terser": "^7.0.2", 87 | "ts-loader": "8.0.4", 88 | "tslib": "^2.0.3", 89 | "typescript": "3.9.2", 90 | "webpack": "4.44.2" 91 | }, 92 | "peerDependencies": { 93 | "react": ">=16.8.0", 94 | "react-dom": ">=16.8.0" 95 | }, 96 | "files": [ 97 | "dist", 98 | "esm", 99 | "umd" 100 | ], 101 | "sideEffects": false, 102 | "auto": { 103 | "baseBranch": "main", 104 | "plugins": [ 105 | [ 106 | "npm", 107 | { 108 | "setRcToken": false 109 | } 110 | ], 111 | "conventional-commits", 112 | "slack" 113 | ] 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /cypress/integration/disableDubbing.spec.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import { Server } from 'mock-socket'; 4 | 5 | import { createAssistantClient } from '../../src'; 6 | import { Message } from '../../src/proto'; 7 | 8 | describe('Проверяем изменение настроек озвучки', () => { 9 | const defaultDubbing = -1; 10 | const configuration = { 11 | settings: { dubbing: defaultDubbing }, 12 | getToken: () => Promise.resolve(''), 13 | url: 'ws://path', 14 | userChannel: '', 15 | userId: '', 16 | version: 5, 17 | }; 18 | 19 | let server: Server; 20 | let assistantClient: ReturnType; 21 | 22 | beforeEach(() => { 23 | server = new Server(configuration.url); 24 | 25 | assistantClient = createAssistantClient(configuration); 26 | }); 27 | 28 | afterEach(() => { 29 | if (server) { 30 | server.stop(); 31 | } 32 | }); 33 | 34 | it('Вызов changeSettings должен отправлять Settings - если соединение активно', (done) => { 35 | let phase: 1 | 2 = 1; 36 | 37 | server.on('connection', (socket) => { 38 | socket.on('message', (data) => { 39 | const message = Message.decode((data as Uint8Array).slice(4)); 40 | if (phase === 1 && message.initialSettings) { 41 | expect(message.initialSettings.settings.dubbing, 'dubbing при старте получен').to.equal( 42 | defaultDubbing, 43 | ); 44 | assistantClient.changeSettings({ disableDubbing: defaultDubbing !== -1 }); 45 | phase = 2; 46 | } else if (phase === 2 && message.settings) { 47 | expect(message.settings.dubbing, 'изменения dubbing получены').to.equal(defaultDubbing * -1); 48 | done(); 49 | } 50 | }); 51 | }); 52 | assistantClient.start(); 53 | }); 54 | 55 | it('Вызов changeSettings не должен менять настройки протокола и отправлять Settings - если dubbing не поменялся', (done) => { 56 | let settingsReceived = false; 57 | 58 | server.on('connection', (socket) => { 59 | socket.on('message', (data) => { 60 | const message = Message.decode((data as Uint8Array).slice(4)); 61 | if (message.initialSettings) { 62 | expect(message.initialSettings.settings.dubbing, 'dubbing при старте получен').to.equal( 63 | defaultDubbing, 64 | ); 65 | assistantClient.changeSettings({ disableDubbing: defaultDubbing === -1 }); 66 | } 67 | 68 | if (message.settings) { 69 | settingsReceived = true; 70 | } 71 | }); 72 | }); 73 | 74 | assistantClient.start(); 75 | cy.wait(1000).then(() => { 76 | assert.isFalse(settingsReceived, 'Настройки были отправлены'); 77 | done(); 78 | }); 79 | }); 80 | 81 | it('Вызов changeSettings должен менять настройки протокола и не отправлять Settings - если соединение неактивно', (done) => { 82 | let settingsReceived = false; 83 | let phase: 1 | 2 = 1; 84 | 85 | server.on('connection', (socket) => { 86 | socket.on('message', (data) => { 87 | const message = Message.decode((data as Uint8Array).slice(4)); 88 | if (message.initialSettings) { 89 | if (phase === 1) { 90 | expect(message.initialSettings.settings.dubbing, 'dubbing при старте получен').to.equal( 91 | defaultDubbing, 92 | ); 93 | server.clients()[0].close(); 94 | assistantClient.changeSettings({ disableDubbing: defaultDubbing !== -1 }); 95 | phase = 2; 96 | return; 97 | } 98 | 99 | expect(message.initialSettings.settings.dubbing, 'dubbing при рестарте получен').to.equal( 100 | defaultDubbing * -1, 101 | ); 102 | done(); 103 | } 104 | 105 | if (message.settings) { 106 | settingsReceived = true; 107 | } 108 | }); 109 | }); 110 | 111 | assistantClient.start(); 112 | cy.wait(1000).then(() => { 113 | assert.isFalse(settingsReceived, 'Настройки были отправлены'); 114 | assistantClient.start(); 115 | }); 116 | }); 117 | }); 118 | -------------------------------------------------------------------------------- /src/NativePanel/NativePanelStyles.ts: -------------------------------------------------------------------------------- 1 | export const NativePanelStyles = ` 2 | .NativePanel { 3 | position: fixed; 4 | bottom: 0; 5 | left: 0; 6 | z-index: 999; 7 | width: 100%; 8 | box-sizing: border-box; 9 | display: flex; 10 | align-items: center; 11 | backdrop-filter: blur(5px); 12 | background: linear-gradient(#ffffff00, #000000); 13 | } 14 | 15 | .NativePanel__sphere { 16 | display: inline-block; 17 | } 18 | .NativePanel__textInputs {} 19 | .NativePanel__touch { 20 | display: inline-block; 21 | margin-left: 16px; 22 | } 23 | .NativePanel__touch .VoiceTouch { 24 | display: none; 25 | } 26 | `; 27 | 28 | export const NativePanelPaddingsSMStyles = ` 29 | .NativePanel { 30 | height: 64px; 31 | padding: 8px 20px; 32 | } 33 | 34 | .NativePanel .Suggests { 35 | padding-left: 20px; 36 | margin-left: -20px; 37 | margin-right: -20px; 38 | } 39 | `; 40 | 41 | export const NativePanelPaddingsMDStyles = ` 42 | .NativePanel { 43 | padding: 28px 12px 12px; 44 | } 45 | 46 | .NativePanel .Suggests { 47 | margin-right: -12px; 48 | } 49 | `; 50 | 51 | export const NativePanelPaddingsLGStyles = ` 52 | .NativePanel { 53 | padding: 36px 64px 24px; 54 | } 55 | 56 | .NativePanel .Suggests { 57 | margin-right: -64px; 58 | } 59 | `; 60 | 61 | export const NativePanelInputOffsetMDStyles = ` 62 | .NativePanel__textInputs { 63 | margin-left: 24px; 64 | } 65 | `; 66 | 67 | export const NativePanelInputOffsetLGStyles = ` 68 | .NativePanel__textInputs { 69 | margin-left: 38px; 70 | } 71 | `; 72 | 73 | // touch 74 | const NativePanelTouchVoiceInputStyles = ` 75 | .NativePanel.voice-input .CarouselTouch { 76 | display: block; 77 | position: absolute; 78 | left: 23px; 79 | top: 50%; 80 | transform: translateY(-50%); 81 | } 82 | .NativePanel.voice-input .NativePanel__sphere { 83 | margin: 0 auto; 84 | } 85 | 86 | .NativePanel.voice-input .NativePanel__textInputs { 87 | position: absolute; 88 | top: -6px; 89 | left: 20px; 90 | right: 20px; 91 | transform: translateY(-100%); 92 | } 93 | 94 | .NativePanel.voice-input .NativePanel__textInputs .TextInput { 95 | display: none; 96 | } 97 | 98 | .NativePanel.voice-input .NativePanel__touch { 99 | position: absolute; 100 | right: 23px; 101 | top: 50%; 102 | transform: translateY(-50%); 103 | } 104 | `; 105 | 106 | const NativePanelTouchTextInputStyles = ` 107 | .NativePanel.text-input .NativePanel__textInputs { 108 | width: 100%; 109 | } 110 | .NativePanel.text-input .NativePanel__textInputs .Suggests { 111 | width: auto; 112 | position: absolute; 113 | top: 0; 114 | left: 20px; 115 | right: 20px; 116 | transform: translateY(-100%); 117 | } 118 | .NativePanel.text-input .NativePanel__sphere { 119 | display: none; 120 | } 121 | .NativePanel.text-input .NativePanel__touch .KeyboardTouch { 122 | display: none; 123 | } 124 | .NativePanel.text-input .NativePanel__touch .VoiceTouch { 125 | display: block; 126 | } 127 | `; 128 | 129 | export const NativePanelTouchStyles = ` 130 | ${NativePanelTouchVoiceInputStyles} 131 | ${NativePanelTouchTextInputStyles} 132 | 133 | .NativePanel.production-mode .Bubble { 134 | display: none; 135 | } 136 | 137 | .NativePanel.has-suggestions.voice-input .Bubble { 138 | top: -54px; 139 | } 140 | 141 | .NativePanel.has-suggestions.text-input .Bubble { 142 | top: -46px; 143 | } 144 | `; 145 | 146 | // desktop 147 | const NativePanelDesktopNotscreenshotModeStyles = ` 148 | .NativePanel:not(.production-mode) .NativePanel__textInputs { 149 | position: relative; 150 | width: 100%; 151 | } 152 | 153 | .NativePanel:not(.production-mode) .NativePanel__textInputs .Suggests { 154 | position: absolute; 155 | top: -13px; 156 | left: 0; 157 | right: 0; 158 | transform: translateY(-100%); 159 | } 160 | `; 161 | 162 | const NativePanelDesktopscreenshotModeStyles = ` 163 | .NativePanel.production-mode .NativePanel__textInputs .TextInput, 164 | .NativePanel.production-mode .Bubble { 165 | display: none; 166 | } 167 | `; 168 | 169 | export const NativePanelDesktopBubblePositionLG = ` 170 | .NativePanel.has-suggestions:not(.production-mode) .Bubble { 171 | top: -33px; 172 | } 173 | `; 174 | 175 | export const NativePanelDesktopBubblePositionMD = ` 176 | .NativePanel.has-suggestions:not(.production-mode) .Bubble { 177 | top: -26px; 178 | } 179 | `; 180 | 181 | export const NativePanelDesktopStyles = ` 182 | ${NativePanelDesktopNotscreenshotModeStyles} 183 | ${NativePanelDesktopscreenshotModeStyles} 184 | 185 | .NativePanel__sphere { 186 | display: inline-block; 187 | } 188 | 189 | .NativePanel__touch { 190 | display: none; 191 | } 192 | 193 | .NativePanel:not(.has-suggestions) .Bubble, 194 | .NativePanel:not(.has-suggestions):not(.production-mode) .Bubble { 195 | top: 12px 196 | } 197 | `; 198 | -------------------------------------------------------------------------------- /src/proto/index.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message Message { 4 | string user_id = 1; //Не используется с версии контракта 3. Даже если будет передано, не будет использовано. Перенесено в InitialSettings 5 | int64 message_id = 2; //Обязательно 6 | int32 last = 3; // false = -1 | 0 - undefined | true = 1 // default=false //Обязательно 7 | string token = 4; //Опционально 8 | string user_channel = 11; //Не используется с версии контракта 3. Даже если будет передано, не будет использовано. Перенесено в InitialSettings 9 | string vps_token = 12; //Устарело с версии 3, должен передаваться в token = 4 10 | repeated DevContext dev_context = 13; ////Устарело с версии 3. 11 | string message_name = 14; //Опционально 12 | int32 version = 15; //Обязательно //версия 3 13 | oneof content { 14 | Voice voice = 5; 15 | Text text = 6; 16 | SystemMessage system_message = 7; // JSON as String. 17 | LegacyDevice legacy_device = 8;//С версии 2 будет отказ, как нарушение контракта 18 | Settings settings = 9;//Отправляется во время сессии (не на старте), если необходимо переопределить настройки. На старте сессии надо использовать InitialSettings 19 | Status status = 10; 20 | Device device = 16; //Не используется с версии контракта 3. Даже если будет передано, не будет использовано. Перенесено в InitialSettings 21 | Bytes bytes = 17;//Массив данных для передачи сервису потребителю, в виде байтов 22 | InitialSettings initial_settings = 18;// Присылается первым сообщением после открытии сессии, они определяют контекст сессии, если не будет прислано будет ошибка 23 | Cancel cancel = 21; 24 | } 25 | int64 timestamp = 19; // время создания чанка 26 | map meta = 20; // для ufsInfo и т.д. 27 | } 28 | 29 | //С версии контракта 3 30 | message InitialSettings { 31 | string user_id = 1;//Обязательно (max 36 символов) 32 | string user_channel = 2;//Обязательно. Пример, SBOL, B2C и т д 33 | Device device = 3;//Обязательно 34 | Settings settings = 4;//Опционально 35 | string locale = 5; //Локаль пользователя (для транслирования user-friendly текстов ошибок на клиента в status.description) 36 | } 37 | 38 | //Присылается только в рамках InitialSettings 39 | message Device { 40 | string platform_type = 1; // Пример, ANDROID 41 | string platform_version = 2; 42 | string surface = 3;////Обязательно. Пример, SBERBOX 43 | string surface_version = 4; 44 | string features = 5;// JSON as String. 45 | string capabilities = 6;// JSON as String. 46 | string device_id = 7; 47 | string device_manufacturer = 8; 48 | string device_model = 9; 49 | string additional_info = 10; // JSON as String, который будет добавлен в блок additionalInfo блока device 50 | string tenant = 11; 51 | } 52 | 53 | //Может присылаться как в рамках InitialSettings, при открытии сессии, так и уже в сессии, тем самым переопределяя дальнейшее поведение сессии 54 | message Settings { 55 | int32 dubbing = 1; // false = -1 | 0 - undefined | true = 1 // default:true 56 | int32 echo = 2; // false = -1 | 0 - undefined | true = 1 // default:false 57 | string tts_engine = 3; // tts engine alias в vps 58 | string asr_engine = 4;// asr engine alias в vps 59 | int32 asr_auto_stop = 5;// 60 | int32 dev_mode = 6;// trace enabler ----Не используется с версии контракт 3. Даже если будет передано, не будет использовано 61 | string auth_connector = 7; //Алиас коннектора для аутентификации. Должен совпадать с ключем в настройках auth:auth-config:КЛЮЧ 62 | string surface = 8; //Не используется с версии контракт 3. Даже если будет передано, не будет использовано. Значение должно передаваться в device 63 | } 64 | 65 | message LegacyDevice { // Не используется с версии контракт 3. Даже если будет передано, будет ошибка 66 | string client_type = 1; 67 | string channel = 2; 68 | string channel_version = 3; 69 | string platform_name = 4; 70 | string platform_version = 5; 71 | string sdk_version = 6; 72 | string protocol_version = 7; 73 | } 74 | 75 | message Voice { 76 | bytes data = 1; 77 | } 78 | 79 | message Text { 80 | string data = 1;//текст 81 | string type = 2;//тип разметки 82 | } 83 | 84 | message SystemMessage { 85 | string data = 1; // JSON as String. 86 | } 87 | 88 | message Status { 89 | int32 code = 1; //код статуса/ошибки 90 | string description = 2; //описание статуса/ошибки 91 | string technical_description = 3; //техническое описание статуса/ошибки 92 | } 93 | 94 | message Bytes { 95 | bytes data = 1;//Массив байтов для передачи сервису потребителю 96 | string desc = 2;//Описание для логов, что за массив байтов 97 | } 98 | 99 | message DevContext { 100 | string name = 1; 101 | int64 timestamp_ms = 2; 102 | string data = 3; 103 | } 104 | 105 | message Cancel {} 106 | -------------------------------------------------------------------------------- /src/assistantSdk/voice/listener/navigatorAudioProvider.ts: -------------------------------------------------------------------------------- 1 | import { createAudioContext } from '../audioContext'; 2 | 3 | /** 4 | * Понижает sample rate c inSampleRate до значения outSampleRate и преобразует Float32Array в ArrayBuffer 5 | * @param buffer Аудио 6 | * @param inSampleRate текущий sample rate 7 | * @param outSampleRate требуемый sample rate 8 | * @returns Аудио со значением sample rate = outSampleRate 9 | */ 10 | const downsampleBuffer = (buffer: Float32Array, inSampleRate: number, outSampleRate: number): ArrayBuffer => { 11 | if (outSampleRate > inSampleRate) { 12 | throw new Error('downsampling rate show be smaller than original sample rate'); 13 | } 14 | const sampleRateRatio = inSampleRate / outSampleRate; 15 | const newLength = Math.round(buffer.length / sampleRateRatio); 16 | const result = new Int16Array(newLength); 17 | 18 | let offsetResult = 0; 19 | let offsetBuffer = 0; 20 | 21 | while (offsetResult < result.length) { 22 | const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio); 23 | let accum = 0; 24 | let count = 0; 25 | for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) { 26 | accum += buffer[i]; 27 | count++; 28 | } 29 | 30 | result[offsetResult] = Math.min(1, accum / count) * 0x7fff; 31 | offsetResult++; 32 | offsetBuffer = nextOffsetBuffer; 33 | } 34 | 35 | return result.buffer; 36 | }; 37 | 38 | const TARGET_SAMPLE_RATE = 16000; 39 | const IS_FIREFOX = typeof window !== 'undefined' && navigator.userAgent.toLowerCase().indexOf('firefox') > -1; 40 | 41 | let context: AudioContext; 42 | let processor: ScriptProcessorNode; 43 | 44 | /** 45 | * Преобразует stream в чанки (кусочки), и передает их в cb, 46 | * будет это делать, пока не будет вызвана функция остановки 47 | * @param stream Аудио-поток 48 | * @param cb callback, куда будут переданы чанки из потока 49 | * @returns Функция, вызов которой остановит передачу чанков 50 | */ 51 | const createAudioRecorder = ( 52 | stream: MediaStream, 53 | cb: (buffer: ArrayBuffer, last: boolean) => void, 54 | ): Promise<() => void> => 55 | new Promise((resolve) => { 56 | let state: 'inactive' | 'recording' = 'inactive'; 57 | let input: MediaStreamAudioSourceNode; 58 | 59 | const stop = () => { 60 | if (state === 'inactive') { 61 | throw new Error("Can't stop inactive recorder"); 62 | } 63 | 64 | state = 'inactive'; 65 | stream.getTracks().forEach((track) => { 66 | track.stop(); 67 | }); 68 | input.disconnect(); 69 | }; 70 | 71 | const start = () => { 72 | if (state !== 'inactive') { 73 | throw new Error("Can't start not inactive recorder"); 74 | } 75 | 76 | state = 'recording'; 77 | 78 | if (!context) { 79 | context = createAudioContext({ 80 | // firefox не умеет выравнивать samplerate, будем делать это самостоятельно 81 | sampleRate: IS_FIREFOX ? undefined : TARGET_SAMPLE_RATE, 82 | }); 83 | } 84 | 85 | input = context.createMediaStreamSource(stream); 86 | 87 | if (!processor) { 88 | processor = context.createScriptProcessor(2048, 1, 1); 89 | } 90 | 91 | const listener = (e: AudioProcessingEvent) => { 92 | const buffer = e.inputBuffer.getChannelData(0); 93 | const data = downsampleBuffer(buffer, context.sampleRate, TARGET_SAMPLE_RATE); 94 | 95 | const last = state === 'inactive'; 96 | cb(data, last); 97 | 98 | if (last) { 99 | processor.removeEventListener('audioprocess', listener); 100 | } 101 | }; 102 | 103 | processor.addEventListener('audioprocess', listener); 104 | processor.addEventListener('audioprocess', () => resolve(stop), { once: true }); 105 | 106 | input.connect(processor); 107 | processor.connect(context.destination); 108 | }; 109 | 110 | start(); 111 | }); 112 | 113 | /** 114 | * Запрашивает у браузера доступ к микрофону и резолвит Promise, если разрешение получено. 115 | * После получения разрешения, чанки с голосом будут передаваться в cb - пока не будет вызвана функция из результата. 116 | * @param cb Callback, куда будут передаваться чанки с голосом пользователя 117 | * @returns Promise, который содержит функцию прерывающую слушание 118 | */ 119 | export const createNavigatorAudioProvider = (cb: (buffer: ArrayBuffer, last: boolean) => void): Promise<() => void> => 120 | navigator.mediaDevices 121 | .getUserMedia({ 122 | audio: true, 123 | }) 124 | .then((stream) => createAudioRecorder(stream, cb)); 125 | -------------------------------------------------------------------------------- /cypress/integration/greetings.spec.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import { Server } from 'mock-socket'; 4 | 5 | import { createAssistantClient } from '../../src'; 6 | import { Message } from '../../src/proto'; 7 | 8 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 9 | type ArgumentsType = T extends (...args: infer U) => any ? U : never; 10 | 11 | describe('Проверяем приветствие', () => { 12 | const configuration = { 13 | settings: {}, 14 | getToken: () => Promise.resolve(''), 15 | url: 'ws://path', 16 | userChannel: '', 17 | userId: '', 18 | version: 5, 19 | }; 20 | 21 | const checkStartAssistant = ( 22 | server: Server, 23 | args: ArgumentsType['start']>, 24 | onMessage: (message: Message) => void, 25 | beforeStart?: (assistant: ReturnType) => void, 26 | ): ReturnType => { 27 | server.on('connection', (socket) => { 28 | assert.isOk('Соединение после старта'); 29 | 30 | socket.on('message', (data) => { 31 | onMessage(Message.decode((data as Uint8Array).slice(4))); 32 | }); 33 | }); 34 | 35 | const assistantClient = createAssistantClient(configuration); 36 | 37 | beforeStart && beforeStart(assistantClient); 38 | 39 | assistantClient.start(...args); 40 | 41 | return assistantClient; 42 | }; 43 | 44 | let server: Server; 45 | 46 | beforeEach(() => { 47 | server = new Server(configuration.url); 48 | }); 49 | 50 | afterEach(() => { 51 | if (server) { 52 | server.stop(); 53 | } 54 | }); 55 | 56 | it('Приветствие включено, первая сессия', (done) => { 57 | checkStartAssistant( 58 | server, 59 | [{ disableGreetings: false, isFirstSession: true }], 60 | ({ messageName, systemMessage }) => { 61 | if (messageName === 'OPEN_ASSISTANT') { 62 | const data = JSON.parse(systemMessage.data); 63 | 64 | expect(data.is_first_session, 'Отправлен "is_first_session"').be.true; 65 | expect(data.meta.current_app.app_info.systemName, 'Отправлен current_app assistant').be.eq( 66 | 'assistant', 67 | ); 68 | 69 | done(); 70 | } 71 | }, 72 | ); 73 | }); 74 | 75 | it('Приветствие включено, не первая сессия', (done) => { 76 | checkStartAssistant( 77 | server, 78 | [{ disableGreetings: false, isFirstSession: false }], 79 | ({ messageName, systemMessage }) => { 80 | if (messageName === 'OPEN_ASSISTANT') { 81 | const data = JSON.parse(systemMessage.data); 82 | 83 | expect(data.is_first_session, 'Не отправлен "is_first_session"').be.eq(undefined); 84 | expect(data.meta.current_app.app_info.systemName, 'Отправлен current_app assistant').be.eq( 85 | 'assistant', 86 | ); 87 | assert.isOk('Отправлен "OPEN_ASSISTANT"'); 88 | 89 | done(); 90 | } 91 | }, 92 | ); 93 | }); 94 | 95 | it('Приветствие включено, текущий апп НЕ assistant', (done) => { 96 | const onMessage = cy.stub(); 97 | 98 | const assistantClient = checkStartAssistant(server, [{ isFirstSession: true }], onMessage, (client) => { 99 | client.setActiveApp({ 100 | projectId: 'test', 101 | applicationId: 'test', 102 | appversionId: 'test', 103 | frontendType: 'WEB_APP', 104 | frontendEndpoint: 'https://example.com', 105 | }); 106 | }); 107 | 108 | // eslint-disable-next-line cypress/no-unnecessary-waiting 109 | cy.wait(500) 110 | .then(() => { 111 | expect(onMessage, 'Нет соединения после старта').to.not.called; 112 | 113 | assistantClient.sendText('text'); 114 | }) 115 | .wait(500) 116 | .then(() => { 117 | expect(onMessage, 'Соединение, отправлен текст').to.called; 118 | }) 119 | .then(done); 120 | }); 121 | 122 | it('Приветствие выключено', (done) => { 123 | const onMessage = cy.stub(); 124 | 125 | const assistantClient = checkStartAssistant( 126 | server, 127 | [{ disableGreetings: true, isFirstSession: false }], 128 | onMessage, 129 | ); 130 | 131 | // eslint-disable-next-line cypress/no-unnecessary-waiting 132 | cy.wait(500) 133 | .then(() => { 134 | expect(onMessage, 'Нет соединения после старта').to.not.called; 135 | 136 | assistantClient.sendText('text'); 137 | }) 138 | .wait(500) 139 | .then(() => { 140 | expect(onMessage, 'Соединение, отправлен текст').to.called; 141 | }) 142 | .then(done); 143 | }); 144 | }); 145 | -------------------------------------------------------------------------------- /cypress/integration/sendAction.spec.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { createAssistant } from '../../src/index'; 3 | 4 | /* eslint-disable @typescript-eslint/camelcase */ 5 | 6 | describe('Проверяем sendAction', () => { 7 | beforeEach(() => { 8 | window.AssistantHost = { 9 | close: cy.stub(), 10 | ready: cy.stub(), 11 | setSuggest: cy.stub(), 12 | }; 13 | }); 14 | 15 | afterEach(() => { 16 | delete window.AssistantHost; 17 | }); 18 | 19 | const state = {}; 20 | const recoveryState = {}; 21 | const requestId = 'custom_requestId'; 22 | const name = 'custom_message_name'; 23 | const action = { type: 'test_action', payload: { test: 'action' } }; 24 | const dataCommand = { type: 'smart_app_data', smart_app_data: { command: 'test_cmd' } }; 25 | const errorCommand = { 26 | type: 'smart_app_error', 27 | smart_app_error: { code: 500, description: 'Some technical problem' }, 28 | }; 29 | const getState = () => state; 30 | const getRecoveryState = () => recoveryState; 31 | const initAssistant = () => createAssistant({ getState, getRecoveryState }); 32 | 33 | it('sendAction должен вызывать assistantHost.sendDataContainer', (done) => { 34 | const assistant = initAssistant(); 35 | 36 | window.AssistantHost.sendDataContainer = (sended) => { 37 | const { data, message_name, requestId } = JSON.parse(sended); 38 | expect(data, 'пришел экшен').to.deep.equal(action); 39 | expect(message_name, 'message_name не заполнен').to.empty; 40 | expect(requestId, 'requestId заполнен').to.not.empty; 41 | done(); 42 | }; 43 | 44 | assistant.sendAction(action); 45 | }); 46 | 47 | it('sendAction должен проксировать name и requestId в assistantHost.sendDataContainer', (done) => { 48 | const assistant = initAssistant(); 49 | 50 | window.AssistantHost.sendDataContainer = (sended) => { 51 | const { data, message_name, requestId } = JSON.parse(sended); 52 | expect(data, 'пришел экшен').to.deep.equal(action); 53 | expect(message_name, 'message_name передан').to.equal(name); 54 | expect(requestId, 'requestId передан').to.equal(requestId); 55 | done(); 56 | }; 57 | 58 | assistant.sendAction(action, undefined, undefined, { name, requestId }); 59 | }); 60 | 61 | it('Вызов assistantClient.onData вызывает обработчики on("data") и on("error")', (done) => { 62 | const status = { data: false, error: false }; 63 | const commands = [dataCommand, errorCommand]; 64 | const assistant = initAssistant(); 65 | 66 | window.AssistantHost.sendDataContainer = (data) => { 67 | const { requestId } = JSON.parse(data); 68 | setTimeout(() => 69 | commands.map((command) => window.AssistantClient.onData({ ...command, sdk_meta: { requestId } })), 70 | ); 71 | }; 72 | 73 | assistant.sendAction( 74 | action, 75 | (data) => { 76 | expect(data).to.deep.equal(dataCommand.smart_app_data); 77 | status.data = true; 78 | if (status.error) { 79 | done(); 80 | } 81 | }, 82 | (error) => { 83 | expect(error).to.deep.equal(errorCommand.smart_app_error); 84 | status.error = true; 85 | if (status.data) { 86 | done(); 87 | } 88 | }, 89 | { requestId }, 90 | ); 91 | }); 92 | 93 | it('Если не передавать onData и onError, должна срабатывать общая подписка', () => { 94 | const commands = [dataCommand, errorCommand]; 95 | const assistant = initAssistant(); 96 | 97 | window.AssistantHost.sendDataContainer = (data) => { 98 | const { requestId } = JSON.parse(data); 99 | setTimeout(() => 100 | commands.map((command, i) => 101 | setTimeout(() => window.AssistantClient.onData({ ...command, sdk_meta: { requestId } }), i), 102 | ), 103 | ); 104 | }; 105 | 106 | assistant.on('data', (data) => { 107 | if (data.smart_app_data) { 108 | expect(data.smart_app_data).to.deep.equal(dataCommand.smart_app_data); 109 | } else { 110 | expect(data.smart_app_error).to.deep.equal(errorCommand.smart_app_error); 111 | } 112 | }); 113 | 114 | assistant.sendAction(action); 115 | }); 116 | 117 | it('После вызова clear обработчики не работают', (done) => { 118 | let counter = 0; 119 | const commands = [dataCommand, dataCommand]; 120 | const assistant = initAssistant(); 121 | 122 | window.AssistantHost.sendDataContainer = (data) => { 123 | const { requestId } = JSON.parse(data); 124 | setTimeout(() => 125 | commands.map((command, i) => 126 | setTimeout(() => window.AssistantClient.onData({ ...command, sdk_meta: { requestId } }), i), 127 | ), 128 | ); 129 | }; 130 | 131 | const clear = assistant.sendAction( 132 | action, 133 | (data) => { 134 | counter++; 135 | clear(); 136 | expect(data).to.deep.equal(dataCommand.smart_app_data); 137 | if (counter > 1) { 138 | throw new Error('Обработчик вызвал больше одного раза'); 139 | } 140 | done(); 141 | }, 142 | undefined, 143 | { requestId }, 144 | ); 145 | }); 146 | }); 147 | -------------------------------------------------------------------------------- /src/mock.ts: -------------------------------------------------------------------------------- 1 | import { createRecordOfflinePlayer } from './record/offline-player'; 2 | import { 3 | AssistantRecord, 4 | AssistantAppState, 5 | AssistantServerAction, 6 | AssistantClientCommand, 7 | AssistantClientCustomizedCommand, 8 | AssistantSmartAppData, 9 | } from './typings'; 10 | 11 | export interface AssistantActionResult { 12 | action: AssistantServerAction; 13 | name?: string | null; 14 | requestId?: string; 15 | state: AssistantAppState; 16 | } 17 | 18 | export interface CommandParams { 19 | onRequest?: () => void; 20 | waitRequest?: boolean; 21 | } 22 | 23 | // сначала создаем mock, затем вызываем createAssistant 24 | export const createAssistantHostMock = ({ context = window }: { context?: Window } = {}) => { 25 | /* eslint-disable-next-line no-spaced-func, func-call-spacing, @typescript-eslint/no-explicit-any */ 26 | const handlers = new Map void>(); 27 | 28 | let currentResolve: ((value: AssistantActionResult) => void) | null = null; 29 | let onReady: () => void; 30 | 31 | const handleAction = (action: AssistantServerAction, name: string | null, requestId?: string) => { 32 | if (!context.AssistantClient || !context.AssistantClient.onRequestState || !context.AssistantClient.onData) { 33 | throw new Error('Assistant not initialized'); 34 | } 35 | 36 | if (currentResolve) { 37 | const resolve = currentResolve; 38 | currentResolve = null; 39 | resolve({ 40 | state: context.AssistantClient.onRequestState(), 41 | name, 42 | action, 43 | requestId, 44 | }); 45 | return; 46 | } 47 | 48 | if ('action_id' in action) { 49 | const actionType = action.action_id.toLowerCase(); 50 | const handler = handlers.has(actionType) ? handlers.get(actionType) : undefined; 51 | if (handler != null) { 52 | handler(action); 53 | } 54 | } 55 | }; 56 | 57 | context.AssistantHost = { 58 | close: () => { 59 | // ничего не делаем 60 | }, 61 | ready: () => { 62 | window.AssistantClient?.onStart && window.AssistantClient?.onStart(); 63 | onReady && onReady(); 64 | }, 65 | sendData: (action: string, message: string | null) => { 66 | handleAction(JSON.parse(action), message); 67 | }, 68 | sendDataContainer: (container: string) => { 69 | const { data: action, message_name: name, requestId } = JSON.parse(container); 70 | 71 | handleAction(action, name, requestId); 72 | }, 73 | setSuggests: () => { 74 | throw new Error('Not implemented method'); 75 | }, 76 | setHints: () => { 77 | throw new Error('Not implemented method'); 78 | }, 79 | sendText: () => { 80 | throw new Error('Not implemented method'); 81 | }, 82 | }; 83 | 84 | /** Добавить обработчик клиентского экшена */ 85 | const addActionHandler = (actionType: string, handler: (action: T) => void) => { 86 | const type = actionType.toLowerCase(); 87 | if (handlers.has(type)) { 88 | throw new Error('Action-handler already exists'); 89 | } 90 | 91 | handlers.set(type, handler); 92 | }; 93 | 94 | /** Удалить обработчик клиентского экшена */ 95 | const removeActionHandler = (actionType: string) => { 96 | const type = actionType.toLowerCase(); 97 | if (handlers.has(type)) { 98 | handlers.delete(type); 99 | } 100 | }; 101 | 102 | /** Вызвать обработчик команды бека */ 103 | const receiveCommand = (command: AssistantClientCustomizedCommand) => { 104 | if (!context.AssistantClient || !context.AssistantClient.onData) { 105 | throw new Error('Assistant not initialized'); 106 | } 107 | 108 | context.AssistantClient.onData(command as AssistantClientCommand); 109 | return new Promise((resolve) => setTimeout(resolve)); 110 | }; 111 | 112 | /** Дождаться и вернуть клиентский экшен и его контекст */ 113 | const waitAction = (onAction?: () => void) => { 114 | return new Promise((resolve) => { 115 | currentResolve = resolve; 116 | onAction && onAction(); 117 | }); 118 | }; 119 | 120 | return { 121 | addActionHandler, 122 | removeActionHandler, 123 | receiveCommand, 124 | waitAction, 125 | onReady: (cb: () => void) => { 126 | onReady = cb; 127 | }, 128 | }; 129 | }; 130 | 131 | export const createAssistantHostMockWithRecord = ({ 132 | context = window, 133 | record, 134 | }: { 135 | context?: Window & typeof globalThis; 136 | record: AssistantRecord; 137 | }) => { 138 | const mock = createAssistantHostMock({ context }); 139 | const player = createRecordOfflinePlayer(record, { context }); 140 | let hasNext = true; 141 | 142 | const next = ({ onRequest, waitRequest = false }: CommandParams = {}) => { 143 | return new Promise((resolve) => { 144 | hasNext = player.continue((command: AssistantClientCommand) => { 145 | if (!waitRequest && onRequest == null) { 146 | resolve(mock.receiveCommand(command)); 147 | return; 148 | } 149 | 150 | return mock.waitAction(onRequest).then((result) => { 151 | // на будущее - неплохо было бы иметь эталон из записи 152 | mock.receiveCommand(command); 153 | resolve(result); 154 | }); 155 | }); 156 | }); 157 | }; 158 | 159 | return { 160 | get hasNext() { 161 | return hasNext; 162 | }, 163 | onReady: mock.onReady, 164 | next, 165 | receiveCommand: mock.receiveCommand, 166 | }; 167 | }; 168 | -------------------------------------------------------------------------------- /src/debug.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/camelcase */ 2 | /* eslint-disable no-underscore-dangle */ 3 | 4 | import axios from 'axios'; 5 | import { v4 } from 'uuid'; 6 | 7 | import { 8 | DPMessage, 9 | AssistantAppState, 10 | Action, 11 | AssistantServerAction, 12 | AssistantCharacterCommand, 13 | AssistantNavigationCommand, 14 | AssistantSmartAppCommand, 15 | } from './typings'; 16 | 17 | const STATE_UPDATE_TIMEOUT = 200; 18 | 19 | interface AppMessage { 20 | name?: string; 21 | data?: string; 22 | text?: string; 23 | } 24 | 25 | interface CreateMessageProps extends AppMessage { 26 | name?: string; 27 | data?: string; 28 | text?: string; 29 | state: AssistantAppState; 30 | applicationId: string; 31 | appVersionId: string; 32 | sessionId: string; 33 | userId: string; 34 | config: IntitializeProps; 35 | } 36 | 37 | const createMessage = (props: CreateMessageProps): DPMessage => { 38 | const messageName: DPMessage['messageName'] = props.data ? 'SERVER_ACTION' : props.name || 'MESSAGE_TO_SKILL'; 39 | 40 | const systemMessage = props.data 41 | ? { 42 | systemMessage: { 43 | data: JSON.stringify({ 44 | app_info: {}, 45 | server_action: JSON.parse(props.data), 46 | }), 47 | }, 48 | } 49 | : {}; 50 | 51 | const payload = { 52 | payload: { 53 | applicationId: props.applicationId, 54 | appversionId: props.appVersionId, 55 | message: props.text 56 | ? { 57 | original_text: props.text, 58 | } 59 | : {}, 60 | device: props.config.device || { 61 | type: 'SBERBOX', 62 | locale: 'ru-RU', 63 | timezone: '+03:00', 64 | install_id: v4(), 65 | }, 66 | }, 67 | }; 68 | 69 | return { 70 | messageName, 71 | sessionId: props.sessionId, 72 | messageId: Math.floor(Math.random() * Math.floor(9999999)), 73 | meta: { 74 | current_app: JSON.stringify({ 75 | state: props.state, 76 | }), 77 | }, 78 | uuid: { 79 | userId: props.userId, 80 | userChannel: 'FAKE', 81 | sub: 'fake_sub', 82 | }, 83 | ...systemMessage, 84 | ...payload, 85 | }; 86 | }; 87 | 88 | export interface IntitializeProps { 89 | request: { 90 | url: string; 91 | method?: 'get' | 'post' | 'put'; 92 | headers?: {}; 93 | }; 94 | device?: DPMessage['device']; 95 | onRequest: (message: DPMessage) => {}; 96 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 97 | onResponse: (res: any) => Action | AssistantServerAction | undefined; 98 | onError?: (e: Error) => void; 99 | } 100 | 101 | const defaultConfig: IntitializeProps = { 102 | request: { 103 | url: 'sberbank.ru', 104 | }, 105 | onRequest: (props) => props, 106 | onResponse: (res) => res, 107 | onError: () => {}, 108 | }; 109 | 110 | export function initializeDebugging(config: IntitializeProps = defaultConfig) { 111 | const currentAppState: AssistantAppState = {}; 112 | const sessionId = v4(); 113 | const userId = v4(); 114 | const applicationId = v4(); 115 | const appVersionId = v4(); 116 | 117 | const createMessageInSession = ( 118 | props: Omit, 119 | ) => 120 | createMessage({ 121 | config, 122 | userId, 123 | sessionId, 124 | applicationId, 125 | appVersionId, 126 | state: currentAppState, 127 | ...props, 128 | }); 129 | 130 | const ask = (props: AppMessage) => 131 | axios({ 132 | method: config.request?.method || 'post', 133 | url: config.request.url, 134 | headers: config.request?.headers, 135 | data: config.onRequest(createMessageInSession(props)), 136 | }) 137 | .then(config.onResponse) 138 | .then((action: unknown) => { 139 | if (action && window.AssistantClient?.onData) { 140 | window.AssistantClient.onData( 141 | action as AssistantCharacterCommand | AssistantNavigationCommand | AssistantSmartAppCommand, 142 | ); 143 | } 144 | }) 145 | .catch(config.onError); 146 | 147 | window.AssistantHost = { 148 | close() {}, 149 | ready() { 150 | setTimeout(() => { 151 | if (window.AssistantClient?.onStart) window.AssistantClient.onStart(); 152 | }, 0); 153 | }, 154 | 155 | sendData(data, name) { 156 | ask({ 157 | data, 158 | name: name || undefined, 159 | }); 160 | }, 161 | 162 | sendDataContainer(container: string) { 163 | const { data, message_name } = JSON.parse(container); 164 | 165 | ask({ 166 | data, 167 | name: message_name || undefined, 168 | }); 169 | }, 170 | setSuggests() {}, 171 | setHints() {}, 172 | sendText() {}, 173 | }; 174 | 175 | window.__dangerouslyGetAssistantAppState = () => ({ ...currentAppState }); 176 | window.__dangerouslySendVoiceMessage = (text) => { 177 | if (window.AssistantClient?.onRequestState) window.AssistantClient.onRequestState(); 178 | 179 | setTimeout( 180 | () => 181 | ask({ 182 | text, 183 | }), 184 | STATE_UPDATE_TIMEOUT, 185 | ); 186 | }; 187 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 188 | window.__dangerouslySendDataMessage = (data: any, name: string | null = null) => 189 | window.AssistantHost?.sendData(JSON.stringify(data), name); 190 | } 191 | -------------------------------------------------------------------------------- /src/assistantSdk/voice/player/trackStream.ts: -------------------------------------------------------------------------------- 1 | import { createChunkQueue } from './chunkQueue'; 2 | 3 | type BytesArraysSizes = { 4 | incomingMessageVoiceDataLength: number; 5 | sourceLen: number; 6 | start: number; 7 | prepend: number | null; 8 | }; 9 | 10 | const from16BitToFloat32 = (incomingData: Int16Array) => { 11 | const l = incomingData.length; 12 | const outputData = new Float32Array(l); 13 | for (let i = 0; i < l; i += 1) { 14 | outputData[i] = incomingData[i] / 32768.0; 15 | } 16 | return outputData; 17 | }; 18 | 19 | /** Возвращает потоковый подгружаемый трек, который умеет себя проигрывать */ 20 | export const createTrackStream = ( 21 | ctx: AudioContext, 22 | { 23 | sampleRate = 24000, 24 | numberOfChannels = 1, 25 | delay = 0, 26 | onPlay, 27 | onEnd, 28 | trackStatus, 29 | }: { 30 | sampleRate?: number; 31 | numberOfChannels?: number; 32 | // минимальная длина кусочка для воспроизведения, сек 33 | delay?: number; 34 | onPlay?: () => void; 35 | onEnd?: () => void; 36 | trackStatus?: 'stop' | 'play' | 'end'; 37 | }, 38 | ) => { 39 | // очередь загруженных чанков (кусочков) трека 40 | const queue = createChunkQueue(); 41 | let extraByte: number | null = null; 42 | let status: 'stop' | 'play' | 'end' = trackStatus || 'stop'; 43 | 44 | let lastChunkOffset = 0; 45 | let startTime = 0; 46 | let firstChunk = true; 47 | 48 | const end = () => { 49 | // останавливаем воспроизведение чанков из очереди воспроизведения 50 | queue.chunks.forEach((chunk) => { 51 | chunk.stop(); 52 | }); 53 | 54 | status = 'end'; 55 | onEnd && onEnd(); 56 | startTime = 0; 57 | lastChunkOffset = 0; 58 | }; 59 | 60 | const play = () => { 61 | if (status === 'end') { 62 | return; 63 | } 64 | 65 | if (status !== 'play') { 66 | status = 'play'; 67 | onPlay && onPlay(); 68 | } 69 | 70 | if (queue.ended) { 71 | end(); 72 | return; 73 | } 74 | 75 | // воспроизводим трек, если он полностью загрузился или длина загруженного больше задержки 76 | if (queue.loaded || queue.duration >= delay) { 77 | startTime = queue.length === 0 ? ctx.currentTime : startTime; 78 | const chunks = queue.popAll(); 79 | chunks.forEach((chunk) => { 80 | queue.toPlay(chunk); 81 | chunk.start(startTime + lastChunkOffset); 82 | lastChunkOffset += chunk.buffer?.duration || 0; 83 | }); 84 | } 85 | }; 86 | 87 | const getExtraBytes = (data: Uint8Array, bytesArraysSizes: BytesArraysSizes) => { 88 | if (extraByte == null && bytesArraysSizes.incomingMessageVoiceDataLength % 2) { 89 | extraByte = data[bytesArraysSizes.incomingMessageVoiceDataLength - 1]; 90 | bytesArraysSizes.incomingMessageVoiceDataLength -= 1; 91 | bytesArraysSizes.sourceLen -= 1; 92 | } else if (extraByte != null) { 93 | bytesArraysSizes.prepend = extraByte; 94 | bytesArraysSizes.start = 1; 95 | if (bytesArraysSizes.incomingMessageVoiceDataLength % 2) { 96 | bytesArraysSizes.incomingMessageVoiceDataLength += 1; 97 | extraByte = null; 98 | } else { 99 | extraByte = data[bytesArraysSizes.incomingMessageVoiceDataLength - 1]; 100 | bytesArraysSizes.sourceLen -= 1; 101 | } 102 | } 103 | }; 104 | 105 | const createChunk = (chunk: Float32Array) => { 106 | const audioBuffer = ctx.createBuffer(numberOfChannels, chunk.length / numberOfChannels, sampleRate); 107 | for (let i = 0; i < numberOfChannels; i++) { 108 | const channelChunk = new Float32Array(chunk.length / numberOfChannels); 109 | let index = 0; 110 | for (let j = i; j < chunk.length; j += numberOfChannels) { 111 | channelChunk[index++] = chunk[j]; 112 | } 113 | 114 | audioBuffer.getChannelData(i).set(channelChunk); 115 | } 116 | const source = ctx.createBufferSource(); 117 | source.buffer = audioBuffer; 118 | source.connect(ctx.destination); 119 | source.onended = () => { 120 | queue.remove(source); 121 | if (queue.ended) { 122 | status = 'end'; 123 | onEnd && onEnd(); 124 | } 125 | }; 126 | return source; 127 | }; 128 | 129 | /** добавляет чанк в очередь на воспроизведение */ 130 | const write = (data: Uint8Array) => { 131 | // 44 байта - заголовок трека 132 | const slicePoint = firstChunk ? 44 : 0; 133 | const bytesArraysSizes: BytesArraysSizes = { 134 | incomingMessageVoiceDataLength: data.length, 135 | sourceLen: data.length, 136 | start: 0, 137 | prepend: null, 138 | }; 139 | 140 | firstChunk = false; 141 | 142 | if (slicePoint >= data.length) { 143 | return; 144 | } 145 | 146 | getExtraBytes(data, bytesArraysSizes); 147 | 148 | const dataBuffer = new ArrayBuffer(bytesArraysSizes.incomingMessageVoiceDataLength); 149 | 150 | const bufferUi8 = new Uint8Array(dataBuffer); 151 | const bufferI16 = new Int16Array(dataBuffer); 152 | 153 | bufferUi8.set(data.slice(0, bytesArraysSizes.sourceLen), bytesArraysSizes.start); 154 | if (bytesArraysSizes.prepend != null) { 155 | bufferUi8[0] = bytesArraysSizes.prepend; 156 | } 157 | 158 | const chunk = createChunk(from16BitToFloat32(bufferI16.slice(slicePoint))); 159 | queue.push(chunk); 160 | 161 | if (status === 'play') { 162 | play(); 163 | } 164 | }; 165 | 166 | return { 167 | get loaded() { 168 | return queue.loaded; 169 | }, 170 | setLoaded: () => { 171 | queue.allLoaded(); 172 | 173 | if (status === 'play') { 174 | play(); 175 | } 176 | }, 177 | write, 178 | get status() { 179 | return status; 180 | }, 181 | play, 182 | stop: end, 183 | }; 184 | }; 185 | -------------------------------------------------------------------------------- /src/NativePanel/NativePanel.tsx: -------------------------------------------------------------------------------- 1 | /* stylelint-disable */ 2 | import React, { useState, useEffect, useCallback } from 'react'; 3 | import { render } from 'react-dom'; 4 | 5 | import { Action, Suggestions, TextAction } from '../typings'; 6 | 7 | import { Bubble } from './components/Bubble'; 8 | import { CarouselTouch } from './components/CarouselTouch'; 9 | import { KeyboardTouch } from './components/KeyboardTouch'; 10 | import { VoiceTouch } from './components/VoiceTouch'; 11 | import { SphereButton } from './components/SphereButton'; 12 | import { TextInput } from './components/TextInput'; 13 | import { Suggests } from './components/Suggests'; 14 | import { styles } from './styles'; 15 | 16 | export interface NativePanelParams { 17 | defaultText?: string; 18 | render?: (props: NativePanelProps) => void; 19 | tabIndex?: number; 20 | screenshotMode?: boolean; 21 | } 22 | 23 | export interface NativePanelProps extends NativePanelParams { 24 | defaultText?: string; 25 | sendServerAction: (action: Record) => void; 26 | sendText: (text: string) => void; 27 | className?: string; 28 | tabIndex?: number; 29 | suggestions: Suggestions['buttons']; 30 | bubbleText: string; 31 | onListen: () => void; 32 | onSubscribeListenStatus: (cb: (type: 'listen' | 'stopped') => void) => () => void; 33 | onSubscribeHypotesis: (cb: (hypotesis: string, last: boolean) => void) => () => void; 34 | } 35 | 36 | export const NativePanel: React.FC = ({ 37 | defaultText = 'Покажи что-нибудь', 38 | sendServerAction, 39 | sendText, 40 | className, 41 | tabIndex, 42 | suggestions, 43 | bubbleText, 44 | onListen, 45 | onSubscribeListenStatus, 46 | onSubscribeHypotesis, 47 | screenshotMode = false, 48 | }) => { 49 | const [value, setValue] = useState(defaultText); 50 | const [recording, setRecording] = useState(false); 51 | const [bubble, setBubble] = useState(bubbleText); 52 | const [prevBubbleText, setPrevBubbleText] = useState(bubbleText); 53 | const [inputType, setInputType] = useState<'text-input' | 'voice-input'>('voice-input'); 54 | 55 | if (bubbleText !== prevBubbleText) { 56 | setPrevBubbleText(bubbleText); 57 | setBubble(bubbleText); 58 | } 59 | 60 | const handleClearBubbleText = () => { 61 | setBubble(''); 62 | }; 63 | 64 | const handleListen = useCallback(() => { 65 | setValue(''); 66 | onListen(); 67 | }, [onListen]); 68 | 69 | const handleToggleInputType = () => { 70 | // eslint-disable-next-line prettier/prettier 71 | setInputType((type) => (type === 'voice-input' ? 'text-input' : 'voice-input')); 72 | }; 73 | 74 | const handleAction = (action: Action) => { 75 | if (typeof action.text !== 'undefined') { 76 | sendText((action as TextAction).text); 77 | } else if (action.type === 'deep_link') { 78 | window.open(action.deep_link, '_blank'); 79 | } else if (action.type === 'server_action') { 80 | sendServerAction(action.server_action); 81 | } else { 82 | // eslint-disable-next-line no-console 83 | console.error('Unsupported action', action); 84 | } 85 | }; 86 | 87 | const handlerEnter = (event: React.KeyboardEvent) => { 88 | if (event.key === 'Enter') { 89 | sendText(value); 90 | setValue(''); 91 | } 92 | }; 93 | 94 | const createSuggestHandler = (suggest: Suggestions['buttons'][0]) => () => { 95 | const { action, actions } = suggest; 96 | 97 | if (action) { 98 | handleAction(action); 99 | } 100 | 101 | if (actions) { 102 | actions.forEach(handleAction); 103 | } 104 | }; 105 | 106 | useEffect(() => { 107 | const unsubscribeStatus = onSubscribeListenStatus((type: 'listen' | 'stopped') => { 108 | setRecording(type === 'listen'); 109 | }); 110 | 111 | const unsubscribeHypotesis = onSubscribeHypotesis((hypotesis: string, last: boolean) => { 112 | setValue(last ? '' : hypotesis); 113 | }); 114 | 115 | return () => { 116 | unsubscribeStatus(); 117 | unsubscribeHypotesis(); 118 | }; 119 | }, [onSubscribeListenStatus, onSubscribeHypotesis]); 120 | 121 | useEffect(() => { 122 | const style = document?.createElement('style'); 123 | style.appendChild(document?.createTextNode(styles)); 124 | document?.getElementsByTagName('head')[0].appendChild(style); 125 | }, []); 126 | 127 | return ( 128 | // eslint-disable-next-line prettier/prettier 129 |
    134 | 135 | 136 | 137 | 138 |
    139 | 140 |
    141 | 142 |
    143 | 144 | ) => setValue(e.currentTarget.value)} 149 | onKeyDown={handlerEnter} 150 | /> 151 |
    152 | 153 |
    154 | 155 | 156 |
    157 |
    158 | ); 159 | }; 160 | 161 | let div: HTMLDivElement | void; 162 | 163 | export const renderNativePanel = (props: NativePanelProps) => { 164 | if (!div) { 165 | div = document.createElement('div'); 166 | document.body.appendChild(div); 167 | } 168 | 169 | if (props.hideNativePanel) { 170 | render(<>, div); 171 | } else { 172 | render(, div); 173 | } 174 | }; 175 | -------------------------------------------------------------------------------- /src/record/mock-recorder.ts: -------------------------------------------------------------------------------- 1 | import cloneDeep from 'lodash.clonedeep'; 2 | 3 | import { ClientLogger, SystemMessageDataType } from '../typings'; 4 | import { Message, SystemMessage } from '../proto'; 5 | 6 | import { createBaseRecorder, Recorder } from './recorder'; 7 | 8 | export interface MockRecorderRecord { 9 | midToRequestKey: Record; 10 | requestKeyToMessages: Record; 11 | } 12 | 13 | type MockRecorderRecordGetter = () => MockRecorderRecord; 14 | interface MockRecorder extends Recorder { 15 | handler: ClientLogger; 16 | getRecord: MockRecorderRecordGetter; 17 | start: () => void; 18 | stop: () => void; 19 | } 20 | 21 | const getDefaultRecord = (): MockRecorderRecord => ({ 22 | midToRequestKey: {}, 23 | requestKeyToMessages: {}, 24 | }); 25 | 26 | export type MockRecorderCreator = (defaultActive?: boolean) => MockRecorder; 27 | 28 | const getMid = (message: Message) => String(message.messageId); 29 | 30 | const getRequestKey = (message: Message) => { 31 | if (message.messageName === 'OPEN_ASSISTANT') { 32 | return message.messageName; 33 | } 34 | 35 | if (message.text && message.text.data) { 36 | return message.text.data; 37 | } 38 | 39 | if (message.initialSettings) { 40 | return `settings.device=${message.initialSettings.device?.surface}`; 41 | } 42 | 43 | const data = message.systemMessage?.data; 44 | 45 | if (typeof data === 'string') { 46 | const systemMessageData: SystemMessageDataType = JSON.parse(data); 47 | 48 | if (systemMessageData.server_action) { 49 | return JSON.stringify(systemMessageData.server_action); 50 | } 51 | } 52 | 53 | // eslint-disable-next-line no-console 54 | console.error('Не получается определить requestKey для запроса', message); 55 | 56 | return undefined; 57 | }; 58 | 59 | const normalizeMessage = (message: Message) => { 60 | if (!message.systemMessage) { 61 | return message; 62 | } 63 | 64 | if (typeof message.systemMessage.data === 'string') { 65 | message.systemMessage.data = JSON.parse(message.systemMessage.data); 66 | } 67 | 68 | return message; 69 | }; 70 | 71 | const serializeMessage = (message: Message) => { 72 | if (message.systemMessage) { 73 | message.systemMessage = SystemMessage.create({ 74 | data: JSON.stringify(message.systemMessage.data), 75 | }); 76 | } 77 | 78 | return message; 79 | }; 80 | 81 | export const createAnswerFromMockByMessageGetter = (record: MockRecorderRecord) => (message: Message) => { 82 | const requestKey = getRequestKey(message); 83 | 84 | if (!requestKey) { 85 | return undefined; 86 | } 87 | 88 | const messagesFromMock = record.requestKeyToMessages[requestKey]; 89 | 90 | if (!messagesFromMock) { 91 | return undefined; 92 | } 93 | 94 | return messagesFromMock.map((messageFromMock) => serializeMessage(cloneDeep(messageFromMock))); 95 | }; 96 | 97 | export const createMockRecorder: MockRecorderCreator = (defaultActive = true) => { 98 | const { prepareHandler, start, stop, getRecord: baseGetRecord, updateRecord } = createBaseRecorder< 99 | MockRecorderRecord 100 | >(defaultActive, getDefaultRecord); 101 | 102 | const getRecord = (): MockRecorderRecord => { 103 | const baseRecord = baseGetRecord(); 104 | 105 | // сортировка нужна для того чтобы при сохранении мока и добавлении нового 106 | // был красивый дифф и было понятно что именно добавили 107 | const requestKeyToMessages = Object.keys(baseRecord.requestKeyToMessages) 108 | .sort() 109 | .reduce((result, key) => { 110 | const value = baseRecord.requestKeyToMessages[key]; 111 | 112 | result[key] = value; 113 | 114 | return result; 115 | }, {}); 116 | 117 | return { 118 | midToRequestKey: baseRecord.midToRequestKey, 119 | requestKeyToMessages, 120 | }; 121 | }; 122 | 123 | const handler = prepareHandler((entry) => { 124 | switch (entry.type) { 125 | case 'outcoming': { 126 | const message = cloneDeep(entry.message); 127 | const mid = getMid(message); 128 | const requestKey = getRequestKey(message); 129 | 130 | if (!requestKey) { 131 | // eslint-disable-next-line no-console 132 | console.error('Не удалось вычислить requestKey для сообщения:', message); 133 | 134 | return; 135 | } 136 | 137 | // eslint-disable-next-line no-console 138 | console.log(`recorder outcoming message with id: ${mid} to requestKey: ${requestKey}`); 139 | 140 | updateRecord((record) => { 141 | record.midToRequestKey[mid] = requestKey; 142 | record.requestKeyToMessages[requestKey] = []; 143 | }); 144 | 145 | break; 146 | } 147 | 148 | case 'incoming': { 149 | updateRecord((record) => { 150 | // скипаем tts, там тяжеловестные бинарные данные от которых мало толка ;) 151 | if (entry.message.messageName === 'TTS') { 152 | return; 153 | } 154 | 155 | const message: Message = cloneDeep(entry.message); 156 | const mid = getMid(message); 157 | const requestKey = record.midToRequestKey[mid]; 158 | 159 | if (!requestKey) { 160 | // eslint-disable-next-line no-console 161 | console.error(`Не удалось получить requestKey по mid=${mid}`); 162 | 163 | return; 164 | } 165 | 166 | const normalizedMessage = normalizeMessage(message); 167 | 168 | // eslint-disable-next-line no-console 169 | console.log(`recorder incoming message with id: ${mid} and requestKey: ${requestKey}`); 170 | 171 | const messagesFromMock = record.requestKeyToMessages[requestKey]; 172 | 173 | if (messagesFromMock) { 174 | messagesFromMock.push(normalizedMessage); 175 | } else { 176 | record.requestKeyToMessages[requestKey] = [normalizedMessage]; 177 | } 178 | 179 | // eslint-disable-next-line no-console 180 | console.log('message', normalizedMessage); 181 | }); 182 | 183 | break; 184 | } 185 | 186 | default: { 187 | break; 188 | } 189 | } 190 | }); 191 | 192 | return { 193 | start, 194 | stop, 195 | getRecord, 196 | handler, 197 | }; 198 | }; 199 | -------------------------------------------------------------------------------- /src/assistantSdk/client/client.ts: -------------------------------------------------------------------------------- 1 | import { createNanoEvents } from '../../nanoevents'; 2 | import { SystemMessageDataType, OriginalMessageType, MessageNames, AppInfo } from '../../typings'; 3 | 4 | import { BatchableMethods, createProtocol } from './protocol'; 5 | 6 | export interface ClientEvents { 7 | voice: (voice: Uint8Array, original: OriginalMessageType) => void; 8 | status: (status: OriginalMessageType['status'], original: OriginalMessageType) => void; 9 | systemMessage: (systemMessage: SystemMessageDataType, original: OriginalMessageType) => void; 10 | } 11 | 12 | export type SystemMessage = SystemMessageDataType & { 13 | messageId: string; 14 | messageName: OriginalMessageType[]; 15 | }; 16 | 17 | export const createClient = ( 18 | protocol: ReturnType, 19 | provideMeta: (() => Promise>>) | undefined = undefined, 20 | ) => { 21 | const { on, emit } = createNanoEvents(); 22 | 23 | /** ждет ответ бека и возвращает данные из этого ответа */ 24 | const waitForAnswer = (messageId: number | Long): Promise => 25 | new Promise((resolve) => { 26 | const off = on('systemMessage', (systemMessageData, originalMessage) => { 27 | if ( 28 | originalMessage.messageId === messageId && 29 | (originalMessage.messageName === MessageNames.ANSWER_TO_USER || 30 | originalMessage.messageName === MessageNames.DO_NOTHING) 31 | ) { 32 | off(); 33 | resolve(systemMessageData); 34 | } 35 | }); 36 | }); 37 | 38 | /** отправляет произвольный systemMessage, не подкладывает мету */ 39 | const sendData = (data: Record, messageName = ''): number | Long => { 40 | const messageId = protocol.getMessageId(); 41 | 42 | protocol.sendSystemMessage( 43 | { 44 | data, 45 | messageName, 46 | }, 47 | true, 48 | messageId, 49 | ); 50 | 51 | return messageId; 52 | }; 53 | 54 | /** отправляет cancel на сообщение */ 55 | const sendCancel = (messageId: number): void => { 56 | protocol.sendCancel({}, true, messageId); 57 | }; 58 | 59 | /** отправляет приветствие */ 60 | const sendOpenAssistant = async ( 61 | { isFirstSession }: { isFirstSession: boolean } = { isFirstSession: false }, 62 | ): Promise => { 63 | // eslint-disable-next-line @typescript-eslint/camelcase 64 | const data = isFirstSession ? { is_first_session: true } : {}; 65 | const meta = provideMeta ? await provideMeta() : {}; 66 | 67 | return waitForAnswer(sendData({ ...meta, ...data }, 'OPEN_ASSISTANT')); 68 | }; 69 | 70 | /** вызывает sendSystemMessage, куда подкладывает мету */ 71 | const sendMeta = async ( 72 | sendSystemMessage: (data: { data: Record; messageName?: string }, last: boolean) => void, 73 | ) => { 74 | const meta = provideMeta ? await provideMeta() : {}; 75 | 76 | if (Object.keys(meta).length) { 77 | sendSystemMessage( 78 | { 79 | data: meta, 80 | messageName: '', 81 | }, 82 | false, 83 | ); 84 | } 85 | }; 86 | 87 | /** отправляет server_action и мету */ 88 | const sendServerAction = async ( 89 | serverAction: unknown, 90 | appInfo: AppInfo, 91 | messageName = 'SERVER_ACTION', 92 | ): Promise => { 93 | const messageId = protocol.getMessageId(); 94 | 95 | // мету и server_action отправляем в одном systemMessage 96 | await sendMeta(({ data }) => { 97 | protocol.sendSystemMessage( 98 | { 99 | // eslint-disable-next-line @typescript-eslint/camelcase 100 | data: { ...data, app_info: appInfo, server_action: serverAction }, 101 | messageName: messageName || 'SERVER_ACTION', 102 | }, 103 | true, 104 | messageId, 105 | ); 106 | }); 107 | 108 | return messageId; 109 | }; 110 | 111 | /** отправляет текст и текущую мету */ 112 | const sendText = async ( 113 | text: string, 114 | isSsml = false, 115 | shouldSendDisableDubbing?: boolean, 116 | ): Promise => { 117 | if (text.trim() === '') { 118 | return undefined; 119 | } 120 | 121 | return protocol.batch(async ({ sendSystemMessage, sendText: clientSendText, sendSettings, messageId }) => { 122 | await sendMeta(sendSystemMessage); 123 | const prevDubbing = protocol.configuration.settings.dubbing; 124 | const sendDisableDubbing = prevDubbing !== -1 && shouldSendDisableDubbing; 125 | 126 | if (sendDisableDubbing) { 127 | await sendSettings({ dubbing: -1 }, false); 128 | } 129 | 130 | isSsml ? clientSendText(text, {}, 'application/ssml') : clientSendText(text, {}); 131 | 132 | if (sendDisableDubbing) { 133 | sendSettings({ dubbing: prevDubbing }); 134 | } 135 | 136 | return messageId; 137 | }); 138 | }; 139 | 140 | /** инициализирует исходящий голосовой поток, факт. передает в callback параметры для отправки голоса, 141 | * отправляет мету */ 142 | const createVoiceStream = ( 143 | callback: ({ 144 | messageId, 145 | sendVoice, 146 | onMessage, 147 | }: Pick & { 148 | onMessage: (cb: (message: OriginalMessageType) => void) => () => void; 149 | }) => Promise, 150 | ): Promise => 151 | protocol.batch(async ({ sendSystemMessage, sendVoice, messageId }) => { 152 | await callback({ 153 | sendVoice, 154 | messageId, 155 | onMessage: (cb: (message: OriginalMessageType) => void) => protocol.on('incoming', cb), 156 | }); 157 | 158 | sendMeta(sendSystemMessage); 159 | }); 160 | 161 | const off = protocol.on('incoming', (message: OriginalMessageType) => { 162 | if (message.voice) { 163 | emit('voice', message.voice.data || new Uint8Array(), message); 164 | } 165 | 166 | if (message.systemMessage?.data) { 167 | emit('systemMessage', JSON.parse(message.systemMessage.data), message); 168 | } 169 | 170 | if (message.status) { 171 | emit('status', message.status, message); 172 | } 173 | }); 174 | 175 | return { 176 | destroy: () => { 177 | off(); 178 | }, 179 | createVoiceStream, 180 | sendData, 181 | sendMeta, 182 | sendOpenAssistant, 183 | sendServerAction, 184 | sendText, 185 | sendCancel, 186 | on, 187 | waitForAnswer, 188 | }; 189 | }; 190 | -------------------------------------------------------------------------------- /cypress/integration/refreshToken.spec.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import { Server } from 'mock-socket'; 4 | 5 | import { createAssistantClient } from '../../src'; 6 | import { Message } from '../../src/proto'; 7 | import { sendMessage } from '../support/helpers/socket.helpers'; 8 | 9 | describe('Проверяем обновление токена', () => { 10 | const token1 = 'token1'; 11 | const token2 = 'token2'; 12 | let currentToken = token1; 13 | 14 | const configuration = { 15 | settings: {}, 16 | getToken: () => Promise.resolve(currentToken), 17 | url: 'ws://path', 18 | userChannel: '', 19 | userId: '', 20 | version: 5, 21 | }; 22 | 23 | let server: Server; 24 | let assistantClient: ReturnType; 25 | 26 | beforeEach(() => { 27 | server = new Server(configuration.url); 28 | 29 | currentToken = token1; 30 | assistantClient = createAssistantClient(configuration); 31 | assistantClient.on('status', (status) => { 32 | // код ошибки валидации токена = -45 33 | if (status.code === -45) { 34 | currentToken = token2; 35 | assistantClient.reconnect(); 36 | } 37 | }); 38 | }); 39 | 40 | afterEach(() => { 41 | if (server) { 42 | server.stop(); 43 | } 44 | }); 45 | 46 | it('Старт с невалидным токеном', (done) => { 47 | const phrase = 'Проверка токена'; 48 | let phase: 1 | 2 | 3 = 1; 49 | 50 | server.on('connection', (socket) => { 51 | socket.on('message', (data) => { 52 | const message = Message.decode((data as Uint8Array).slice(4)); 53 | if (phase === 1 && message.initialSettings) { 54 | // 1. старт, отвечаем что токен невалиден 55 | expect(message.token, 'первый токен получен').to.deep.equal(token1); 56 | phase = 2; 57 | server.clients()[0].close(); 58 | sendMessage(socket, 1, { statusData: { code: -45 } }, { messageName: 'VPS_CLIENT' }); 59 | } else if (phase === 2 && message.initialSettings) { 60 | // 2. ожидаем обновленный токен 61 | expect(message.token, 'второй токен получен').to.deep.equal(token2); 62 | phase = 3; 63 | } else if (phase === 3 && message.text) { 64 | // 3. после токена должно прийти изначальное сообщение 65 | expect(message.text.data, 'Текст получен').to.deep.equal(phrase); 66 | done(); 67 | } 68 | }); 69 | }); 70 | 71 | assistantClient.sendText(phrase); 72 | }); 73 | 74 | it('Токен становится невалиден после реконнекта', (done) => { 75 | const phrase1 = 'Проверка токена 1'; 76 | const phrase2 = 'Проверка токена 2'; 77 | let phase: 1 | 2 | 3 | 4 | 5 = 1; 78 | 79 | server.on('connection', (socket) => { 80 | socket.on('message', (data) => { 81 | const message = Message.decode((data as Uint8Array).slice(4)); 82 | if (phase === 1 && message.initialSettings) { 83 | // 1. первый старт, считаем токен валидным 84 | expect(message.token, 'первый токен получен - валиден').to.deep.equal(token1); 85 | phase = 2; 86 | } else if (phase === 2 && message.text?.data) { 87 | // 2. должен прийти phrase1, закрываем сокет и отправляем phrase2 88 | expect(message.text.data, 'Текст получен').to.deep.equal(phrase1); 89 | phase = 3; 90 | server.clients()[0].close(); 91 | setTimeout(() => assistantClient.sendText(phrase2), 100); 92 | } else if (phase === 3 && message.initialSettings) { 93 | // 3. считаем первый токен невалидным 94 | expect(message.token, 'первый токен получен - невалиден').to.deep.equal(token1); 95 | phase = 4; 96 | sendMessage(socket, 1, { statusData: { code: -45 } }, { messageName: 'VPS_CLIENT' }); 97 | } else if (phase === 4 && message.initialSettings) { 98 | // 4. ожидаем новый токен 99 | expect(message.token, 'второй токен получен').to.deep.equal(token2); 100 | phase = 5; 101 | } else if (phase === 5 && message.text?.data) { 102 | // 5. ожидаем phrase2 103 | expect(message.text.data, 'Текст получен').to.deep.equal(phrase2); 104 | done(); 105 | } 106 | }); 107 | }); 108 | 109 | assistantClient.sendText(phrase1); 110 | }); 111 | 112 | it('Обновление токена, должно поднимать сокет, если он был закрыт', (done) => { 113 | const phrase = 'Проверка токена'; 114 | let phase: 1 | 2 | 3 = 1; 115 | 116 | server.on('connection', (socket) => { 117 | socket.on('message', (data) => { 118 | const message = Message.decode((data as Uint8Array).slice(4)); 119 | if (phase === 1 && message.initialSettings) { 120 | // 1. первый старт, считаем токен валидным 121 | expect(message.token, 'первый токен получен').to.deep.equal(token1); 122 | phase = 2; 123 | } else if (phase === 2 && message.text?.data) { 124 | // 2. должен прийти phrase, отвечаем, что токен невалиден, закрываем сокет 125 | expect(message.text.data, 'Текст получен').to.deep.equal(phrase); 126 | expect(message.token, 'первый токен получен - невалиден').to.deep.equal(token1); 127 | phase = 3; 128 | server.clients()[0].close(); 129 | sendMessage(socket, 1, { statusData: { code: -45 } }, { messageName: 'VPS_CLIENT' }); 130 | } else if (phase === 3 && message.initialSettings) { 131 | // 3. Ожидаем новый токен 132 | expect(message.token, 'второй токен получен').to.deep.equal(token2); 133 | done(); 134 | } 135 | }); 136 | }); 137 | 138 | assistantClient.sendText(phrase); 139 | }); 140 | 141 | it('getToken возвращает исключение', (done) => { 142 | let phase: 1 | 2 = 1; 143 | assistantClient = createAssistantClient({ 144 | ...configuration, 145 | getToken: () => { 146 | if (phase === 1) { 147 | throw new Error('unknown error'); 148 | } else { 149 | return Promise.resolve(token1); 150 | } 151 | }, 152 | }); 153 | 154 | server.on('connection', (socket) => { 155 | socket.on('message', (data) => { 156 | if (phase === 1) { 157 | assert.fail('Если токен не был разрезолвлен, сообщения не должны приходить'); 158 | } 159 | 160 | const message = Message.decode((data as Uint8Array).slice(4)); 161 | if (phase === 2 && message.initialSettings) { 162 | expect(message.token, ' токен получен').to.deep.equal(token1); 163 | done(); 164 | } 165 | }); 166 | }); 167 | 168 | assistantClient.on('error', (error) => { 169 | expect(error.type, 'Получен тип ошибки').to.equal('GET_TOKEN_ERROR'); 170 | expect(error.message, 'Получен текст ошибки').to.include('unknown error'); 171 | phase = 2; 172 | setTimeout(assistantClient.reconnect, 500); 173 | }); 174 | 175 | assistantClient.start(); 176 | }); 177 | }); 178 | -------------------------------------------------------------------------------- /src/assistantSdk/client/methods.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Settings, 3 | SystemMessage, 4 | Device, 5 | IDevice, 6 | ISettings, 7 | Text, 8 | Voice, 9 | LegacyDevice, 10 | ILegacyDevice, 11 | InitialSettings, 12 | IInitialSettings, 13 | Cancel, 14 | ICancel, 15 | IMessage, 16 | } from '../../proto'; 17 | import { VpsVersion } from '../../typings'; 18 | 19 | export type BatchableMethods = { 20 | sendText: ( 21 | data: string, 22 | params?: { 23 | messageId?: number; 24 | last?: 1 | -1; 25 | messageName?: string; 26 | vpsToken?: string; 27 | userId?: string; 28 | token?: string; 29 | userChannel?: string; 30 | version?: VpsVersion; 31 | meta?: { [k: string]: string }; 32 | }, 33 | type?: string, 34 | messageId?: number, 35 | ) => void; 36 | sendSystemMessage: ( 37 | data: { data: Record; messageName?: string }, 38 | last: boolean, 39 | params?: { 40 | meta?: { [k: string]: string }; 41 | }, 42 | ) => void; 43 | sendVoice: ( 44 | data: Uint8Array, 45 | last: boolean, 46 | messageName?: string, 47 | params?: { 48 | meta?: { [k: string]: string }; 49 | }, 50 | ) => void; 51 | sendSettings: (data: ISettings, last?: boolean, messageId?: number) => void; 52 | messageId: number; 53 | }; 54 | 55 | export const createClientMethods = ({ 56 | getMessageId, 57 | sendMessage, 58 | }: { 59 | getMessageId: () => number; 60 | sendMessage: (message: IMessage) => void; 61 | }) => { 62 | const send = ({ 63 | payload, 64 | messageId, 65 | ...other 66 | }: { 67 | payload: ( 68 | | { settings: Settings } 69 | | { device: Device } 70 | | { systemMessage: SystemMessage } 71 | | { text: Text } 72 | | { voice: Voice } 73 | | { legacyDevice: LegacyDevice } 74 | | { initialSettings: InitialSettings } 75 | | { cancel: Cancel } 76 | ) & { 77 | last: 1 | -1; 78 | messageName?: string; 79 | meta?: { [k: string]: string }; 80 | }; 81 | messageId: number; 82 | }) => { 83 | sendMessage({ 84 | messageName: '', 85 | ...payload, 86 | messageId, 87 | ...other, 88 | }); 89 | }; 90 | 91 | const sendDevice = (data: IDevice, last = true, messageId = getMessageId()) => { 92 | return send({ 93 | payload: { 94 | device: Device.create(data), 95 | last: last ? 1 : -1, 96 | }, 97 | messageId, 98 | }); 99 | }; 100 | 101 | const sendInitialSettings = ( 102 | data: IInitialSettings, 103 | last = true, 104 | messageId = getMessageId(), 105 | params: { meta?: { [k: string]: string } } = {}, 106 | ) => { 107 | return send({ 108 | payload: { 109 | initialSettings: InitialSettings.create(data), 110 | last: last ? 1 : -1, 111 | ...params, 112 | }, 113 | messageId, 114 | }); 115 | }; 116 | 117 | const sendCancel = (data: ICancel, last = true, messageId = getMessageId()) => { 118 | return send({ 119 | payload: { 120 | cancel: Cancel.create(data), 121 | last: last ? 1 : -1, 122 | }, 123 | messageId, 124 | }); 125 | }; 126 | 127 | const sendLegacyDevice = (data: ILegacyDevice, last = true, messageId = getMessageId()) => { 128 | return send({ 129 | payload: { 130 | legacyDevice: LegacyDevice.create(data), 131 | last: last ? 1 : -1, 132 | }, 133 | messageId, 134 | }); 135 | }; 136 | 137 | const sendSettings = (data: ISettings, last = true, messageId = getMessageId()) => { 138 | return send({ 139 | payload: { 140 | settings: Settings.create(data), 141 | last: last ? 1 : -1, 142 | }, 143 | messageId, 144 | }); 145 | }; 146 | 147 | const sendText = ( 148 | data: string, 149 | params: { 150 | messageId?: number; 151 | last?: 1 | -1; 152 | messageName?: string; 153 | vpsToken?: string; 154 | userId?: string; 155 | token?: string; 156 | userChannel?: string; 157 | version?: VpsVersion; 158 | meta?: { [k: string]: string }; 159 | } = {}, 160 | type = '', 161 | messageId = getMessageId(), 162 | ) => { 163 | const text = type ? { data, type } : { data }; 164 | send({ 165 | payload: { 166 | text: Text.create(text), 167 | last: params.last ?? 1, 168 | }, 169 | messageId, 170 | ...params, 171 | }); 172 | }; 173 | 174 | const sendSystemMessage = ( 175 | { data, messageName: mesName = '' }: { data: Record; messageName?: string }, 176 | last = true, 177 | messageId = getMessageId(), 178 | params: { 179 | meta?: { [k: string]: string }; 180 | } = {}, 181 | ) => { 182 | send({ 183 | payload: { 184 | systemMessage: SystemMessage.create({ 185 | data: JSON.stringify(data), 186 | }), 187 | messageName: mesName, 188 | last: last ? 1 : -1, 189 | ...params, 190 | }, 191 | messageId, 192 | }); 193 | }; 194 | 195 | const sendVoice = ( 196 | data: Uint8Array, 197 | last = true, 198 | messageId = getMessageId(), 199 | mesName?: string, 200 | params: { 201 | meta?: { [k: string]: string }; 202 | } = {}, 203 | ) => { 204 | return send({ 205 | payload: { 206 | voice: Voice.create({ 207 | data: new Uint8Array(data), 208 | }), 209 | messageName: mesName, 210 | last: last ? 1 : -1, 211 | ...params, 212 | }, 213 | messageId, 214 | }); 215 | }; 216 | 217 | const batch = (cb: (methods: BatchableMethods) => T): T => { 218 | const batchingMessageId = getMessageId(); 219 | let lastMessageSent = false; 220 | const checkLastMessageStatus = (last?: boolean) => { 221 | if (lastMessageSent) { 222 | if (last) { 223 | throw new Error("Can't send two last items in batch"); 224 | } else { 225 | throw new Error("Can't send messages in batch after last message have been sent"); 226 | } 227 | } else if (last) { 228 | lastMessageSent = true; 229 | } 230 | }; 231 | 232 | const upgradedSendText: typeof sendText = (...[data, params, type]) => { 233 | checkLastMessageStatus(params?.last === 1); 234 | return sendText(data, params, type, batchingMessageId); 235 | }; 236 | 237 | const upgradedSendSystemMessage: ( 238 | data: { data: Record; messageName?: string }, 239 | last: boolean, 240 | params?: { 241 | meta?: { [k: string]: string }; 242 | }, 243 | ) => ReturnType = (data, last, params) => { 244 | checkLastMessageStatus(last); 245 | return sendSystemMessage(data, last, batchingMessageId, params); 246 | }; 247 | 248 | const upgradedSendVoice: ( 249 | data: Uint8Array, 250 | last: boolean, 251 | messageName?: string, 252 | params?: { 253 | meta?: { [k: string]: string }; 254 | }, 255 | ) => ReturnType = (data, last, mesName, params) => { 256 | checkLastMessageStatus(last); 257 | return sendVoice(data, last, batchingMessageId, mesName, params); 258 | }; 259 | 260 | const upgradedSendSettings: ( 261 | data: ISettings, 262 | last?: boolean, 263 | messageId?: number, 264 | ) => ReturnType = (data, last, messageId) => { 265 | checkLastMessageStatus(last); 266 | return sendSettings(data, last, messageId); 267 | }; 268 | 269 | return cb({ 270 | sendText: upgradedSendText, 271 | sendSystemMessage: upgradedSendSystemMessage, 272 | sendVoice: upgradedSendVoice, 273 | sendSettings: upgradedSendSettings, 274 | messageId: batchingMessageId, 275 | }); 276 | }; 277 | 278 | return { 279 | sendDevice, 280 | sendInitialSettings, 281 | sendCancel, 282 | sendLegacyDevice, 283 | sendSettings, 284 | sendText, 285 | sendSystemMessage, 286 | sendVoice, 287 | batch, 288 | }; 289 | }; 290 | --------------------------------------------------------------------------------