├── .github └── workflows │ ├── build.yml │ └── pack.yml ├── .gitignore ├── .pkg-cache └── v3.4 │ └── built-v14.20.0-win-x64 ├── .prettierrc.json ├── .vscode └── launch.json ├── LICENSE ├── README.md ├── assets ├── Ave#0.png ├── Ave#1.png ├── Ave#2.png ├── ave.ico ├── echo.ico ├── echo.png ├── measure.png ├── moon.png ├── snow-rotate.png ├── snow.png └── sun.png ├── ave.config.ts ├── docs └── images │ ├── echo-usage.png │ └── logo.png ├── package-lock.json ├── package.json ├── pkg.config.json ├── rollup.config.js ├── src ├── app.tsx ├── asr │ ├── asr.ts │ ├── base.ts │ ├── index.ts │ └── postasr.ts ├── common │ └── index.ts ├── config │ └── index.ts ├── layout │ └── index.ts ├── nlp │ ├── base.ts │ ├── helsinki-nlp.ts │ └── index.ts ├── resource │ └── index.ts ├── server │ └── index.ts └── shadow │ ├── common.ts │ ├── display.ts │ ├── index.ts │ ├── measure.ts │ └── translate.ts ├── tsconfig.json └── web-ui ├── .gitignore ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── public ├── favicon.ico ├── index.html ├── logo.png ├── manifest.json └── robots.txt ├── src ├── App.css ├── App.test.tsx ├── App.tsx ├── index.css ├── index.tsx ├── lib │ └── index.ts ├── logo.svg ├── pages │ ├── home.module.css │ └── home.tsx ├── react-app-env.d.ts ├── reportWebVitals.ts └── setupTests.ts └── tsconfig.json /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: windows-latest 8 | 9 | strategy: 10 | matrix: 11 | node-version: [14.x] 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Use Node.js ${{ matrix.node-version }} 16 | uses: actions/setup-node@v1 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - name: npm install, build 20 | run: | 21 | npm ci 22 | npm run build 23 | env: 24 | CI: true 25 | -------------------------------------------------------------------------------- /.github/workflows/pack.yml: -------------------------------------------------------------------------------- 1 | name: pack 2 | 3 | on: [push] 4 | 5 | jobs: 6 | pack: 7 | runs-on: windows-latest 8 | 9 | strategy: 10 | matrix: 11 | node-version: [14.x] 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Use Node.js ${{ matrix.node-version }} 16 | uses: actions/setup-node@v1 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - name: npm install and pack app 20 | run: | 21 | npm ci 22 | npm run release 23 | env: 24 | CI: true 25 | - uses: actions/upload-artifact@v3 26 | with: 27 | name: echo 28 | path: "bin" 29 | if-no-files-found: error 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | bin 4 | # .pkg-cache 5 | dist 6 | /*.traineddata 7 | 8 | asr-server 9 | asr-server-* 10 | nlp-server 11 | nlp-server-* 12 | nlp-gpu-server 13 | nlp-gpu-server-* 14 | web-ui.png 15 | web-ui-*.png 16 | /config.json 17 | subtitle 18 | perf 19 | log 20 | *.exe 21 | 22 | echo-web-ui 23 | echo-web-ui-* -------------------------------------------------------------------------------- /.pkg-cache/v3.4/built-v14.20.0-win-x64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/.pkg-cache/v3.4/built-v14.20.0-win-x64 -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "always", 3 | "bracketSpacing": true, 4 | "printWidth": 10000000, 5 | "proseWrap": "never", 6 | "useTabs": true, 7 | "tabWidth": 4 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Launch", 5 | "type": "node", 6 | "request": "launch", 7 | "args": [ 8 | "./src/app.tsx" 9 | ], 10 | "runtimeArgs": [ 11 | "--nolazy", 12 | "-r", 13 | "ts-node/register" 14 | ], 15 | "sourceMaps": true, 16 | "cwd": "${workspaceRoot}", 17 | "protocol": "inspector" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 qber-soft 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 |
6 | 7 | [![build](https://github.com/rerender2021/echo/actions/workflows/build.yml/badge.svg?branch=main&event=push)](https://github.com/rerender2021/echo/actions/workflows/build.yml) [![pack](https://github.com/rerender2021/echo/actions/workflows/pack.yml/badge.svg?branch=main&event=push)](https://github.com/rerender2021/echo/actions/workflows/pack.yml) 8 | 9 |
10 | 11 | # 简介 12 | 13 | 回声 (Echo) 是一个简单的翻译器，原理： 14 | 15 | - 使用语音识别，获得文字用于翻译。目前支持离线情况下，英文翻译成中文。 16 | - GUI 部分则是使用 [Ave React](https://qber-soft.github.io/Ave-React-Docs/) 开发的。 17 | 18 | ![echo-usage](./docs/images/echo-usage.png) 19 | 20 | 演示视频见: 21 | 22 | - v1.0.0: [回声：实时英语语音翻译](https://www.bilibili.com/video/BV11L411d7HE/) 23 | 24 | - v1.1.0: [回声更新：支持使用GPU & 长句分解](https://www.bilibili.com/video/BV1Qa4y1M7jV/) 25 | 26 | - v1.2.0: [回声更新：支持历史字幕 & 自助问题排查](https://www.bilibili.com/video/BV1XN411g7tF/) 27 | 28 | # 使用说明 29 | 30 | - 软件首页：https://rerender2021.github.io/products/echo/ 31 | 32 | # 开发者向 33 | 34 | ## 本地开发 35 | 36 | ```bash 37 | > npm install 38 | > npm run dev 39 | ``` 40 | 41 | 开发过程中需要确保本机启动了语音识别服务器和翻译服务器。 42 | 43 | - 语音识别服务器：[ASR-API 1.1.0](https://github.com/rerender2021/ASR-API/releases/download/1.1.0/asr-server-v1.1.0.zip) 44 | - 翻译服务器：[NLP-API 1.0.1](https://github.com/rerender2021/NLP-API/releases/download/1.0.1/NLP-API-v1.0.1.zip) 45 | 46 | 下载它们，并解压到项目下，确保项目目录结构如下： 47 | 48 | ``` 49 | - nlp-server 50 | - NLP-API.exe 51 | - ... 52 | - asr-server-v1.1.0 53 | - ASR-API.exe 54 | - ... 55 | - src 56 | - ... 57 | - package.json 58 | ``` 59 | 60 | 如需使用GPU： 61 | 62 | - GPU翻译服务器：下载链接中的2个压缩分卷并解压缩（文件太大，只能分卷压缩上传） 63 | - [NLP-GPU-API 1.0.0](https://github.com/rerender2021/NLP-GPU-API/releases/tag/1.0.0) 64 | 65 | 66 | 下载后，解压到项目下，确保项目目录结构如下： 67 | 68 | ``` 69 | - nlp-gpu-server 70 | - NLP-GPU-API.exe 71 | - ... 72 | - asr-server-v1.1.0 73 | - ASR-API.exe 74 | - ... 75 | - src 76 | - ... 77 | - package.json 78 | ``` 79 | 80 | ## 功能扩展 81 | 82 | 运行过程中，语音识别和翻译会请求本地接口，因此，不使用以上离线服务器，而是自己起一个服务器对接在线 API，也可正常使用。 83 | 84 | 相关接口和数据结构约定见代码： 85 | 86 | - 语音识别: [./src/asr/asr.ts](./src/asr/asr.ts) 87 | - 翻译: [./src/nlp/helsinki-nlp.ts](./src/nlp/helsinki-nlp.ts) 88 | 89 | ## 打包发布 90 | 91 | - 生成 exe 92 | 93 | ```bash 94 | > npm run release 95 | ``` 96 | 97 | # 开源协议 98 | 99 | [MIT](./LICENSE) 100 | 101 | # 赞赏 102 | 103 | `:)` 如果此软件值得赞赏，可以请作者看小说，一元足足可看八章呢。 104 | 105 |

106 | 107 |

108 | -------------------------------------------------------------------------------- /assets/Ave#0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/Ave#0.png -------------------------------------------------------------------------------- /assets/Ave#1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/Ave#1.png -------------------------------------------------------------------------------- /assets/Ave#2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/Ave#2.png -------------------------------------------------------------------------------- /assets/ave.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/ave.ico -------------------------------------------------------------------------------- /assets/echo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/echo.ico -------------------------------------------------------------------------------- /assets/echo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/echo.png -------------------------------------------------------------------------------- /assets/measure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/measure.png -------------------------------------------------------------------------------- /assets/moon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/moon.png -------------------------------------------------------------------------------- /assets/snow-rotate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/snow-rotate.png -------------------------------------------------------------------------------- /assets/snow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/snow.png -------------------------------------------------------------------------------- /assets/sun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/assets/sun.png -------------------------------------------------------------------------------- /ave.config.ts: -------------------------------------------------------------------------------- 1 | import { IPackConfig } from "ave-pack"; 2 | 3 | const config: IPackConfig = { 4 | build: { 5 | projectRoot: __dirname, 6 | target: "node14-win-x64", 7 | input: "./dist/_/_/app.js", 8 | output: "./bin/echo.exe", 9 | // set DEBUG_PKG=1 10 | debug: false, 11 | edit: false 12 | }, 13 | resource: { 14 | icon: "./assets/echo.ico", 15 | productVersion: "1.0.0", 16 | productName: "Echo", 17 | fileVersion: "1.0.0", 18 | companyName: "QberSoft", 19 | fileDescription: "A simple asr translator powered by avernakis react.", 20 | LegalCopyright: `© ${new Date().getFullYear()} QberSoft Copyright.`, 21 | }, 22 | }; 23 | 24 | export default config; 25 | -------------------------------------------------------------------------------- /docs/images/echo-usage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/docs/images/echo-usage.png -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/docs/images/logo.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "echo", 3 | "version": "1.0.0", 4 | "description": "A simple asr translator powered by avernakis react.", 5 | "main": "dist/_/_/app.js", 6 | "private": "true", 7 | "scripts": { 8 | "dev": "tsnd --respawn ./src/app.tsx", 9 | "dev:once": "ts-node ./src/app.tsx", 10 | "prebuild": "del-cli ./build && del-cli ./dist", 11 | "build": "tsc", 12 | "postbuild": "npm run copy && npm run bundle", 13 | "copy": "copyfiles -f ./node_modules/ave-ui/lib/* ./dist/lib && copyfiles ./assets/* ./dist", 14 | "bundle": "rollup --config rollup.config.js", 15 | "prerelease": "npm run build", 16 | "release": "ave-pack pack" 17 | }, 18 | "author": "ivjtk; rerender2021", 19 | "license": "MIT", 20 | "devDependencies": { 21 | "@rollup/plugin-commonjs": "^23.0.2", 22 | "@rollup/plugin-json": "^5.0.1", 23 | "@rollup/plugin-node-resolve": "^15.0.1", 24 | "@types/debounce": "^1.2.1", 25 | "@types/express": "^4.17.21", 26 | "@types/node": "^17.0.21", 27 | "@types/react": "^17.0.0", 28 | "ave-pack": "^0.9.4", 29 | "copyfiles": "^2.4.1", 30 | "del-cli": "^4.0.1", 31 | "rollup": "^2.78.0", 32 | "ts-node-dev": "^1.1.8", 33 | "typescript": "^4.6.2" 34 | }, 35 | "dependencies": { 36 | "ave-react": "^0.1.4", 37 | "axios": "^1.3.2", 38 | "debounce": "^1.2.1", 39 | "express": "^4.18.2", 40 | "react": "^17.0.0", 41 | "sentence-splitter": "^4.2.0", 42 | "socket.io": "^4.7.2" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pkg.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "assets": [ 3 | "dist\\assets\\**\\*", 4 | "dist\\lib\\*.node", 5 | "dist\\_\\_\\build\\*" 6 | ], 7 | "scripts": [ 8 | ] 9 | } -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import { nodeResolve } from '@rollup/plugin-node-resolve'; 2 | import commonjs from '@rollup/plugin-commonjs'; 3 | import json from '@rollup/plugin-json'; 4 | 5 | export default [{ 6 | input: 'build/src/app.js', 7 | output: { 8 | file: 'dist/_/_/app.js', 9 | format: 'cjs' 10 | }, 11 | plugins: [json(), nodeResolve(), commonjs({ ignoreDynamicRequires: true })], 12 | }] -------------------------------------------------------------------------------- /src/app.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback, useEffect, useMemo, useState } from "react"; 2 | import { Image, AveRenderer, Grid, Window, getAppContext, IIconResource, IWindowComponentProps, Button, CheckBox, ICheckBoxComponentProps, ScrollBar, Label, IScrollBarComponentProps, Hyperlink } from "ave-react"; 3 | import { App, ThemePredefined_Dark, CheckValue } from "ave-ui"; 4 | import { VoskAsrEngine } from "./asr"; 5 | import { HelsinkiNlpEngine } from "./nlp"; 6 | import { containerLayout, controlLayout } from "./layout"; 7 | import { iconResource } from "./resource"; 8 | import { onMeasure, onTranslate, safe, shadowRelated } from "./shadow"; 9 | import { AsrConfig, getWebUiConfig, NlpConfig } from "./config"; 10 | import axios from "axios"; 11 | import { emitFlushEvent, isInitError, shutdown, startEchoWebUI } from "./server"; 12 | import { assetsPath, runtimeAssetsPath } from "./common"; 13 | 14 | function onInit(app: App) { 15 | const context = getAppContext(); 16 | context.setIconResource(iconResource as unknown as IIconResource); 17 | } 18 | 19 | function initTheme() { 20 | const context = getAppContext(); 21 | const themeImage = context.getThemeImage(); 22 | const themeDark = new ThemePredefined_Dark(); 23 | themeDark.SetStyle(themeImage, 0); 24 | } 25 | 26 | enum ButtonText { 27 | Measure = "设置字幕区", 28 | Recognize = "语音识别", 29 | BreakLongText = "长句分解", 30 | SetTopMost = "字幕置顶", 31 | SubtitleEn = "英文字幕", 32 | SubtitleZh = "中文字幕", 33 | } 34 | 35 | export function Echo() { 36 | const asrEngine = useMemo( 37 | () => 38 | new VoskAsrEngine({ 39 | ...AsrConfig, 40 | }), 41 | [] 42 | ); 43 | const nlpEngine = useMemo( 44 | () => 45 | new HelsinkiNlpEngine({ 46 | ...NlpConfig, 47 | }), 48 | [] 49 | ); 50 | const onClose = useCallback(() => { 51 | asrEngine.destroy(); 52 | nlpEngine.destroy(); 53 | shutdown(); 54 | }, []); 55 | 56 | const onSetTopMost = useCallback((sender) => { 57 | let shouldTopMost = true; 58 | 59 | const checkValue = sender.GetValue(); 60 | if (checkValue === CheckValue.Unchecked) { 61 | shouldTopMost = false; 62 | } else if (checkValue === CheckValue.Checked) { 63 | shouldTopMost = true; 64 | } 65 | 66 | shadowRelated.displayWindow?.SetTopMost(shouldTopMost); 67 | if (!shadowRelated.displayWindow) { 68 | shadowRelated.defaultTopMost = shouldTopMost; 69 | } 70 | }, []); 71 | 72 | const onSetRecognize = useCallback((sender) => { 73 | shadowRelated.subtitleQueue = []; 74 | 75 | let shouldRecognize = false; 76 | 77 | const checkValue = sender.GetValue(); 78 | if (checkValue === CheckValue.Unchecked) { 79 | shouldRecognize = false; 80 | emitFlushEvent(); 81 | shadowRelated.onUpdateTranslationResult({ en: "", zh: "" }); 82 | } else if (checkValue === CheckValue.Checked) { 83 | shouldRecognize = true; 84 | } 85 | 86 | shadowRelated.shouldRecognize = shouldRecognize; 87 | }, []); 88 | 89 | const onSetBreakLongText = useCallback((sender) => { 90 | let value = false; 91 | 92 | const checkValue = sender.GetValue(); 93 | if (checkValue === CheckValue.Unchecked) { 94 | value = false; 95 | } else if (checkValue === CheckValue.Checked) { 96 | value = true; 97 | } 98 | 99 | shadowRelated.shouldBreakLongText = value; 100 | }, []); 101 | 102 | const onSetDisplaySubtitle = useCallback((sender) => { 103 | const checkValue = sender.GetValue(); 104 | const text = sender.GetText(); 105 | const isChecked = checkValue === CheckValue.Checked; 106 | if (text === ButtonText.SubtitleEn) { 107 | shadowRelated.subtitleConfig.en = isChecked; 108 | } else if (text === ButtonText.SubtitleZh) { 109 | shadowRelated.subtitleConfig.zh = isChecked; 110 | } 111 | shadowRelated.onUpdateTranslationConfig(); 112 | }, []); 113 | 114 | const [fontSize, setFontSize] = useState(16); 115 | const onSetFontSize = useCallback((sender) => { 116 | const fontSize = sender.GetValue(); 117 | shadowRelated.onUpdateFontSize(fontSize); 118 | setFontSize(fontSize); 119 | }, []); 120 | 121 | const [title, setTitle] = useState("Echo"); 122 | const [asrReady, setAsrReady] = useState(false); 123 | const [isError, setIsError] = useState(false); 124 | 125 | useEffect(() => { 126 | initTheme(); 127 | asrEngine 128 | .init() 129 | .then( 130 | safe(() => { 131 | setAsrReady(true); 132 | setIsError(isInitError()); 133 | }) 134 | ) 135 | .catch((error) => { 136 | console.error(error?.message); 137 | setIsError(true); 138 | }); 139 | nlpEngine 140 | .init() 141 | .then( 142 | safe(async () => { 143 | const port = NlpConfig.nlpPort; 144 | const response = await axios.get(`http://localhost:${port}/gpu`); 145 | if (response.data.gpu === "True") { 146 | console.log("great! use gpu"); 147 | setTitle("Echo (GPU)"); 148 | } else { 149 | console.log("gpu is not available"); 150 | } 151 | setIsError(isInitError()); 152 | }) 153 | ) 154 | .catch((error) => { 155 | console.error(error?.message); 156 | setIsError(true); 157 | }); 158 | onTranslate(asrEngine, nlpEngine); 159 | }, []); 160 | 161 | const webUiLink = `http://localhost:${getWebUiConfig().port}`; 162 | 163 | const defaultHomeIconPath = assetsPath("snow.png"); 164 | const defaultHomeRotateIconPath = assetsPath("snow-rotate.png"); 165 | const customHomeIconPath = runtimeAssetsPath("./web-ui.png"); 166 | const customHomeRotateIconPath = runtimeAssetsPath("./web-ui-hover.png"); 167 | console.log("icon path", { 168 | customHomeIconPath, 169 | customHomeRotateIconPath 170 | }); 171 | const [imgSrc, setImgSrc] = useState(customHomeIconPath ?? defaultHomeIconPath); 172 | const onEnterImage = () => { 173 | setImgSrc(customHomeRotateIconPath ?? defaultHomeRotateIconPath); 174 | }; 175 | const onLeaveImage = () => { 176 | setImgSrc(customHomeIconPath ?? defaultHomeIconPath); 177 | }; 178 | const gotoWebUi = () => { 179 | // https://stackoverflow.com/a/49013356 180 | const url = webUiLink; 181 | const start = "start"; 182 | require("child_process").exec(start + " " + url); 183 | }; 184 | 185 | return ( 186 | 187 | 188 | 189 | 190 | 191 | 192 | {asrReady && !isError ? ( 193 | <> 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | ) : asrReady && isError ? ( 223 | 224 | `} onClick={gotoWebUi} /> 225 | 226 | ) : ( 227 | 228 | 229 | 230 | )} 231 | 232 | 233 | 234 | ); 235 | } 236 | 237 | AveRenderer.render(); 238 | 239 | startEchoWebUI(); 240 | -------------------------------------------------------------------------------- /src/asr/asr.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import path from "path"; 3 | import fs from "fs"; 4 | import childProcess from "child_process"; 5 | import { IAsrEngine, IAsrEngineOptions, ISentence } from "./base"; 6 | import { emptySentence, shadowRelated } from "../shadow"; 7 | import { postasr } from "./postasr"; 8 | import { inspectLog, ErrorEvent } from "../server"; 9 | 10 | enum AsrVersion { 11 | v100, 12 | v110, 13 | v120, 14 | } 15 | 16 | export class VoskAsrEngine implements IAsrEngine { 17 | private options: IAsrEngineOptions; 18 | private asr: childProcess.ChildProcessWithoutNullStreams; 19 | private version: AsrVersion; 20 | 21 | constructor(options: IAsrEngineOptions) { 22 | this.options = options; 23 | this.version = AsrVersion.v100; 24 | } 25 | 26 | getAsrPath() { 27 | const port = this.options.asrPort; 28 | const voskPort = this.options.asrSocketPort; 29 | 30 | const v120 = path.resolve(process.cwd(), "asr-server-v1.2.0"); 31 | if (fs.existsSync(v120)) { 32 | this.version = AsrVersion.v120; 33 | console.log("use asr-server-v1.2.0"); 34 | return { asrDir: v120, exePath: path.resolve(v120, "./ASR-API.exe"), args: [`--port=${port}`, `--vosk-port=${voskPort}`] }; 35 | } 36 | 37 | const v110 = path.resolve(process.cwd(), "asr-server-v1.1.0"); 38 | if (fs.existsSync(v110)) { 39 | this.version = AsrVersion.v110; 40 | console.log("use asr-server-v1.1.0"); 41 | return { asrDir: v110, exePath: path.resolve(v110, "./ASR-API.exe") }; 42 | } 43 | 44 | const v100 = path.resolve(process.cwd(), "asr-server"); 45 | if (fs.existsSync(v100)) { 46 | console.log("use asr-server-v1.0.0"); 47 | return { asrDir: v100, exePath: path.resolve(v100, "./ASR-API.exe") }; 48 | } 49 | 50 | return { asrDir: "", exePath: "" }; 51 | } 52 | 53 | async init() { 54 | console.log("try to init vosk asr engine"); 55 | const { asrDir, exePath, args = [] } = this.getAsrPath(); 56 | if (asrDir && exePath) { 57 | return new Promise((resolve, reject) => { 58 | console.log("asrDir exists, start asr server", asrDir); 59 | 60 | const asr = childProcess.spawn(exePath, args, { windowsHide: true, detached: false /** hide console */ }); 61 | this.asr = asr; 62 | asr.stdout.on("data", (data) => { 63 | const isError = inspectLog(data?.toString()); 64 | if(isError) { 65 | reject(false); 66 | } 67 | console.log(`stdout: ${data}`); 68 | if (data.includes("has been started")) { 69 | console.log("asr server started"); 70 | resolve(true); 71 | } 72 | }); 73 | 74 | asr.stderr.on("data", (data) => { 75 | const isError = inspectLog(data?.toString()); 76 | if(isError) { 77 | reject(false); 78 | } 79 | console.error(`stderr: ${data}`); 80 | }); 81 | 82 | asr.on("close", (code) => { 83 | console.log(`asr server exit: ${code}`); 84 | reject(false); 85 | }); 86 | }); 87 | } else { 88 | console.log(ErrorEvent.AsrServerNotExist.log); 89 | inspectLog(ErrorEvent.AsrServerNotExist.log); 90 | } 91 | } 92 | 93 | async destroy() { 94 | if (this.asr) { 95 | console.log("exit asr server process"); 96 | this.asr.kill(); 97 | process.kill(this.asr?.pid); 98 | process.exit(); 99 | } 100 | } 101 | 102 | private async asrApi(): Promise { 103 | const port = this.options.asrPort; 104 | 105 | if (this.version === AsrVersion.v100) { 106 | const response = await axios.post(`http://localhost:${port}/asr`, {}, { timeout: 2000 }); 107 | const result = response?.data?.result; 108 | const data = JSON.parse(result || "{}"); 109 | const asrText = data.partial || ""; 110 | return asrText; 111 | } else { 112 | const response = await axios.post(`http://localhost:${port}/asr_queue`, {}, { timeout: 1000 }); 113 | const result = response?.data?.result; 114 | const data = JSON.parse(result[result.length - 1] || "{}"); 115 | const asrText = data.partial || ""; 116 | return asrText; 117 | } 118 | } 119 | 120 | async getAsrResult(): Promise { 121 | let asrResult = ""; 122 | try { 123 | asrResult = await this.asrApi(); 124 | } catch (error) { 125 | console.log(`asr failed: ${error.message}`); 126 | } finally { 127 | return asrResult; 128 | } 129 | } 130 | 131 | async recognize(): Promise { 132 | let sentence: ISentence = emptySentence; 133 | try { 134 | const asrText = await this.getAsrResult(); 135 | 136 | if (shadowRelated.shouldBreakLongText) { 137 | const result = await postasr(asrText); 138 | sentence = { text: result, asr: asrText }; 139 | } else { 140 | sentence = { text: asrText, asr: asrText }; 141 | } 142 | } catch (error) { 143 | console.log(`asr failed: ${error.message}`); 144 | this.options?.onError(error.message); 145 | } finally { 146 | return sentence; 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/asr/base.ts: -------------------------------------------------------------------------------- 1 | export interface ISentence { 2 | text: string; 3 | asr: string 4 | } 5 | 6 | export interface IAsrEngineOptions { 7 | timeout: number 8 | asrPort: number 9 | asrSocketPort: number 10 | onRecognize?: OnRecognize; 11 | onError?: OnError; 12 | } 13 | 14 | export interface IAsrEngineConstructor { 15 | new (options: IAsrEngineOptions): IAsrEngine; 16 | } 17 | 18 | export interface IAsrEngine { 19 | recognize(): Promise; 20 | init(): void; 21 | destroy(): void; 22 | } 23 | 24 | export type OnRecognize = (progress: number) => void; 25 | export type OnError = (message: string) => void; 26 | -------------------------------------------------------------------------------- /src/asr/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./asr"; 2 | export * from "./base"; -------------------------------------------------------------------------------- /src/asr/postasr.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { split } from "sentence-splitter"; 3 | import { AsrConfig } from "../config"; 4 | 5 | class SessionManager { 6 | private prevTextLength: number = Number.MAX_SAFE_INTEGER; 7 | isNewSession(asrText: string) { 8 | const result = asrText.length < this.prevTextLength; 9 | return result; 10 | } 11 | 12 | update(asrText: string) { 13 | this.prevTextLength = asrText.length; 14 | } 15 | } 16 | 17 | const sessionManager = new SessionManager(); 18 | const maxTextLength = 100; 19 | const longTextLength = 60; 20 | let tokenIndex = 0; 21 | 22 | async function getTextToPunct(asrText: string) { 23 | if (asrText.length >= maxTextLength) { 24 | const port = AsrConfig.asrPort; 25 | const punctResponse = await axios.post(`http://localhost:${port}/punct`, { text: asrText }, { timeout: 1000 }); 26 | const withPunct = punctResponse?.data?.text || ""; 27 | return withPunct; 28 | } else { 29 | return asrText; 30 | } 31 | } 32 | 33 | function getSubarray(array: any[], from: number, to: number) { 34 | return array.slice(from, to + 1); 35 | } 36 | 37 | function getSentences(withPunct: string) { 38 | const raw = split(withPunct) 39 | ?.map((each) => each?.raw?.trim()) 40 | .filter((each) => Boolean(each)); 41 | const sentences = []; 42 | raw.forEach((each) => { 43 | if (each.length >= longTextLength) { 44 | sentences.push(...each.split(",").map((each) => each?.trim())); 45 | } else { 46 | sentences.push(each); 47 | } 48 | }); 49 | return sentences; 50 | } 51 | 52 | async function punctText(text: string) { 53 | const withPunct = await getTextToPunct(text); 54 | const sentences = getSentences(withPunct); 55 | 56 | // prettier-ignore 57 | const toIgnore = sentences.length === 1 ? [] : ( 58 | sentences.length >= 3 ? 59 | getSubarray(sentences, 0, sentences.length - 2) : 60 | [sentences[0]] 61 | ); 62 | 63 | let offset = 0; 64 | toIgnore.forEach((each) => { 65 | offset += each.split(" ").length; 66 | }); 67 | 68 | const allToken = text.split(" "); 69 | const theRest = getSubarray(allToken, offset, allToken.length - 1); 70 | const lastUnstable = theRest.join(" "); 71 | return { lastUnstable, offset, sentences }; 72 | } 73 | 74 | export async function postasr(asrText: string) { 75 | try { 76 | if (asrText) { 77 | const isNewSession = sessionManager.isNewSession(asrText); 78 | sessionManager.update(asrText); 79 | if (isNewSession) { 80 | tokenIndex = 0; 81 | } 82 | 83 | let result = asrText; 84 | 85 | if (asrText.length >= maxTextLength) { 86 | if (tokenIndex === 0) { 87 | const { lastUnstable, offset } = await punctText(asrText); 88 | tokenIndex = offset; 89 | result = lastUnstable; 90 | } else { 91 | const allToken = asrText.split(" "); 92 | const theRest = getSubarray(allToken, tokenIndex, allToken.length - 1); 93 | const currentUnstable = theRest.join(" "); 94 | result = currentUnstable; 95 | 96 | // still long text, punct it again 97 | if (currentUnstable.length >= maxTextLength) { 98 | const { lastUnstable, offset } = await punctText(currentUnstable); 99 | tokenIndex += offset; 100 | result = lastUnstable; 101 | } 102 | } 103 | } 104 | return result; 105 | } 106 | } catch (error) { 107 | console.log(`postasr failed: ${error?.message}`); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/common/index.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import fs from "fs"; 3 | 4 | export function assetsPath(name: string) { 5 | const root = path.resolve(__dirname, "../../assets"); 6 | return path.resolve(root, `./${name}`); 7 | } 8 | 9 | export function runtimeAssetsPath(name: string) { 10 | const root = process.cwd(); 11 | const filePath = path.resolve(root, `./${name}`); 12 | return fs.existsSync(filePath) ? filePath : undefined; 13 | } -------------------------------------------------------------------------------- /src/config/index.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import fs from "fs"; 3 | import { IAsrEngineOptions } from "../asr/base"; 4 | import { INlpEngineOptions } from "../nlp/base"; 5 | 6 | const defaultConfig = { 7 | /** timeout for asr and translate api call*/ 8 | timeout: 3500, 9 | asrPort: 8200, 10 | asrSocketPort: 8210, 11 | nlpPort: 8100, 12 | webUiPort: 8350 13 | }; 14 | 15 | export function getConfig() { 16 | const configPath = path.resolve(process.cwd(), "./config.json"); 17 | if (!fs.existsSync(configPath)) { 18 | console.log(`config not exist at ${configPath}, create it!`); 19 | fs.writeFileSync(configPath, JSON.stringify(defaultConfig, null, 4), "utf-8"); 20 | } 21 | 22 | try { 23 | const configJson = JSON.parse(fs.readFileSync(configPath, "utf-8")); 24 | console.log(`parse config succeed, use it`); 25 | return configJson; 26 | } catch (error) { 27 | console.log(`parse config failed, ${error?.message}, use default config`); 28 | return defaultConfig; 29 | } 30 | } 31 | 32 | export function getWebUiConfig() { 33 | const config = getConfig(); 34 | return { 35 | port: config?.webUiPort ?? defaultConfig.webUiPort 36 | } 37 | } 38 | 39 | function getAsrConfig(): IAsrEngineOptions { 40 | const config = getConfig(); 41 | return { 42 | timeout: config?.timeout || defaultConfig.timeout, 43 | asrPort: config?.asrPort || defaultConfig.asrPort, 44 | asrSocketPort: config?.asrSocketPort || defaultConfig.asrSocketPort 45 | }; 46 | } 47 | 48 | export const AsrConfig: IAsrEngineOptions = getAsrConfig(); 49 | 50 | function getNlpConfig(): INlpEngineOptions { 51 | const config = getConfig(); 52 | return { 53 | timeout: config?.timeout || defaultConfig.timeout, 54 | nlpPort: config?.nlpPort || defaultConfig.nlpPort 55 | }; 56 | } 57 | 58 | export const NlpConfig: INlpEngineOptions = getNlpConfig(); -------------------------------------------------------------------------------- /src/layout/index.ts: -------------------------------------------------------------------------------- 1 | export const containerLayout = { 2 | columns: `16dpx 1 16dpx`, 3 | rows: `1`, 4 | areas: { 5 | control: { row: 0, column: 1 }, 6 | }, 7 | }; 8 | 9 | export const controlLayout = { 10 | columns: `1 1 1 1 1`, 11 | rows: `16dpx 32dpx 16dpx 32dpx 16dpx 32dpx 16dpx 32dpx 16dpx 32dpx 16dpx 16dpx 4dpx 1 32dpx 4dpx 64dpx 8dpx`, 12 | areas: { 13 | measure: { row: 1, column: 0, columnSpan: 5 }, 14 | recognize: { row: 3, column: 0, columnSpan: 5 }, 15 | breakLongText: { row: 5, column: 0, columnSpan: 2 }, 16 | topmost: { row: 7, column: 0, columnSpan: 2 }, 17 | zh: { row: 9, column: 0, columnSpan: 2 }, 18 | en: { row: 9, column: 2, columnSpan: 2 }, 19 | fontSizeLabel: { row: 11, column: 0 }, 20 | fontSize: { row: 11, column: 1, columnSpan: 3 }, 21 | fontSizeValue: { row: 11, column: 4 }, 22 | snow: { row: 16, column: 4 }, 23 | }, 24 | }; -------------------------------------------------------------------------------- /src/nlp/base.ts: -------------------------------------------------------------------------------- 1 | export interface ITranslateResult { 2 | text: string; 3 | } 4 | 5 | export interface INlpEngineOptions { 6 | timeout: number 7 | nlpPort: number 8 | onTranslate?: OnTranslate; 9 | onError?: OnError; 10 | } 11 | 12 | export interface INlpEngineConstructor { 13 | new (options: INlpEngineOptions): INlpEngine; 14 | } 15 | 16 | export interface INlpEngine { 17 | translate(text: string): Promise; 18 | init(): void; 19 | destroy(): void; 20 | } 21 | 22 | export type OnTranslate = (text: string) => string; 23 | export type OnError = (message: string) => void; 24 | -------------------------------------------------------------------------------- /src/nlp/helsinki-nlp.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import axios from "axios"; 4 | import childProcess from "child_process"; 5 | import { INlpEngine, INlpEngineOptions, ITranslateResult } from "./base"; 6 | import { ErrorEvent, inspectLog } from "../server"; 7 | 8 | export class HelsinkiNlpEngine implements INlpEngine { 9 | private options: INlpEngineOptions; 10 | private nlp: childProcess.ChildProcessWithoutNullStreams; 11 | private cache: Record; 12 | constructor(options: INlpEngineOptions) { 13 | this.options = options; 14 | this.cache = {}; 15 | } 16 | 17 | getNlpPath() { 18 | const gpu = path.resolve(process.cwd(), "nlp-gpu-server"); 19 | if (fs.existsSync(gpu)) { 20 | console.log("nlp-gpu-server exists! use it"); 21 | return { nlpDir: gpu, exePath: path.resolve(gpu, "./NLP-GPU-API.exe") }; 22 | } 23 | 24 | const cpu = path.resolve(process.cwd(), "nlp-server"); 25 | if (fs.existsSync(cpu)) { 26 | console.log("use nlp-server"); 27 | return { nlpDir: cpu, exePath: path.resolve(cpu, "./NLP-API.exe") }; 28 | } 29 | 30 | return { nlpDir: "", exePath: "" }; 31 | } 32 | 33 | async init() { 34 | console.log("try to init nlp engine"); 35 | const { nlpDir, exePath } = this.getNlpPath(); 36 | if (nlpDir && exePath) { 37 | return new Promise((resolve, reject) => { 38 | console.log("nlpDir exists, start nlp server", nlpDir); 39 | 40 | const port = this.options.nlpPort; 41 | const nlp = childProcess.spawn(exePath, [`--lang-from=en`, `--lang-to=zh`, `--model-dir=.\\model`, `--port=${port}`], { windowsHide: true, detached: false /** hide console */ }); 42 | this.nlp = nlp; 43 | nlp.stdout.on("data", (data) => { 44 | console.log(`stdout: ${data}`); 45 | const isError = inspectLog(data?.toString()); 46 | if(isError) { 47 | reject(false); 48 | } 49 | if (data.includes("has been started")) { 50 | console.log("nlp server started"); 51 | resolve(true); 52 | } 53 | }); 54 | 55 | nlp.stderr.on("data", (data) => { 56 | const isError = inspectLog(data?.toString()); 57 | if(isError) { 58 | reject(false); 59 | } 60 | console.error(`stderr: ${data}`); 61 | }); 62 | 63 | nlp.on("close", (code) => { 64 | console.log(`nlp server exit: ${code}`); 65 | reject(false); 66 | }); 67 | }); 68 | } else { 69 | console.log(ErrorEvent.NlpServerNotExist.log); 70 | inspectLog(ErrorEvent.NlpServerNotExist.log); 71 | } 72 | } 73 | 74 | async destroy() { 75 | if (this.nlp) { 76 | console.log("exit nlp server process"); 77 | this.nlp.kill(); 78 | process.kill(this.nlp?.pid); 79 | process.exit(); 80 | } 81 | } 82 | 83 | async translate(text: string): Promise { 84 | try { 85 | if (this.cache[text]) { 86 | return { text: this.cache[text] }; 87 | } 88 | const timeout = this.options.timeout; 89 | const port = this.options.nlpPort; 90 | const translated = await axios.post( 91 | `http://localhost:${port}/translate`, 92 | { 93 | text, 94 | }, 95 | { timeout } 96 | ); 97 | const result = translated.data.result[0].translation_text; 98 | this.cache[text] = result; 99 | return { text: result }; 100 | } catch (error) { 101 | console.log(`translate failed: ${error.message}`); 102 | return { text: "" }; 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/nlp/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./Helsinki-nlp"; -------------------------------------------------------------------------------- /src/resource/index.ts: -------------------------------------------------------------------------------- 1 | import { assetsPath } from "../common"; 2 | 3 | const iconResource = { 4 | size: [16], 5 | path: { 6 | windowIcon: [assetsPath("echo.png")], 7 | "measure": [assetsPath("measure.png")], 8 | "theme-light": [assetsPath("sun.png")], 9 | "theme-dark": [assetsPath("moon.png")] 10 | }, 11 | } as const; 12 | 13 | export { iconResource }; 14 | 15 | export type IconResourceMapType = Record; 16 | -------------------------------------------------------------------------------- /src/server/index.ts: -------------------------------------------------------------------------------- 1 | import express from "express"; 2 | import path from "path"; 3 | import { getWebUiConfig } from "../config"; 4 | import { Server } from "socket.io"; 5 | import http from "http"; 6 | 7 | const app = express(); 8 | const server = http.createServer(app); 9 | const io = new Server(server, { 10 | cors: { 11 | origin: "*", 12 | }, 13 | }); 14 | 15 | const sockets = new Map(); 16 | 17 | type SubtitleType = { zh: string; en: string }; 18 | const cachedSubtitles: SubtitleType[] = []; 19 | 20 | type ErrorEventType = { log: string; message: string; link?: string }; 21 | const cachedErrorEvent: ErrorEventType[] = []; 22 | const emitedError = new Set(); 23 | const logHistory: string[] = []; 24 | 25 | export function isInitError() { 26 | return emitedError.size !== 0; 27 | } 28 | 29 | export const ErrorEvent = { 30 | NlpServerNotExist: { 31 | log: "[ERROR] nlp server not exist", 32 | message: "没有找到 NLP 服务器, 请检查目录结构。", 33 | link: "https://rerender2021.github.io/products/echo/#%E4%B8%8B%E8%BD%BD%E5%AE%89%E8%A3%85", 34 | }, 35 | AsrServerNotExist: { 36 | log: "[ERROR] asr server not exist", 37 | message: "没有找到语音服务器, 请检查目录结构。", 38 | link: "https://rerender2021.github.io/products/echo/#%E4%B8%8B%E8%BD%BD%E5%AE%89%E8%A3%85", 39 | }, 40 | ChineseInPath: { 41 | log: "[ERROR] chinese found in path", 42 | message: "请检查软件路径是否包含中文, 若包含, 需修改为英文。", 43 | }, 44 | AsrNotWork: { 45 | log: "[ERROR] asr config error", 46 | message: "语音服务器启动失败, 请检查立体声混音相关配置。", 47 | link: "https://rerender2021.github.io/products/echo/#%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98", 48 | }, 49 | PortUsed: { 50 | log: "[ERROR] port used", 51 | message: "端口被占用, 需解除端口占用后再运行。", 52 | link: "https://www.runoob.com/w3cnote/windows-finds-port-usage.html", 53 | }, 54 | }; 55 | 56 | export function inspectLog(log: string) { 57 | // console.log("inspect log", { log }); 58 | logHistory.push(log); 59 | if (log === ErrorEvent.NlpServerNotExist.log) { 60 | emitErorrEvent(ErrorEvent.NlpServerNotExist); 61 | return true; 62 | } else if (log === ErrorEvent.AsrServerNotExist.log) { 63 | emitErorrEvent(ErrorEvent.AsrServerNotExist); 64 | return true; 65 | } else if (log.includes("WinError 1225") || log.includes("character maps to ")) { 66 | emitErorrEvent(ErrorEvent.ChineseInPath); 67 | return true; 68 | } else if (log.includes("websockets.server:connection open")) { 69 | const asrDone = logHistory.find((each) => each.includes("VoskAPI") && each.includes("Done")); 70 | if (!asrDone) { 71 | emitErorrEvent(ErrorEvent.AsrNotWork); 72 | return true; 73 | } 74 | } else if (log.includes("error while attempting to bind on address")) { 75 | const port = log?.split("127.0.0.1', ")?.[1]?.substring(0, 4); 76 | if (port) { 77 | emitErorrEvent({ 78 | log: `${port} ${ErrorEvent.PortUsed.log}`, 79 | message: `${port} ${ErrorEvent.PortUsed.message}`, 80 | link: ErrorEvent.PortUsed.link, 81 | }); 82 | return true; 83 | } 84 | } 85 | 86 | return false; 87 | } 88 | 89 | function emitErorrEvent(event: ErrorEventType) { 90 | if (sockets.size === 0) { 91 | cachedErrorEvent.push(event); 92 | console.log("[EMIT] cache error event", { event }); 93 | } else { 94 | // emit cached 95 | if (cachedErrorEvent.length !== 0) { 96 | emitCachedErrorEvent(); 97 | } 98 | 99 | // emit current 100 | emitEchoError(event); 101 | 102 | // send log history 103 | io.emit("log-history", { logHistory }); 104 | } 105 | } 106 | 107 | function emitEchoError(event: ErrorEventType) { 108 | if (!emitedError.has(event.log)) { 109 | io.emit("echo-error", event); 110 | console.log("[EMIT] emit error event", { event }); 111 | emitedError.add(event.log); 112 | } 113 | } 114 | 115 | function emitCachedErrorEvent() { 116 | console.log("[EMIT] emit cached error event"); 117 | cachedErrorEvent.forEach((event) => { 118 | emitEchoError(event); 119 | }); 120 | cachedErrorEvent.splice(0, cachedErrorEvent.length); 121 | } 122 | 123 | // https://socket.io/get-started/chat#integrating-socketio 124 | export function emitSubtitleEvent(subtitle: SubtitleType) { 125 | if (sockets.size === 0) { 126 | cachedSubtitles.push(subtitle); 127 | } else { 128 | if (cachedSubtitles.length !== 0) { 129 | emitCachedSubtitleEvent(); 130 | } 131 | io.emit("subtitle", subtitle); 132 | } 133 | } 134 | 135 | function emitCachedSubtitleEvent() { 136 | cachedSubtitles.forEach((subtitle) => { 137 | io.emit("subtitle", subtitle); 138 | }); 139 | cachedSubtitles.splice(0, cachedSubtitles.length); 140 | } 141 | 142 | export function emitFlushEvent() { 143 | io.emit("flush"); 144 | } 145 | 146 | export function shutdown() { 147 | server.close(); 148 | io.close(); 149 | process.exit(); 150 | } 151 | 152 | export function startEchoWebUI() { 153 | const root = path.resolve(process.cwd(), "./echo-web-ui-v1.2.0"); 154 | app.use(express.static(root)); 155 | 156 | const { port } = getWebUiConfig(); 157 | 158 | app.get("/", (req, res) => { 159 | res.send("Hello Echo!"); 160 | }); 161 | 162 | io.on("connection", (socket) => { 163 | console.log("a client connected"); 164 | sockets.set(socket, { 165 | connected: true, 166 | }); 167 | 168 | if (cachedErrorEvent.length !== 0) { 169 | emitCachedErrorEvent(); 170 | } 171 | 172 | if (cachedSubtitles.length !== 0) { 173 | emitCachedSubtitleEvent(); 174 | } 175 | 176 | socket.on("disconnect", (reason) => { 177 | sockets.delete(socket); 178 | }); 179 | }); 180 | 181 | process.on('exit', function (){ 182 | shutdown(); 183 | console.log('process end!'); 184 | }); 185 | 186 | try { 187 | server.listen(port, () => { 188 | console.log(`echo web ui server listening on port ${port}`); 189 | }); 190 | 191 | server.on('error', (error) => { 192 | shutdown(); 193 | console.error(error); 194 | }); 195 | 196 | } catch (error) { 197 | console.error(error); 198 | } 199 | 200 | } 201 | -------------------------------------------------------------------------------- /src/shadow/common.ts: -------------------------------------------------------------------------------- 1 | import { getAppContext } from "ave-react"; 2 | import { IGridControl, Vec2, Grid as NativeGrid, Window as NativeWindow } from "ave-ui"; 3 | import { ISentence } from "../asr"; 4 | 5 | export interface ISubtitle { 6 | zh: string; 7 | en: string; 8 | } 9 | 10 | export interface ISubtitleConfig { 11 | zh: boolean; 12 | en: boolean; 13 | } 14 | 15 | export type ShadowRelatedType = { 16 | prevSentence: ISentence; 17 | prevTranslation: string; 18 | shouldTranslate: boolean; 19 | shouldRecognize: boolean; 20 | shouldBreakLongText: boolean; 21 | subtitleQueue: Array; 22 | subtitleConfig: ISubtitleConfig; 23 | measureWindow: NativeWindow; 24 | selected: IGridControl; 25 | start: Vec2; 26 | end: Vec2; 27 | current: Vec2; 28 | selectedArea: { 29 | start: Vec2; 30 | end: Vec2; 31 | }; 32 | displayWindow: NativeWindow; 33 | defaultTopMost: boolean; 34 | selectedAreaIsEmpty(): boolean; 35 | onUpdateTranslationResult: (subtitle: ISubtitle) => void; 36 | onUpdateTranslationConfig: () => void; 37 | onUpdateFontSize: (size: number) => void; 38 | }; 39 | 40 | export const emptySentence: ISentence = { text: "", asr: "" }; 41 | 42 | export const shadowRelated: ShadowRelatedType = { 43 | prevSentence: emptySentence, 44 | prevTranslation: "", 45 | shouldTranslate: false, 46 | shouldRecognize: false, 47 | shouldBreakLongText: false, 48 | subtitleQueue: [], 49 | subtitleConfig: { 50 | en: true, 51 | zh: true, 52 | }, 53 | measureWindow: null, 54 | selected: null, 55 | start: null, 56 | end: null, 57 | current: null, 58 | selectedArea: { 59 | start: new Vec2(0, 0), 60 | end: new Vec2(0, 0), 61 | }, 62 | displayWindow: null, 63 | defaultTopMost: true, 64 | selectedAreaIsEmpty(this: ShadowRelatedType) { 65 | return this.selectedArea.start.x === 0 && this.selectedArea.start.y === 0 && this.selectedArea.end.x === 0 && this.selectedArea.end.x === 0; 66 | }, 67 | onUpdateTranslationResult: () => {}, 68 | onUpdateTranslationConfig: () => {}, 69 | onUpdateFontSize: () => {}, 70 | }; 71 | 72 | globalThis.shadowRelated = shadowRelated; 73 | 74 | export function safe(callback: Function) { 75 | return (...args: any[]) => { 76 | try { 77 | return callback(...args); 78 | } catch (error) { 79 | console.error(error); 80 | } 81 | }; 82 | } 83 | 84 | export function getPrimaryMonitor() { 85 | const context = getAppContext(); 86 | const window = context.getWindow(); 87 | const platform = window.GetPlatform(); 88 | const monitors = platform.MonitorEnumerate(); 89 | const primary = monitors.find((each) => each.Primary); 90 | return primary; 91 | } 92 | 93 | export async function sleep(time: number) { 94 | return new Promise((resolve) => setTimeout(resolve, time)); 95 | } 96 | -------------------------------------------------------------------------------- /src/shadow/display.ts: -------------------------------------------------------------------------------- 1 | import { safe, shadowRelated } from "./common"; 2 | import { WindowFramePart, DpiMargin, RichLabelTextColor, Byo2Font, AlignType, RichLabel as NativeRichLabel, DpiSize, DockMode, Vec2, Vec4, Grid as NativeGrid, Window as NativeWindow, WindowFlag, WindowCreation } from "ave-ui"; 3 | import { emitSubtitleEvent } from "../server"; 4 | 5 | export const onDisplay = safe(async function () { 6 | if (!shadowRelated.displayWindow) { 7 | console.log("display window not initialized, init it"); 8 | const cp = new WindowCreation(); 9 | cp.Flag |= WindowFlag.Layered; 10 | cp.Title = "Display"; 11 | 12 | const width = shadowRelated.selectedArea.end.x - shadowRelated.selectedArea.start.x; 13 | const height = shadowRelated.selectedArea.end.y - shadowRelated.selectedArea.start.y; 14 | 15 | cp.Layout.Size = new Vec2(width || 300, height || 120); 16 | cp.Layout.Position = new Vec2(shadowRelated.selectedArea.start.x, shadowRelated.selectedArea.start.y); 17 | 18 | shadowRelated.displayWindow = new NativeWindow(cp); 19 | } 20 | 21 | if (!shadowRelated.displayWindow.IsWindowCreated()) { 22 | console.log("display window not created, create it"); 23 | 24 | shadowRelated.displayWindow.OnCreateContent( 25 | safe(() => { 26 | console.log("display window create content callback"); 27 | 28 | shadowRelated.displayWindow.SetBackground(false); 29 | shadowRelated.displayWindow.SetTopMost(shadowRelated.defaultTopMost); 30 | 31 | const frame = shadowRelated.displayWindow.GetFrame(); 32 | frame.SetCaptionVisible(false); 33 | frame.OnNcHitTest( 34 | safe((sender, pos, part) => { 35 | if (part == WindowFramePart.Client) return WindowFramePart.Caption; 36 | return part; 37 | }) 38 | ); 39 | 40 | const container = new NativeGrid(shadowRelated.displayWindow); 41 | { 42 | const content = new NativeGrid(shadowRelated.displayWindow); 43 | const color = new Vec4(0, 0, 0, 255); 44 | content.SetBackColor(color); 45 | content.SetOpacity(0.5); 46 | container.ControlAdd(content).SetDock(DockMode.Fill); 47 | 48 | function createSubtitle() { 49 | const fd = shadowRelated.displayWindow.GetTheme().GetFont(); 50 | fd.Size = 16; 51 | const fontDef = new Byo2Font(shadowRelated.displayWindow, fd); 52 | 53 | const textColor = new RichLabelTextColor(); 54 | textColor.Text.Color = new Vec4(255, 255, 255, 255); 55 | 56 | const label = new NativeRichLabel(shadowRelated.displayWindow); 57 | label.FmSetDefaultFont(fontDef); 58 | label.FmSetDefaultTextColor(textColor); 59 | 60 | label.SetAlignHorz(AlignType.Near); 61 | label.SetAlignVert(AlignType.Center); 62 | return label; 63 | } 64 | 65 | // TODO: crash when use "" 66 | const en = createSubtitle(); 67 | const zh = createSubtitle(); 68 | en.SetText(" "); 69 | zh.SetText(" "); 70 | 71 | const subtitle = new NativeGrid(shadowRelated.displayWindow); 72 | subtitle.RowAddSlice(...[1]); 73 | subtitle.RowAddDpx(...[2]); 74 | subtitle.RowAddSlice(...[1]); 75 | subtitle.ColAddSlice(...[1]); 76 | 77 | const margin = new DpiMargin( 78 | DpiSize.FromPixelScaled(50), // margin left 79 | DpiSize.FromPixelScaled(5), // margin top 80 | DpiSize.FromPixelScaled(50), // margin right 81 | DpiSize.FromPixelScaled(5) // margin bottom 82 | ); 83 | const enGrid = subtitle.ControlAdd(en).SetGrid(0, 0).SetMargin(margin); 84 | const zhGrid = subtitle.ControlAdd(zh).SetGrid(0, 2).SetMargin(margin); 85 | container.ControlAdd(subtitle).SetGrid(0, 0); 86 | 87 | shadowRelated.onUpdateFontSize = safe((size: number) => { 88 | const fd = shadowRelated.displayWindow.GetTheme().GetFont(); 89 | fd.Size = size; 90 | const fontDef = new Byo2Font(shadowRelated.displayWindow, fd); 91 | 92 | en.FmSetDefaultFont(fontDef); 93 | zh.FmSetDefaultFont(fontDef); 94 | shadowRelated.displayWindow.Redraw(); 95 | }); 96 | shadowRelated.onUpdateTranslationResult = safe((subtitle: { zh: string; en: string }) => { 97 | emitSubtitleEvent(subtitle); 98 | en.SetText(subtitle.en || " "); 99 | zh.SetText(subtitle.zh || " "); 100 | console.log("update subtitle", { subtitle }); 101 | shadowRelated.displayWindow.Redraw(); 102 | }); 103 | 104 | shadowRelated.onUpdateTranslationConfig = safe(() => { 105 | const config = shadowRelated.subtitleConfig; 106 | if (config.en && !config.zh) { 107 | enGrid.SetGrid(0, 0, 1, 3); 108 | en.SetOpacity(1); 109 | zh.SetOpacity(0); 110 | } else if (!config.en && config.zh) { 111 | zhGrid.SetGrid(0, 0, 1, 3); 112 | en.SetOpacity(0); 113 | zh.SetOpacity(1); 114 | } else if (!config.en && !config.zh) { 115 | en.SetOpacity(0); 116 | enGrid.SetGrid(0, 0); 117 | zh.SetOpacity(0); 118 | zhGrid.SetGrid(0, 2); 119 | } else if (config.en && config.zh) { 120 | en.SetOpacity(1); 121 | enGrid.SetGrid(0, 0); 122 | zh.SetOpacity(1); 123 | zhGrid.SetGrid(0, 2); 124 | } 125 | 126 | shadowRelated.displayWindow.Redraw(); 127 | }); 128 | } 129 | 130 | shadowRelated.displayWindow.SetSize(new Vec2(shadowRelated.selectedArea.end.x - shadowRelated.selectedArea.start.x, shadowRelated.selectedArea.end.y - shadowRelated.selectedArea.start.y)); 131 | shadowRelated.displayWindow.SetPosition(new Vec2(shadowRelated.selectedArea.start.x, shadowRelated.selectedArea.start.y)); 132 | shadowRelated.displayWindow.SetContent(container); 133 | 134 | console.log("display window set content"); 135 | return true; 136 | }) 137 | ); 138 | 139 | const result = shadowRelated.displayWindow.CreateWindow(shadowRelated.measureWindow); 140 | console.log("display window create result", result); 141 | } 142 | 143 | if (shadowRelated.displayWindow.IsWindowCreated()) { 144 | console.log("display window reset pos"); 145 | 146 | shadowRelated.displayWindow.SetSize(new Vec2(shadowRelated.selectedArea.end.x - shadowRelated.selectedArea.start.x, shadowRelated.selectedArea.end.y - shadowRelated.selectedArea.start.y)); 147 | shadowRelated.displayWindow.SetPosition(new Vec2(shadowRelated.selectedArea.start.x, shadowRelated.selectedArea.start.y)); 148 | 149 | console.log("display window created, activate it"); 150 | 151 | shadowRelated.displayWindow.SetVisible(true); 152 | shadowRelated.displayWindow.Activate(); 153 | 154 | console.log("activate display window done"); 155 | } 156 | }); 157 | -------------------------------------------------------------------------------- /src/shadow/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./common"; 2 | export * from "./measure"; 3 | export * from "./translate"; -------------------------------------------------------------------------------- /src/shadow/measure.ts: -------------------------------------------------------------------------------- 1 | import { DpiSize_2, DpiSize, CursorType, DockMode, Vec2, Vec4, Grid as NativeGrid, Window as NativeWindow, WindowFlag, WindowCreation } from "ave-ui"; 2 | import { sleep, getPrimaryMonitor, safe, shadowRelated } from "./common"; 3 | import { onDisplay } from "./display"; 4 | 5 | export const onReset = safe(async function () { 6 | if (!shadowRelated.displayWindow) { 7 | return; 8 | } 9 | 10 | const rect = shadowRelated.displayWindow.GetRect(); 11 | shadowRelated.selectedArea.start = rect.Position.Clone(); 12 | shadowRelated.selectedArea.end = new Vec2(rect.Position.x + rect.Size.x, rect.Position.y + rect.Size.y); 13 | }); 14 | 15 | export const onMeasure = safe(async function () { 16 | if (!shadowRelated.measureWindow) { 17 | console.log("measure window not initialized, init it"); 18 | const cp = new WindowCreation(); 19 | cp.Flag |= WindowFlag.Layered; 20 | cp.Flag |= WindowFlag.Indicator; 21 | cp.Title = "Measure"; 22 | 23 | cp.Layout.Position = new Vec2(0, 0); 24 | const primary = getPrimaryMonitor(); 25 | cp.Layout.Size = primary.AreaFull.Size; 26 | 27 | shadowRelated.measureWindow = new NativeWindow(cp); 28 | } 29 | 30 | if (!shadowRelated.measureWindow.IsWindowCreated()) { 31 | console.log("measure window not created, create it"); 32 | 33 | shadowRelated.measureWindow.OnCreateContent( 34 | safe(() => { 35 | console.log("measure window create content callback"); 36 | 37 | shadowRelated.measureWindow.SetBackground(false); 38 | shadowRelated.measureWindow.OnPointerCursor(() => CursorType.Cross); 39 | 40 | const container = new NativeGrid(shadowRelated.measureWindow); 41 | { 42 | const content = new NativeGrid(shadowRelated.measureWindow); 43 | const color = new Vec4(0, 0, 0, 255); 44 | content.SetBackColor(color); 45 | content.SetOpacity(0.7); 46 | container.ControlAdd(content).SetDock(DockMode.Fill); 47 | 48 | let moveRectGrid: NativeGrid = null; 49 | 50 | container.OnPointerPress( 51 | safe((sender: NativeWindow, mp) => { 52 | if (shadowRelated.selected) { 53 | return; 54 | } 55 | 56 | shadowRelated.start = mp.Position.Clone(); 57 | shadowRelated.end = null; 58 | shadowRelated.current = null; 59 | 60 | moveRectGrid = new NativeGrid(shadowRelated.measureWindow); 61 | moveRectGrid.SetBackColor(new Vec4(255, 255, 255, 255)); 62 | moveRectGrid.SetOpacity(0.5); 63 | 64 | shadowRelated.selected = container.ControlAdd(moveRectGrid); 65 | moveRectGrid.SetVisible(false); 66 | shadowRelated.selected.SetPos(new DpiSize_2(DpiSize.FromPixel(shadowRelated.start.x), DpiSize.FromPixel(shadowRelated.start.y))); 67 | shadowRelated.selected.SetSize(new DpiSize_2(DpiSize.FromPixel(1), DpiSize.FromPixel(1))); 68 | }) 69 | ); 70 | 71 | container.OnPointerRelease( 72 | safe(async (sender: NativeWindow, mp) => { 73 | shadowRelated.end = mp.Position.Clone(); 74 | 75 | // 76 | shadowRelated.selected.SetPos(new DpiSize_2(DpiSize.FromPixel(0), DpiSize.FromPixel(0))); 77 | shadowRelated.selected.SetSize(new DpiSize_2(DpiSize.FromPixel(0), DpiSize.FromPixel(0))); 78 | 79 | moveRectGrid.SetBackColor(new Vec4(255, 0, 0, 255)); 80 | container.ControlRemove(moveRectGrid); 81 | 82 | moveRectGrid = null; 83 | shadowRelated.selected = null; 84 | 85 | shadowRelated.measureWindow.Redraw(); 86 | await sleep(100); 87 | shadowRelated.measureWindow.SetVisible(false); 88 | 89 | shadowRelated.selectedArea.start = new Vec2(Math.min(shadowRelated.start.x, shadowRelated.end.x), Math.min(shadowRelated.start.y, shadowRelated.end.y)); 90 | shadowRelated.selectedArea.end = new Vec2(Math.max(shadowRelated.start.x, shadowRelated.end.x), Math.max(shadowRelated.start.y, shadowRelated.end.y)); 91 | 92 | // 93 | shadowRelated.start = null; 94 | shadowRelated.end = null; 95 | shadowRelated.current = null; 96 | 97 | // 98 | console.log("measure reuslt selected area", shadowRelated.selectedArea); 99 | 100 | // 101 | onDisplay(); 102 | }) 103 | ); 104 | 105 | container.OnPointerMove( 106 | safe((sender: NativeWindow, mp) => { 107 | shadowRelated.current = mp.Position.Clone(); 108 | // console.log("mesaure move", shadowRelated.current.x, shadowRelated.current.y); 109 | 110 | if (moveRectGrid && shadowRelated.start && shadowRelated.current.x !== shadowRelated.start.x && shadowRelated.current.y !== shadowRelated.start.y) { 111 | moveRectGrid.SetVisible(true); 112 | } else if (moveRectGrid) { 113 | moveRectGrid.SetVisible(false); 114 | } 115 | 116 | if (shadowRelated.start && shadowRelated.selected) { 117 | const x = Math.min(shadowRelated.start.x, shadowRelated.current.x); 118 | const y = Math.min(shadowRelated.start.y, shadowRelated.current.y); 119 | const width = Math.abs(shadowRelated.current.x - shadowRelated.start.x); 120 | const height = Math.abs(shadowRelated.current.y - shadowRelated.start.y); 121 | if (width > 1 && height > 1) { 122 | // console.log("draw measure area", x, y, width, height); 123 | shadowRelated.selected.SetPos(new DpiSize_2(DpiSize.FromPixel(x), DpiSize.FromPixel(y))); 124 | shadowRelated.selected.SetSize(new DpiSize_2(DpiSize.FromPixel(width), DpiSize.FromPixel(height))); 125 | } 126 | } 127 | }) 128 | ); 129 | } 130 | 131 | console.log("measure window set content"); 132 | shadowRelated.measureWindow.SetContent(container); 133 | return true; 134 | }) 135 | ); 136 | const result = shadowRelated.measureWindow.CreateWindow(); 137 | console.log("measure window create result", result); 138 | } 139 | 140 | if (shadowRelated.measureWindow.IsWindowCreated()) { 141 | console.log("measure window created, activate it"); 142 | 143 | shadowRelated.displayWindow?.SetVisible(false); 144 | 145 | shadowRelated.measureWindow.SetVisible(true); 146 | shadowRelated.measureWindow.Activate(); 147 | 148 | console.log("activate measure window done"); 149 | } 150 | }); 151 | -------------------------------------------------------------------------------- /src/shadow/translate.ts: -------------------------------------------------------------------------------- 1 | import { sleep, shadowRelated } from "./common"; 2 | import { IAsrEngine } from "../asr/base"; 3 | import { INlpEngine } from "../nlp/base"; 4 | 5 | export const onTranslate = async function (asrEngine: IAsrEngine, nlpEngine: INlpEngine) { 6 | _onRecognize(asrEngine); 7 | _onTranslate(nlpEngine); 8 | _onUpdateSubtitle(); 9 | }; 10 | 11 | let prevLength = Number.MAX_SAFE_INTEGER; 12 | let lastUpdateTime = Date.now(); 13 | const subtitleDelay = 1000; 14 | 15 | const _onUpdateSubtitle = async function () { 16 | try { 17 | if (!shadowRelated.shouldRecognize) { 18 | shadowRelated.subtitleQueue = []; 19 | return; 20 | } 21 | const current = shadowRelated.subtitleQueue.shift(); 22 | if (current) { 23 | const now = Date.now(); 24 | if (current.en.length < prevLength) { 25 | // length change, a new subtitle found! 26 | const dt = now - lastUpdateTime; 27 | if (dt <= subtitleDelay) { 28 | await sleep(Math.abs(subtitleDelay - dt)); 29 | } 30 | } 31 | shadowRelated.onUpdateTranslationResult(current); 32 | prevLength = current.en.length || 0; 33 | lastUpdateTime = Date.now(); 34 | } 35 | } catch (error) { 36 | console.error("recognize failed", error); 37 | } finally { 38 | setTimeout(() => { 39 | _onUpdateSubtitle(); 40 | }, 0); 41 | } 42 | }; 43 | 44 | const _onRecognize = async function (asrEngine: IAsrEngine) { 45 | try { 46 | if (!shadowRelated.shouldRecognize) { 47 | return; 48 | } 49 | const asrStart = Date.now(); 50 | const sentence = await asrEngine.recognize(); 51 | const asrEnd = Date.now(); 52 | 53 | if (sentence.text && sentence.text !== shadowRelated.prevSentence.text) { 54 | console.log(`asr end in ${asrEnd - asrStart}ms`); 55 | shadowRelated.prevSentence = sentence; 56 | shadowRelated.shouldTranslate = true; 57 | } 58 | } catch (error) { 59 | console.error("recognize failed", error); 60 | } finally { 61 | setTimeout(() => { 62 | _onRecognize(asrEngine); 63 | }, 100); 64 | } 65 | }; 66 | const _onTranslate = async function (nlpEngine: INlpEngine) { 67 | try { 68 | if (!shadowRelated.shouldRecognize) { 69 | return; 70 | } 71 | 72 | if (!shadowRelated.subtitleConfig.zh) { 73 | const { text, ...rest } = shadowRelated.prevSentence; 74 | shadowRelated.subtitleQueue.push({ zh: "", en: text, ...rest }); 75 | return; 76 | } 77 | 78 | if (shadowRelated.shouldTranslate) { 79 | shadowRelated.shouldTranslate = false; 80 | console.log("will translate"); 81 | const translateStart = Date.now(); 82 | const { text: enText, ...rest } = shadowRelated.prevSentence; 83 | const input = enText; 84 | const { text } = await nlpEngine.translate(input); 85 | shadowRelated.prevTranslation = text; 86 | const translateEnd = Date.now(); 87 | console.log(`translate end in ${translateEnd - translateStart}ms`, { enText: input, text }); 88 | shadowRelated.subtitleQueue.push({ zh: shadowRelated.prevTranslation, en: input, ...rest }); 89 | } 90 | } catch (error) { 91 | console.error("translate failed", error); 92 | } finally { 93 | setTimeout(() => { 94 | _onTranslate(nlpEngine); 95 | }, 500); 96 | } 97 | }; 98 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "moduleResolution": "node", 6 | "outDir": "build/src", 7 | "jsx": "react", 8 | "esModuleInterop" : true, 9 | }, 10 | "include": [ 11 | "./src" 12 | ] 13 | } -------------------------------------------------------------------------------- /web-ui/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /web-ui/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 rerender2021 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /web-ui/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/web-ui/README.md -------------------------------------------------------------------------------- /web-ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "echo-web-ui", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@monaco-editor/react": "^4.6.0", 7 | "@testing-library/jest-dom": "^5.17.0", 8 | "@testing-library/react": "^13.4.0", 9 | "@testing-library/user-event": "^13.5.0", 10 | "@types/jest": "^27.5.2", 11 | "@types/node": "^16.18.61", 12 | "@types/react": "^18.2.37", 13 | "@types/react-dom": "^18.2.15", 14 | "antd": "^5.1.0", 15 | "axios": "^1.4.0", 16 | "monaco-editor": "^0.44.0", 17 | "react": "^18.2.0", 18 | "react-dom": "^18.2.0", 19 | "react-scripts": "5.0.1", 20 | "socket.io-client": "^4.7.2", 21 | "typescript": "^4.9.5", 22 | "web-vitals": "^2.1.4" 23 | }, 24 | "scripts": { 25 | "start": "react-scripts start", 26 | "build": "cross-env GENERATE_SOURCEMAP=false react-scripts build", 27 | "test": "react-scripts test", 28 | "eject": "react-scripts eject" 29 | }, 30 | "eslintConfig": { 31 | "extends": [ 32 | "react-app", 33 | "react-app/jest" 34 | ] 35 | }, 36 | "browserslist": { 37 | "production": [ 38 | ">0.2%", 39 | "not dead", 40 | "not op_mini all" 41 | ], 42 | "development": [ 43 | "last 1 chrome version", 44 | "last 1 firefox version", 45 | "last 1 safari version" 46 | ] 47 | }, 48 | "devDependencies": { 49 | "cross-env": "^7.0.3" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /web-ui/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/web-ui/public/favicon.ico -------------------------------------------------------------------------------- /web-ui/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | Echo 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /web-ui/public/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/web-ui/public/logo.png -------------------------------------------------------------------------------- /web-ui/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Echo", 3 | "name": "Echo", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | } 10 | ], 11 | "start_url": ".", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /web-ui/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /web-ui/src/App.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rerender2021/echo/349597a46017c524bc7217b7ed17e78527593b14/web-ui/src/App.css -------------------------------------------------------------------------------- /web-ui/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render, screen } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | render(); 7 | const linkElement = screen.getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /web-ui/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import './App.css'; 3 | import Home from './pages/home'; 4 | 5 | function App() { 6 | return ( 7 | 8 | ); 9 | } 10 | 11 | export default App; 12 | -------------------------------------------------------------------------------- /web-ui/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /web-ui/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import './index.css'; 4 | import App from './App'; 5 | // import reportWebVitals from './reportWebVitals'; 6 | 7 | const root = ReactDOM.createRoot( 8 | document.getElementById('root') as HTMLElement 9 | ); 10 | root.render( 11 | // 12 | 13 | // 14 | ); 15 | 16 | // If you want to start measuring performance in your app, pass a function 17 | // to log results (for example: reportWebVitals(console.log)) 18 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 19 | // reportWebVitals(); 20 | -------------------------------------------------------------------------------- /web-ui/src/lib/index.ts: -------------------------------------------------------------------------------- 1 | import { io } from "socket.io-client"; 2 | 3 | // https://socket.io/how-to/use-with-react 4 | export const socket = io("http://localhost:8350", { 5 | autoConnect: true, 6 | }); 7 | -------------------------------------------------------------------------------- /web-ui/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /web-ui/src/pages/home.module.css: -------------------------------------------------------------------------------- 1 | .container { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | margin-bottom: 20px; 6 | } 7 | 8 | .header { 9 | width: 100%; 10 | height: 64px; 11 | box-shadow: inset 0 -1px 0 #E3E5E7; 12 | display: flex; 13 | align-items: center; 14 | } 15 | 16 | .errors { 17 | width: 80%; 18 | margin-top: 20px; 19 | } 20 | 21 | .logo { 22 | height: 48px; 23 | margin-left: 6px; 24 | } 25 | 26 | .subtitleList { 27 | margin-top: 20px; 28 | width: 80%; 29 | height: 600px; 30 | } -------------------------------------------------------------------------------- /web-ui/src/pages/home.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef, useState } from "react"; 2 | import { socket } from "../lib"; 3 | import style from "./home.module.css"; 4 | import * as monaco from "monaco-editor"; 5 | import { loader, Editor } from "@monaco-editor/react"; 6 | import { Alert, Button, Card, Space } from "antd"; 7 | 8 | loader.config({ monaco }); 9 | 10 | type ErrorEventType = { log: string; message: string; link?: string }; 11 | 12 | export default function Home() { 13 | const refEditor = useRef(null); 14 | const refSubtitleList = useRef([]); 15 | const refErrors = useRef([]); 16 | const refIsConnectError = useRef(false); 17 | const refLogHistory = useRef([]); 18 | const [updateKey, setUpdateKey] = useState(Date.now()); 19 | 20 | useEffect(() => { 21 | const editor = refEditor.current as any; 22 | if(refErrors.current.length === 0) { 23 | editor?.revealLine(editor.getModel().getLineCount() + 10); 24 | } 25 | }, [updateKey]); 26 | 27 | useEffect(() => { 28 | socket.on("connect", () => { 29 | console.log("connect"); 30 | refIsConnectError.current = false; 31 | setUpdateKey(Date.now()); 32 | }); 33 | 34 | socket.on("connect_error", (error) => { 35 | console.error("connect error", { error }); 36 | refIsConnectError.current = true; 37 | refErrors.current = []; 38 | setUpdateKey(Date.now()); 39 | }); 40 | 41 | const update = (text: string) => { 42 | if ( 43 | refSubtitleList.current[refSubtitleList.current.length - 1] !== text 44 | ) { 45 | const newSubtitleList = [...refSubtitleList.current, text]; 46 | refSubtitleList.current = newSubtitleList; 47 | setUpdateKey(Date.now()); 48 | } 49 | }; 50 | 51 | let count = Number.MAX_SAFE_INTEGER; 52 | let prevSubtitle = { zh: "", en: "" }; 53 | let timer: any; 54 | socket.on("subtitle", (value: { zh: string; en: string }) => { 55 | const newCount = value?.en?.length ?? 0; 56 | const isDecreasing = newCount < count; 57 | console.log("check", { isDecreasing, newCount, count }); 58 | if (isDecreasing && prevSubtitle.en !== value.en) { 59 | if (prevSubtitle?.en || prevSubtitle?.zh) { 60 | console.log("new subtitle", { value }); 61 | update(`${prevSubtitle?.en}\n${prevSubtitle?.zh}`); 62 | clearTimeout(timer); 63 | } 64 | } 65 | count = newCount; 66 | prevSubtitle = value; 67 | }); 68 | 69 | socket.on("flush", () => { 70 | update(`${prevSubtitle?.en}\n${prevSubtitle?.zh}`); 71 | }); 72 | 73 | socket.on("echo-error", (value: ErrorEventType) => { 74 | console.error("echo error", { value }); 75 | refErrors.current = [...refErrors.current, value]; 76 | setUpdateKey(Date.now()); 77 | }); 78 | 79 | socket.on("log-history", (value: { logHistory: string[] }) => { 80 | console.error("log history", { value }); 81 | refLogHistory.current = [...value.logHistory]; 82 | setUpdateKey(Date.now()); 83 | }); 84 | 85 | }, []); 86 | 87 | function handleEditorDidMount(editor: any) { 88 | refEditor.current = editor; 89 | } 90 | 91 | const editorValue = refErrors.current.length === 0 ? refSubtitleList.current.join("\n\n"): refLogHistory.current.join("\n\n"); 92 | 93 | return ( 94 |
95 |
96 | 97 | logo 98 | 99 |
100 |
101 | 106 | {refIsConnectError.current && ( 107 | 112 | )} 113 | {refErrors.current.map((each) => { 114 | return ( 115 | 119 | {each.message} 120 | {each?.link && ( 121 | 124 | )} 125 | 126 | } 127 | type="error" 128 | showIcon 129 | /> 130 | ); 131 | })} 132 | 133 |
134 | 135 | 144 | 145 |
146 | ); 147 | } 148 | -------------------------------------------------------------------------------- /web-ui/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /web-ui/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | }; 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /web-ui/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom'; 6 | -------------------------------------------------------------------------------- /web-ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "moduleResolution": "node", 18 | "resolveJsonModule": true, 19 | "isolatedModules": true, 20 | "noEmit": true, 21 | "jsx": "react-jsx" 22 | }, 23 | "include": [ 24 | "src" 25 | ] 26 | } 27 | --------------------------------------------------------------------------------