├── .eslintignore ├── examples ├── carplay-node-without-audio │ ├── .gitignore │ ├── client │ │ ├── index.tsx │ │ ├── config.ts │ │ └── App.tsx │ ├── public │ │ └── index.html │ ├── tsconfig.json │ ├── README.md │ ├── package.json │ └── server │ │ └── index.ts └── carplay-web-app │ ├── src │ ├── react-app-env.d.ts │ ├── setupProxy.js │ ├── setupTests.ts │ ├── App.test.tsx │ ├── worker │ │ ├── utils.ts │ │ ├── render │ │ │ ├── RenderEvents.ts │ │ │ ├── lib │ │ │ │ ├── utils.ts │ │ │ │ └── h264-utils.ts │ │ │ ├── Render.worker.ts │ │ │ ├── WebGLRenderer.ts │ │ │ ├── WebGL2Renderer.ts │ │ │ └── WebGPURenderer.ts │ │ ├── types.ts │ │ └── CarPlay.worker.ts │ ├── index.css │ ├── reportWebVitals.ts │ ├── index.tsx │ ├── App.css │ ├── useCarplayTouch.ts │ ├── logo.svg │ ├── useCarplayAudio.ts │ └── App.tsx │ ├── public │ ├── robots.txt │ ├── favicon.ico │ ├── logo192.png │ ├── logo512.png │ ├── manifest.json │ └── index.html │ ├── .gitignore │ ├── README.md │ ├── config-overrides.js │ ├── tsconfig.json │ ├── types │ └── ringbuf.js │ │ └── index.d.ts │ └── package.json ├── .prettierrc ├── src ├── modules │ ├── messages │ │ ├── index.ts │ │ ├── utils.ts │ │ ├── __tests__ │ │ │ ├── sendable.test.ts │ │ │ ├── common.test.ts │ │ │ └── readable.test.ts │ │ ├── common.ts │ │ ├── readable.ts │ │ └── sendable.ts │ ├── index.ts │ ├── __tests__ │ │ ├── mocks │ │ │ └── usbMocks.ts │ │ └── DongleDriver.test.ts │ └── DongleDriver.ts ├── web │ ├── tsconfig.json │ ├── index.ts │ ├── WebMicrophone.ts │ ├── recorder.worklet.ts │ └── CarplayWeb.ts └── node │ ├── index.ts │ ├── NodeMicrophone.ts │ └── CarplayNode.ts ├── tsconfig.build.json ├── .editorconfig ├── jest.config.js ├── tsconfig.json ├── .eslintrc.json ├── scripts ├── startNode.ts └── configIcon.ts ├── LICENSE ├── .github └── workflows │ └── build_and_test.yml ├── package.json ├── .gitignore └── README.md /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/.gitignore: -------------------------------------------------------------------------------- 1 | public/bundle.js 2 | server/index.js 3 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "arrowParens": "avoid" 5 | } -------------------------------------------------------------------------------- /examples/carplay-web-app/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /src/modules/messages/index.ts: -------------------------------------------------------------------------------- 1 | export * from './readable.js' 2 | export * from './common.js' 3 | export * from './sendable.js' 4 | -------------------------------------------------------------------------------- /examples/carplay-web-app/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rhysmorgan134/node-CarPlay/HEAD/examples/carplay-web-app/public/favicon.ico -------------------------------------------------------------------------------- /examples/carplay-web-app/public/logo192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rhysmorgan134/node-CarPlay/HEAD/examples/carplay-web-app/public/logo192.png -------------------------------------------------------------------------------- /examples/carplay-web-app/public/logo512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rhysmorgan134/node-CarPlay/HEAD/examples/carplay-web-app/public/logo512.png -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "src/web/*", "src/**/*.test.ts", "src/**/mocks"] 4 | } 5 | -------------------------------------------------------------------------------- /src/modules/index.ts: -------------------------------------------------------------------------------- 1 | export * from './messages/index.js' 2 | export { 3 | HandDriveType, 4 | DongleConfig, 5 | DEFAULT_CONFIG, 6 | DongleDriver, 7 | } from './DongleDriver.js' 8 | -------------------------------------------------------------------------------- /src/web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "lib": ["es2023", "DOM", "WebWorker"] 5 | }, 6 | "include": ["*"], 7 | "exclude": [] 8 | } 9 | -------------------------------------------------------------------------------- /src/node/index.ts: -------------------------------------------------------------------------------- 1 | import CarplayNode from './CarplayNode.js' 2 | 3 | export * from '../modules/index.js' 4 | export { default as NodeMicrophone } from './NodeMicrophone.js' 5 | export default CarplayNode 6 | -------------------------------------------------------------------------------- /src/web/index.ts: -------------------------------------------------------------------------------- 1 | import CarplayWeb from './CarplayWeb.js' 2 | 3 | export * from '../modules/index.js' 4 | export { default as WebMicrophone } from './WebMicrophone.js' 5 | export * from './CarplayWeb.js' 6 | export default CarplayWeb 7 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/client/index.tsx: -------------------------------------------------------------------------------- 1 | import ReactDOM from 'react-dom/client' 2 | import App from './App.js' 3 | 4 | const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement) 5 | root.render() 6 | -------------------------------------------------------------------------------- /src/modules/messages/utils.ts: -------------------------------------------------------------------------------- 1 | export const clamp = (number: number, min: number, max: number) => { 2 | return Math.max(min, Math.min(number, max)) 3 | } 4 | 5 | export function getCurrentTimeInMs() { 6 | return Math.round(Date.now() / 1000) 7 | } 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | indent_size = 2 6 | indent_style = space 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | charset = utf-8 10 | 11 | [*.md] 12 | indent_size = 2 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/setupProxy.js: -------------------------------------------------------------------------------- 1 | module.exports = function (app) { 2 | app.use(function (req, res, next) { 3 | res.setHeader('Cross-Origin-Opener-Policy', 'same-origin') 4 | res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp') 5 | next() 6 | }) 7 | } 8 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/setupTests.ts: -------------------------------------------------------------------------------- 1 | // jest-dom adds custom jest matchers for asserting on DOM nodes. 2 | // allows you to do things like: 3 | // expect(element).toHaveTextContent(/react/i) 4 | // learn more: https://github.com/testing-library/jest-dom 5 | import '@testing-library/jest-dom' 6 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/App.test.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { render, screen } from '@testing-library/react' 3 | import App from './App' 4 | 5 | test('renders learn react link', () => { 6 | render() 7 | const linkElement = screen.getByText(/learn react/i) 8 | expect(linkElement).toBeInTheDocument() 9 | }) 10 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | export default { 3 | preset: 'ts-jest/presets/default-esm', // or other ESM presets 4 | moduleNameMapper: { 5 | '^(\\.{1,2}/.*)\\.js$': '$1', 6 | }, 7 | testEnvironment: 'node', 8 | modulePathIgnorePatterns: ['/examples/', 'mocks', 'dist'], 9 | } 10 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/utils.ts: -------------------------------------------------------------------------------- 1 | import { decodeTypeMap } from 'node-carplay/web' 2 | import { AudioPlayerKey } from './types' 3 | 4 | export const createAudioPlayerKey = (decodeType: number, audioType: number) => { 5 | const format = decodeTypeMap[decodeType] 6 | const audioKey = [format.frequency, format.channel, audioType].join('_') 7 | return audioKey as AudioPlayerKey 8 | } 9 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/client/config.ts: -------------------------------------------------------------------------------- 1 | import type { DongleConfig } from 'node-carplay/node' 2 | 3 | export const config: Partial & { 4 | width: DongleConfig['width'] 5 | height: DongleConfig['height'] 6 | } = { 7 | audioTransferMode: true, 8 | 9 | // NOTE: Change the following for your usecase 10 | fps: 60, 11 | width: 1024, 12 | height: 600, 13 | } 14 | -------------------------------------------------------------------------------- /examples/carplay-web-app/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "dist", 4 | "target": "es2022", 5 | "lib": ["ES2023"], 6 | "module": "ES2022", 7 | "strict": true, 8 | "esModuleInterop": true, 9 | "skipLibCheck": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "moduleResolution": "Bundler", 12 | "sourceMap": true, 13 | "declaration": true 14 | }, 15 | "exclude": ["node_modules", "src/web/*"], 16 | "include": ["src/**/*"] 17 | } 18 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "parserOptions": { 4 | "ecmaVersion": 12, 5 | "sourceType": "module" 6 | }, 7 | "plugins": ["@typescript-eslint", "import"], 8 | "extends": [ 9 | "eslint:recommended", 10 | "plugin:@typescript-eslint/recommended", 11 | "prettier" 12 | ], 13 | "rules": { 14 | "import/extensions": ["error", "ignorePackages"] 15 | }, 16 | "env": { 17 | "browser": true, 18 | "es2021": true 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals' 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry) 7 | getFID(onPerfEntry) 8 | getFCP(onPerfEntry) 9 | getLCP(onPerfEntry) 10 | getTTFB(onPerfEntry) 11 | }) 12 | } 13 | } 14 | 15 | export default reportWebVitals 16 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | CarPlay App 8 | 9 | 10 | 11 |
12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /examples/carplay-web-app/README.md: -------------------------------------------------------------------------------- 1 | # Running this example 2 | 3 | In the root folder of node-CarPlay run: 4 | ``` 5 | npm i 6 | ``` 7 | 8 | Then in this folder: 9 | ``` 10 | npm i 11 | npm start 12 | ``` 13 | 14 | Make sure chrome has "Experimental Web Platform features" enabled in chrome://flags. 15 | 16 | On Raspberry Pi make sure to also grant plugdev permissions to usb devices 17 | ``` 18 | echo "SUBSYSTEM==\"usb\", ATTR{idVendor}==\"1314\", ATTR{idProduct}==\"152*\", MODE=\"0660\", GROUP=\"plugdev\"" | sudo tee /etc/udev/rules.d/52-nodecarplay.rules 19 | ``` 20 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import './index.css' 4 | import App from './App' 5 | import reportWebVitals from './reportWebVitals' 6 | 7 | const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement) 8 | root.render() 9 | 10 | // If you want to start measuring performance in your app, pass a function 11 | // to log results (for example: reportWebVitals(console.log)) 12 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 13 | reportWebVitals() 14 | -------------------------------------------------------------------------------- /examples/carplay-web-app/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "64x64 32x32 24x24 16x16", 8 | "type": "image/x-icon" 9 | }, 10 | { 11 | "src": "logo192.png", 12 | "type": "image/png", 13 | "sizes": "192x192" 14 | }, 15 | { 16 | "src": "logo512.png", 17 | "type": "image/png", 18 | "sizes": "512x512" 19 | } 20 | ], 21 | "start_url": ".", 22 | "display": "standalone", 23 | "theme_color": "#000000", 24 | "background_color": "#ffffff" 25 | } 26 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "esModuleInterop": true, 8 | "allowSyntheticDefaultImports": true, 9 | "strict": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "module": "es2022", 13 | "moduleResolution": "bundler", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react-jsx" 18 | }, 19 | "include": ["server/*.ts", "client"] 20 | } 21 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/RenderEvents.ts: -------------------------------------------------------------------------------- 1 | export type WorkerEventType = 'init' | 'frame' | 'renderDone' 2 | 3 | export type Renderer = 'webgl' | 'webgl2' | 'webgpu' 4 | 5 | export interface WorkerEvent { 6 | type: WorkerEventType 7 | } 8 | 9 | export class RenderEvent implements WorkerEvent { 10 | type: WorkerEventType = 'frame' 11 | 12 | constructor(public frameData: ArrayBuffer) {} 13 | } 14 | 15 | export class InitEvent implements WorkerEvent { 16 | type: WorkerEventType = 'init' 17 | 18 | constructor( 19 | public canvas: OffscreenCanvas, 20 | public videoPort: MessagePort, 21 | public renderer: Renderer = 'webgl', 22 | public reportFps: boolean = false, 23 | ) {} 24 | } 25 | -------------------------------------------------------------------------------- /examples/carplay-web-app/config-overrides.js: -------------------------------------------------------------------------------- 1 | const webpack = require('webpack') 2 | 3 | module.exports = function override(config, env) { 4 | //do stuff with the webpack config... 5 | config.resolve.fallback = { 6 | ...config.resolve.fallback, 7 | stream: require.resolve('stream-browserify'), 8 | buffer: require.resolve('buffer'), 9 | } 10 | config.resolve.extensions = [...config.resolve.extensions, '.ts', '.js'] 11 | config.plugins = [ 12 | ...config.plugins, 13 | new webpack.ProvidePlugin({ 14 | process: 'process/browser', 15 | Buffer: ['buffer', 'Buffer'], 16 | }), 17 | ] 18 | // console.log(config.resolve) 19 | // console.log(config.plugins) 20 | 21 | return config 22 | } 23 | -------------------------------------------------------------------------------- /examples/carplay-web-app/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": ["dom", "dom.iterable", "esnext", "WebWorker"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "esModuleInterop": true, 8 | "allowSyntheticDefaultImports": true, 9 | "strict": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "module": "esnext", 13 | "moduleResolution": "Bundler", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react-jsx", 18 | "types": ["@webgpu/types"], 19 | "typeRoots": ["types", "node_modules/@types"] 20 | }, 21 | "include": ["src", "types"] 22 | } 23 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #282c34; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /scripts/startNode.ts: -------------------------------------------------------------------------------- 1 | import { DongleConfig } from '../src/modules/DongleDriver.js' 2 | import { PhoneType } from '../src/modules/index.js' 3 | import CarplayNode from '../src/node/CarplayNode.js' 4 | const config: DongleConfig = { 5 | dpi: 160, 6 | nightMode: false, 7 | hand: 0, 8 | boxName: 'nodePlay', 9 | width: 800, 10 | height: 600, 11 | fps: 20, 12 | mediaDelay: 300, 13 | audioTransferMode: false, 14 | format: 5, 15 | iBoxVersion: 2, 16 | packetMax: 49152, 17 | phoneWorkMode: 2, 18 | wifiType: '5ghz', 19 | micType: 'os', 20 | phoneConfig: { 21 | [PhoneType.CarPlay]: { 22 | frameInterval: 5000, 23 | }, 24 | [PhoneType.AndroidAuto]: { 25 | frameInterval: null, 26 | }, 27 | }, 28 | } 29 | 30 | const carplay = new CarplayNode(config) 31 | carplay.start() 32 | -------------------------------------------------------------------------------- /examples/carplay-web-app/types/ringbuf.js/index.d.ts: -------------------------------------------------------------------------------- 1 | type TypedArray = 2 | | Int8Array 3 | | Uint8Array 4 | | Uint8ClampedArray 5 | | Int16Array 6 | | Uint16Array 7 | | Int32Array 8 | | Uint32Array 9 | | Float32Array 10 | | Float64Array 11 | 12 | type TypedArrayConstructor = { 13 | new (): T 14 | new (size: number): T 15 | new (buffer: ArrayBuffer, byteOffset?: number, length?: number): T 16 | BYTES_PER_ELEMENT: number 17 | } 18 | 19 | declare module 'ringbuf.js' { 20 | export class RingBuffer { 21 | constructor(sab: SharedArrayBuffer, type: TypedArrayConstructor) 22 | buf: SharedArrayBuffer 23 | push(elements: T, length?: number, offset = 0): number 24 | pop(elements: TypedArray, length: number, offset = 0): number 25 | empty(): boolean 26 | full(): boolean 27 | capacity(): number 28 | availableRead(): number 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/README.md: -------------------------------------------------------------------------------- 1 | # Running this example 2 | 3 | **Note:** This example does not have audio processing, and relies on a separate bluetooth audio output device (optional) for audio. 4 | If there's no audio dongle connected, audio will play from iPhone itself. 5 | 6 | In the root folder of node-CarPlay run: 7 | ``` 8 | npm i 9 | ``` 10 | 11 | Then in this folder: 12 | ``` 13 | npm i 14 | npm start 15 | ``` 16 | 17 | You may open `http://localhost:3000` in the browser of your choice (not just Chrome). The included client handles video and touch inputs. 18 | If you would like to access the raw video feed, it is available at `http://localhost:3000/stream/video` 19 | 20 | On Raspberry Pi make sure to also grant plugdev permissions to usb devices 21 | ``` 22 | echo "SUBSYSTEM==\"usb\", ATTR{idVendor}==\"1314\", ATTR{idProduct}==\"152*\", MODE=\"0660\", GROUP=\"plugdev\"" | sudo tee /etc/udev/rules.d/52-nodecarplay.rules 23 | ``` 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Rhys Morgan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/node/NodeMicrophone.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events' 2 | import Mic from 'node-microphone' 3 | 4 | export default class NodeMicrophone extends EventEmitter { 5 | private _active: boolean 6 | private _mic: Mic 7 | private _timeout: NodeJS.Timeout | null = null 8 | 9 | constructor() { 10 | super() 11 | this._active = false 12 | this._mic = new Mic() 13 | const micEmitter = this._mic as unknown as EventEmitter 14 | micEmitter.on('data', data => { 15 | if (this._active) { 16 | this.emit('data', data) 17 | } 18 | }) 19 | micEmitter.on('info', info => { 20 | console.error(info) 21 | }) 22 | 23 | micEmitter.on('error', error => { 24 | console.error(error) 25 | }) 26 | } 27 | 28 | start() { 29 | console.debug('starting mic') 30 | this._mic.startRecording() 31 | this._active = true 32 | this._timeout = setTimeout(() => { 33 | this.stop() 34 | }, 10000) 35 | } 36 | 37 | stop() { 38 | this._active = false 39 | if (this._timeout) { 40 | clearTimeout(this._timeout) 41 | } 42 | this._mic.stopRecording() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/types.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DongleConfig, 3 | CarplayMessage, 4 | AudioData, 5 | TouchAction, 6 | } from 'node-carplay/web' 7 | 8 | export type AudioPlayerKey = string & { __brand: 'AudioPlayerKey' } 9 | 10 | export type CarplayWorkerMessage = 11 | | { data: CarplayMessage } 12 | | { data: { type: 'requestBuffer'; message: AudioData } } 13 | 14 | export type InitialisePayload = { 15 | videoPort: MessagePort 16 | microphonePort: MessagePort 17 | } 18 | 19 | export type AudioPlayerPayload = { 20 | sab: SharedArrayBuffer 21 | decodeType: number 22 | audioType: number 23 | } 24 | 25 | export type StartPayload = { 26 | config: Partial 27 | } 28 | 29 | export type Command = 30 | | { type: 'frame' } 31 | | { type: 'stop' } 32 | | { type: 'initialise'; payload: InitialisePayload } 33 | | { type: 'audioBuffer'; payload: AudioPlayerPayload } 34 | | { type: 'start'; payload: StartPayload } 35 | | { 36 | type: 'touch' 37 | payload: { x: number; y: number; action: TouchAction } 38 | } 39 | 40 | export interface CarPlayWorker 41 | extends Omit { 42 | postMessage(message: Command, transfer?: Transferable[]): void 43 | onmessage: ((this: Worker, ev: CarplayWorkerMessage) => any) | null 44 | } 45 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/useCarplayTouch.ts: -------------------------------------------------------------------------------- 1 | import { useState, useCallback } from 'react' 2 | import { TouchAction } from 'node-carplay/web' 3 | import { CarPlayWorker } from './worker/types' 4 | 5 | export const useCarplayTouch = ( 6 | worker: CarPlayWorker, 7 | width: number, 8 | height: number, 9 | ) => { 10 | const [pointerdown, setPointerDown] = useState(false) 11 | 12 | const sendTouchEvent: React.PointerEventHandler = useCallback( 13 | e => { 14 | let action = TouchAction.Up 15 | if (e.type === 'pointerdown') { 16 | action = TouchAction.Down 17 | setPointerDown(true) 18 | } else if (pointerdown) { 19 | switch (e.type) { 20 | case 'pointermove': 21 | action = TouchAction.Move 22 | break 23 | case 'pointerup': 24 | case 'pointercancel': 25 | case 'pointerout': 26 | setPointerDown(false) 27 | action = TouchAction.Up 28 | break 29 | } 30 | } else { 31 | return 32 | } 33 | 34 | const { offsetX: x, offsetY: y } = e.nativeEvent 35 | worker.postMessage({ 36 | type: 'touch', 37 | payload: { x: x / width, y: y / height, action }, 38 | }) 39 | }, 40 | [pointerdown, worker, width, height], 41 | ) 42 | 43 | return sendTouchEvent 44 | } 45 | -------------------------------------------------------------------------------- /scripts/configIcon.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import { webusb } from 'usb' 3 | import { 4 | SendFile, 5 | FileAddress, 6 | DongleDriver, 7 | SendIconConfig, 8 | } from '../src/modules/index.js' 9 | import process from 'process' 10 | 11 | const { knownDevices } = DongleDriver 12 | 13 | const driver = new DongleDriver() 14 | 15 | const config = async () => { 16 | const device = await webusb.requestDevice({ filters: knownDevices }) 17 | if (!device) { 18 | console.log('No device found - Connect dongle to USB and try again') 19 | } else { 20 | await device.open() 21 | await driver.initialise(device) 22 | 23 | const iconLabel = process.argv[2] 24 | if (iconLabel) { 25 | console.log(`Setting custom icon label ${iconLabel}`) 26 | await driver.send(new SendIconConfig({ label: iconLabel })) 27 | } 28 | 29 | const iconPath = process.argv[3] 30 | if (fs.existsSync(iconPath)) { 31 | console.log('Found Icon - Copying to dongle') 32 | const iconBuff = fs.readFileSync(iconPath) 33 | await driver.send(new SendFile(iconBuff, FileAddress.ICON_120)) 34 | await driver.send(new SendFile(iconBuff, FileAddress.ICON_180)) 35 | await driver.send(new SendFile(iconBuff, FileAddress.ICON_250)) 36 | console.log('Done copying Icon') 37 | } 38 | 39 | await driver.close() 40 | } 41 | } 42 | 43 | config() 44 | -------------------------------------------------------------------------------- /src/web/WebMicrophone.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events' 2 | 3 | export default class WebMicrophone extends EventEmitter { 4 | private active = false 5 | private sampleRate = 16000 6 | private audioContext: AudioContext 7 | private inputStream: MediaStreamAudioSourceNode 8 | private recorder: AudioWorkletNode | null = null 9 | 10 | constructor(mediaStream: MediaStream, messagePort: MessagePort) { 11 | super() 12 | const audioContext = new AudioContext({ sampleRate: this.sampleRate }) 13 | this.inputStream = audioContext.createMediaStreamSource(mediaStream) 14 | this.audioContext = audioContext 15 | audioContext.audioWorklet 16 | .addModule(new URL('./recorder.worklet.js', import.meta.url)) 17 | .then(() => { 18 | this.recorder = new AudioWorkletNode(audioContext, 'recorder.worklet') 19 | this.recorder.port.postMessage(messagePort, [messagePort]) 20 | }) 21 | } 22 | 23 | async start() { 24 | if (!this.recorder) return 25 | console.debug('starting mic') 26 | this.active = true 27 | this.inputStream 28 | .connect(this.recorder) 29 | .connect(this.audioContext.destination) 30 | } 31 | 32 | stop() { 33 | if (!this.recorder) return 34 | console.debug('stopping mic') 35 | this.active = false 36 | this.inputStream.disconnect() 37 | this.recorder.disconnect() 38 | } 39 | 40 | destroy() { 41 | if (!this.recorder) return 42 | this.inputStream.disconnect() 43 | this.recorder.disconnect() 44 | this.recorder = null 45 | this.audioContext.close() 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /.github/workflows/build_and_test.yml: -------------------------------------------------------------------------------- 1 | name: build_and_test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master # TODO: Remove this in favor of main eventually? 8 | - next 9 | pull_request: 10 | 11 | jobs: 12 | build_and_test: 13 | runs-on: ubuntu-latest 14 | 15 | container: 16 | image: node:18 # Tracking current LTS 17 | 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v3 21 | 22 | - name: Cache Node packages 23 | uses: actions/cache@v3 24 | with: 25 | path: '**/node_modules' 26 | key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} 27 | restore-keys: | 28 | ${{ runner.os }}-npm- 29 | 30 | - name: Packager Version 31 | run: npm version 32 | 33 | - name: Install Dependencies 34 | run: (npm install) && (cd examples/carplay-web-app && npm install) 35 | 36 | - name: Typecheck sources 37 | run: npm run typecheck 38 | 39 | - name: Lint sources 40 | run: npm run lint 41 | 42 | - name: Run Tests 43 | run: npm run test 44 | 45 | - name: Build example 46 | run: (cd examples/carplay-web-app && npm run build) 47 | 48 | - name: Zip up the built example 49 | run: (cd examples/carplay-web-app/build && tar -cvzf ../../../carplay-web-app.tar.gz *) 50 | 51 | - name: Upload built example 52 | uses: actions/upload-artifact@v4 53 | with: 54 | name: carplay-web-app.tar.gz 55 | path: carplay-web-app.tar.gz 56 | retention-days: 30 57 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/lib/utils.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/codewithpassion/foxglove-studio-h264-extension/tree/main 2 | // MIT License 3 | import { Bitstream, NALUStream, SPS } from './h264-utils' 4 | 5 | type GetNaluResult = { type: NaluTypes; nalu: Uint8Array; rawNalu: Uint8Array } 6 | 7 | enum NaluTypes { 8 | NDR = 1, 9 | IDR = 5, 10 | SEI = 6, 11 | SPS = 7, 12 | PPS = 8, 13 | AUD = 9, 14 | } 15 | 16 | function getNaluFromStream( 17 | buffer: Uint8Array, 18 | type: NaluTypes, 19 | ): GetNaluResult | null { 20 | const stream = new NALUStream(buffer, { type: 'annexB' }) 21 | 22 | for (const nalu of stream.nalus()) { 23 | if (nalu?.nalu) { 24 | const bitstream = new Bitstream(nalu.nalu) 25 | bitstream.seek(3) 26 | const nal_unit_type = bitstream.u(5) 27 | if (nal_unit_type === type) { 28 | return { type: nal_unit_type, ...nalu } 29 | } 30 | } 31 | } 32 | 33 | return null 34 | } 35 | 36 | function isKeyFrame(frameData: Uint8Array): boolean { 37 | const idr = getNaluFromStream(frameData, NaluTypes.IDR) 38 | return Boolean(idr) 39 | } 40 | 41 | function getDecoderConfig(frameData: Uint8Array): VideoDecoderConfig | null { 42 | const spsNalu = getNaluFromStream(frameData, NaluTypes.SPS) 43 | if (spsNalu) { 44 | const sps = new SPS(spsNalu.nalu) 45 | const decoderConfig: VideoDecoderConfig = { 46 | codec: sps.MIME, 47 | codedHeight: sps.picHeight, 48 | codedWidth: sps.picWidth, 49 | } 50 | return decoderConfig 51 | } 52 | return null 53 | } 54 | 55 | export { getDecoderConfig, isKeyFrame } 56 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "carplay-node-server", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "cors": "^2.8.5", 7 | "express": "^4.18.2", 8 | "node-carplay": "file:../../", 9 | "ws": "^8.14.0" 10 | }, 11 | "devDependencies": { 12 | "@types/cors": "^2.8.14", 13 | "@types/express": "^4.17.17", 14 | "@types/jmuxer": "^2.0.3", 15 | "@types/react": "^18.2.21", 16 | "@types/react-dom": "^18.2.7", 17 | "@types/ws": "^8.5.5", 18 | "buffer": "^6.0.3", 19 | "esbuild": "^0.19.2", 20 | "events": "^3.3.0", 21 | "jmuxer": "^2.0.5", 22 | "react": "^18.2.0", 23 | "react-dom": "^18.2.0", 24 | "stream-browserify": "^3.0.0" 25 | }, 26 | "scripts": { 27 | "build": "npm run build:server && npm run build:client", 28 | "build:server": "esbuild server/index.ts --bundle --outfile=server/index.js --define:process.env.NODE_ENV=\\\"production\\\" --platform=node --external:usb --external:node-microphone --external:ws --external:express --external:cors --tree-shaking=true", 29 | "build:client": "esbuild client/index.tsx --bundle --outfile=public/bundle.js --define:process.env.NODE_ENV=\\\"production\\\" --alias:events=events --alias:stream=stream-browserify --alias:buffer=buffer --tree-shaking=true", 30 | "watch:server": "npm run build:server -- --watch", 31 | "watch:client": "npm run build:client -- --watch", 32 | "start": "node server/index.js", 33 | "prepare": "npm run build" 34 | }, 35 | "optionalDependencies": { 36 | "bufferutil": "^4.0.7", 37 | "utf-8-validate": "^6.0.3" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /examples/carplay-web-app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "carplay-app-worker", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@testing-library/jest-dom": "^5.17.0", 7 | "@testing-library/react": "^13.4.0", 8 | "@testing-library/user-event": "^13.5.0", 9 | "@types/jest": "^27.5.2", 10 | "@types/node": "^16.18.40", 11 | "@types/react": "^18.2.20", 12 | "@types/react-dom": "^18.2.7", 13 | "@webgpu/types": "^0.1.37", 14 | "buffer": "^6.0.3", 15 | "http-proxy-middleware": "^2.0.6", 16 | "node-carplay": "file:../../", 17 | "pcm-ringbuf-player": "0.0.6", 18 | "process": "^0.11.10", 19 | "react": "^18.2.0", 20 | "react-dom": "^18.2.0", 21 | "react-loader-spinner": "^5.3.4", 22 | "react-scripts": "5.0.1", 23 | "stream-browserify": "^3.0.0", 24 | "typescript": "^4.9.5", 25 | "web-vitals": "^3.4.0" 26 | }, 27 | "devDependencies": { 28 | "@babel/plugin-proposal-private-property-in-object": "^7.21.11", 29 | "@types/jmuxer": "^2.0.3", 30 | "react-app-rewired": "^2.2.1" 31 | }, 32 | "scripts": { 33 | "start": "react-app-rewired start", 34 | "build": "react-app-rewired build", 35 | "test": "react-app-rewired test", 36 | "eject": "react-app-rewired eject" 37 | }, 38 | "eslintConfig": { 39 | "extends": [ 40 | "react-app", 41 | "react-app/jest" 42 | ] 43 | }, 44 | "browserslist": { 45 | "production": [ 46 | "last 1 chrome version", 47 | "last 1 firefox version", 48 | "last 1 safari version" 49 | ], 50 | "development": [ 51 | "last 1 chrome version", 52 | "last 1 firefox version", 53 | "last 1 safari version" 54 | ] 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/web/recorder.worklet.ts: -------------------------------------------------------------------------------- 1 | class RecorderProcessor extends AudioWorkletProcessor { 2 | bufferSize = 2048 3 | _bytesWritten = 0 4 | _buffer = new Float32Array(this.bufferSize) 5 | _outPort: MessagePort | undefined 6 | 7 | constructor() { 8 | super() 9 | this.initBuffer() 10 | this.port.onmessage = ev => { 11 | this._outPort = ev.data 12 | } 13 | } 14 | 15 | private floatTo16BitPCM(input: Float32Array) { 16 | let i = input.length 17 | const output = new Int16Array(i) 18 | while (i--) { 19 | const s = Math.max(-1, Math.min(1, input[i])) 20 | output[i] = s < 0 ? s * 0x8000 : s * 0x7fff 21 | } 22 | return output 23 | } 24 | 25 | initBuffer() { 26 | this._bytesWritten = 0 27 | } 28 | 29 | isBufferEmpty() { 30 | return this._bytesWritten === 0 31 | } 32 | 33 | isBufferFull() { 34 | return this._bytesWritten === this.bufferSize 35 | } 36 | 37 | /** 38 | * @param {Float32Array[][]} inputs 39 | * @returns {boolean} 40 | */ 41 | process(inputs: Float32Array[][]) { 42 | this.append(inputs[0][0]) 43 | 44 | return true 45 | } 46 | 47 | /** 48 | * 49 | * @param {Float32Array} channelData 50 | */ 51 | append(channelData: Float32Array) { 52 | if (this.isBufferFull()) { 53 | this.flush() 54 | } 55 | 56 | if (!channelData) return 57 | 58 | for (let i = 0; i < channelData.length; i++) { 59 | this._buffer[this._bytesWritten++] = channelData[i] 60 | } 61 | } 62 | 63 | flush() { 64 | const data = this.floatTo16BitPCM( 65 | this._bytesWritten < this.bufferSize 66 | ? this._buffer.slice(0, this._bytesWritten) 67 | : this._buffer, 68 | ) 69 | if (this._outPort) { 70 | this._outPort.postMessage(data, [data.buffer]) 71 | } 72 | this.initBuffer() 73 | } 74 | } 75 | 76 | registerProcessor('recorder.worklet', RecorderProcessor) 77 | -------------------------------------------------------------------------------- /examples/carplay-web-app/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 17 | 18 | 27 | React App 28 | 29 | 30 | 31 |
32 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-carplay", 3 | "version": "4.3.0", 4 | "description": "Carplay dongle driver for Node.js & Browser", 5 | "type": "module", 6 | "exports": { 7 | "./node": "./dist/node/index.js", 8 | "./web": "./dist/web/index.js" 9 | }, 10 | "scripts": { 11 | "prepare": "npm run build", 12 | "start:node": "ts-node --esm ./scripts/startNode.ts", 13 | "config:icon": "ts-node --esm ./scripts/configIcon.ts", 14 | "test": "jest --silent", 15 | "coverage": "jest --silent --coverage", 16 | "build": "tsc --build ./tsconfig.build.json ./src/web/tsconfig.json", 17 | "watch": "tsc --build ./tsconfig.build.json ./src/web/tsconfig.json --watch", 18 | "clean": "tsc --build --clean ./ ./src/web/tsconfig.json", 19 | "typecheck": "tsc --noEmit", 20 | "lint": "eslint --ext .js,.ts,.tsx .", 21 | "format": "prettier --ignore-path .gitignore --write \"**/*.+(js|ts|tsx|json)\"" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "git+https://github.com/rhysmorgan134/node-CarPlay.git" 26 | }, 27 | "author": "rhysmorgan134", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/rhysmorgan134/node-CarPlay/issues" 31 | }, 32 | "homepage": "https://github.com/rhysmorgan134/node-CarPlay#readme", 33 | "dependencies": { 34 | "node-microphone": "https://github.com/rhysmorgan134/node-microphone.git", 35 | "usb": "^2.10.0" 36 | }, 37 | "devDependencies": { 38 | "@types/audioworklet": "^0.0.50", 39 | "@types/jest": "^29.5.4", 40 | "@types/node": "^18.11.9", 41 | "@types/node-microphone": "^0.1.1", 42 | "@types/w3c-web-usb": "^1.0.6", 43 | "@types/webaudioapi": "^0.0.27", 44 | "@typescript-eslint/eslint-plugin": "^6.7.0", 45 | "@typescript-eslint/parser": "^6.7.0", 46 | "eslint": "^8.49.0", 47 | "eslint-config-prettier": "^9.0.0", 48 | "eslint-plugin-import": "^2.28.1", 49 | "jest": "^29.7.0", 50 | "prettier": "^3.0.3", 51 | "ts-jest": "^29.1.1", 52 | "ts-node": "^10.9.1", 53 | "typescript": "^5.2.2" 54 | }, 55 | "files": [ 56 | "*.md", 57 | "dist/" 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /src/modules/__tests__/mocks/usbMocks.ts: -------------------------------------------------------------------------------- 1 | import { jest } from '@jest/globals' 2 | 3 | export const usbEndpoint: Partial = { 4 | endpointNumber: 1, 5 | type: 'bulk', 6 | packetSize: 512, 7 | } 8 | 9 | export const usbInterface: USBInterface = { 10 | interfaceNumber: 0, 11 | claimed: false, 12 | alternates: [], 13 | alternate: { 14 | alternateSetting: 0, 15 | interfaceClass: 0, 16 | interfaceProtocol: 0, 17 | interfaceSubclass: 0, 18 | endpoints: [ 19 | { 20 | ...usbEndpoint, 21 | direction: 'in', 22 | } as USBEndpoint, 23 | { 24 | ...usbEndpoint, 25 | direction: 'out', 26 | } as USBEndpoint, 27 | ], 28 | }, 29 | } 30 | 31 | export const deviceConfig: USBConfiguration = { 32 | configurationValue: 1, 33 | interfaces: [usbInterface], 34 | } 35 | 36 | export const usbDeviceFactory = ( 37 | devicePartial: Partial = {}, 38 | ): USBDevice => { 39 | return { 40 | opened: false, 41 | usbVersionMajor: 2, 42 | usbVersionMinor: 0, 43 | usbVersionSubminor: 0, 44 | deviceClass: 0xff, 45 | deviceSubclass: 0xff, 46 | deviceProtocol: 0xff, 47 | vendorId: 0x1234, 48 | productId: 0xabcd, 49 | deviceVersionMajor: 1, 50 | deviceVersionMinor: 0, 51 | deviceVersionSubminor: 0, 52 | configurations: [deviceConfig], 53 | configuration: deviceConfig, 54 | open: jest.fn<() => Promise>(), 55 | close: jest.fn<() => Promise>(), 56 | forget: jest.fn<() => Promise>(), 57 | selectConfiguration: jest.fn<() => Promise>(), 58 | claimInterface: jest.fn<() => Promise>(), 59 | releaseInterface: jest.fn<() => Promise>(), 60 | selectAlternateInterface: jest.fn<() => Promise>(), 61 | controlTransferIn: jest.fn<() => Promise>(), 62 | controlTransferOut: jest.fn<() => Promise>(), 63 | clearHalt: jest.fn<() => Promise>(), 64 | transferIn: jest.fn<() => Promise>(), 65 | transferOut: jest.fn<() => Promise>(), 66 | isochronousTransferIn: 67 | jest.fn<() => Promise>(), 68 | isochronousTransferOut: 69 | jest.fn<() => Promise>(), 70 | reset: jest.fn<() => Promise>(), 71 | ...devicePartial, 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | .apk 10 | *.apk 11 | .idea 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 15 | 16 | # Runtime data 17 | pids 18 | *.pid 19 | *.seed 20 | *.pid.lock 21 | 22 | # Directory for instrumented libs generated by jscoverage/JSCover 23 | lib-cov 24 | 25 | # Coverage directory used by tools like istanbul 26 | coverage 27 | *.lcov 28 | 29 | # nyc test coverage 30 | .nyc_output 31 | 32 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 33 | .grunt 34 | 35 | # Bower dependency directory (https://bower.io/) 36 | bower_components 37 | 38 | # node-waf configuration 39 | .lock-wscript 40 | 41 | # Compiled binary addons (https://nodejs.org/api/addons.html) 42 | build/Release 43 | 44 | # Dependency directories 45 | node_modules/ 46 | jspm_packages/ 47 | 48 | # Snowpack dependency directory (https://snowpack.dev/) 49 | web_modules/ 50 | 51 | # TypeScript cache 52 | *.tsbuildinfo 53 | 54 | # Optional npm cache directory 55 | .npm 56 | 57 | # Optional eslint cache 58 | .eslintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variables file 76 | .env 77 | .env.test 78 | .env.production 79 | 80 | # parcel-bundler cache (https://parceljs.org/) 81 | .cache 82 | .parcel-cache 83 | 84 | # Next.js build output 85 | .next 86 | out 87 | 88 | # Nuxt.js build / generate output 89 | .nuxt 90 | dist 91 | 92 | # Gatsby files 93 | .cache/ 94 | # Comment in the public line in if your project uses Gatsby and not Next.js 95 | # https://nextjs.org/blog/next-9-1#public-directory-support 96 | # public 97 | 98 | # vuepress build output 99 | .vuepress/dist 100 | 101 | # Serverless directories 102 | .serverless/ 103 | 104 | # FuseBox cache 105 | .fusebox/ 106 | 107 | # DynamoDB Local files 108 | .dynamodb/ 109 | 110 | # TernJS port file 111 | .tern-port 112 | 113 | # Stores VSCode versions used for testing VSCode extensions 114 | .vscode-test 115 | 116 | # yarn v2 117 | .yarn/cache 118 | .yarn/unplugged 119 | .yarn/build-state.yml 120 | .yarn/install-state.gz 121 | .pnp.* 122 | 123 | # macOS Misc 124 | .DS_Store 125 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/server/index.ts: -------------------------------------------------------------------------------- 1 | import cors from 'cors' 2 | import http from 'http' 3 | import express from 'express' 4 | import WebSocket, { WebSocketServer } from 'ws' 5 | 6 | import CarPlayNode, { SendTouch } from 'node-carplay/node' 7 | import path from 'path' 8 | 9 | import { config } from '../client/config.js' 10 | 11 | const PORT = parseInt(process.env.PORT ?? '', 10) || 3000 12 | const PATH_STATIC = path.join(__dirname, '..', 'public') 13 | 14 | async function main() { 15 | const carPlay: CarPlayNode = new CarPlayNode(config) 16 | 17 | const clientsVideo = new Set() 18 | 19 | carPlay.onmessage = function ({ type, message }) { 20 | if (type === 'plugged') { 21 | console.log('statusChange plugged') 22 | } else if (type === 'unplugged') { 23 | console.log('statusChange unplugged') 24 | } else if (type === 'video') { 25 | if (message.data != null) { 26 | clientsVideo.forEach(client => { 27 | client.write(message.data) 28 | }) 29 | } 30 | } 31 | } 32 | 33 | const app = express() 34 | 35 | app.use(cors()) 36 | 37 | app.get('/stream/video', (req, res) => { 38 | res.writeHead(200, { 39 | 'Content-Type': 'video/h264', 40 | 'Access-Control-Allow-Origin': req.headers.origin ?? '*', 41 | }) 42 | clientsVideo.add(res) 43 | res.on('end', () => { 44 | clientsVideo.delete(res) 45 | }) 46 | }) 47 | 48 | app.use(express.static(PATH_STATIC)) 49 | 50 | const server = app.listen(PORT) 51 | 52 | const wss = new WebSocketServer({ server }) 53 | 54 | wss.on('connection', (ws: WebSocket) => { 55 | ws.on('message', (chunk: Buffer) => { 56 | let message = null 57 | 58 | try { 59 | message = JSON.parse(chunk.toString('utf8')) 60 | } catch (_) { 61 | // Ignore malformed messages 62 | console.debug('Malformed message', message) 63 | return 64 | } 65 | 66 | if (typeof message !== 'object' || message == null) { 67 | // Ignore malformed messages 68 | return 69 | } 70 | 71 | if (message.type === 'touch') { 72 | const { x, y, action } = message 73 | carPlay?.dongleDriver.send( 74 | new SendTouch(x / config.width, y / config.height, action), 75 | ) 76 | } 77 | }) 78 | }) 79 | 80 | console.log(`Server listening on http://localhost:${PORT}/`) 81 | 82 | await carPlay.start() 83 | } 84 | 85 | main().catch(err => { 86 | console.error(`Error initializing server`, { err }) 87 | process.exit(1) 88 | }) 89 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/modules/messages/__tests__/sendable.test.ts: -------------------------------------------------------------------------------- 1 | import { MessageHeader, MessageType, CommandMapping } from '../common.js' 2 | import { SendCommand, SendTouch, TouchAction } from '../sendable.js' 3 | 4 | describe('Sendable Messages', () => { 5 | describe('SendTouch Message', () => { 6 | it('constructs message with correct values', () => { 7 | const message = new SendTouch(0.1, 0.2, TouchAction.Up) 8 | expect(message.x).toBe(0.1) 9 | expect(message.y).toBe(0.2) 10 | expect(message.action).toBe(TouchAction.Up) 11 | }) 12 | 13 | it('serialises message correctly', () => { 14 | const expectedPayload = Buffer.alloc(16) 15 | expectedPayload.writeUInt32LE(TouchAction.Up, 0) 16 | expectedPayload.writeUInt32LE(0.1 * 10000, 4) 17 | expectedPayload.writeUInt32LE(0.2 * 10000, 8) 18 | 19 | const expectedHeader = MessageHeader.asBuffer( 20 | MessageType.Touch, 21 | Buffer.byteLength(expectedPayload), 22 | ) 23 | 24 | const message = new SendTouch(0.1, 0.2, TouchAction.Up) 25 | 26 | const data = message.serialise() 27 | expect(data).toStrictEqual( 28 | Buffer.concat([expectedHeader, expectedPayload]), 29 | ) 30 | }) 31 | 32 | it('serialises message with clamped values (0-10000)', () => { 33 | const expectedPayload = Buffer.alloc(16) 34 | expectedPayload.writeUInt32LE(TouchAction.Up, 0) 35 | expectedPayload.writeUInt32LE(0, 4) 36 | expectedPayload.writeUInt32LE(10000, 8) 37 | 38 | const expectedHeader = MessageHeader.asBuffer( 39 | MessageType.Touch, 40 | Buffer.byteLength(expectedPayload), 41 | ) 42 | 43 | const message = new SendTouch(-0.5, 2, TouchAction.Up) 44 | const data = message.serialise() 45 | expect(data).toStrictEqual( 46 | Buffer.concat([expectedHeader, expectedPayload]), 47 | ) 48 | }) 49 | }) 50 | 51 | describe('Command Message', () => { 52 | it('constructs message with correct values', () => { 53 | const message = new SendCommand('siri') 54 | expect(message.value).toBe(CommandMapping.siri) 55 | }) 56 | 57 | it('serialises message correctly', () => { 58 | const expectedPayload = Buffer.alloc(4) 59 | expectedPayload.writeUInt32LE(CommandMapping.siri, 0) 60 | 61 | const expectedHeader = MessageHeader.asBuffer( 62 | MessageType.Command, 63 | Buffer.byteLength(expectedPayload), 64 | ) 65 | 66 | const message = new SendCommand('siri') 67 | 68 | const data = message.serialise() 69 | expect(data).toStrictEqual( 70 | Buffer.concat([expectedHeader, expectedPayload]), 71 | ) 72 | }) 73 | }) 74 | }) 75 | -------------------------------------------------------------------------------- /src/modules/messages/__tests__/common.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | MessageHeader, 3 | MessageType, 4 | HeaderBuildError, 5 | CommandMapping, 6 | } from '../common.js' 7 | import { Command, Unplugged } from '../readable.js' 8 | 9 | describe('MessageHeader', () => { 10 | describe('constructor', () => { 11 | it('Constructs instance with type and length', () => { 12 | const header = new MessageHeader(10, MessageType.Command) 13 | expect(header.type).toBe(MessageType.Command) 14 | expect(header.length).toBe(10) 15 | }) 16 | }) 17 | 18 | describe('fromBuffer', () => { 19 | it('Constructs instance with type and length', () => { 20 | const buffer = Buffer.alloc(16) 21 | buffer.writeUInt32LE(MessageHeader.magic, 0) 22 | buffer.writeUInt32LE(10, 4) 23 | buffer.writeUInt32LE(MessageType.Command, 8) 24 | buffer.writeUInt32LE(((MessageType.Command ^ -1) & 0xffffffff) >>> 0, 12) 25 | const header = MessageHeader.fromBuffer(buffer) 26 | expect(header.type).toBe(MessageType.Command) 27 | expect(header.length).toBe(10) 28 | }) 29 | 30 | it('throws if buffer length is wrong', () => { 31 | const buffer = Buffer.alloc(10) 32 | expect(() => MessageHeader.fromBuffer(buffer)).toThrow( 33 | new HeaderBuildError('Invalid buffer size - Expecting 16, got 10'), 34 | ) 35 | }) 36 | 37 | it('throws if magic number check fails', () => { 38 | const buffer = Buffer.alloc(16) 39 | buffer.writeUInt32LE(12345, 0) 40 | expect(() => MessageHeader.fromBuffer(buffer)).toThrow( 41 | new HeaderBuildError('Invalid magic number, received 12345'), 42 | ) 43 | }) 44 | 45 | it('throws if type check fails', () => { 46 | const buffer = Buffer.alloc(16) 47 | buffer.writeUInt32LE(MessageHeader.magic, 0) 48 | buffer.writeUInt32LE(10, 4) 49 | buffer.writeUInt32LE(MessageType.Command, 8) 50 | buffer.writeUInt32LE(12345, 12) 51 | expect(() => MessageHeader.fromBuffer(buffer)).toThrow( 52 | new HeaderBuildError('Invalid type check, received 12345'), 53 | ) 54 | }) 55 | }) 56 | 57 | describe('toMessage', () => { 58 | it('constructs message based on type with no data', () => { 59 | const header = new MessageHeader(0, MessageType.Unplugged) 60 | expect(header.toMessage() instanceof Unplugged).toBeTruthy() 61 | }) 62 | 63 | it('constructs message based on type with data', () => { 64 | const header = new MessageHeader(4, MessageType.Command) 65 | const data = Buffer.alloc(4) 66 | data.writeUInt32LE(CommandMapping.siri, 0) 67 | const message = header.toMessage(data) 68 | expect(message instanceof Command).toBeTruthy() 69 | expect((message as Command).value).toBe(CommandMapping.siri) 70 | }) 71 | }) 72 | }) 73 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/useCarplayAudio.ts: -------------------------------------------------------------------------------- 1 | import { useCallback, useEffect, useState } from 'react' 2 | import './App.css' 3 | import { 4 | AudioCommand, 5 | AudioData, 6 | WebMicrophone, 7 | decodeTypeMap, 8 | } from 'node-carplay/web' 9 | import { PcmPlayer } from 'pcm-ringbuf-player' 10 | import { AudioPlayerKey, CarPlayWorker } from './worker/types' 11 | import { createAudioPlayerKey } from './worker/utils' 12 | 13 | //TODO: allow to configure 14 | const defaultAudioVolume = 1 15 | const defaultNavVolume = 0.5 16 | 17 | const useCarplayAudio = ( 18 | worker: CarPlayWorker, 19 | microphonePort: MessagePort, 20 | ) => { 21 | const [mic, setMic] = useState(null) 22 | const [audioPlayers] = useState(new Map()) 23 | 24 | const getAudioPlayer = useCallback( 25 | (audio: AudioData): PcmPlayer => { 26 | const { decodeType, audioType } = audio 27 | const format = decodeTypeMap[decodeType] 28 | const audioKey = createAudioPlayerKey(decodeType, audioType) 29 | let player = audioPlayers.get(audioKey) 30 | if (player) return player 31 | player = new PcmPlayer(format.frequency, format.channel) 32 | audioPlayers.set(audioKey, player) 33 | player.volume(defaultAudioVolume) 34 | player.start() 35 | worker.postMessage({ 36 | type: 'audioBuffer', 37 | payload: { 38 | sab: player.getRawBuffer(), 39 | decodeType, 40 | audioType, 41 | }, 42 | }) 43 | return player 44 | }, 45 | [audioPlayers, worker], 46 | ) 47 | 48 | const processAudio = useCallback( 49 | (audio: AudioData) => { 50 | if (audio.volumeDuration) { 51 | const { volume, volumeDuration } = audio 52 | const player = getAudioPlayer(audio) 53 | player.volume(volume, volumeDuration) 54 | } else if (audio.command) { 55 | switch (audio.command) { 56 | case AudioCommand.AudioNaviStart: 57 | const navPlayer = getAudioPlayer(audio) 58 | navPlayer.volume(defaultNavVolume) 59 | break 60 | case AudioCommand.AudioMediaStart: 61 | case AudioCommand.AudioOutputStart: 62 | const mediaPlayer = getAudioPlayer(audio) 63 | mediaPlayer.volume(defaultAudioVolume) 64 | break 65 | } 66 | } 67 | }, 68 | [getAudioPlayer], 69 | ) 70 | 71 | // audio init 72 | useEffect(() => { 73 | const initMic = async () => { 74 | try { 75 | const mediaStream = await navigator.mediaDevices.getUserMedia({ 76 | audio: true, 77 | }) 78 | const mic = new WebMicrophone(mediaStream, microphonePort) 79 | setMic(mic) 80 | } catch (err) { 81 | console.error('Failed to init microphone', err) 82 | } 83 | } 84 | 85 | initMic() 86 | 87 | return () => { 88 | audioPlayers.forEach(p => p.stop()) 89 | } 90 | }, [audioPlayers, worker, microphonePort]) 91 | 92 | const startRecording = useCallback(() => { 93 | mic?.start() 94 | }, [mic]) 95 | 96 | const stopRecording = useCallback(() => { 97 | mic?.stop() 98 | }, [mic]) 99 | 100 | return { processAudio, getAudioPlayer, startRecording, stopRecording } 101 | } 102 | 103 | export default useCarplayAudio 104 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/CarPlay.worker.ts: -------------------------------------------------------------------------------- 1 | import CarplayWeb, { 2 | CarplayMessage, 3 | DongleConfig, 4 | SendAudio, 5 | SendCommand, 6 | SendTouch, 7 | findDevice, 8 | } from 'node-carplay/web' 9 | import { AudioPlayerKey, Command } from './types' 10 | import { RenderEvent } from './render/RenderEvents' 11 | import { RingBuffer } from 'ringbuf.js' 12 | import { createAudioPlayerKey } from './utils' 13 | 14 | let carplayWeb: CarplayWeb | null = null 15 | let videoPort: MessagePort | null = null 16 | let microphonePort: MessagePort | null = null 17 | let config: Partial | null = null 18 | const audioBuffers: Record> = {} 19 | const pendingAudio: Record = {} 20 | 21 | const handleMessage = (message: CarplayMessage) => { 22 | const { type, message: payload } = message 23 | if (type === 'video' && videoPort) { 24 | videoPort.postMessage(new RenderEvent(payload.data), [payload.data.buffer]) 25 | } else if (type === 'audio' && payload.data) { 26 | const { decodeType, audioType } = payload 27 | const audioKey = createAudioPlayerKey(decodeType, audioType) 28 | if (audioBuffers[audioKey]) { 29 | audioBuffers[audioKey].push(payload.data) 30 | } else { 31 | if (!pendingAudio[audioKey]) { 32 | pendingAudio[audioKey] = [] 33 | } 34 | pendingAudio[audioKey].push(payload.data) 35 | payload.data = undefined 36 | 37 | const getPlayerMessage = { 38 | type: 'getAudioPlayer', 39 | message: { 40 | ...payload, 41 | }, 42 | } 43 | postMessage(getPlayerMessage) 44 | } 45 | } else { 46 | postMessage(message) 47 | } 48 | } 49 | 50 | onmessage = async (event: MessageEvent) => { 51 | switch (event.data.type) { 52 | case 'initialise': 53 | if (carplayWeb) return 54 | videoPort = event.data.payload.videoPort 55 | microphonePort = event.data.payload.microphonePort 56 | microphonePort.onmessage = ev => { 57 | if (carplayWeb) { 58 | const data = new SendAudio(ev.data) 59 | carplayWeb.dongleDriver.send(data) 60 | } 61 | } 62 | break 63 | case 'audioBuffer': 64 | const { sab, decodeType, audioType } = event.data.payload 65 | const audioKey = createAudioPlayerKey(decodeType, audioType) 66 | audioBuffers[audioKey] = new RingBuffer(sab, Int16Array) 67 | if (pendingAudio[audioKey]) { 68 | pendingAudio[audioKey].forEach(buf => { 69 | audioBuffers[audioKey].push(buf) 70 | }) 71 | pendingAudio[audioKey] = [] 72 | } 73 | break 74 | case 'start': 75 | if (carplayWeb) return 76 | config = event.data.payload.config 77 | const device = await findDevice() 78 | if (device) { 79 | carplayWeb = new CarplayWeb(config) 80 | carplayWeb.onmessage = handleMessage 81 | carplayWeb.start(device) 82 | } 83 | break 84 | case 'touch': 85 | if (config && carplayWeb) { 86 | const { x, y, action } = event.data.payload 87 | const data = new SendTouch(x, y, action) 88 | carplayWeb.dongleDriver.send(data) 89 | } 90 | break 91 | case 'stop': 92 | await carplayWeb?.stop() 93 | carplayWeb = null 94 | break 95 | case 'frame': 96 | if (carplayWeb) { 97 | const data = new SendCommand('frame') 98 | carplayWeb.dongleDriver.send(data) 99 | } 100 | break 101 | } 102 | } 103 | 104 | export {} 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

Node Carplay

3 |

4 | Carplay dongle driver for Node.js & Browser 5 |

6 | 7 | ## Join us on Slack 8 | https://join.slack.com/t/automotive-pis/shared_invite/zt-27n3manj4-v0Q35NWPyHUAtrHl7sScjQ 9 | 10 | 11 |
12 | Table of Contents 13 |
    14 |
  1. 15 | About The Project 16 | 19 |
  2. 20 |
  3. 21 | Getting Started 22 | 26 |
  4. 27 |
  5. Usage
  6. 28 |
  7. Contributing
  8. 29 |
  9. License
  10. 30 |
31 |
32 | 33 | 34 | 35 | 36 | ## About The Project 37 | 38 | ![Node Carplay in Chrome](https://github.com/rhysmorgan134/node-CarPlay/assets/4278113/3cbb5cab-fd62-4282-9fad-1b1aed90ad33) 39 | 40 | [Example Video (outdated)](https://youtu.be/mBeYd7RNw1w) 41 | 42 | This repository contains the npm package `node-carplay` that can be used on the Web or in Node.js. It allows interfacing with the [Carlinkit USB adapter](https://amzn.to/3X6OaF9) and stream audio/video on your computer. The package can be used in the Node.js environment using native USB bindings ([`libudev-dev` required](https://github.com/node-usb/node-usb#prerequisites)), or in Chrome (or equivalent browsers) using [`WebUSB` API](https://developer.mozilla.org/en-US/docs/Web/API/WebUSB_API). 43 | 44 | There are multiple Carplay dongles on the market, the ones that convert wired to wireless carplay WILL NOT WORK. You need one that converts android/factory infotainment systems into Carplay (CPC200-Autokit or CPC200-CCPA etc). The package forwards video feed in h264, and PCM audio coming in from the USB dongle. 45 | 46 | There's an included example `carplay-web-app` that runs in the browser and renders the Carplay environment. It supports mic input and audio output through Chrome audio stack as well as touch / mouse input. 47 | 48 | ### Acknowledgements 49 | 50 | This project is inspired by the work of @electric-monk on the Python version. 51 | 52 | * [PyCarplay](https://github.com/electric-monk/pycarplay) by @electric-monk 53 | * [Node-USB](https://github.com/node-usb/node-usb) 54 | * [jMuxer](https://github.com/samirkumardas/jmuxer) 55 | 56 | 57 | ## Getting Started 58 | 59 | ### Prerequisites 60 | 61 | If you are on macOS and want to use the microphone in `node` environment, you need `sox` 62 | 63 | ```shell 64 | brew install sox 65 | ``` 66 | 67 | If you are on Linux, you need `libudev-dev` for USB support in `node` environment 68 | 69 | ```shell 70 | sudo apt install -y libudev-dev 71 | ``` 72 | 73 | ### Installation 74 | 75 | ```javascript 76 | npm install node-carplay 77 | ``` 78 | 79 | ## Usage 80 | 81 | There is an included example (not in the NPM package, but in the [Git repository](https://github.com/rhysmorgan134/node-CarPlay)). It is recommended to take the example and modify your way out of it. 82 | 83 | 84 | ## Contributing 85 | 86 | Contributions are what make the open source community such an amazing place to be learn, inspire, and create. Any contributions you make are **greatly appreciated**. 87 | 88 | 1. Fork the Project 89 | 2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`) 90 | 3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) 91 | 4. Push to the Branch (`git push origin feature/AmazingFeature`) 92 | 5. Open a Pull Request 93 | 94 | 95 | ## License 96 | 97 | The contents of this repository are licensed under the terms of the MIT License. 98 | See the `LICENSE` file for more info. 99 | -------------------------------------------------------------------------------- /examples/carplay-node-without-audio/client/App.tsx: -------------------------------------------------------------------------------- 1 | import { useCallback, useEffect, useState, useRef } from 'react' 2 | import JMuxer from 'jmuxer' 3 | import { TouchAction } from 'node-carplay/web' 4 | 5 | import { config } from './config.js' 6 | 7 | const RETRY_DELAY_MS = 6000 8 | // ^ Note: This retry delay is lower than carplay-web-app 9 | // because the dongle is handled on server side, and it is 10 | // higher than 5 seconds because that's the default "frame" 11 | // time. 12 | 13 | function App() { 14 | const [pointerdown, setPointerDown] = useState(false) 15 | 16 | const connectionRef = useRef(null) 17 | const retryTimeoutRef = useRef(null) 18 | 19 | const clearRetryTimeout = useCallback(() => { 20 | if (retryTimeoutRef.current) { 21 | clearTimeout(retryTimeoutRef.current) 22 | retryTimeoutRef.current = null 23 | } 24 | }, []) 25 | 26 | useEffect(() => { 27 | if (connectionRef.current) { 28 | return 29 | } 30 | const jmuxer = new JMuxer({ 31 | node: 'video', 32 | mode: 'video', 33 | fps: config.fps, 34 | flushingTime: 0, 35 | debug: false, 36 | }) 37 | 38 | const connectionUrl = new URL('/', window.location.href) 39 | connectionUrl.protocol = connectionUrl.protocol.replace('http', 'ws') 40 | 41 | const connection = new WebSocket(connectionUrl.href) 42 | connectionRef.current = connection 43 | 44 | fetch(`/stream/video`) 45 | .then(async response => { 46 | const reader = response.body!.getReader() 47 | 48 | // eslint-disable-next-line no-constant-condition 49 | while (true) { 50 | const { value, done } = await reader.read() 51 | if (done) break 52 | jmuxer.feed({ 53 | video: value, 54 | duration: 0, 55 | }) 56 | clearRetryTimeout() 57 | } 58 | }) 59 | .catch(err => { 60 | console.error('Error in video stream', err) 61 | if (retryTimeoutRef.current == null) { 62 | console.error(`Reloading page in ${RETRY_DELAY_MS}ms`) 63 | retryTimeoutRef.current = setTimeout(() => { 64 | window.location.reload() 65 | }, RETRY_DELAY_MS) 66 | } 67 | }) 68 | }, []) 69 | 70 | const sendTouchEvent: React.PointerEventHandler = useCallback( 71 | e => { 72 | let action = TouchAction.Up 73 | if (e.type === 'pointerdown') { 74 | action = TouchAction.Down 75 | setPointerDown(true) 76 | } else if (pointerdown) { 77 | switch (e.type) { 78 | case 'pointermove': 79 | action = TouchAction.Move 80 | break 81 | case 'pointerup': 82 | case 'pointercancel': 83 | case 'pointerout': 84 | setPointerDown(false) 85 | action = TouchAction.Up 86 | break 87 | } 88 | } else { 89 | return 90 | } 91 | 92 | const { offsetX: x, offsetY: y } = e.nativeEvent 93 | 94 | connectionRef.current?.send( 95 | JSON.stringify({ 96 | type: 'touch', 97 | x, 98 | y, 99 | action, 100 | }), 101 | ) 102 | }, 103 | [pointerdown], 104 | ) 105 | 106 | return ( 107 |
112 |
127 |
129 |
130 | ) 131 | } 132 | 133 | export default App 134 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/Render.worker.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/codewithpassion/foxglove-studio-h264-extension/tree/main 2 | // MIT License 3 | import { getDecoderConfig, isKeyFrame } from './lib/utils' 4 | import { InitEvent, RenderEvent, WorkerEvent } from './RenderEvents' 5 | import { WebGL2Renderer } from './WebGL2Renderer' 6 | import { WebGLRenderer } from './WebGLRenderer' 7 | import { WebGPURenderer } from './WebGPURenderer' 8 | 9 | export interface FrameRenderer { 10 | draw(data: VideoFrame): void 11 | } 12 | 13 | // eslint-disable-next-line no-restricted-globals 14 | const scope = self as unknown as Worker 15 | 16 | type HostType = Window & typeof globalThis 17 | 18 | export class RenderWorker { 19 | constructor(private host: HostType) {} 20 | 21 | private renderer: FrameRenderer | null = null 22 | private videoPort: MessagePort | null = null 23 | private pendingFrame: VideoFrame | null = null 24 | private startTime: number | null = null 25 | private frameCount = 0 26 | private timestamp = 0 27 | private fps = 0 28 | 29 | private onVideoDecoderOutput = (frame: VideoFrame) => { 30 | // Update statistics. 31 | if (this.startTime == null) { 32 | this.startTime = performance.now() 33 | } else { 34 | const elapsed = (performance.now() - this.startTime) / 1000 35 | this.fps = ++this.frameCount / elapsed 36 | } 37 | 38 | // Schedule the frame to be rendered. 39 | this.renderFrame(frame) 40 | } 41 | 42 | private renderFrame = (frame: VideoFrame) => { 43 | if (!this.pendingFrame) { 44 | // Schedule rendering in the next animation frame. 45 | requestAnimationFrame(this.renderAnimationFrame) 46 | } else { 47 | // Close the current pending frame before replacing it. 48 | this.pendingFrame.close() 49 | } 50 | // Set or replace the pending frame. 51 | this.pendingFrame = frame 52 | } 53 | 54 | private renderAnimationFrame = () => { 55 | if (this.pendingFrame) { 56 | this.renderer?.draw(this.pendingFrame) 57 | this.pendingFrame = null 58 | } 59 | } 60 | 61 | private onVideoDecoderOutputError = (err: Error) => { 62 | console.error(`H264 Render worker decoder error`, err) 63 | } 64 | 65 | private decoder = new VideoDecoder({ 66 | output: this.onVideoDecoderOutput, 67 | error: this.onVideoDecoderOutputError, 68 | }) 69 | 70 | init = (event: InitEvent) => { 71 | switch (event.renderer) { 72 | case 'webgl': 73 | this.renderer = new WebGLRenderer(event.canvas) 74 | break 75 | case 'webgl2': 76 | this.renderer = new WebGL2Renderer(event.canvas) 77 | break 78 | case 'webgpu': 79 | this.renderer = new WebGPURenderer(event.canvas) 80 | break 81 | } 82 | this.videoPort = event.videoPort 83 | this.videoPort.onmessage = ev => { 84 | this.onFrame(ev.data as RenderEvent) 85 | } 86 | 87 | if (event.reportFps) { 88 | setInterval(() => { 89 | if (this.decoder.state === 'configured') { 90 | console.debug(`FPS: ${this.fps}`) 91 | } 92 | }, 5000) 93 | } 94 | } 95 | 96 | onFrame = (event: RenderEvent) => { 97 | const frameData = new Uint8Array(event.frameData) 98 | 99 | if (this.decoder.state === 'unconfigured') { 100 | const decoderConfig = getDecoderConfig(frameData) 101 | if (decoderConfig) { 102 | this.decoder.configure(decoderConfig) 103 | console.log(decoderConfig) 104 | } 105 | } 106 | if (this.decoder.state === 'configured') { 107 | try { 108 | this.decoder.decode( 109 | new EncodedVideoChunk({ 110 | type: isKeyFrame(frameData) ? 'key' : 'delta', 111 | data: frameData, 112 | timestamp: this.timestamp++, 113 | }), 114 | ) 115 | } catch (e) { 116 | console.error(`H264 Render Worker decode error`, e) 117 | } 118 | } 119 | } 120 | } 121 | 122 | // eslint-disable-next-line no-restricted-globals 123 | const worker = new RenderWorker(self) 124 | scope.addEventListener('message', (event: MessageEvent) => { 125 | if (event.data.type === 'init') { 126 | worker.init(event.data as InitEvent) 127 | } 128 | }) 129 | 130 | export {} 131 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/WebGLRenderer.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/w3c/webcodecs/blob/main/samples/video-decode-display/renderer_webgl.js 2 | // License: https://www.w3.org/copyright/software-license-2023/ 3 | 4 | import { FrameRenderer } from './Render.worker' 5 | 6 | export class WebGLRenderer implements FrameRenderer { 7 | #canvas: OffscreenCanvas | null = null 8 | #ctx: WebGLRenderingContext | null = null 9 | 10 | static vertexShaderSource = ` 11 | attribute vec2 xy; 12 | 13 | varying highp vec2 uv; 14 | 15 | void main(void) { 16 | gl_Position = vec4(xy, 0.0, 1.0); 17 | // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1). 18 | // UV coordinates are Y-flipped relative to vertex coordinates. 19 | uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0); 20 | } 21 | ` 22 | 23 | static fragmentShaderSource = ` 24 | varying highp vec2 uv; 25 | 26 | uniform sampler2D texture; 27 | 28 | void main(void) { 29 | gl_FragColor = texture2D(texture, uv); 30 | } 31 | ` 32 | 33 | constructor(canvas: OffscreenCanvas) { 34 | this.#canvas = canvas 35 | const gl = (this.#ctx = canvas.getContext('webgl')) 36 | if (!gl) { 37 | throw Error('WebGL context is null') 38 | } 39 | 40 | const vertexShader = gl.createShader(gl.VERTEX_SHADER) 41 | 42 | if (!vertexShader) { 43 | throw Error('VertexShader is null') 44 | } 45 | 46 | gl.shaderSource(vertexShader, WebGLRenderer.vertexShaderSource) 47 | gl.compileShader(vertexShader) 48 | 49 | if (gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS) == null) { 50 | throw gl.getShaderInfoLog(vertexShader) 51 | } 52 | 53 | const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER) 54 | if (!fragmentShader) { 55 | throw Error('FragmentShader is null') 56 | } 57 | gl.shaderSource(fragmentShader, WebGLRenderer.fragmentShaderSource) 58 | gl.compileShader(fragmentShader) 59 | if (gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS) == null) { 60 | throw gl.getShaderInfoLog(fragmentShader) 61 | } 62 | 63 | const shaderProgram = gl.createProgram() 64 | if (!shaderProgram) { 65 | throw Error('ShaderProgram is null') 66 | } 67 | gl.attachShader(shaderProgram, vertexShader) 68 | gl.attachShader(shaderProgram, fragmentShader) 69 | gl.linkProgram(shaderProgram) 70 | if (gl.getProgramParameter(shaderProgram, gl.LINK_STATUS) == null) { 71 | throw gl.getProgramInfoLog(shaderProgram) 72 | } 73 | gl.useProgram(shaderProgram) 74 | 75 | // Vertex coordinates, clockwise from bottom-left. 76 | const vertexBuffer = gl.createBuffer() 77 | gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer) 78 | gl.bufferData( 79 | gl.ARRAY_BUFFER, 80 | new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), 81 | gl.STATIC_DRAW, 82 | ) 83 | 84 | const xyLocation = gl.getAttribLocation(shaderProgram, 'xy') 85 | gl.vertexAttribPointer(xyLocation, 2, gl.FLOAT, false, 0, 0) 86 | gl.enableVertexAttribArray(xyLocation) 87 | 88 | // Create one texture to upload frames to. 89 | const texture = gl.createTexture() 90 | gl.bindTexture(gl.TEXTURE_2D, texture) 91 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST) 92 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST) 93 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) 94 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) 95 | } 96 | 97 | draw(frame: VideoFrame): void { 98 | if (this.#canvas) { 99 | this.#canvas.width = frame.displayWidth 100 | this.#canvas.height = frame.displayHeight 101 | } 102 | 103 | const gl = this.#ctx! 104 | 105 | // Upload the frame. 106 | gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame) 107 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 108 | frame.close() 109 | 110 | // Configure and clear the drawing area. 111 | gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight) 112 | gl.clearColor(1.0, 0.0, 0.0, 1.0) 113 | gl.clear(gl.COLOR_BUFFER_BIT) 114 | 115 | // Draw the frame. 116 | gl.drawArrays(gl.TRIANGLE_FAN, 0, 4) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/WebGL2Renderer.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/w3c/webcodecs/blob/main/samples/video-decode-display/renderer_webgl.js 2 | // License: https://www.w3.org/copyright/software-license-2023/ 3 | 4 | import { FrameRenderer } from './Render.worker' 5 | 6 | export class WebGL2Renderer implements FrameRenderer { 7 | #canvas: OffscreenCanvas | null = null 8 | #ctx: WebGL2RenderingContext | null = null 9 | 10 | static vertexShaderSource = ` 11 | attribute vec2 xy; 12 | 13 | varying highp vec2 uv; 14 | 15 | void main(void) { 16 | gl_Position = vec4(xy, 0.0, 1.0); 17 | // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1). 18 | // UV coordinates are Y-flipped relative to vertex coordinates. 19 | uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0); 20 | } 21 | ` 22 | 23 | static fragmentShaderSource = ` 24 | varying highp vec2 uv; 25 | 26 | uniform sampler2D texture; 27 | 28 | void main(void) { 29 | gl_FragColor = texture2D(texture, uv); 30 | } 31 | ` 32 | 33 | constructor(canvas: OffscreenCanvas) { 34 | this.#canvas = canvas 35 | const gl = (this.#ctx = canvas.getContext('webgl2')) 36 | if (!gl) { 37 | throw Error('WebGL context is null') 38 | } 39 | 40 | const vertexShader = gl.createShader(gl.VERTEX_SHADER) 41 | 42 | if (!vertexShader) { 43 | throw Error('VertexShader is null') 44 | } 45 | 46 | gl.shaderSource(vertexShader, WebGL2Renderer.vertexShaderSource) 47 | gl.compileShader(vertexShader) 48 | 49 | if (gl.getShaderParameter(vertexShader, gl.COMPILE_STATUS) == null) { 50 | throw gl.getShaderInfoLog(vertexShader) 51 | } 52 | 53 | const fragmentShader = gl.createShader(gl.FRAGMENT_SHADER) 54 | if (!fragmentShader) { 55 | throw Error('FragmentShader is null') 56 | } 57 | gl.shaderSource(fragmentShader, WebGL2Renderer.fragmentShaderSource) 58 | gl.compileShader(fragmentShader) 59 | if (gl.getShaderParameter(fragmentShader, gl.COMPILE_STATUS) == null) { 60 | throw gl.getShaderInfoLog(fragmentShader) 61 | } 62 | 63 | const shaderProgram = gl.createProgram() 64 | if (!shaderProgram) { 65 | throw Error('ShaderProgram is null') 66 | } 67 | gl.attachShader(shaderProgram, vertexShader) 68 | gl.attachShader(shaderProgram, fragmentShader) 69 | gl.linkProgram(shaderProgram) 70 | if (gl.getProgramParameter(shaderProgram, gl.LINK_STATUS) == null) { 71 | throw gl.getProgramInfoLog(shaderProgram) 72 | } 73 | gl.useProgram(shaderProgram) 74 | 75 | // Vertex coordinates, clockwise from bottom-left. 76 | const vertexBuffer = gl.createBuffer() 77 | gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer) 78 | gl.bufferData( 79 | gl.ARRAY_BUFFER, 80 | new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), 81 | gl.STATIC_DRAW, 82 | ) 83 | 84 | const xyLocation = gl.getAttribLocation(shaderProgram, 'xy') 85 | gl.vertexAttribPointer(xyLocation, 2, gl.FLOAT, false, 0, 0) 86 | gl.enableVertexAttribArray(xyLocation) 87 | 88 | // Create one texture to upload frames to. 89 | const texture = gl.createTexture() 90 | gl.bindTexture(gl.TEXTURE_2D, texture) 91 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST) 92 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST) 93 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) 94 | gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) 95 | } 96 | 97 | draw(frame: VideoFrame): void { 98 | if (this.#canvas) { 99 | this.#canvas.width = frame.displayWidth 100 | this.#canvas.height = frame.displayHeight 101 | } 102 | 103 | const gl = this.#ctx! 104 | 105 | // Upload the frame. 106 | gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame) 107 | // eslint-disable-next-line @typescript-eslint/no-unsafe-call 108 | frame.close() 109 | 110 | // Configure and clear the drawing area. 111 | gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight) 112 | gl.clearColor(1.0, 0.0, 0.0, 1.0) 113 | gl.clear(gl.COLOR_BUFFER_BIT) 114 | 115 | // Draw the frame. 116 | gl.drawArrays(gl.TRIANGLE_FAN, 0, 4) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/web/CarplayWeb.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Message, 3 | Plugged, 4 | Unplugged, 5 | VideoData, 6 | AudioData, 7 | MediaData, 8 | SendCommand, 9 | Command, 10 | DongleDriver, 11 | DongleConfig, 12 | DEFAULT_CONFIG, 13 | } from '../modules/index.js' 14 | 15 | const { knownDevices } = DongleDriver 16 | 17 | export type CarplayMessage = 18 | | { type: 'plugged'; message?: undefined } 19 | | { type: 'unplugged'; message?: undefined } 20 | | { type: 'failure'; message?: undefined } 21 | | { type: 'audio'; message: AudioData } 22 | | { type: 'video'; message: VideoData } 23 | | { type: 'media'; message: MediaData } 24 | | { type: 'command'; message: Command } 25 | 26 | export const isCarplayDongle = (device: USBDevice) => { 27 | const known = knownDevices.some( 28 | kd => kd.productId === device.productId && kd.vendorId === device.vendorId, 29 | ) 30 | return known 31 | } 32 | 33 | export const findDevice = async (): Promise => { 34 | try { 35 | const devices = await navigator.usb.getDevices() 36 | return ( 37 | devices.find(d => { 38 | return isCarplayDongle(d) ? d : undefined 39 | }) || null 40 | ) 41 | } catch (err) { 42 | return null 43 | } 44 | } 45 | 46 | export const requestDevice = async (): Promise => { 47 | try { 48 | const { knownDevices } = DongleDriver 49 | const device = await navigator.usb.requestDevice({ 50 | filters: knownDevices, 51 | }) 52 | return device 53 | } catch (err) { 54 | return null 55 | } 56 | } 57 | 58 | export default class CarplayWeb { 59 | private _started: boolean = false 60 | private _pairTimeout: NodeJS.Timeout | null = null 61 | private _frameInterval: NodeJS.Timer | null = null 62 | private _config: DongleConfig 63 | public dongleDriver: DongleDriver 64 | 65 | constructor(config: Partial) { 66 | this._config = Object.assign({}, DEFAULT_CONFIG, config) 67 | const driver = new DongleDriver() 68 | driver.on('message', (message: Message) => { 69 | if (message instanceof Plugged) { 70 | this.clearPairTimeout() 71 | this.clearFrameInterval() 72 | 73 | const phoneTypeConfg = this._config.phoneConfig[message.phoneType] 74 | if (phoneTypeConfg?.frameInterval) { 75 | this._frameInterval = setInterval( 76 | () => { 77 | this.dongleDriver.send(new SendCommand('frame')) 78 | }, 79 | phoneTypeConfg?.frameInterval, 80 | ) 81 | } 82 | this.onmessage?.({ type: 'plugged' }) 83 | } else if (message instanceof Unplugged) { 84 | this.onmessage?.({ type: 'unplugged' }) 85 | } else if (message instanceof VideoData) { 86 | this.clearPairTimeout() 87 | this.onmessage?.({ type: 'video', message }) 88 | } else if (message instanceof AudioData) { 89 | this.clearPairTimeout() 90 | this.onmessage?.({ type: 'audio', message }) 91 | } else if (message instanceof MediaData) { 92 | this.clearPairTimeout() 93 | this.onmessage?.({ type: 'media', message }) 94 | } else if (message instanceof Command) { 95 | this.onmessage?.({ type: 'command', message }) 96 | } 97 | }) 98 | driver.on('failure', () => { 99 | this.onmessage?.({ type: 'failure' }) 100 | }) 101 | this.dongleDriver = driver 102 | } 103 | 104 | private clearPairTimeout() { 105 | if (this._pairTimeout) { 106 | clearTimeout(this._pairTimeout) 107 | this._pairTimeout = null 108 | } 109 | } 110 | 111 | private clearFrameInterval() { 112 | if (this._frameInterval) { 113 | clearInterval(this._frameInterval) 114 | this._pairTimeout = null 115 | } 116 | } 117 | 118 | public onmessage: ((ev: CarplayMessage) => void) | null = null 119 | 120 | start = async (usbDevice: USBDevice) => { 121 | if (this._started) return 122 | const { initialise, start, send } = this.dongleDriver 123 | 124 | console.debug('opening device') 125 | await usbDevice.open() 126 | await usbDevice.reset() 127 | 128 | await initialise(usbDevice) 129 | await start(this._config) 130 | this._pairTimeout = setTimeout(() => { 131 | console.debug('no device, sending pair') 132 | send(new SendCommand('wifiPair')) 133 | }, 15000) 134 | this._started = true 135 | } 136 | 137 | stop = async () => { 138 | try { 139 | this.clearFrameInterval() 140 | this.clearPairTimeout() 141 | await this.dongleDriver.close() 142 | } catch (err) { 143 | console.error(err) 144 | } finally { 145 | this._started = false 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/WebGPURenderer.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/w3c/webcodecs/blob/main/samples/video-decode-display/renderer_webgpu.js 2 | // License: https://www.w3.org/copyright/software-license-2023/ 3 | 4 | import { FrameRenderer } from './Render.worker' 5 | 6 | export class WebGPURenderer implements FrameRenderer { 7 | #canvas: OffscreenCanvas | null = null 8 | #ctx: GPUCanvasContext | null = null 9 | 10 | #started: Promise | null = null 11 | 12 | // WebGPU state shared between setup and drawing. 13 | #format: GPUTextureFormat | null = null 14 | #device: GPUDevice | null = null 15 | #pipeline: GPURenderPipeline | null = null 16 | #sampler: GPUSamplerDescriptor | null = null 17 | 18 | // Generates two triangles covering the whole canvas. 19 | static vertexShaderSource = ` 20 | struct VertexOutput { 21 | @builtin(position) Position: vec4, 22 | @location(0) uv: vec2, 23 | } 24 | 25 | @vertex 26 | fn vert_main(@builtin(vertex_index) VertexIndex: u32) -> VertexOutput { 27 | var pos = array, 6>( 28 | vec2( 1.0, 1.0), 29 | vec2( 1.0, -1.0), 30 | vec2(-1.0, -1.0), 31 | vec2( 1.0, 1.0), 32 | vec2(-1.0, -1.0), 33 | vec2(-1.0, 1.0) 34 | ); 35 | 36 | var uv = array, 6>( 37 | vec2(1.0, 0.0), 38 | vec2(1.0, 1.0), 39 | vec2(0.0, 1.0), 40 | vec2(1.0, 0.0), 41 | vec2(0.0, 1.0), 42 | vec2(0.0, 0.0) 43 | ); 44 | 45 | var output : VertexOutput; 46 | output.Position = vec4(pos[VertexIndex], 0.0, 1.0); 47 | output.uv = uv[VertexIndex]; 48 | return output; 49 | } 50 | ` 51 | 52 | // Samples the external texture using generated UVs. 53 | static fragmentShaderSource = ` 54 | @group(0) @binding(1) var mySampler: sampler; 55 | @group(0) @binding(2) var myTexture: texture_external; 56 | 57 | @fragment 58 | fn frag_main(@location(0) uv : vec2) -> @location(0) vec4 { 59 | return textureSampleBaseClampToEdge(myTexture, mySampler, uv); 60 | } 61 | ` 62 | 63 | constructor(canvas: OffscreenCanvas) { 64 | this.#canvas = canvas 65 | this.#started = this.#start() 66 | } 67 | 68 | async #start() { 69 | const adapter = await navigator.gpu.requestAdapter() 70 | if (!adapter) { 71 | throw Error('WebGPU Adapter is null') 72 | } 73 | this.#device = await adapter.requestDevice() 74 | this.#format = navigator.gpu.getPreferredCanvasFormat() 75 | 76 | if (!this.#canvas) { 77 | throw Error('Canvas is null') 78 | } 79 | 80 | this.#ctx = this.#canvas.getContext('webgpu') 81 | 82 | if (!this.#ctx) { 83 | throw Error('Context is null') 84 | } 85 | 86 | this.#ctx.configure({ 87 | device: this.#device, 88 | format: this.#format, 89 | alphaMode: 'opaque', 90 | }) 91 | 92 | this.#pipeline = this.#device.createRenderPipeline({ 93 | layout: 'auto', 94 | vertex: { 95 | module: this.#device.createShaderModule({ 96 | code: WebGPURenderer.vertexShaderSource, 97 | }), 98 | entryPoint: 'vert_main', 99 | }, 100 | fragment: { 101 | module: this.#device.createShaderModule({ 102 | code: WebGPURenderer.fragmentShaderSource, 103 | }), 104 | entryPoint: 'frag_main', 105 | targets: [{ format: this.#format }], 106 | }, 107 | primitive: { 108 | topology: 'triangle-list', 109 | }, 110 | }) 111 | 112 | // Default sampler configuration is nearset + clamp. 113 | this.#sampler = this.#device.createSampler({}) 114 | } 115 | 116 | async draw(frame: VideoFrame): Promise { 117 | // Don't try to draw any frames until the context is configured. 118 | await this.#started 119 | 120 | if (!this.#ctx) { 121 | throw Error('Context is null') 122 | } 123 | if (!this.#canvas) { 124 | throw Error('Canvas is null') 125 | } 126 | if (!this.#device) { 127 | throw Error('Device is null') 128 | } 129 | if (!this.#pipeline) { 130 | throw Error('Pipeline is null') 131 | } 132 | 133 | this.#canvas.width = frame.displayWidth 134 | this.#canvas.height = frame.displayHeight 135 | 136 | const uniformBindGroup = this.#device.createBindGroup({ 137 | layout: this.#pipeline.getBindGroupLayout(0), 138 | entries: [ 139 | { binding: 1, resource: this.#sampler }, 140 | { 141 | binding: 2, 142 | resource: this.#device.importExternalTexture({ source: frame }), 143 | }, 144 | ] as any, // TODO: fix typing 145 | }) 146 | 147 | const commandEncoder = this.#device.createCommandEncoder() 148 | const textureView = this.#ctx.getCurrentTexture().createView() 149 | const renderPassDescriptor: GPURenderPassDescriptor = { 150 | colorAttachments: [ 151 | { 152 | view: textureView, 153 | clearValue: [1.0, 0.0, 0.0, 1.0], 154 | loadOp: 'clear', 155 | storeOp: 'store', 156 | }, 157 | ], 158 | } 159 | 160 | const passEncoder = commandEncoder.beginRenderPass(renderPassDescriptor) 161 | passEncoder.setPipeline(this.#pipeline) 162 | passEncoder.setBindGroup(0, uniformBindGroup) 163 | passEncoder.draw(6, 1, 0, 0) 164 | passEncoder.end() 165 | this.#device.queue.submit([commandEncoder.finish()]) 166 | 167 | frame.close() 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /src/node/CarplayNode.ts: -------------------------------------------------------------------------------- 1 | import { webusb } from 'usb' 2 | import NodeMicrophone from './NodeMicrophone.js' 3 | import { 4 | AudioData, 5 | MediaData, 6 | Message, 7 | Plugged, 8 | SendAudio, 9 | SendCommand, 10 | SendTouch, 11 | Unplugged, 12 | VideoData, 13 | DongleDriver, 14 | DongleConfig, 15 | DEFAULT_CONFIG, 16 | CommandValue, 17 | Command, 18 | AudioCommand, 19 | } from '../modules/index.js' 20 | 21 | const USB_WAIT_PERIOD_MS = 3000 22 | 23 | export type CarplayMessage = 24 | | { type: 'plugged'; message?: undefined } 25 | | { type: 'unplugged'; message?: undefined } 26 | | { type: 'failure'; message?: undefined } 27 | | { type: 'audio'; message: AudioData } 28 | | { type: 'video'; message: VideoData } 29 | | { type: 'media'; message: MediaData } 30 | | { type: 'command'; message: Command } 31 | 32 | export default class CarplayNode { 33 | private _pairTimeout: NodeJS.Timeout | null = null 34 | private _frameInterval: NodeJS.Timer | null = null 35 | private _config: DongleConfig 36 | public dongleDriver: DongleDriver 37 | 38 | constructor(config: Partial) { 39 | this._config = Object.assign({}, DEFAULT_CONFIG, config) 40 | const mic = new NodeMicrophone() 41 | const driver = new DongleDriver() 42 | mic.on('data', data => { 43 | driver.send(new SendAudio(data)) 44 | }) 45 | driver.on('message', (message: Message) => { 46 | if (message instanceof Plugged) { 47 | this.clearPairTimeout() 48 | this.clearFrameInterval() 49 | const phoneTypeConfg = this._config.phoneConfig?.[message.phoneType] 50 | if (phoneTypeConfg?.frameInterval) { 51 | this._frameInterval = setInterval( 52 | () => { 53 | this.dongleDriver.send(new SendCommand('frame')) 54 | }, 55 | phoneTypeConfg?.frameInterval, 56 | ) 57 | } 58 | this.onmessage?.({ type: 'plugged' }) 59 | } else if (message instanceof Unplugged) { 60 | this.onmessage?.({ type: 'unplugged' }) 61 | } else if (message instanceof VideoData) { 62 | this.clearPairTimeout() 63 | this.onmessage?.({ type: 'video', message }) 64 | } else if (message instanceof AudioData) { 65 | this.clearPairTimeout() 66 | this.onmessage?.({ type: 'audio', message }) 67 | } else if (message instanceof MediaData) { 68 | this.clearPairTimeout() 69 | this.onmessage?.({ type: 'media', message }) 70 | } else if (message instanceof Command) { 71 | this.onmessage?.({ type: 'command', message }) 72 | } 73 | 74 | // Trigger internal event logic 75 | if (message instanceof AudioData && message.command != null) { 76 | switch (message.command) { 77 | case AudioCommand.AudioSiriStart: 78 | case AudioCommand.AudioPhonecallStart: 79 | mic.start() 80 | break 81 | case AudioCommand.AudioSiriStop: 82 | case AudioCommand.AudioPhonecallStop: 83 | mic.stop() 84 | break 85 | } 86 | } 87 | }) 88 | driver.on('failure', () => { 89 | this.onmessage?.({ type: 'failure' }) 90 | }) 91 | this.dongleDriver = driver 92 | } 93 | 94 | private async findDevice() { 95 | let device: USBDevice | null = null 96 | 97 | while (device == null) { 98 | try { 99 | device = await webusb.requestDevice({ 100 | filters: DongleDriver.knownDevices, 101 | }) 102 | } catch (err) { 103 | // ^ requestDevice throws an error when no device is found, so keep retrying 104 | } 105 | 106 | if (device == null) { 107 | console.log('No device found, retrying') 108 | await new Promise(resolve => setTimeout(resolve, USB_WAIT_PERIOD_MS)) 109 | } 110 | } 111 | 112 | return device 113 | } 114 | 115 | start = async () => { 116 | // Find device to "reset" first 117 | let device = await this.findDevice() 118 | await device.open() 119 | await device.reset() 120 | await device.close() 121 | // Resetting the device causes an unplug event in node-usb 122 | // so subsequent writes fail with LIBUSB_ERROR_NO_DEVICE 123 | // or LIBUSB_TRANSFER_ERROR 124 | 125 | console.log('Reset device, finding again...') 126 | await new Promise(resolve => setTimeout(resolve, USB_WAIT_PERIOD_MS)) 127 | // ^ Device disappears after reset for 1-3 seconds 128 | 129 | device = await this.findDevice() 130 | console.log('found & opening') 131 | 132 | await device.open() 133 | 134 | let initialised = false 135 | try { 136 | const { initialise, start, send } = this.dongleDriver 137 | await initialise(device) 138 | await start(this._config) 139 | this._pairTimeout = setTimeout(() => { 140 | console.debug('no device, sending pair') 141 | send(new SendCommand('wifiPair')) 142 | }, 15000) 143 | initialised = true 144 | } catch (err) { 145 | console.error(err) 146 | } 147 | 148 | if (!initialised) { 149 | console.log('carplay not initialised, retrying in 2s') 150 | setTimeout(this.start, 2000) 151 | } 152 | } 153 | 154 | stop = async () => { 155 | try { 156 | this.clearPairTimeout() 157 | this.clearFrameInterval() 158 | await this.dongleDriver.close() 159 | } catch (err) { 160 | console.error(err) 161 | } 162 | } 163 | 164 | private clearPairTimeout() { 165 | if (this._pairTimeout) { 166 | clearTimeout(this._pairTimeout) 167 | this._pairTimeout = null 168 | } 169 | } 170 | 171 | private clearFrameInterval() { 172 | if (this._frameInterval) { 173 | clearInterval(this._frameInterval) 174 | this._pairTimeout = null 175 | } 176 | } 177 | 178 | sendKey = (action: CommandValue) => { 179 | this.dongleDriver.send(new SendCommand(action)) 180 | } 181 | sendTouch = ({ type, x, y }: { type: number; x: number; y: number }) => { 182 | this.dongleDriver.send(new SendTouch(x, y, type)) 183 | } 184 | 185 | public onmessage: ((ev: CarplayMessage) => void) | null = null 186 | } 187 | -------------------------------------------------------------------------------- /src/modules/messages/common.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Message, 3 | AudioData, 4 | VideoData, 5 | MediaData, 6 | BluetoothAddress, 7 | BluetoothDeviceName, 8 | BluetoothPIN, 9 | ManufacturerInfo, 10 | SoftwareVersion, 11 | Command, 12 | Plugged, 13 | WifiDeviceName, 14 | HiCarLink, 15 | BluetoothPairedList, 16 | Opened, 17 | BoxInfo, 18 | Unplugged, 19 | Phase, 20 | } from './readable.js' 21 | 22 | export enum CommandMapping { 23 | invalid = 0, //'invalid', 24 | startRecordAudio = 1, 25 | stopRecordAudio = 2, 26 | requestHostUI = 3, //'Carplay Interface My Car button clicked', 27 | siri = 5, //'Siri Button', 28 | mic = 7, //'Car Microphone', 29 | boxMic = 15, //'Box Microphone', 30 | enableNightMode = 16, // night mode 31 | disableNightMode = 17, // disable night mode 32 | wifi24g = 24, //'2.4G Wifi', 33 | wifi5g = 25, //'5G Wifi', 34 | left = 100, //'Button Left', 35 | right = 101, //'Button Right', 36 | frame = 12, 37 | audioTransferOn = 22, // Phone will Stream audio directly to car system and not dongle 38 | audioTransferOff = 23, // DEFAULT - Phone will stream audio to the dongle and it will send it over the link 39 | selectDown = 104, //'Button Select Down', 40 | selectUp = 105, //'Button Select Up', 41 | back = 106, //'Button Back', 42 | up = 113, //'Button Up', 43 | down = 114, //'Button Down', 44 | home = 200, //'Button Home', 45 | play = 201, //'Button Play', 46 | pause = 202, //'Button Pause', 47 | playOrPause = 203, //'Button Switch Play/Pause', 48 | next = 204, //'Button Next Track', 49 | prev = 205, //'Button Prev Track', 50 | acceptPhone = 300, //'Accept Phone Call', 51 | rejectPhone = 301, //'Reject Phone Call', 52 | requestVideoFocus = 500, 53 | releaseVideoFocus = 501, 54 | wifiEnable = 1000, 55 | autoConnetEnable = 1001, 56 | wifiConnect = 1002, 57 | scanningDevice = 1003, 58 | deviceFound = 1004, 59 | deviceNotFound = 1005, 60 | connectDeviceFailed = 1006, 61 | btConnected = 1007, 62 | btDisconnected = 1008, 63 | wifiConnected = 1009, 64 | wifiDisconnected = 1010, 65 | btPairStart = 1011, 66 | wifiPair = 1012, 67 | } 68 | 69 | export type CommandValue = keyof typeof CommandMapping 70 | 71 | export enum MessageType { 72 | Open = 0x01, 73 | Plugged = 0x02, 74 | Phase = 0x03, 75 | Unplugged = 0x04, 76 | Touch = 0x05, 77 | VideoData = 0x06, 78 | AudioData = 0x07, 79 | Command = 0x08, 80 | LogoType = 0x09, 81 | DisconnectPhone = 0xf, 82 | CloseDongle = 0x15, 83 | BluetoothAddress = 0x0a, 84 | BluetoothPIN = 0x0c, 85 | BluetoothDeviceName = 0x0d, 86 | WifiDeviceName = 0x0e, 87 | BluetoothPairedList = 0x12, 88 | ManufacturerInfo = 0x14, 89 | MultiTouch = 0x17, 90 | HiCarLink = 0x18, 91 | BoxSettings = 0x19, 92 | MediaData = 0x2a, 93 | SendFile = 0x99, 94 | HeartBeat = 0xaa, 95 | SoftwareVersion = 0xcc, 96 | } 97 | 98 | export class HeaderBuildError extends Error {} 99 | 100 | export class MessageHeader { 101 | length: number 102 | type: MessageType 103 | 104 | public constructor(length: number, type: MessageType) { 105 | this.length = length 106 | this.type = type 107 | } 108 | 109 | static fromBuffer(data: Buffer): MessageHeader { 110 | if (data.length !== 16) { 111 | throw new HeaderBuildError( 112 | `Invalid buffer size - Expecting 16, got ${data.length}`, 113 | ) 114 | } 115 | const magic = data.readUInt32LE(0) 116 | if (magic !== MessageHeader.magic) { 117 | throw new HeaderBuildError(`Invalid magic number, received ${magic}`) 118 | } 119 | const length = data.readUInt32LE(4) 120 | const msgType: MessageType = data.readUInt32LE(8) 121 | const typeCheck = data.readUInt32LE(12) 122 | if (typeCheck != ((msgType ^ -1) & 0xffffffff) >>> 0) { 123 | throw new HeaderBuildError(`Invalid type check, received ${typeCheck}`) 124 | } 125 | return new MessageHeader(length, msgType) 126 | } 127 | 128 | static asBuffer(messageType: MessageType, byeLength: number): Buffer { 129 | const dataLen = Buffer.alloc(4) 130 | dataLen.writeUInt32LE(byeLength) 131 | const type = Buffer.alloc(4) 132 | type.writeUInt32LE(messageType) 133 | const typeCheck = Buffer.alloc(4) 134 | typeCheck.writeUInt32LE(((messageType ^ -1) & 0xffffffff) >>> 0) 135 | const magicNumber = Buffer.alloc(4) 136 | magicNumber.writeUInt32LE(MessageHeader.magic) 137 | return Buffer.concat([magicNumber, dataLen, type, typeCheck]) 138 | } 139 | 140 | toMessage(data?: Buffer): Message | null { 141 | const { type } = this 142 | if (data) { 143 | switch (type) { 144 | case MessageType.AudioData: 145 | return new AudioData(this, data) 146 | case MessageType.VideoData: 147 | return new VideoData(this, data) 148 | case MessageType.MediaData: 149 | return new MediaData(this, data) 150 | case MessageType.BluetoothAddress: 151 | return new BluetoothAddress(this, data) 152 | case MessageType.BluetoothDeviceName: 153 | return new BluetoothDeviceName(this, data) 154 | case MessageType.BluetoothPIN: 155 | return new BluetoothPIN(this, data) 156 | case MessageType.ManufacturerInfo: 157 | return new ManufacturerInfo(this, data) 158 | case MessageType.SoftwareVersion: 159 | return new SoftwareVersion(this, data) 160 | case MessageType.Command: 161 | return new Command(this, data) 162 | case MessageType.Plugged: 163 | return new Plugged(this, data) 164 | case MessageType.WifiDeviceName: 165 | return new WifiDeviceName(this, data) 166 | case MessageType.HiCarLink: 167 | return new HiCarLink(this, data) 168 | case MessageType.BluetoothPairedList: 169 | return new BluetoothPairedList(this, data) 170 | case MessageType.Open: 171 | return new Opened(this, data) 172 | case MessageType.BoxSettings: 173 | return new BoxInfo(this, data) 174 | case MessageType.Phase: 175 | return new Phase(this, data) 176 | default: 177 | console.debug( 178 | `Unknown message type: ${type}, data: ${data.toString()}`, 179 | ) 180 | return null 181 | } 182 | } else { 183 | switch (type) { 184 | case MessageType.Unplugged: 185 | return new Unplugged(this) 186 | default: 187 | console.debug(`Unknown message type without data: ${type}`) 188 | return null 189 | } 190 | } 191 | } 192 | 193 | static dataLength = 16 194 | static magic = 0x55aa55aa 195 | } 196 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | useCallback, 3 | useEffect, 4 | useLayoutEffect, 5 | useMemo, 6 | useRef, 7 | useState, 8 | } from 'react' 9 | import { RotatingLines } from 'react-loader-spinner' 10 | import './App.css' 11 | import { 12 | findDevice, 13 | requestDevice, 14 | DongleConfig, 15 | CommandMapping, 16 | } from 'node-carplay/web' 17 | import { CarPlayWorker } from './worker/types' 18 | import useCarplayAudio from './useCarplayAudio' 19 | import { useCarplayTouch } from './useCarplayTouch' 20 | import { InitEvent } from './worker/render/RenderEvents' 21 | 22 | const width = window.innerWidth 23 | const height = window.innerHeight 24 | 25 | const videoChannel = new MessageChannel() 26 | const micChannel = new MessageChannel() 27 | 28 | const config: Partial = { 29 | width, 30 | height, 31 | fps: 60, 32 | mediaDelay: 300, 33 | } 34 | 35 | const RETRY_DELAY_MS = 30000 36 | 37 | function App() { 38 | const [isPlugged, setPlugged] = useState(false) 39 | const [deviceFound, setDeviceFound] = useState(null) 40 | const retryTimeoutRef = useRef(null) 41 | 42 | const canvasRef = useRef(null) 43 | const [canvasElement, setCanvasElement] = useState( 44 | null, 45 | ) 46 | 47 | const renderWorker = useMemo(() => { 48 | if (!canvasElement) return 49 | 50 | const worker = new Worker( 51 | new URL('./worker/render/Render.worker.ts', import.meta.url), 52 | ) 53 | const canvas = canvasElement.transferControlToOffscreen() 54 | worker.postMessage(new InitEvent(canvas, videoChannel.port2), [ 55 | canvas, 56 | videoChannel.port2, 57 | ]) 58 | return worker 59 | }, [canvasElement]) 60 | 61 | useLayoutEffect(() => { 62 | if (canvasRef.current) { 63 | setCanvasElement(canvasRef.current) 64 | } 65 | }, []) 66 | 67 | const carplayWorker = useMemo(() => { 68 | const worker = new Worker( 69 | new URL('./worker/CarPlay.worker.ts', import.meta.url), 70 | ) as CarPlayWorker 71 | const payload = { 72 | videoPort: videoChannel.port1, 73 | microphonePort: micChannel.port1, 74 | } 75 | worker.postMessage({ type: 'initialise', payload }, [ 76 | videoChannel.port1, 77 | micChannel.port1, 78 | ]) 79 | return worker 80 | }, []) 81 | 82 | const { processAudio, getAudioPlayer, startRecording, stopRecording } = 83 | useCarplayAudio(carplayWorker, micChannel.port2) 84 | 85 | const clearRetryTimeout = useCallback(() => { 86 | if (retryTimeoutRef.current) { 87 | clearTimeout(retryTimeoutRef.current) 88 | retryTimeoutRef.current = null 89 | } 90 | }, []) 91 | 92 | // subscribe to worker messages 93 | useEffect(() => { 94 | carplayWorker.onmessage = ev => { 95 | const { type } = ev.data 96 | switch (type) { 97 | case 'plugged': 98 | setPlugged(true) 99 | break 100 | case 'unplugged': 101 | setPlugged(false) 102 | break 103 | case 'requestBuffer': 104 | clearRetryTimeout() 105 | getAudioPlayer(ev.data.message) 106 | break 107 | case 'audio': 108 | clearRetryTimeout() 109 | processAudio(ev.data.message) 110 | break 111 | case 'media': 112 | //TODO: implement 113 | break 114 | case 'command': 115 | const { 116 | message: { value }, 117 | } = ev.data 118 | switch (value) { 119 | case CommandMapping.startRecordAudio: 120 | startRecording() 121 | break 122 | case CommandMapping.stopRecordAudio: 123 | stopRecording() 124 | break 125 | } 126 | break 127 | case 'failure': 128 | if (retryTimeoutRef.current == null) { 129 | console.error( 130 | `Carplay initialization failed -- Reloading page in ${RETRY_DELAY_MS}ms`, 131 | ) 132 | retryTimeoutRef.current = setTimeout(() => { 133 | window.location.reload() 134 | }, RETRY_DELAY_MS) 135 | } 136 | break 137 | } 138 | } 139 | }, [ 140 | carplayWorker, 141 | clearRetryTimeout, 142 | getAudioPlayer, 143 | processAudio, 144 | renderWorker, 145 | startRecording, 146 | stopRecording, 147 | ]) 148 | 149 | const checkDevice = useCallback( 150 | async (request: boolean = false) => { 151 | const device = request ? await requestDevice() : await findDevice() 152 | if (device) { 153 | setDeviceFound(true) 154 | const payload = { 155 | config, 156 | } 157 | carplayWorker.postMessage({ type: 'start', payload }) 158 | } else { 159 | setDeviceFound(false) 160 | } 161 | }, 162 | [carplayWorker], 163 | ) 164 | 165 | // usb connect/disconnect handling and device check 166 | useEffect(() => { 167 | navigator.usb.onconnect = async () => { 168 | checkDevice() 169 | } 170 | 171 | navigator.usb.ondisconnect = async () => { 172 | const device = await findDevice() 173 | if (!device) { 174 | carplayWorker.postMessage({ type: 'stop' }) 175 | setDeviceFound(false) 176 | } 177 | } 178 | 179 | checkDevice() 180 | }, [carplayWorker, checkDevice]) 181 | 182 | const onClick = useCallback(() => { 183 | checkDevice(true) 184 | }, [checkDevice]) 185 | 186 | const sendTouchEvent = useCarplayTouch(carplayWorker, width, height) 187 | 188 | const isLoading = !isPlugged 189 | 190 | return ( 191 |
196 | {isLoading && ( 197 |
207 | {deviceFound === false && ( 208 | 211 | )} 212 | {deviceFound === true && ( 213 | 220 | )} 221 |
222 | )} 223 |
238 | 243 |
244 |
245 | ) 246 | } 247 | 248 | export default App 249 | -------------------------------------------------------------------------------- /src/modules/DongleDriver.ts: -------------------------------------------------------------------------------- 1 | import EventEmitter from 'events' 2 | import { MessageHeader, HeaderBuildError } from './messages/common.js' 3 | import { PhoneType } from './messages/readable.js' 4 | import { 5 | SendableMessage, 6 | SendNumber, 7 | FileAddress, 8 | SendOpen, 9 | SendBoolean, 10 | SendString, 11 | SendBoxSettings, 12 | SendCommand, 13 | HeartBeat, 14 | } from './messages/sendable.js' 15 | 16 | const CONFIG_NUMBER = 1 17 | const MAX_ERROR_COUNT = 5 18 | 19 | export enum HandDriveType { 20 | LHD = 0, 21 | RHD = 1, 22 | } 23 | 24 | export type PhoneTypeConfig = { 25 | frameInterval: number | null 26 | } 27 | 28 | type PhoneTypeConfigMap = { 29 | [K in PhoneType]: PhoneTypeConfig 30 | } 31 | 32 | export type DongleConfig = { 33 | androidWorkMode?: boolean 34 | width: number 35 | height: number 36 | fps: number 37 | dpi: number 38 | format: number 39 | iBoxVersion: number 40 | packetMax: number 41 | phoneWorkMode: number 42 | nightMode: boolean 43 | boxName: string 44 | hand: HandDriveType 45 | mediaDelay: number 46 | audioTransferMode: boolean 47 | wifiType: '2.4ghz' | '5ghz' 48 | micType: 'box' | 'os' 49 | phoneConfig: Partial 50 | } 51 | 52 | export const DEFAULT_CONFIG: DongleConfig = { 53 | width: 800, 54 | height: 640, 55 | fps: 20, 56 | dpi: 160, 57 | format: 5, 58 | iBoxVersion: 2, 59 | phoneWorkMode: 2, 60 | packetMax: 49152, 61 | boxName: 'nodePlay', 62 | nightMode: false, 63 | hand: HandDriveType.LHD, 64 | mediaDelay: 300, 65 | audioTransferMode: false, 66 | wifiType: '5ghz', 67 | micType: 'os', 68 | phoneConfig: { 69 | [PhoneType.CarPlay]: { 70 | frameInterval: 5000, 71 | }, 72 | [PhoneType.AndroidAuto]: { 73 | frameInterval: null, 74 | }, 75 | }, 76 | } 77 | 78 | export class DriverStateError extends Error {} 79 | 80 | export class DongleDriver extends EventEmitter { 81 | private _heartbeatInterval: NodeJS.Timer | null = null 82 | private _device: USBDevice | null = null 83 | private _inEP: USBEndpoint | null = null 84 | private _outEP: USBEndpoint | null = null 85 | private errorCount = 0 86 | 87 | static knownDevices = [ 88 | { vendorId: 0x1314, productId: 0x1520 }, 89 | { vendorId: 0x1314, productId: 0x1521 }, 90 | ] 91 | 92 | // NB! Make sure to reset the device outside of this class 93 | // Resetting device through node-usb can cause transfer issues 94 | // and for it to "disappear" 95 | 96 | initialise = async (device: USBDevice) => { 97 | if (this._device) { 98 | return 99 | } 100 | 101 | try { 102 | this._device = device 103 | 104 | console.debug('initializing') 105 | if (!device.opened) { 106 | throw new DriverStateError('Illegal state - device not opened') 107 | } 108 | await this._device.selectConfiguration(CONFIG_NUMBER) 109 | 110 | if (!this._device.configuration) { 111 | throw new DriverStateError( 112 | 'Illegal state - device has no configuration', 113 | ) 114 | } 115 | 116 | console.debug('getting interface') 117 | const { 118 | interfaceNumber, 119 | alternate: { endpoints }, 120 | } = this._device.configuration.interfaces[0] 121 | 122 | const inEndpoint = endpoints.find(e => e.direction === 'in') 123 | const outEndpoint = endpoints.find(e => e.direction === 'out') 124 | 125 | if (!inEndpoint) { 126 | throw new DriverStateError('Illegal state - no IN endpoint found') 127 | } 128 | 129 | if (!outEndpoint) { 130 | throw new DriverStateError('Illegal state - no OUT endpoint found') 131 | } 132 | this._inEP = inEndpoint 133 | this._outEP = outEndpoint 134 | 135 | console.debug('claiming') 136 | await this._device.claimInterface(interfaceNumber) 137 | 138 | console.debug(this._device) 139 | } catch (err) { 140 | this.close() 141 | throw err 142 | } 143 | } 144 | 145 | send = async (message: SendableMessage) => { 146 | if (!this._device?.opened) { 147 | return null 148 | } 149 | 150 | try { 151 | const payload = message.serialise() 152 | const transferResult = await this._device?.transferOut( 153 | this._outEP!.endpointNumber, 154 | payload, 155 | ) 156 | if (transferResult.status !== 'ok') { 157 | console.error(transferResult) 158 | return false 159 | } 160 | return true 161 | } catch (err) { 162 | console.error('Failure sending message to dongle', err) 163 | return false 164 | } 165 | } 166 | 167 | private readLoop = async () => { 168 | while (this._device?.opened) { 169 | // If we error out - stop loop, emit failure 170 | if (this.errorCount >= MAX_ERROR_COUNT) { 171 | this.close() 172 | this.emit('failure') 173 | return 174 | } 175 | 176 | try { 177 | const headerData = await this._device?.transferIn( 178 | this._inEP!.endpointNumber, 179 | MessageHeader.dataLength, 180 | ) 181 | const data = headerData?.data?.buffer 182 | if (!data) { 183 | throw new HeaderBuildError('Failed to read header data') 184 | } 185 | const header = MessageHeader.fromBuffer(Buffer.from(data)) 186 | let extraData: Buffer | undefined = undefined 187 | if (header.length) { 188 | const extraDataRes = ( 189 | await this._device?.transferIn( 190 | this._inEP!.endpointNumber, 191 | header.length, 192 | ) 193 | )?.data?.buffer 194 | if (!extraDataRes) { 195 | console.error('Failed to read extra data') 196 | return 197 | } 198 | extraData = Buffer.from(extraDataRes) 199 | } 200 | 201 | const message = header.toMessage(extraData) 202 | if (message) this.emit('message', message) 203 | } catch (error) { 204 | if (error instanceof HeaderBuildError) { 205 | console.error(`Error parsing header for data`, error) 206 | } else { 207 | console.error(`Unexpected Error parsing header for data`, error) 208 | } 209 | this.errorCount++ 210 | } 211 | } 212 | } 213 | 214 | start = async (config: DongleConfig) => { 215 | if (!this._device) { 216 | throw new DriverStateError('No device set - call initialise first') 217 | } 218 | if (!this._device?.opened) { 219 | return 220 | } 221 | 222 | this.errorCount = 0 223 | const { 224 | dpi: _dpi, 225 | nightMode: _nightMode, 226 | boxName: _boxName, 227 | audioTransferMode, 228 | wifiType, 229 | micType, 230 | } = config 231 | const initMessages = [ 232 | new SendNumber(_dpi, FileAddress.DPI), 233 | new SendOpen(config), 234 | new SendBoolean(_nightMode, FileAddress.NIGHT_MODE), 235 | new SendNumber(config.hand, FileAddress.HAND_DRIVE_MODE), 236 | new SendBoolean(true, FileAddress.CHARGE_MODE), 237 | new SendString(_boxName, FileAddress.BOX_NAME), 238 | new SendBoxSettings(config), 239 | new SendCommand('wifiEnable'), 240 | new SendCommand(wifiType === '5ghz' ? 'wifi5g' : 'wifi24g'), 241 | new SendCommand(micType === 'box' ? 'boxMic' : 'mic'), 242 | new SendCommand( 243 | audioTransferMode ? 'audioTransferOn' : 'audioTransferOff', 244 | ), 245 | ] 246 | if (config.androidWorkMode) { 247 | initMessages.push( 248 | new SendBoolean(config.androidWorkMode, FileAddress.ANDROID_WORK_MODE), 249 | ) 250 | } 251 | await Promise.all(initMessages.map(this.send)) 252 | setTimeout(() => { 253 | this.send(new SendCommand('wifiConnect')) 254 | }, 1000) 255 | 256 | this.readLoop() 257 | 258 | this._heartbeatInterval = setInterval(() => { 259 | this.send(new HeartBeat()) 260 | }, 2000) 261 | } 262 | 263 | close = async () => { 264 | if (!this._device) { 265 | return 266 | } 267 | if (this._heartbeatInterval) { 268 | clearInterval(this._heartbeatInterval) 269 | this._heartbeatInterval = null 270 | } 271 | await this._device.close() 272 | this._device = null 273 | this._inEP = null 274 | this._outEP = null 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /src/modules/messages/__tests__/readable.test.ts: -------------------------------------------------------------------------------- 1 | import { MessageHeader, MessageType, CommandMapping } from '../common.js' 2 | import { 3 | AudioCommand, 4 | AudioData, 5 | BluetoothAddress, 6 | BluetoothDeviceName, 7 | BluetoothPIN, 8 | BluetoothPairedList, 9 | Command, 10 | HiCarLink, 11 | ManufacturerInfo, 12 | Plugged, 13 | SoftwareVersion, 14 | VideoData, 15 | WifiDeviceName, 16 | } from '../readable.js' 17 | 18 | describe('Readable Messages', () => { 19 | describe('Command Message', () => { 20 | it('constructs message with correct value', () => { 21 | const header = new MessageHeader(4, MessageType.Command) 22 | const data = Buffer.alloc(4) 23 | data.writeUInt32LE(CommandMapping.frame, 0) 24 | const message = header.toMessage(data) 25 | expect(message instanceof Command).toBeTruthy() 26 | expect((message as Command).value).toBe(CommandMapping.frame) 27 | }) 28 | }) 29 | 30 | describe('ManufacturerInfo Message', () => { 31 | it('constructs message with correct values', () => { 32 | const header = new MessageHeader(8, MessageType.ManufacturerInfo) 33 | const data = Buffer.alloc(8) 34 | data.writeUInt32LE(1, 0) 35 | data.writeUInt32LE(3, 4) 36 | const message = header.toMessage(data) 37 | expect(message instanceof ManufacturerInfo).toBeTruthy() 38 | expect((message as ManufacturerInfo).a).toBe(1) 39 | expect((message as ManufacturerInfo).b).toBe(3) 40 | }) 41 | }) 42 | 43 | describe('SoftwareVersion Message', () => { 44 | it('constructs message with correct values', () => { 45 | const data = Buffer.from('version 1') 46 | const header = new MessageHeader(data.length, MessageType.SoftwareVersion) 47 | const message = header.toMessage(data) 48 | expect(message instanceof SoftwareVersion).toBeTruthy() 49 | expect((message as SoftwareVersion).version).toBe('version 1') 50 | }) 51 | }) 52 | 53 | describe('BluetoothAddress Message', () => { 54 | it('constructs message with correct values', () => { 55 | const data = Buffer.from('00:11:22:33:FF:EE') 56 | const header = new MessageHeader( 57 | data.length, 58 | MessageType.BluetoothAddress, 59 | ) 60 | const message = header.toMessage(data) 61 | expect(message instanceof BluetoothAddress).toBeTruthy() 62 | expect((message as BluetoothAddress).address).toBe('00:11:22:33:FF:EE') 63 | }) 64 | }) 65 | 66 | describe('BluetoothPIN Message', () => { 67 | it('constructs message with correct values', () => { 68 | const data = Buffer.from('1234') 69 | const header = new MessageHeader(data.length, MessageType.BluetoothPIN) 70 | const message = header.toMessage(data) 71 | expect(message instanceof BluetoothPIN).toBeTruthy() 72 | expect((message as BluetoothPIN).pin).toBe('1234') 73 | }) 74 | }) 75 | 76 | describe('BluetoothDeviceName Message', () => { 77 | it('constructs message with correct values', () => { 78 | const data = Buffer.from('device 1') 79 | const header = new MessageHeader( 80 | data.length, 81 | MessageType.BluetoothDeviceName, 82 | ) 83 | const message = header.toMessage(data) 84 | expect(message instanceof BluetoothDeviceName).toBeTruthy() 85 | expect((message as BluetoothDeviceName).name).toBe('device 1') 86 | }) 87 | }) 88 | 89 | describe('HiCarLink Message', () => { 90 | it('constructs message with correct values', () => { 91 | const data = Buffer.from('hicar://some-link') 92 | const header = new MessageHeader(data.length, MessageType.HiCarLink) 93 | const message = header.toMessage(data) 94 | expect(message instanceof HiCarLink).toBeTruthy() 95 | expect((message as HiCarLink).link).toBe('hicar://some-link') 96 | }) 97 | }) 98 | 99 | describe('BluetoothPairedList Message', () => { 100 | it('constructs message with correct values', () => { 101 | const data = Buffer.from('00:11:22:33:FF:EETest') 102 | const header = new MessageHeader( 103 | data.length, 104 | MessageType.BluetoothPairedList, 105 | ) 106 | const message = header.toMessage(data) 107 | expect(message instanceof BluetoothPairedList).toBeTruthy() 108 | expect((message as BluetoothPairedList).data).toBe( 109 | '00:11:22:33:FF:EETest', 110 | ) 111 | }) 112 | }) 113 | 114 | describe('WifiDeviceName Message', () => { 115 | it('constructs message with correct values', () => { 116 | const data = Buffer.from('00:11:22:33:FF:EE') 117 | const header = new MessageHeader(data.length, MessageType.WifiDeviceName) 118 | const message = header.toMessage(data) 119 | expect(message instanceof WifiDeviceName).toBeTruthy() 120 | expect((message as WifiDeviceName).name).toBe('00:11:22:33:FF:EE') 121 | }) 122 | }) 123 | 124 | describe('Plugged Message', () => { 125 | it('constructs message with wifi when it has 8 bytes of data', () => { 126 | const data = Buffer.alloc(8) 127 | data.writeUInt32LE(3, 0) 128 | data.writeUInt32LE(1, 4) 129 | const header = new MessageHeader(data.length, MessageType.Plugged) 130 | const message = header.toMessage(data) 131 | expect(message instanceof Plugged).toBeTruthy() 132 | expect((message as Plugged).phoneType).toBe(3) 133 | expect((message as Plugged).wifi).toBe(1) 134 | }) 135 | 136 | it('constructs message with no wifi if data is not 8 bytes', () => { 137 | const data = Buffer.alloc(4) 138 | data.writeUInt32LE(3, 0) 139 | const header = new MessageHeader(data.length, MessageType.Plugged) 140 | const message = header.toMessage(data) 141 | expect(message instanceof Plugged).toBeTruthy() 142 | expect((message as Plugged).phoneType).toBe(3) 143 | expect((message as Plugged).wifi).toBeUndefined() 144 | }) 145 | }) 146 | 147 | describe('AudioData Message', () => { 148 | it('constructs message with raw audio data', () => { 149 | const data = Buffer.alloc(512) 150 | data.writeUInt32LE(1, 0) 151 | data.writeFloatLE(0.5, 4) 152 | data.writeUInt32LE(1, 8) 153 | const header = new MessageHeader(data.length, MessageType.AudioData) 154 | const message = header.toMessage(data) 155 | expect(message instanceof AudioData).toBeTruthy() 156 | expect((message as AudioData).decodeType).toBe(1) 157 | expect((message as AudioData).volume).toBe(0.5) 158 | expect((message as AudioData).audioType).toBe(1) 159 | expect((message as AudioData).data).toStrictEqual( 160 | new Int16Array(data.buffer, 12), 161 | ) 162 | }) 163 | 164 | it('constructs message with volume duration', () => { 165 | const data = Buffer.alloc(16) 166 | data.writeUInt32LE(1, 0) 167 | data.writeFloatLE(0.5, 4) 168 | data.writeUInt32LE(1, 8) 169 | data.writeFloatLE(0.5, 12) 170 | const header = new MessageHeader(data.length, MessageType.AudioData) 171 | const message = header.toMessage(data) 172 | expect(message instanceof AudioData).toBeTruthy() 173 | expect((message as AudioData).decodeType).toBe(1) 174 | expect((message as AudioData).volume).toBe(0.5) 175 | expect((message as AudioData).audioType).toBe(1) 176 | expect((message as AudioData).volumeDuration).toBe(0.5) 177 | expect((message as AudioData).data).toBeUndefined() 178 | }) 179 | 180 | it('constructs message with command', () => { 181 | const data = Buffer.alloc(13) 182 | data.writeUInt32LE(1, 0) 183 | data.writeFloatLE(0.5, 4) 184 | data.writeUInt32LE(1, 8) 185 | data.writeInt8(1, 12) 186 | const header = new MessageHeader(data.length, MessageType.AudioData) 187 | const message = header.toMessage(data) 188 | expect(message instanceof AudioData).toBeTruthy() 189 | expect((message as AudioData).decodeType).toBe(1) 190 | expect((message as AudioData).volume).toBe(0.5) 191 | expect((message as AudioData).audioType).toBe(1) 192 | expect((message as AudioData).command).toBe(AudioCommand.AudioOutputStart) 193 | expect((message as AudioData).volumeDuration).toBeUndefined() 194 | expect((message as AudioData).data).toBeUndefined() 195 | }) 196 | }) 197 | 198 | describe('VideoData Message', () => { 199 | it('constructs message with correct values', () => { 200 | const data = Buffer.alloc(512) 201 | data.writeUInt32LE(800, 0) 202 | data.writeUInt32LE(600, 4) 203 | data.writeUInt32LE(1, 8) 204 | data.writeUInt32LE(10, 12) 205 | data.writeUInt32LE(2, 16) 206 | const header = new MessageHeader(data.length, MessageType.VideoData) 207 | const message = header.toMessage(data) 208 | expect(message instanceof VideoData).toBeTruthy() 209 | expect((message as VideoData).width).toBe(800) 210 | expect((message as VideoData).height).toBe(600) 211 | expect((message as VideoData).flags).toBe(1) 212 | expect((message as VideoData).length).toBe(10) 213 | expect((message as VideoData).unknown).toBe(2) 214 | expect((message as VideoData).data).toStrictEqual(data.subarray(20)) 215 | }) 216 | }) 217 | }) 218 | -------------------------------------------------------------------------------- /src/modules/messages/readable.ts: -------------------------------------------------------------------------------- 1 | import { MessageHeader, CommandMapping } from './common.js' 2 | 3 | export enum AudioCommand { 4 | AudioOutputStart = 1, 5 | AudioOutputStop = 2, 6 | AudioInputConfig = 3, 7 | AudioPhonecallStart = 4, 8 | AudioPhonecallStop = 5, 9 | AudioNaviStart = 6, 10 | AudioNaviStop = 7, 11 | AudioSiriStart = 8, 12 | AudioSiriStop = 9, 13 | AudioMediaStart = 10, 14 | AudioMediaStop = 11, 15 | AudioAlertStart = 12, 16 | AudioAlertStop = 13, 17 | } 18 | 19 | export abstract class Message { 20 | header: MessageHeader 21 | 22 | constructor(header: MessageHeader) { 23 | this.header = header 24 | } 25 | } 26 | 27 | export class Command extends Message { 28 | value: CommandMapping 29 | 30 | constructor(header: MessageHeader, data: Buffer) { 31 | super(header) 32 | this.value = data.readUInt32LE(0) 33 | } 34 | } 35 | 36 | export class ManufacturerInfo extends Message { 37 | a: number 38 | b: number 39 | 40 | constructor(header: MessageHeader, data: Buffer) { 41 | super(header) 42 | this.a = data.readUInt32LE(0) 43 | this.b = data.readUInt32LE(4) 44 | } 45 | } 46 | 47 | export class SoftwareVersion extends Message { 48 | version: string 49 | 50 | constructor(header: MessageHeader, data: Buffer) { 51 | super(header) 52 | this.version = data.toString('ascii') 53 | } 54 | } 55 | 56 | export class BluetoothAddress extends Message { 57 | address: string 58 | 59 | constructor(header: MessageHeader, data: Buffer) { 60 | super(header) 61 | this.address = data.toString('ascii') 62 | } 63 | } 64 | 65 | export class BluetoothPIN extends Message { 66 | pin: string 67 | 68 | constructor(header: MessageHeader, data: Buffer) { 69 | super(header) 70 | this.pin = data.toString('ascii') 71 | } 72 | } 73 | 74 | export class BluetoothDeviceName extends Message { 75 | name: string 76 | 77 | constructor(header: MessageHeader, data: Buffer) { 78 | super(header) 79 | this.name = data.toString('ascii') 80 | } 81 | } 82 | 83 | export class WifiDeviceName extends Message { 84 | name: string 85 | 86 | constructor(header: MessageHeader, data: Buffer) { 87 | super(header) 88 | this.name = data.toString('ascii') 89 | } 90 | } 91 | 92 | export class HiCarLink extends Message { 93 | link: string 94 | 95 | constructor(header: MessageHeader, data: Buffer) { 96 | super(header) 97 | this.link = data.toString('ascii') 98 | } 99 | } 100 | 101 | export class BluetoothPairedList extends Message { 102 | data: string 103 | 104 | constructor(header: MessageHeader, data: Buffer) { 105 | super(header) 106 | this.data = data.toString('ascii') 107 | } 108 | } 109 | 110 | export enum PhoneType { 111 | AndroidMirror = 1, 112 | CarPlay = 3, 113 | iPhoneMirror = 4, 114 | AndroidAuto = 5, 115 | HiCar = 6, 116 | } 117 | 118 | export class Plugged extends Message { 119 | phoneType: PhoneType 120 | wifi?: number 121 | 122 | constructor(header: MessageHeader, data: Buffer) { 123 | super(header) 124 | const wifiAvail = Buffer.byteLength(data) === 8 125 | if (wifiAvail) { 126 | this.phoneType = data.readUInt32LE(0) 127 | this.wifi = data.readUInt32LE(4) 128 | console.debug( 129 | 'wifi avail, phone type: ', 130 | PhoneType[this.phoneType], 131 | ' wifi: ', 132 | this.wifi, 133 | ) 134 | } else { 135 | this.phoneType = data.readUInt32LE(0) 136 | console.debug('no wifi avail, phone type: ', PhoneType[this.phoneType]) 137 | } 138 | } 139 | } 140 | 141 | export class Unplugged extends Message { 142 | constructor(header: MessageHeader) { 143 | super(header) 144 | } 145 | } 146 | 147 | export type AudioFormat = { 148 | frequency: 48000 | 44100 | 24000 | 16000 | 8000 149 | channel: 1 | 2 150 | bitDepth: number 151 | format?: string 152 | mimeType?: string 153 | } 154 | 155 | type DecodeTypeMapping = { 156 | [key: number]: AudioFormat 157 | } 158 | 159 | export const decodeTypeMap: DecodeTypeMapping = { 160 | 1: { 161 | frequency: 44100, 162 | channel: 2, 163 | bitDepth: 16, 164 | format: "S16LE", 165 | mimeType: "audio/L16; rate=44100; channels=2" 166 | }, 167 | 2: { 168 | frequency: 44100, 169 | channel: 2, 170 | bitDepth: 16, 171 | format: "S16LE", 172 | mimeType: "audio/L16; rate=44100; channels=2" 173 | }, 174 | 3: { 175 | frequency: 8000, 176 | channel: 1, 177 | bitDepth: 16, 178 | format: "S16LE", 179 | mimeType: "audio/L16; rate=8000; channels=1" 180 | }, 181 | 4: { 182 | frequency: 48000, 183 | channel: 2, 184 | bitDepth: 16, 185 | format: "S16LE", 186 | mimeType: "audio/L16; rate=48000; channels=2" 187 | }, 188 | 5: { 189 | frequency: 16000, 190 | channel: 1, 191 | bitDepth: 16, 192 | format: "S16LE", 193 | mimeType: "audio/L16; rate=16000; channels=1" 194 | }, 195 | 6: { 196 | frequency: 24000, 197 | channel: 1, 198 | bitDepth: 16, 199 | format: "S16LE", 200 | mimeType: "audio/L16; rate=24000; channels=1" 201 | }, 202 | 7: { 203 | frequency: 16000, 204 | channel: 2, 205 | bitDepth: 16, 206 | format: "S16LE", 207 | mimeType: "audio/L16; rate=16000; channels=2" 208 | }, 209 | } 210 | 211 | export class AudioData extends Message { 212 | command?: AudioCommand 213 | decodeType: number 214 | volume: number 215 | volumeDuration?: number 216 | audioType: number 217 | data?: Int16Array 218 | 219 | constructor(header: MessageHeader, data: Buffer) { 220 | super(header) 221 | this.decodeType = data.readUInt32LE(0) 222 | this.volume = data.readFloatLE(4) 223 | this.audioType = data.readUInt32LE(8) 224 | const amount = data.length - 12 225 | if (amount === 1) { 226 | this.command = data.readInt8(12) 227 | } else if (amount === 4) { 228 | this.volumeDuration = data.readFloatLE(12) 229 | } else { 230 | this.data = new Int16Array(data.buffer, 12) 231 | } 232 | } 233 | } 234 | 235 | export class VideoData extends Message { 236 | width: number 237 | height: number 238 | flags: number 239 | length: number 240 | unknown: number 241 | data: Buffer 242 | 243 | constructor(header: MessageHeader, data: Buffer) { 244 | super(header) 245 | this.width = data.readUInt32LE(0) 246 | this.height = data.readUInt32LE(4) 247 | this.flags = data.readUInt32LE(8) 248 | this.length = data.readUInt32LE(12) 249 | this.unknown = data.readUInt32LE(16) 250 | this.data = data.subarray(20) 251 | } 252 | } 253 | 254 | enum MediaType { 255 | Data = 1, 256 | AlbumCover = 3, 257 | } 258 | 259 | export class MediaData extends Message { 260 | payload?: 261 | | { 262 | type: MediaType.Data 263 | media: { 264 | MediaSongName?: string 265 | MediaAlbumName?: string 266 | MediaArtistName?: string 267 | MediaAPPName?: string 268 | MediaSongDuration?: number 269 | MediaSongPlayTime?: number 270 | } 271 | } 272 | | { type: MediaType.AlbumCover; base64Image: string } 273 | 274 | constructor(header: MessageHeader, data: Buffer) { 275 | super(header) 276 | const type = data.readUInt32LE(0) 277 | if (type === MediaType.AlbumCover) { 278 | const imageData = data.subarray(4) 279 | this.payload = { 280 | type, 281 | base64Image: imageData.toString('base64'), 282 | } 283 | } else if (type === MediaType.Data) { 284 | const mediaData = data.subarray(4, data.length - 1) 285 | this.payload = { 286 | type, 287 | media: JSON.parse(mediaData.toString('utf8')), 288 | } 289 | } else { 290 | console.info(`Unexpected media type: ${type}`) 291 | } 292 | } 293 | } 294 | 295 | export class Opened extends Message { 296 | width: number 297 | height: number 298 | fps: number 299 | format: number 300 | packetMax: number 301 | iBox: number 302 | phoneMode: number 303 | 304 | constructor(header: MessageHeader, data: Buffer) { 305 | super(header) 306 | this.width = data.readUInt32LE(0) 307 | this.height = data.readUInt32LE(4) 308 | this.fps = data.readUInt32LE(8) 309 | this.format = data.readUInt32LE(12) 310 | this.packetMax = data.readUInt32LE(16) 311 | this.iBox = data.readUInt32LE(20) 312 | this.phoneMode = data.readUInt32LE(24) 313 | } 314 | } 315 | export class BoxInfo extends Message { 316 | settings: 317 | | { 318 | HiCar: number 319 | OemName: string 320 | WiFiChannel: number 321 | boxType: string 322 | hwVersion: string 323 | productType: string 324 | uuid: string 325 | } 326 | | { 327 | MDLinkType: string 328 | MDModel: string 329 | MDOSVersion: string 330 | MDLinkVersion: string 331 | cpuTemp: number 332 | } 333 | constructor(header: MessageHeader, data: Buffer) { 334 | super(header) 335 | this.settings = JSON.parse(data.toString()) 336 | } 337 | } 338 | 339 | export class Phase extends Message { 340 | phase: number 341 | 342 | constructor(header: MessageHeader, data: Buffer) { 343 | super(header) 344 | //TODO: find correct mapped values 345 | this.phase = data.readUInt32LE(0) 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /src/modules/__tests__/DongleDriver.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from '@jest/globals' 2 | import { 3 | DongleDriver, 4 | DEFAULT_CONFIG, 5 | DriverStateError, 6 | } from '../DongleDriver.js' 7 | import { 8 | FileAddress, 9 | HeartBeat, 10 | SendBoolean, 11 | SendBoxSettings, 12 | SendCommand, 13 | SendNumber, 14 | SendOpen, 15 | SendString, 16 | SendableMessage, 17 | } from '../messages/index.js' 18 | import { 19 | usbDeviceFactory, 20 | deviceConfig, 21 | usbInterface, 22 | usbEndpoint, 23 | } from './mocks/usbMocks.js' 24 | 25 | const expectMessageSent = (device: USBDevice, message: SendableMessage) => { 26 | expect(device.transferOut).toHaveBeenCalledWith(1, message.serialise()) 27 | } 28 | 29 | describe('DongleDriver', () => { 30 | beforeEach(() => { 31 | jest.resetAllMocks() 32 | }) 33 | 34 | describe('Initialise method', () => { 35 | it('fails if device is not open', async () => { 36 | const driver = new DongleDriver() 37 | 38 | const device = usbDeviceFactory() 39 | 40 | await expect(driver.initialise(device)).rejects.toThrow( 41 | new DriverStateError('Illegal state - device not opened'), 42 | ) 43 | }) 44 | 45 | it('fails if device has no config', async () => { 46 | const driver = new DongleDriver() 47 | const device = usbDeviceFactory({ 48 | opened: true, 49 | configuration: undefined, 50 | }) 51 | await expect(driver.initialise(device)).rejects.toThrow( 52 | new DriverStateError('Illegal state - device has no configuration'), 53 | ) 54 | }) 55 | 56 | it('fails if device config has no IN endpoint', async () => { 57 | const driver = new DongleDriver() 58 | const device = usbDeviceFactory({ 59 | opened: true, 60 | configuration: { 61 | ...deviceConfig, 62 | interfaces: [ 63 | { 64 | ...usbInterface, 65 | alternate: { 66 | ...usbInterface.alternate, 67 | endpoints: [ 68 | { 69 | ...usbEndpoint, 70 | direction: 'out', 71 | } as USBEndpoint, 72 | ], 73 | }, 74 | }, 75 | ], 76 | }, 77 | }) 78 | await expect(driver.initialise(device)).rejects.toThrow( 79 | new DriverStateError('Illegal state - no IN endpoint found'), 80 | ) 81 | }) 82 | 83 | it('fails if device config has no out endpoint', async () => { 84 | const driver = new DongleDriver() 85 | const device = usbDeviceFactory({ 86 | opened: true, 87 | configuration: { 88 | ...deviceConfig, 89 | interfaces: [ 90 | { 91 | ...usbInterface, 92 | alternate: { 93 | ...usbInterface.alternate, 94 | endpoints: [ 95 | { 96 | ...usbEndpoint, 97 | direction: 'in', 98 | } as USBEndpoint, 99 | ], 100 | }, 101 | }, 102 | ], 103 | }, 104 | }) 105 | await expect(driver.initialise(device)).rejects.toThrow( 106 | new DriverStateError('Illegal state - no OUT endpoint found'), 107 | ) 108 | }) 109 | 110 | it('returns when device is initialised correctly', async () => { 111 | const driver = new DongleDriver() 112 | const device = usbDeviceFactory({ opened: true }) 113 | await driver.initialise(device) 114 | expect(device.selectConfiguration).toHaveBeenCalledTimes(1) 115 | expect(device.claimInterface).toHaveBeenCalledTimes(1) 116 | expect(device.claimInterface).toHaveBeenCalledWith( 117 | device.configuration?.interfaces[0].interfaceNumber, 118 | ) 119 | }) 120 | }) 121 | 122 | describe('Start method', () => { 123 | it('fails if driver is not initialised with device', async () => { 124 | const driver = new DongleDriver() 125 | await expect(driver.start(DEFAULT_CONFIG)).rejects.toThrow( 126 | new DriverStateError('No device set - call initialise first'), 127 | ) 128 | }) 129 | 130 | it('returns without sending data to device if device is not open', async () => { 131 | const driver = new DongleDriver() 132 | const device = usbDeviceFactory({ opened: true }) 133 | await driver.initialise(device) 134 | Object.defineProperty(device, 'opened', { value: false }) 135 | await driver.start(DEFAULT_CONFIG) 136 | expect(device.transferOut).toHaveBeenCalledTimes(0) 137 | }) 138 | 139 | it('returns and sends open commands to device when device is open', async () => { 140 | jest.useFakeTimers() 141 | jest.spyOn(global, 'setTimeout') 142 | jest.spyOn(global, 'setInterval') 143 | 144 | const driver = new DongleDriver() 145 | const device = usbDeviceFactory({ opened: true }) 146 | await driver.initialise(device) 147 | await driver.start(DEFAULT_CONFIG) 148 | 149 | expectMessageSent( 150 | device, 151 | new SendNumber(DEFAULT_CONFIG.dpi, FileAddress.DPI), 152 | ) 153 | expectMessageSent(device, new SendOpen(DEFAULT_CONFIG)) 154 | expectMessageSent( 155 | device, 156 | new SendBoolean(DEFAULT_CONFIG.nightMode, FileAddress.NIGHT_MODE), 157 | ) 158 | expectMessageSent( 159 | device, 160 | new SendBoolean(false, FileAddress.HAND_DRIVE_MODE), 161 | ) 162 | expectMessageSent(device, new SendBoolean(true, FileAddress.CHARGE_MODE)) 163 | expectMessageSent( 164 | device, 165 | new SendString(DEFAULT_CONFIG.boxName, FileAddress.BOX_NAME), 166 | ) 167 | expectMessageSent(device, new SendBoxSettings(DEFAULT_CONFIG)) 168 | expectMessageSent(device, new SendCommand('wifiEnable')) 169 | expectMessageSent(device, new SendCommand('audioTransferOff')) 170 | 171 | jest.runOnlyPendingTimers() 172 | 173 | // delayed wifi connect and interval messages 174 | expectMessageSent(device, new SendCommand('wifiConnect')) 175 | expectMessageSent(device, new HeartBeat()) 176 | }) 177 | 178 | it('sets up correct timeouts and intervals when device is open', async () => { 179 | jest.useFakeTimers() 180 | jest.spyOn(global, 'setTimeout') 181 | jest.spyOn(global, 'setInterval') 182 | 183 | const driver = new DongleDriver() 184 | const device = usbDeviceFactory({ opened: true }) 185 | await driver.initialise(device) 186 | await driver.start(DEFAULT_CONFIG) 187 | jest.runOnlyPendingTimers() 188 | // wifi connect 189 | expect(setTimeout).toHaveBeenCalledTimes(1) 190 | expect(setTimeout).toHaveBeenLastCalledWith(expect.any(Function), 1000) 191 | // heartbeat interval 192 | expect(setInterval).toHaveBeenCalledTimes(1) 193 | expect(setInterval).toHaveBeenCalledWith(expect.any(Function), 2000) 194 | }) 195 | }) 196 | 197 | describe('Send method', () => { 198 | it('returns null if there is no sevice', async () => { 199 | const driver = new DongleDriver() 200 | const res = await driver.send(new SendCommand('frame')) 201 | expect(res).toBeNull() 202 | }) 203 | 204 | it('returns null if device is not open', async () => { 205 | const driver = new DongleDriver() 206 | const device = usbDeviceFactory({ opened: true }) 207 | await driver.initialise(device) 208 | Object.defineProperty(device, 'opened', { value: false }) 209 | const res = await driver.send(new SendCommand('frame')) 210 | expect(res).toBeNull() 211 | expect(device.transferOut).toHaveBeenCalledTimes(0) 212 | }) 213 | 214 | it('returns true and calls transferOut with correct data if device is set and initialised', async () => { 215 | const driver = new DongleDriver() 216 | 217 | const device = usbDeviceFactory({ 218 | opened: true, 219 | transferOut: jest 220 | .fn<() => Promise>() 221 | .mockResolvedValue({ 222 | status: 'ok', 223 | bytesWritten: 1, 224 | }), 225 | }) 226 | 227 | await driver.initialise(device) 228 | const message = new SendCommand('frame') 229 | const res = await driver.send(message) 230 | expect(res).toBeTruthy() 231 | expect(device.transferOut).toHaveBeenCalledTimes(1) 232 | expect(device.transferOut).toBeCalledWith(1, message.serialise()) 233 | }) 234 | 235 | it('returns false and if transferOut indicates failure', async () => { 236 | const driver = new DongleDriver() 237 | const device = usbDeviceFactory({ 238 | opened: true, 239 | transferOut: jest 240 | .fn<() => Promise>() 241 | .mockResolvedValue({ 242 | status: 'stall', 243 | bytesWritten: 0, 244 | }), 245 | }) 246 | 247 | await driver.initialise(device) 248 | const message = new SendCommand('frame') 249 | const res = await driver.send(message) 250 | expect(res).toBeFalsy() 251 | expect(device.transferOut).toHaveBeenCalledTimes(1) 252 | expect(device.transferOut).toBeCalledWith(1, message.serialise()) 253 | }) 254 | }) 255 | 256 | describe('Close method', () => { 257 | it('returns if no device is set', async () => { 258 | const driver = new DongleDriver() 259 | await driver.close() 260 | }) 261 | 262 | it('closes device when set', async () => { 263 | const driver = new DongleDriver() 264 | const device = usbDeviceFactory({ opened: true }) 265 | await driver.initialise(device) 266 | await driver.close() 267 | expect(device.close).toHaveBeenCalledTimes(1) 268 | }) 269 | }) 270 | }) 271 | -------------------------------------------------------------------------------- /src/modules/messages/sendable.ts: -------------------------------------------------------------------------------- 1 | import { DongleConfig } from '../DongleDriver.js' 2 | import { 3 | MessageType, 4 | MessageHeader, 5 | CommandMapping, 6 | CommandValue, 7 | } from './common.js' 8 | import { clamp, getCurrentTimeInMs } from './utils.js' 9 | 10 | export abstract class SendableMessage { 11 | abstract type: MessageType 12 | 13 | serialise() { 14 | return MessageHeader.asBuffer(this.type, 0) 15 | } 16 | } 17 | 18 | export abstract class SendableMessageWithPayload extends SendableMessage { 19 | abstract type: MessageType 20 | 21 | abstract getPayload(): Buffer 22 | 23 | override serialise() { 24 | const data = this.getPayload() 25 | const byteLength = Buffer.byteLength(data) 26 | const header = MessageHeader.asBuffer(this.type, byteLength) 27 | return Buffer.concat([header, data]) 28 | } 29 | } 30 | 31 | export class SendCommand extends SendableMessageWithPayload { 32 | type = MessageType.Command 33 | value: CommandMapping 34 | 35 | getPayload(): Buffer { 36 | const data = Buffer.alloc(4) 37 | data.writeUInt32LE(this.value) 38 | return data 39 | } 40 | 41 | constructor(value: CommandValue) { 42 | super() 43 | this.value = CommandMapping[value] 44 | } 45 | } 46 | 47 | export enum TouchAction { 48 | Down = 14, 49 | Move = 15, 50 | Up = 16, 51 | } 52 | 53 | export class SendTouch extends SendableMessageWithPayload { 54 | type = MessageType.Touch 55 | x: number 56 | y: number 57 | action: TouchAction 58 | 59 | getPayload(): Buffer { 60 | const actionB = Buffer.alloc(4) 61 | const xB = Buffer.alloc(4) 62 | const yB = Buffer.alloc(4) 63 | const flags = Buffer.alloc(4) 64 | actionB.writeUInt32LE(this.action) 65 | 66 | const finalX = clamp(10000 * this.x, 0, 10000) 67 | const finalY = clamp(10000 * this.y, 0, 10000) 68 | 69 | xB.writeUInt32LE(finalX) 70 | yB.writeUInt32LE(finalY) 71 | const data = Buffer.concat([actionB, xB, yB, flags]) 72 | return data 73 | } 74 | 75 | constructor(x: number, y: number, action: TouchAction) { 76 | super() 77 | this.x = x 78 | this.y = y 79 | this.action = action 80 | } 81 | } 82 | 83 | export enum MultiTouchAction { 84 | Down = 1, 85 | Move = 2, 86 | Up = 0, 87 | } 88 | 89 | class TouchItem { 90 | x: number 91 | y: number 92 | action: MultiTouchAction 93 | id: number 94 | 95 | constructor(x: number, y: number, action: MultiTouchAction, id: number) { 96 | this.x = x 97 | this.y = y 98 | this.action = action 99 | this.id = id 100 | } 101 | 102 | getPayload(): Buffer { 103 | const actionB = Buffer.alloc(4) 104 | const xB = Buffer.alloc(4) 105 | const yB = Buffer.alloc(4) 106 | const idB = Buffer.alloc(4) 107 | actionB.writeUInt32LE(this.action) 108 | idB.writeUInt32LE(this.id) 109 | 110 | //const finalX = clamp(10000 * this.x, 0, 10000) 111 | //const finalY = clamp(10000 * this.y, 0, 10000) 112 | 113 | xB.writeFloatLE(this.x) 114 | yB.writeFloatLE(this.y) 115 | const data = Buffer.concat([xB, yB, actionB, idB]) 116 | return data 117 | } 118 | } 119 | export class SendMultiTouch extends SendableMessageWithPayload { 120 | type = MessageType.MultiTouch 121 | touches: TouchItem[] 122 | 123 | getPayload(): Buffer { 124 | const data = Buffer.concat(this.touches.map(i => i.getPayload())) 125 | return data 126 | } 127 | 128 | constructor(touchData: { x: number; y: number; action: MultiTouchAction }[]) { 129 | super() 130 | this.touches = touchData.map(({ x, y, action }, index) => { 131 | return new TouchItem(x, y, action, index) 132 | }) 133 | } 134 | } 135 | 136 | export class SendAudio extends SendableMessageWithPayload { 137 | type = MessageType.AudioData 138 | data: Int16Array 139 | 140 | getPayload(): Buffer { 141 | const audioData = Buffer.alloc(12) 142 | audioData.writeUInt32LE(5, 0) 143 | audioData.writeFloatLE(0.0, 4) 144 | audioData.writeUInt32LE(3, 8) 145 | return Buffer.concat([audioData, Buffer.from(this.data.buffer)]) 146 | } 147 | 148 | constructor(data: Int16Array) { 149 | super() 150 | this.data = data 151 | } 152 | } 153 | 154 | export class SendFile extends SendableMessageWithPayload { 155 | type = MessageType.SendFile 156 | content: Buffer 157 | fileName: string 158 | 159 | private getFileName = (name: string) => { 160 | return Buffer.from(name + '\0', 'ascii') 161 | } 162 | 163 | private getLength = (data: Buffer) => { 164 | const buffer = Buffer.alloc(4) 165 | buffer.writeUInt32LE(Buffer.byteLength(data)) 166 | return buffer 167 | } 168 | 169 | getPayload(): Buffer { 170 | const newFileName = this.getFileName(this.fileName) 171 | const nameLength = this.getLength(newFileName) 172 | const contentLength = this.getLength(this.content) 173 | const message = [nameLength, newFileName, contentLength, this.content] 174 | const data = Buffer.concat(message) 175 | return data 176 | } 177 | 178 | constructor(content: Buffer, fileName: string) { 179 | super() 180 | this.content = content 181 | this.fileName = fileName 182 | } 183 | } 184 | 185 | export enum FileAddress { 186 | DPI = '/tmp/screen_dpi', 187 | NIGHT_MODE = '/tmp/night_mode', 188 | HAND_DRIVE_MODE = '/tmp/hand_drive_mode', 189 | CHARGE_MODE = '/tmp/charge_mode', 190 | BOX_NAME = '/etc/box_name', 191 | OEM_ICON = '/etc/oem_icon.png', 192 | AIRPLAY_CONFIG = '/etc/airplay.conf', 193 | ICON_120 = '/etc/icon_120x120.png', 194 | ICON_180 = '/etc/icon_180x180.png', 195 | ICON_250 = '/etc/icon_256x256.png', 196 | ANDROID_WORK_MODE = '/etc/android_work_mode', 197 | } 198 | 199 | export class SendNumber extends SendFile { 200 | constructor(content: number, file: FileAddress) { 201 | const message = Buffer.alloc(4) 202 | message.writeUInt32LE(content) 203 | super(message, file) 204 | } 205 | } 206 | 207 | export class SendBoolean extends SendNumber { 208 | constructor(content: boolean, file: FileAddress) { 209 | super(Number(content), file) 210 | } 211 | } 212 | 213 | export class SendString extends SendFile { 214 | constructor(content: string, file: FileAddress) { 215 | if (content.length > 16) { 216 | console.error('string too long') 217 | } 218 | const message = Buffer.from(content, 'ascii') 219 | super(message, file) 220 | } 221 | } 222 | 223 | export class HeartBeat extends SendableMessage { 224 | type = MessageType.HeartBeat 225 | } 226 | 227 | export class SendOpen extends SendableMessageWithPayload { 228 | type = MessageType.Open 229 | config: DongleConfig 230 | 231 | getPayload(): Buffer { 232 | const { config } = this 233 | const width = Buffer.alloc(4) 234 | width.writeUInt32LE(config.width) 235 | const height = Buffer.alloc(4) 236 | height.writeUInt32LE(config.height) 237 | const fps = Buffer.alloc(4) 238 | fps.writeUInt32LE(config.fps) 239 | const format = Buffer.alloc(4) 240 | format.writeUInt32LE(config.format) 241 | const packetMax = Buffer.alloc(4) 242 | packetMax.writeUInt32LE(config.packetMax) 243 | const iBox = Buffer.alloc(4) 244 | iBox.writeUInt32LE(config.iBoxVersion) 245 | const phoneMode = Buffer.alloc(4) 246 | phoneMode.writeUInt32LE(config.phoneWorkMode) 247 | return Buffer.concat([ 248 | width, 249 | height, 250 | fps, 251 | format, 252 | packetMax, 253 | iBox, 254 | phoneMode, 255 | ]) 256 | } 257 | 258 | constructor(config: DongleConfig) { 259 | super() 260 | this.config = config 261 | } 262 | } 263 | 264 | export class SendBoxSettings extends SendableMessageWithPayload { 265 | type = MessageType.BoxSettings 266 | private syncTime: number | null 267 | private config: DongleConfig 268 | 269 | getPayload(): Buffer { 270 | // Intentionally using "syncTime" from now to avoid any drift 271 | // & delay between constructor() and getData() 272 | 273 | return Buffer.from( 274 | JSON.stringify({ 275 | mediaDelay: this.config.mediaDelay, 276 | syncTime: this.syncTime ?? getCurrentTimeInMs(), 277 | androidAutoSizeW: this.config.width, 278 | androidAutoSizeH: this.config.height, 279 | }), 280 | 'ascii', 281 | ) 282 | } 283 | 284 | constructor(config: DongleConfig, syncTime: number | null = null) { 285 | super() 286 | this.config = config 287 | this.syncTime = syncTime 288 | } 289 | } 290 | 291 | export enum LogoType { 292 | HomeButton = 1, 293 | Siri = 2, 294 | } 295 | 296 | export class SendLogoType extends SendableMessageWithPayload { 297 | type = MessageType.LogoType 298 | logoType: LogoType 299 | 300 | getPayload(): Buffer { 301 | const data = Buffer.alloc(4) 302 | data.writeUInt32LE(this.logoType) 303 | return data 304 | } 305 | 306 | constructor(logoType: LogoType) { 307 | super() 308 | this.logoType = logoType 309 | } 310 | } 311 | 312 | export class SendIconConfig extends SendFile { 313 | constructor(config: { label?: string }) { 314 | const valueMap: { 315 | oemIconVisible: number 316 | name: string 317 | model: string 318 | oemIconPath: string 319 | oemIconLabel?: string 320 | } = { 321 | oemIconVisible: 1, 322 | name: 'AutoBox', 323 | model: 'Magic-Car-Link-1.00', 324 | oemIconPath: FileAddress.OEM_ICON, 325 | } 326 | 327 | if (config.label) { 328 | valueMap.oemIconLabel = config.label 329 | } 330 | 331 | const fileData = Object.entries(valueMap) 332 | .map(e => `${e[0]} = ${e[1]}`) 333 | .join('\n') 334 | 335 | super(Buffer.from(fileData + '\n', 'ascii'), FileAddress.AIRPLAY_CONFIG) 336 | } 337 | } 338 | 339 | // Disconnects phone and closes dongle - need to send open command again 340 | export class SendCloseDongle extends SendableMessage { 341 | type = MessageType.CloseDongle 342 | } 343 | 344 | // Disconnects phone session - dongle is still open and phone can re-connect 345 | export class SendDisconnectPhone extends SendableMessage { 346 | type = MessageType.DisconnectPhone 347 | } 348 | -------------------------------------------------------------------------------- /examples/carplay-web-app/src/worker/render/lib/h264-utils.ts: -------------------------------------------------------------------------------- 1 | // Based on https://github.com/OllieJones/h264-interp-utils 2 | // MIT License 3 | 4 | function byte2hex(val: number) { 5 | return ('00' + val.toString(16)).slice(-2) 6 | } 7 | 8 | const validNaluTypes = new Set(['packet', 'annexB', 'unknown']) 9 | 10 | export const profileNames: Map = new Map([ 11 | [66, 'BASELINE'], 12 | [77, 'MAIN'], 13 | [88, 'EXTENDED'], 14 | [100, 'FREXT_HP'], 15 | [110, 'FREXT_Hi10P'], 16 | [122, 'FREXT_Hi422'], 17 | [244, 'FREXT_Hi444'], 18 | [44, 'FREXT_CAVLC444'], 19 | ]) 20 | 21 | export const chromaFormatValues = { 22 | 0: 'YUV400', 23 | 1: 'YUV420', 24 | 2: 'YUV422', 25 | 3: 'YUV444', 26 | } 27 | 28 | // noinspection DuplicatedCode 29 | /** 30 | * Tools for handling general bitstream issues. 31 | */ 32 | export class RawBitstream { 33 | ptr: number 34 | buffer: Uint8Array 35 | protected originalByteLength: number 36 | protected max: number 37 | 38 | /** 39 | * Construct a bitstream 40 | * @param stream Buffer containing the stream, or length in bits 41 | */ 42 | constructor(stream: Uint8Array | number) { 43 | this.ptr = 0 44 | if (typeof stream === 'number') { 45 | this.buffer = new Uint8Array((stream + 7) >> 3) 46 | this.originalByteLength = this.buffer.byteLength 47 | this.max = stream 48 | } else if (typeof stream === 'undefined') { 49 | this.buffer = new Uint8Array(8192) 50 | this.originalByteLength = this.buffer.byteLength 51 | this.max = 8192 << 3 52 | } else { 53 | this.buffer = new Uint8Array(stream, 0, stream.byteLength) 54 | this.max = this.buffer.byteLength << 3 55 | this.originalByteLength = stream.byteLength 56 | } 57 | } 58 | 59 | /** 60 | * utility / debugging function to examine next 16 bits of stream 61 | * @returns {string} Remaining unconsumed bits in the stream 62 | * (Careful: getters cannot have side-effects like advancing a pointer) 63 | */ 64 | get peek16(): string { 65 | let n = 16 66 | let p = this.ptr 67 | if (n + p > this.remaining) n = this.remaining 68 | const bitstrings = [] 69 | const hexstrings = [] 70 | /* nibble accumulators */ 71 | const bits = [] 72 | let nibble = 0 73 | for (let i = 0; i < n; i++) { 74 | const q = p >> 3 75 | const o = 0x07 - (p & 0x07) 76 | const bit = (this.buffer[q]! >> o) & 0x01 77 | nibble = (nibble << 1) | bit 78 | bits.push(bit) 79 | p++ 80 | if (i === n - 1 || i % 4 === 3) { 81 | hexstrings.push(nibble.toString(16)) 82 | let bitstring = '' 83 | bits.forEach(bit => { 84 | bitstring += bit === 0 ? '0' : '1' 85 | }) 86 | bitstrings.push(bitstring) 87 | bits.length = 0 88 | nibble = 0 89 | } 90 | } 91 | return bitstrings.join(' ') + ' ' + hexstrings.join('') 92 | } 93 | 94 | /** 95 | * number of bits remaining in the present stream 96 | * @returns {number} 97 | */ 98 | get remaining() { 99 | return this.max - this.ptr 100 | } 101 | 102 | /** 103 | * number of bits already consumed in the present stream 104 | * @returns {number} 105 | */ 106 | get consumed() { 107 | return this.ptr 108 | } 109 | 110 | seek(pos = 0) { 111 | if (pos > this.max) throw new Error('cannot seek beyond end') 112 | this.ptr = pos 113 | } 114 | 115 | reallocate(size: number) { 116 | if (this.ptr + size <= this.max) return 117 | const newSize = (0xff + Math.floor((this.max + size) * 1.25)) & ~0xff 118 | const newBuf = new Uint8Array((newSize + 7) >>> 3) 119 | this.max = newSize 120 | newBuf.set(this.buffer) 121 | this.buffer = newBuf 122 | } 123 | 124 | /** 125 | * copy bits from some other bitstream to this one 126 | * @param {RawBitstream} from the source of the copy 127 | * @param {number} ptr the starting bit position of the copy in "from" 128 | * @param {number} count the number of bits to copy 129 | * @param {number|undefined} to the starting bit position to receive the copy, or the current pointer 130 | */ 131 | copyBits( 132 | from: RawBitstream, 133 | ptr: number, 134 | count: number, 135 | to: number | undefined, 136 | ) { 137 | /* this is a little intricate for the sake of performance */ 138 | this.reallocate(count) 139 | /* handle pointer saving. */ 140 | const savedFromPtr = from.ptr 141 | const savedToPtr = this.ptr 142 | from.ptr = ptr 143 | if (typeof to === 'number') this.ptr = to 144 | 145 | /* split the copy into a starting fragment of < 8 bits, 146 | * a multiple of 8 bits, 147 | * and an ending fragment of less than 8 bits 148 | */ 149 | const firstFragLen = (8 - this.ptr) & 0x07 150 | const lastFragLen = (count - firstFragLen) & 0x07 151 | const byteCopyLen = count - (firstFragLen + lastFragLen) 152 | 153 | /* copy first fragment bit by bit */ 154 | for (let i = 0; i < firstFragLen; i++) { 155 | const b = from.u_1() 156 | this.put_u_1(b) 157 | } 158 | 159 | /* copy whole bytes byte-by-byte */ 160 | const thisbuf = this.buffer 161 | const frombuf = from.buffer 162 | let q = this.ptr >> 3 163 | const byteLen = byteCopyLen >> 3 164 | const lshift = from.ptr & 0x07 165 | if (lshift === 0) { 166 | /* byte-aligned source and dest */ 167 | let r = from.ptr >> 3 168 | 169 | /* four-way loop unroll */ 170 | let n = byteLen & 0x03 171 | while (n-- > 0) { 172 | thisbuf[q++] = frombuf[r++]! 173 | } 174 | n = byteLen >> 2 175 | while (n-- > 0) { 176 | thisbuf[q++] = frombuf[r++]! 177 | thisbuf[q++] = frombuf[r++]! 178 | thisbuf[q++] = frombuf[r++]! 179 | thisbuf[q++] = frombuf[r++]! 180 | } 181 | } else { 182 | /* unaligned source, retrieve it with masks and shifts */ 183 | const rshift = 8 - lshift 184 | const mask = (0xff << rshift) & 0xff 185 | let p = (from.ptr >> 3) + 1 186 | let v1 = frombuf[p - 1]! 187 | let v2 = frombuf[p]! 188 | 189 | /* 8-way loop unroll. 190 | * This is a hot path when changing pic_parameter_set_ids in Slices. */ 191 | let n = byteLen & 0x07 192 | while (n-- > 0) { 193 | thisbuf[q++] = ((v1 & ~mask) << lshift) | ((v2 & mask) >> rshift) 194 | v1 = v2 195 | v2 = frombuf[++p]! 196 | } 197 | n = byteLen >> 3 198 | while (n-- > 0) { 199 | /* flip back and forth between v1 and v2 */ 200 | thisbuf[q++] = ((v1 & ~mask) << lshift) | ((v2 & mask) >> rshift) 201 | v1 = frombuf[++p]! 202 | 203 | thisbuf[q++] = ((v2 & ~mask) << lshift) | ((v1 & mask) >> rshift) 204 | v2 = frombuf[++p]! 205 | 206 | thisbuf[q++] = ((v1 & ~mask) << lshift) | ((v2 & mask) >> rshift) 207 | v1 = frombuf[++p]! 208 | 209 | thisbuf[q++] = ((v2 & ~mask) << lshift) | ((v1 & mask) >> rshift) 210 | v2 = frombuf[++p]! 211 | 212 | thisbuf[q++] = ((v1 & ~mask) << lshift) | ((v2 & mask) >> rshift) 213 | v1 = frombuf[++p]! 214 | 215 | thisbuf[q++] = ((v2 & ~mask) << lshift) | ((v1 & mask) >> rshift) 216 | v2 = frombuf[++p]! 217 | 218 | thisbuf[q++] = ((v1 & ~mask) << lshift) | ((v2 & mask) >> rshift) 219 | v1 = frombuf[++p]! 220 | 221 | thisbuf[q++] = ((v2 & ~mask) << lshift) | ((v1 & mask) >> rshift) 222 | v2 = frombuf[++p]! 223 | } 224 | } 225 | from.ptr += byteCopyLen 226 | this.ptr += byteCopyLen 227 | 228 | /* copy the last fragment bit by bit */ 229 | for (let i = 0; i < lastFragLen; i++) { 230 | const b = from.u_1() 231 | this.put_u_1(b) 232 | } 233 | 234 | /* restore saved pointers */ 235 | from.ptr = savedFromPtr 236 | if (typeof to === 'number') this.ptr = savedToPtr 237 | } 238 | 239 | /** 240 | * put one bit 241 | */ 242 | put_u_1(b: number) { 243 | if (this.ptr + 1 > this.max) 244 | throw new Error('NALUStream error: bitstream exhausted') 245 | const p = this.ptr >> 3 246 | const o = 0x07 - (this.ptr & 0x07) 247 | const val = b << o 248 | const mask = ~(1 << o) 249 | this.buffer[p] = (this.buffer[p]! & mask) | val 250 | this.ptr++ 251 | return val 252 | } 253 | 254 | /** 255 | * get one bit 256 | * @returns {number} 257 | */ 258 | u_1() { 259 | if (this.ptr + 1 > this.max) 260 | throw new Error('NALUStream error: bitstream exhausted') 261 | const p = this.ptr >> 3 262 | const o = 0x07 - (this.ptr & 0x07) 263 | const val = (this.buffer[p]! >> o) & 0x01 264 | this.ptr++ 265 | return val 266 | } 267 | 268 | /** 269 | * get two bits 270 | * @returns {number} 271 | */ 272 | u_2() { 273 | return (this.u_1() << 1) | this.u_1() 274 | } 275 | 276 | /** 277 | * get three bits 278 | * @returns {number} 279 | */ 280 | u_3() { 281 | return (this.u_1() << 2) | (this.u_1() << 1) | this.u_1() 282 | } 283 | 284 | /** 285 | * get n bits 286 | * @param n 287 | * @returns {number} 288 | */ 289 | u(n: number) { 290 | if (n === 8) return this.u_8() 291 | if (this.ptr + n >= this.max) 292 | throw new Error('NALUStream error: bitstream exhausted') 293 | let val = 0 294 | for (let i = 0; i < n; i++) { 295 | val = (val << 1) | this.u_1() 296 | } 297 | return val 298 | } 299 | 300 | /** 301 | * get one byte (as an unsigned number) 302 | * @returns {number} 303 | */ 304 | u_8() { 305 | if (this.ptr + 8 > this.max) 306 | throw new Error('NALUStream error: bitstream exhausted') 307 | const o = this.ptr & 0x07 308 | if (o === 0) { 309 | const val = this.buffer[this.ptr >> 3] 310 | this.ptr += 8 311 | return val 312 | } else { 313 | const n = 8 - o 314 | const rmask = (0xff << n) & 0xff 315 | const lmask = ~rmask & 0xff 316 | const p = this.ptr >> 3 317 | this.ptr += 8 318 | return ( 319 | ((this.buffer[p]! & lmask) << o) | ((this.buffer[p + 1]! & rmask) >> n) 320 | ) 321 | } 322 | } 323 | 324 | /** 325 | * get an unsigned H.264-style variable-bit number 326 | * in exponential Golomb format 327 | * @returns {number} 328 | */ 329 | ue_v() { 330 | let zeros = 0 331 | while (!this.u_1()) zeros++ 332 | let val = 1 << zeros 333 | for (let i = zeros - 1; i >= 0; i--) { 334 | val |= this.u_1() << i 335 | } 336 | return val - 1 337 | } 338 | 339 | put_u8(val: number) { 340 | this.reallocate(8) 341 | if ((this.ptr & 0x07) === 0) { 342 | this.buffer[this.ptr >> 3] = val 343 | this.ptr += 8 344 | return 345 | } 346 | this.put_u(val, 8) 347 | } 348 | 349 | put_u(val: number, count: number) { 350 | this.reallocate(count) 351 | if (count === 0) return 352 | while (count > 0) { 353 | count-- 354 | this.put_u_1((val >> count) & 0x01) 355 | } 356 | } 357 | 358 | /** 359 | * Put an exponential-Golomb coded unsigned integer into the bitstream 360 | * https://en.wikipedia.org/wiki/Exponential-Golomb_coding 361 | * @param {number} val to insert 362 | * @returns {number} count of bits inserted 363 | */ 364 | put_ue_v(val: number) { 365 | const v = val + 1 366 | let v1 = v 367 | let z = -1 368 | do { 369 | z++ 370 | v1 = v1 >> 1 371 | } while (v1 !== 0) 372 | this.put_u(0, z) 373 | this.put_u(v, z + 1) 374 | return z + z + 1 375 | } 376 | 377 | /** 378 | * when done putting into a buffer, mark it complete, 379 | * rewind it to the beginning, and shorten its contents, 380 | * as if it had just been loaded. 381 | * @returns {number} the number of bits in the buffer 382 | */ 383 | put_complete() { 384 | const newLength = this.ptr 385 | const newByteLength = (newLength + 7) >> 3 386 | this.buffer = this.buffer.subarray(0, newByteLength) 387 | this.originalByteLength = newByteLength 388 | this.max = newLength 389 | this.ptr = 0 390 | return newLength 391 | } 392 | 393 | /** 394 | * get a signed h.264-style variable bit number 395 | * in exponential Golomb format 396 | * @returns {number} (without negative zeros) 397 | */ 398 | se_v() { 399 | const codeword = this.ue_v() 400 | const result = codeword & 0x01 ? 1 + (codeword >> 1) : -(codeword >> 1) 401 | return result === 0 ? 0 : result 402 | } 403 | 404 | /** 405 | * Put an exponential-Golomb coded signed integer into the bitstream 406 | * https://en.wikipedia.org/wiki/Exponential-Golomb_coding#Extension_to_negative_numbers 407 | * @param {number} val to insert 408 | * @returns {number} count of bits inserted 409 | */ 410 | put_se_v(val: number) { 411 | const cw = val <= 0 ? -val << 1 : (val << 1) - 1 412 | return this.put_ue_v(cw) 413 | } 414 | } 415 | 416 | /** 417 | * Tools for handling h264 bitstream issues. 418 | */ 419 | export class Bitstream extends RawBitstream { 420 | deemulated: boolean = false 421 | /** 422 | * Construct a bitstream 423 | * @param stream Buffer containing the stream, or length in bits 424 | */ 425 | constructor(stream: Uint8Array | number) { 426 | super(stream) 427 | if (typeof stream !== 'number' && typeof stream !== 'undefined') { 428 | this.deemulated = this.hasEmulationPrevention(this.buffer) 429 | this.buffer = this.deemulated ? this.deemulate(this.buffer) : this.buffer 430 | this.max = this.buffer.byteLength << 3 431 | } 432 | } 433 | 434 | get stream() { 435 | return this.deemulated ? this.reemulate(this.buffer) : this.buffer 436 | } 437 | 438 | copyBits(from: Bitstream, ptr: number, count: number, to: number) { 439 | this.deemulated = from.deemulated 440 | super.copyBits(from, ptr, count, to) 441 | } 442 | 443 | /** 444 | * add emulation prevention bytes 445 | * @param {Uint8Array} buf 446 | * @returns {Uint8Array} 447 | */ 448 | reemulate(buf: Uint8Array) { 449 | const size = Math.floor(this.originalByteLength * 1.2) 450 | const stream = new Uint8Array(size) 451 | const len = buf.byteLength - 1 452 | let q = 0 453 | let p = 0 454 | stream[p++] = buf[q++]! 455 | stream[p++] = buf[q++]! 456 | while (q < len) { 457 | if (buf[q - 2] === 0 && buf[q - 1] === 0 && buf[q]! <= 3) { 458 | stream[p++] = 3 459 | stream[p++] = buf[q++]! 460 | } 461 | stream[p++]! = buf[q++]! 462 | } 463 | stream[p++] = buf[q++]! 464 | return stream.subarray(0, p) 465 | } 466 | 467 | hasEmulationPrevention(stream: Uint8Array) { 468 | /* maybe no need to remove emulation protection? scan for 00 00 */ 469 | for (let i = 1; i < stream.byteLength; i++) { 470 | if (stream[i - 1] === 0 && stream[i] === 0) { 471 | return true 472 | } 473 | } 474 | return false 475 | } 476 | 477 | /** 478 | * remove the emulation prevention bytes 479 | * @param {Uint8Array} stream 480 | * @returns {Uint8Array} 481 | */ 482 | deemulate(stream: Uint8Array) { 483 | const buf = new Uint8Array(stream.byteLength) 484 | let p = 0 485 | let q = 0 486 | const len = stream.byteLength - 1 487 | buf[q++] = stream[p++]! 488 | buf[q++] = stream[p++]! 489 | /* remove emulation prevention: 00 00 03 00 means 00 00 00, 00 00 03 01 means 00 00 01 */ 490 | while (p < len) { 491 | if ( 492 | stream[p - 2] === 0 && 493 | stream[p - 1] === 0 && 494 | stream[p] === 3 && 495 | stream[p]! <= 3 496 | ) 497 | p++ 498 | else buf[q++] = stream[p++]! 499 | } 500 | buf[q++] = stream[p++]! 501 | return buf.subarray(0, q) 502 | } 503 | } 504 | 505 | export type StreamType = 'packet' | 'annexB' | 'unknown' 506 | 507 | type NextPackageResult = { n: number; s: number; e: number; message?: string } 508 | 509 | export class NALUStream { 510 | strict: boolean 511 | type: StreamType | null 512 | buf: Uint8Array 513 | boxSize: number | null 514 | cursor: number 515 | nextPacket: 516 | | ((buf: Uint8Array, p: number, boxSize: number) => NextPackageResult) 517 | | undefined 518 | 519 | /** 520 | * Construct a NALUStream from a buffer, figuring out what kind of stream it 521 | * is when the options are omitted. 522 | * @param {Uint8Array} buf buffer with a sequence of one or more NALUs 523 | * @param options Pareser options 524 | * @param {boolean} options.strict "Throw additional exceptions" 525 | * @param {number} options.boxSize box size 526 | * @param {number} options.boxSizeMinusOne 527 | * @param {"packet"|"annexB"|"unknown"} options.type type 528 | * 529 | * { strict: boolean, boxSize: number, boxSizeMinusOne: number , type='packet' |'annexB'}, 530 | */ 531 | constructor( 532 | buf: Uint8Array, 533 | options: { 534 | strict?: boolean 535 | boxSize?: number 536 | boxSizeMinusOne?: number 537 | type: StreamType 538 | }, 539 | ) { 540 | this.strict = false 541 | this.type = null 542 | // this.buf = null; 543 | this.boxSize = null 544 | this.cursor = 0 545 | this.nextPacket = undefined 546 | 547 | if (options) { 548 | if (typeof options.strict === 'boolean') 549 | this.strict = Boolean(options.strict) 550 | if (options.boxSizeMinusOne) this.boxSize = options.boxSizeMinusOne + 1 551 | if (options.boxSize) this.boxSize = options.boxSize 552 | if (options.type) this.type = options.type 553 | if (this.type && !validNaluTypes.has(this.type)) 554 | throw new Error('NALUStream error: type must be packet or annexB') 555 | } 556 | 557 | if (this.strict && this.boxSize && (this.boxSize < 2 || this.boxSize > 6)) 558 | throw new Error('NALUStream error: invalid boxSize') 559 | 560 | /* don't copy this.buf from input, just project it */ 561 | this.buf = new Uint8Array(buf, 0, buf.length) 562 | 563 | if (!this.type || !this.boxSize) { 564 | const { type, boxSize } = this.getType(4) 565 | this.type = type 566 | this.boxSize = boxSize 567 | } 568 | this.nextPacket = 569 | this.type === 'packet' 570 | ? this.nextLengthCountedPacket 571 | : this.nextAnnexBPacket 572 | } 573 | 574 | get boxSizeMinusOne() { 575 | return this.boxSize! - 1 576 | } 577 | 578 | /** 579 | * getter for number of NALUs in the stream 580 | * @returns {number} 581 | */ 582 | get packetCount() { 583 | return this.iterate() 584 | } 585 | 586 | /** 587 | * Returns an array of NALUs 588 | * NOTE WELL: this yields subarrays of the NALUs in the stream, not copies. 589 | * so changing the NALU contents also changes the stream. Beware. 590 | * @returns {[]} 591 | */ 592 | get packets() { 593 | const pkts: Uint8Array[] = [] 594 | this.iterate((buf, first, last) => { 595 | const pkt = buf.subarray(first, last) 596 | pkts.push(pkt) 597 | }) 598 | return pkts 599 | } 600 | 601 | /** 602 | * read an n-byte unsigned number 603 | * @param buff 604 | * @param ptr 605 | * @param boxSize 606 | * @returns {number} 607 | */ 608 | static readUIntNBE(buff: Uint8Array, ptr: number, boxSize: number) { 609 | if (!boxSize) throw new Error('readUIntNBE error: need a boxsize') 610 | let result = 0 | 0 611 | for (let i = ptr; i < ptr + boxSize; i++) { 612 | result = (result << 8) | buff[i]! 613 | } 614 | return result 615 | } 616 | 617 | static array2hex(array: Uint8Array) { 618 | // buffer is an ArrayBuffer 619 | return Array.prototype.map 620 | .call(new Uint8Array(array, 0, array.byteLength), x => 621 | ('00' + x.toString(16)).slice(-2), 622 | ) 623 | .join(' ') 624 | } 625 | 626 | /** 627 | * Iterator allowing 628 | * for (const nalu of stream) { } 629 | * Yields, space-efficiently, the elements of the stream 630 | * NOTE WELL: this yields subarrays of the NALUs in the stream, not copies. 631 | * so changing the NALU contents also changes the stream. Beware. 632 | * @returns {{next: next}} 633 | */ 634 | [Symbol.iterator]() { 635 | let delim = { n: 0, s: 0, e: 0 } 636 | return { 637 | next: () => { 638 | if (this.type === 'unknown' || this.boxSize! < 1 || delim.n < 0) 639 | return { value: undefined, done: true } 640 | delim = this.nextPacket?.(this.buf, delim.n, this.boxSize!) ?? { 641 | n: 0, 642 | s: 0, 643 | e: 0, 644 | } 645 | while (true) { 646 | if (delim.e > delim.s) { 647 | const pkt = this.buf.subarray(delim.s, delim.e) 648 | return { value: pkt, done: false } 649 | } 650 | if (delim.n < 0) break 651 | delim = this.nextPacket?.(this.buf, delim.n, this.boxSize!) ?? { 652 | n: 0, 653 | s: 0, 654 | e: 0, 655 | } 656 | } 657 | return { value: undefined, done: true } 658 | }, 659 | } 660 | } 661 | 662 | /** 663 | * Iterator allowing 664 | * for (const n of stream.nalus()) { 665 | * const {rawNalu, nalu} = n 666 | * } 667 | * Yields, space-efficiently, the elements of the stream 668 | * NOTE WELL: this yields subarrays of the NALUs in the stream, not copies. 669 | * so changing the NALU contents also changes the stream. Beware. 670 | 671 | */ 672 | nalus() { 673 | return { 674 | [Symbol.iterator]: () => { 675 | let delim = { n: 0, s: 0, e: 0 } 676 | return { 677 | next: () => { 678 | if (this.type === 'unknown' || this.boxSize! < 1 || delim.n < 0) 679 | return { value: undefined, done: true } 680 | delim = this.nextPacket?.(this.buf, delim.n, this.boxSize!) ?? { 681 | n: 0, 682 | s: 0, 683 | e: 0, 684 | } 685 | while (true) { 686 | if (delim.e > delim.s) { 687 | const nalu = this.buf.subarray(delim.s, delim.e) 688 | const rawNalu = this.buf.subarray( 689 | delim.s - this.boxSize!, 690 | delim.e, 691 | ) 692 | return { value: { nalu, rawNalu }, done: false } 693 | } 694 | if (delim.n < 0) break 695 | delim = this.nextPacket?.(this.buf, delim.n, this.boxSize!) ?? { 696 | n: 0, 697 | s: 0, 698 | e: 0, 699 | } 700 | } 701 | return { value: undefined, done: true } 702 | }, 703 | } 704 | }, 705 | } 706 | } 707 | 708 | /** 709 | * Convert an annexB stream to a packet stream in place, overwriting the buffer 710 | * @returns {NALUStream} 711 | */ 712 | convertToPacket() { 713 | if (this.type === 'packet') return this 714 | /* change 00 00 00 01 delimiters to packet lengths */ 715 | if (this.type === 'annexB' && this.boxSize === 4) { 716 | this.iterate((buff, first, last) => { 717 | let p = first - 4 718 | if (p < 0) throw new Error('NALUStream error: Unexpected packet format') 719 | const len = last - first 720 | buff[p++] = 0xff & (len >> 24) 721 | buff[p++] = 0xff & (len >> 16) 722 | buff[p++] = 0xff & (len >> 8) 723 | buff[p++] = 0xff & len 724 | }) 725 | } else if (this.type === 'annexB' && this.boxSize === 3) { 726 | /* change 00 00 01 delimiters to packet lengths */ 727 | this.iterate((buff, first, last) => { 728 | let p = first - 3 729 | if (p < 0) throw new Error('Unexpected packet format') 730 | const len = last - first 731 | if (this.strict && 0xff && len >> 24 !== 0) 732 | throw new Error( 733 | 'NALUStream error: Packet too long to store length when boxLenMinusOne is 2', 734 | ) 735 | buff[p++] = 0xff & (len >> 16) 736 | buff[p++] = 0xff & (len >> 8) 737 | buff[p++] = 0xff & len 738 | }) 739 | } 740 | this.type = 'packet' 741 | this.nextPacket = this.nextLengthCountedPacket 742 | 743 | return this 744 | } 745 | 746 | convertToAnnexB() { 747 | if (this.type === 'annexB') return this 748 | 749 | if (this.type === 'packet' && this.boxSize === 4) { 750 | this.iterate((buff, first) => { 751 | let p = first - 4 752 | if (p < 0) throw new Error('NALUStream error: Unexpected packet format') 753 | buff[p++] = 0xff & 0 754 | buff[p++] = 0xff & 0 755 | buff[p++] = 0xff & 0 756 | buff[p++] = 0xff & 1 757 | }) 758 | } else if (this.type === 'packet' && this.boxSize === 3) { 759 | this.iterate((buff, first) => { 760 | let p = first - 3 761 | if (p < 0) throw new Error('Unexpected packet format') 762 | buff[p++] = 0xff & 0 763 | buff[p++] = 0xff & 0 764 | buff[p++] = 0xff & 1 765 | }) 766 | } 767 | this.type = 'annexB' 768 | this.nextPacket = this.nextAnnexBPacket 769 | 770 | return this 771 | } 772 | 773 | iterate( 774 | callback: 775 | | ((buf: Uint8Array, s: number, e: number) => void) 776 | | undefined = undefined, 777 | ) { 778 | if (this.type === 'unknown') return 0 779 | if (this.boxSize! < 1) return 0 780 | let packetCount = 0 781 | let delim = this.nextPacket?.(this.buf, 0, this.boxSize!) ?? { 782 | n: 0, 783 | s: 0, 784 | e: 0, 785 | } 786 | while (true) { 787 | if (delim.e > delim.s) { 788 | packetCount++ 789 | if (typeof callback === 'function') callback(this.buf, delim.s, delim.e) 790 | } 791 | if (delim.n < 0) break 792 | delim = this.nextPacket?.(this.buf, delim.n, this.boxSize!) ?? { 793 | n: 0, 794 | s: 0, 795 | e: 0, 796 | } 797 | } 798 | return packetCount 799 | } 800 | 801 | /** 802 | * iterator helper for delimited streams either 00 00 01 or 00 00 00 01 803 | * @param buf 804 | * @param p 805 | * @returns iterator 806 | */ 807 | nextAnnexBPacket(buf: Uint8Array, p: number, _: number) { 808 | const buflen = buf.byteLength 809 | const start = p 810 | if (p === buflen) return { n: -1, s: start, e: p } 811 | while (p < buflen) { 812 | if (p + 2 > buflen) return { n: -1, s: start, e: buflen } 813 | if (buf[p] === 0 && buf[p + 1] === 0) { 814 | const d = buf[p + 2] 815 | if (d === 1) { 816 | /* 00 00 01 found */ 817 | return { n: p + 3, s: start, e: p } 818 | } else if (d === 0) { 819 | if (p + 3 > buflen) return { n: -1, s: start, e: buflen } 820 | const e = buf[p + 3] 821 | if (e === 1) { 822 | /* 00 00 00 01 found */ 823 | return { n: p + 4, s: start, e: p } 824 | } 825 | } 826 | } 827 | p++ 828 | } 829 | return { n: -1, s: start, e: p } 830 | } 831 | 832 | /** 833 | * iterator helper for length-counted data 834 | * @param buf 835 | * @param p 836 | * @param boxSize 837 | * @returns {{s: *, e: *, n: *}|{s: number, e: number, message: string, n: number}} 838 | */ 839 | nextLengthCountedPacket(buf: Uint8Array, p: number, boxSize: number) { 840 | const buflen = buf.byteLength 841 | if (p < buflen) { 842 | const plength = NALUStream.readUIntNBE(buf, p, boxSize) 843 | if (plength < 2 || plength > buflen + boxSize) { 844 | return { n: -2, s: 0, e: 0, message: 'bad length' } 845 | } 846 | return { 847 | n: p + boxSize + plength, 848 | s: p + boxSize, 849 | e: p + boxSize + plength, 850 | } 851 | } 852 | return { n: -1, s: 0, e: 0, message: 'end of buffer' } 853 | } 854 | 855 | /** 856 | * figure out type of data stream 857 | * @returns {{boxSize: number, type: string}} 858 | */ 859 | getType = (scanLimit: number): { boxSize: number; type: StreamType } => { 860 | if (this.type && this.boxSize) 861 | return { type: this.type, boxSize: this.boxSize } 862 | /* start with a delimiter? */ 863 | if (!this.type || this.type === 'annexB') { 864 | if (this.buf[0] === 0 && this.buf[1] === 0 && this.buf[2] === 1) { 865 | return { type: 'annexB', boxSize: 3 } 866 | } else if ( 867 | this.buf[0] === 0 && 868 | this.buf[1] === 0 && 869 | this.buf[2] === 0 && 870 | this.buf[3] === 1 871 | ) { 872 | return { type: 'annexB', boxSize: 4 } 873 | } 874 | } 875 | /* possibly packet stream with lengths */ 876 | /* try various boxSize values */ 877 | for (let boxSize = 4; boxSize >= 1; boxSize--) { 878 | let packetCount = 0 879 | if (this.buf.length <= boxSize) { 880 | packetCount = -1 881 | break 882 | } 883 | let delim = this.nextLengthCountedPacket(this.buf, 0, boxSize) 884 | while (true) { 885 | if (delim.n < -1) { 886 | packetCount = -1 887 | break 888 | } 889 | if (delim.e - delim.s) { 890 | packetCount++ 891 | if (scanLimit && packetCount >= scanLimit) break 892 | } 893 | if (delim.n < 0) break 894 | delim = this.nextLengthCountedPacket(this.buf, delim.n, boxSize) 895 | } 896 | if (packetCount > 0) { 897 | return { type: 'packet', boxSize: boxSize } 898 | } 899 | } 900 | if (this.strict) 901 | throw new Error( 902 | 'NALUStream error: cannot determine stream type or box size', 903 | ) 904 | return { type: 'unknown', boxSize: -1 } 905 | } 906 | } 907 | 908 | export class SPS { 909 | bitstream: Bitstream 910 | // private buffer: Uint8Array; 911 | nal_ref_id: number 912 | nal_unit_type: number | undefined 913 | profile_idc: number 914 | profileName: string 915 | constraint_set0_flag: number 916 | constraint_set1_flag: number 917 | constraint_set2_flag: number 918 | constraint_set3_flag: number 919 | constraint_set4_flag: number 920 | constraint_set5_flag: number 921 | level_idc: number 922 | seq_parameter_set_id: number 923 | has_no_chroma_format_idc: boolean 924 | chroma_format_idc: number | undefined 925 | bit_depth_luma_minus8: number | undefined 926 | separate_colour_plane_flag: number | undefined 927 | chromaArrayType: number | undefined 928 | bitDepthLuma: number | undefined 929 | bit_depth_chroma_minus8: number | undefined 930 | lossless_qpprime_flag: number | undefined 931 | bitDepthChroma: number | undefined 932 | seq_scaling_matrix_present_flag: number | undefined 933 | seq_scaling_list_present_flag: Array | undefined 934 | seq_scaling_list: Array | undefined 935 | log2_max_frame_num_minus4: number | undefined 936 | maxFrameNum: number 937 | pic_order_cnt_type: number 938 | log2_max_pic_order_cnt_lsb_minus4: number | undefined 939 | maxPicOrderCntLsb: number | undefined 940 | delta_pic_order_always_zero_flag: number | undefined 941 | offset_for_non_ref_pic: number | undefined 942 | offset_for_top_to_bottom_field: number | undefined 943 | num_ref_frames_in_pic_order_cnt_cycle: number | undefined 944 | offset_for_ref_frame: Array | undefined 945 | max_num_ref_frames: number 946 | gaps_in_frame_num_value_allowed_flag: number 947 | pic_width_in_mbs_minus_1: number 948 | picWidth: number 949 | pic_height_in_map_units_minus_1: number 950 | frame_mbs_only_flag: number 951 | interlaced: boolean 952 | mb_adaptive_frame_field_flag: number | undefined 953 | picHeight: number 954 | direct_8x8_inference_flag: number 955 | frame_cropping_flag: number 956 | frame_cropping_rect_left_offset: number | undefined 957 | frame_cropping_rect_right_offset: number | undefined 958 | frame_cropping_rect_top_offset: number | undefined 959 | frame_cropping_rect_bottom_offset: number | undefined 960 | cropRect: { x: number; y: number; width: number; height: number } 961 | vui_parameters_present_flag: number 962 | aspect_ratio_info_present_flag: number | undefined 963 | aspect_ratio_idc: number | undefined 964 | sar_width: number | undefined 965 | sar_height: number | undefined 966 | overscan_info_present_flag: number | undefined 967 | overscan_appropriate_flag: number | undefined 968 | video_signal_type_present_flag: number | undefined 969 | video_format: number | undefined 970 | video_full_range_flag: number | undefined 971 | color_description_present_flag: number | undefined 972 | color_primaries: number | undefined 973 | transfer_characteristics: number | undefined 974 | matrix_coefficients: number | undefined 975 | chroma_loc_info_present_flag: number | undefined 976 | chroma_sample_loc_type_top_field: number | undefined 977 | chroma_sample_loc_type_bottom_field: number | undefined 978 | timing_info_present_flag: number | undefined 979 | num_units_in_tick: number | undefined 980 | time_scale: number | undefined 981 | fixed_frame_rate_flag: number | undefined 982 | framesPerSecond: number | undefined 983 | nal_hrd_parameters_present_flag: number | undefined 984 | 985 | success: boolean 986 | 987 | constructor(SPS: Uint8Array) { 988 | const bitstream = new Bitstream(SPS) 989 | this.bitstream = bitstream 990 | // this.buffer = bitstream.buffer 991 | 992 | const forbidden_zero_bit = bitstream.u_1() 993 | if (forbidden_zero_bit) throw new Error('NALU error: invalid NALU header') 994 | this.nal_ref_id = bitstream.u_2() 995 | this.nal_unit_type = bitstream.u(5) 996 | if (this.nal_unit_type !== 7) throw new Error('SPS error: not SPS') 997 | 998 | this.profile_idc = bitstream.u_8()! 999 | if (profileNames.has(this.profile_idc)) { 1000 | this.profileName = profileNames.get(this.profile_idc)! 1001 | } else { 1002 | throw new Error('SPS error: invalid profile_idc') 1003 | } 1004 | 1005 | this.constraint_set0_flag = bitstream.u_1() 1006 | this.constraint_set1_flag = bitstream.u_1() 1007 | this.constraint_set2_flag = bitstream.u_1() 1008 | this.constraint_set3_flag = bitstream.u_1() 1009 | this.constraint_set4_flag = bitstream.u_1() 1010 | this.constraint_set5_flag = bitstream.u_1() 1011 | const reserved_zero_2bits = bitstream.u_2() 1012 | if (reserved_zero_2bits !== 0) 1013 | throw new Error('SPS error: reserved_zero_2bits must be zero') 1014 | 1015 | this.level_idc = bitstream.u_8()! 1016 | 1017 | this.seq_parameter_set_id = bitstream.ue_v() 1018 | if (this.seq_parameter_set_id > 31) 1019 | throw new Error('SPS error: seq_parameter_set_id must be 31 or less') 1020 | 1021 | this.has_no_chroma_format_idc = 1022 | this.profile_idc === 66 || 1023 | this.profile_idc === 77 || 1024 | this.profile_idc === 88 1025 | 1026 | if (!this.has_no_chroma_format_idc) { 1027 | this.chroma_format_idc = bitstream.ue_v() 1028 | if (this.bit_depth_luma_minus8 && this.bit_depth_luma_minus8 > 3) 1029 | throw new Error('SPS error: chroma_format_idc must be 3 or less') 1030 | if (this.chroma_format_idc === 3) { 1031 | /* 3 = YUV444 */ 1032 | this.separate_colour_plane_flag = bitstream.u_1() 1033 | this.chromaArrayType = this.separate_colour_plane_flag 1034 | ? 0 1035 | : this.chroma_format_idc 1036 | } 1037 | this.bit_depth_luma_minus8 = bitstream.ue_v() 1038 | if (this.bit_depth_luma_minus8 > 6) 1039 | throw new Error('SPS error: bit_depth_luma_minus8 must be 6 or less') 1040 | this.bitDepthLuma = this.bit_depth_luma_minus8 + 8 1041 | this.bit_depth_chroma_minus8 = bitstream.ue_v() 1042 | if (this.bit_depth_chroma_minus8 > 6) 1043 | throw new Error('SPS error: bit_depth_chroma_minus8 must be 6 or less') 1044 | this.lossless_qpprime_flag = bitstream.u_1() 1045 | this.bitDepthChroma = this.bit_depth_chroma_minus8 + 8 1046 | this.seq_scaling_matrix_present_flag = bitstream.u_1() 1047 | if (this.seq_scaling_matrix_present_flag) { 1048 | const n_ScalingList = this.chroma_format_idc !== 3 ? 8 : 12 1049 | this.seq_scaling_list_present_flag = [] 1050 | this.seq_scaling_list = [] 1051 | for (let i = 0; i < n_ScalingList; i++) { 1052 | const seqScalingListPresentFlag = bitstream.u_1() 1053 | this.seq_scaling_list_present_flag.push(seqScalingListPresentFlag) 1054 | if (seqScalingListPresentFlag) { 1055 | const sizeOfScalingList = i < 6 ? 16 : 64 1056 | let nextScale = 8 1057 | let lastScale = 8 1058 | const delta_scale = [] 1059 | for (let j = 0; j < sizeOfScalingList; j++) { 1060 | if (nextScale !== 0) { 1061 | const deltaScale = bitstream.se_v() 1062 | delta_scale.push(deltaScale) 1063 | nextScale = (lastScale + deltaScale + 256) % 256 1064 | } 1065 | lastScale = nextScale === 0 ? lastScale : nextScale 1066 | this.seq_scaling_list.push(delta_scale) 1067 | } 1068 | } 1069 | } 1070 | } 1071 | } 1072 | 1073 | this.log2_max_frame_num_minus4 = bitstream.ue_v() 1074 | if (this.log2_max_frame_num_minus4 > 12) 1075 | throw new Error('SPS error: log2_max_frame_num_minus4 must be 12 or less') 1076 | this.maxFrameNum = 1 << (this.log2_max_frame_num_minus4 + 4) 1077 | 1078 | this.pic_order_cnt_type = bitstream.ue_v() 1079 | if (this.pic_order_cnt_type > 2) 1080 | throw new Error('SPS error: pic_order_cnt_type must be 2 or less') 1081 | 1082 | // let expectedDeltaPerPicOrderCntCycle = 0 1083 | switch (this.pic_order_cnt_type) { 1084 | case 0: 1085 | this.log2_max_pic_order_cnt_lsb_minus4 = bitstream.ue_v() 1086 | if (this.log2_max_pic_order_cnt_lsb_minus4 > 12) 1087 | throw new Error( 1088 | 'SPS error: log2_max_pic_order_cnt_lsb_minus4 must be 12 or less', 1089 | ) 1090 | this.maxPicOrderCntLsb = 1091 | 1 << (this.log2_max_pic_order_cnt_lsb_minus4 + 4) 1092 | break 1093 | case 1: 1094 | this.delta_pic_order_always_zero_flag = bitstream.u_1() 1095 | this.offset_for_non_ref_pic = bitstream.se_v() 1096 | this.offset_for_top_to_bottom_field = bitstream.se_v() 1097 | this.num_ref_frames_in_pic_order_cnt_cycle = bitstream.ue_v() 1098 | this.offset_for_ref_frame = [] 1099 | for (let i = 0; i < this.num_ref_frames_in_pic_order_cnt_cycle; i++) { 1100 | const offsetForRefFrame = bitstream.se_v() 1101 | this.offset_for_ref_frame.push(offsetForRefFrame) 1102 | // eslint-disable-next-line no-unused-vars 1103 | // expectedDeltaPerPicOrderCntCycle += offsetForRefFrame 1104 | } 1105 | break 1106 | case 2: 1107 | /* there is nothing for case 2 */ 1108 | break 1109 | } 1110 | 1111 | this.max_num_ref_frames = bitstream.ue_v() 1112 | this.gaps_in_frame_num_value_allowed_flag = bitstream.u_1() 1113 | this.pic_width_in_mbs_minus_1 = bitstream.ue_v() 1114 | this.picWidth = (this.pic_width_in_mbs_minus_1 + 1) << 4 1115 | this.pic_height_in_map_units_minus_1 = bitstream.ue_v() 1116 | this.frame_mbs_only_flag = bitstream.u_1() 1117 | this.interlaced = !this.frame_mbs_only_flag 1118 | if (this.frame_mbs_only_flag === 0) { 1119 | /* 1 if frames rather than fields (no interlacing) */ 1120 | this.mb_adaptive_frame_field_flag = bitstream.u_1() 1121 | } 1122 | this.picHeight = 1123 | ((2 - this.frame_mbs_only_flag) * 1124 | (this.pic_height_in_map_units_minus_1 + 1)) << 1125 | 4 1126 | 1127 | this.direct_8x8_inference_flag = bitstream.u_1() 1128 | this.frame_cropping_flag = bitstream.u_1() 1129 | if (this.frame_cropping_flag) { 1130 | this.frame_cropping_rect_left_offset = bitstream.ue_v() 1131 | this.frame_cropping_rect_right_offset = bitstream.ue_v() 1132 | this.frame_cropping_rect_top_offset = bitstream.ue_v() 1133 | this.frame_cropping_rect_bottom_offset = bitstream.ue_v() 1134 | this.cropRect = { 1135 | x: this.frame_cropping_rect_left_offset, 1136 | y: this.frame_cropping_rect_top_offset, 1137 | width: 1138 | this.picWidth - 1139 | (this.frame_cropping_rect_left_offset + 1140 | this.frame_cropping_rect_right_offset), 1141 | height: 1142 | this.picHeight - 1143 | (this.frame_cropping_rect_top_offset + 1144 | this.frame_cropping_rect_bottom_offset), 1145 | } 1146 | } else { 1147 | this.cropRect = { 1148 | x: 0, 1149 | y: 0, 1150 | width: this.picWidth, 1151 | height: this.picHeight, 1152 | } 1153 | } 1154 | this.vui_parameters_present_flag = bitstream.u_1() 1155 | if (this.vui_parameters_present_flag) { 1156 | this.aspect_ratio_info_present_flag = bitstream.u_1() 1157 | if (this.aspect_ratio_info_present_flag) { 1158 | this.aspect_ratio_idc = bitstream.u_8() 1159 | if (this.aspect_ratio_idc) { 1160 | this.sar_width = bitstream.u(16) 1161 | this.sar_height = bitstream.u(16) 1162 | } 1163 | } 1164 | 1165 | this.overscan_info_present_flag = bitstream.u_1() 1166 | if (this.overscan_info_present_flag) 1167 | this.overscan_appropriate_flag = bitstream.u_1() 1168 | this.video_signal_type_present_flag = bitstream.u_1() 1169 | if (this.video_signal_type_present_flag) { 1170 | this.video_format = bitstream.u(3) 1171 | this.video_full_range_flag = bitstream.u_1() 1172 | this.color_description_present_flag = bitstream.u_1() 1173 | if (this.color_description_present_flag) { 1174 | this.color_primaries = bitstream.u_8() 1175 | this.transfer_characteristics = bitstream.u_8() 1176 | this.matrix_coefficients = bitstream.u_8() 1177 | } 1178 | } 1179 | this.chroma_loc_info_present_flag = bitstream.u_1() 1180 | if (this.chroma_loc_info_present_flag) { 1181 | this.chroma_sample_loc_type_top_field = bitstream.ue_v() 1182 | this.chroma_sample_loc_type_bottom_field = bitstream.ue_v() 1183 | } 1184 | this.timing_info_present_flag = bitstream.u_1() 1185 | if (this.timing_info_present_flag) { 1186 | this.num_units_in_tick = bitstream.u(32) 1187 | this.time_scale = bitstream.u(32) 1188 | this.fixed_frame_rate_flag = bitstream.u_1() 1189 | if ( 1190 | this.num_units_in_tick && 1191 | this.time_scale && 1192 | this.num_units_in_tick > 0 1193 | ) { 1194 | this.framesPerSecond = this.time_scale / (2 * this.num_units_in_tick) 1195 | } 1196 | } 1197 | this.nal_hrd_parameters_present_flag = bitstream.u_1() 1198 | } 1199 | this.success = true 1200 | } 1201 | 1202 | get stream() { 1203 | return this.bitstream.stream 1204 | } 1205 | 1206 | get profile_compatibility() { 1207 | let v = this.constraint_set0_flag << 7 1208 | v |= this.constraint_set1_flag << 6 1209 | v |= this.constraint_set2_flag << 5 1210 | v |= this.constraint_set3_flag << 4 1211 | v |= this.constraint_set4_flag << 3 1212 | v |= this.constraint_set5_flag << 1 1213 | return v 1214 | } 1215 | 1216 | /** 1217 | * getter for the MIME type encoded in this avcC 1218 | * @returns {string} 1219 | */ 1220 | get MIME() { 1221 | const f = [] 1222 | f.push('avc1.') 1223 | f.push(byte2hex(this.profile_idc).toUpperCase()) 1224 | f.push(byte2hex(this.profile_compatibility).toUpperCase()) 1225 | f.push(byte2hex(this.level_idc).toUpperCase()) 1226 | return f.join('') 1227 | } 1228 | } 1229 | --------------------------------------------------------------------------------