├── src
    ├── transformers
    │   ├── index.ts
    │   ├── VideoTransformer.ts
    │   ├── types.ts
    │   └── BackgroundTransformer.ts
    ├── webgl
    │   ├── shader-programs
    │   │   ├── vertexShader.ts
    │   │   ├── boxBlurShader.ts
    │   │   ├── compositeShader.ts
    │   │   ├── downSampler.ts
    │   │   └── blurShader.ts
    │   ├── utils.ts
    │   └── index.ts
    ├── utils.ts
    ├── logger.ts
    ├── index.ts
    └── ProcessorWrapper.ts
├── .changeset
    ├── twenty-coats-hang.md
    ├── cool-views-fall.md
    └── config.json
├── example
    ├── ali-kazal-tbw_KQE3Cbg-unsplash.jpg
    ├── samantha-gades-BlIhVfXbi9s-unsplash.jpg
    ├── vite.config.js
    ├── styles.css
    ├── index.html
    └── sample.ts
├── jest.config.js
├── vite.config.mjs
├── tsconfig.eslint.json
├── tsup.config.ts
├── .prettierrc
├── .eslintrc.cjs
├── NOTICE
├── .github
    └── workflows
    │   └── release.yaml
├── tsconfig.json
├── package.json
├── .gitignore
├── README.md
├── CHANGELOG.md
└── LICENSE


/src/transformers/index.ts:
--------------------------------------------------------------------------------
1 | export * from './BackgroundTransformer';
2 | export * from './types';
3 | 


--------------------------------------------------------------------------------
/.changeset/twenty-coats-hang.md:
--------------------------------------------------------------------------------
1 | ---
2 | '@livekit/track-processors': patch
3 | ---
4 | 
5 | Exports BackgroundProcessorWrapper and adds docs
6 | 


--------------------------------------------------------------------------------
/example/ali-kazal-tbw_KQE3Cbg-unsplash.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/livekit/track-processors-js/HEAD/example/ali-kazal-tbw_KQE3Cbg-unsplash.jpg


--------------------------------------------------------------------------------
/.changeset/cool-views-fall.md:
--------------------------------------------------------------------------------
1 | ---
2 | '@livekit/track-processors': patch
3 | ---
4 | 
5 | Update getLogger to return explicit StructuredLogger type
6 | 


--------------------------------------------------------------------------------
/example/samantha-gades-BlIhVfXbi9s-unsplash.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/livekit/track-processors-js/HEAD/example/samantha-gades-BlIhVfXbi9s-unsplash.jpg


--------------------------------------------------------------------------------
/jest.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 |   clearMocks: true,
3 |   modulePathIgnorePatterns: ['<rootDir>/dist/'],
4 |   preset: 'ts-jest',
5 |   testEnvironment: 'node',
6 | };
7 | 


--------------------------------------------------------------------------------
/vite.config.mjs:
--------------------------------------------------------------------------------
 1 | import { defineConfig } from 'vite';
 2 | 
 3 | export default defineConfig({
 4 |   server: {
 5 |     port: 8080,
 6 |     open: true,
 7 |     fs: {
 8 |       strict: false,
 9 |     },
10 |   },
11 | });
12 | 


--------------------------------------------------------------------------------
/tsconfig.eslint.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "extends": "./tsconfig.json",
 3 |   "include": [
 4 |     "src/**/*.ts",
 5 |     "src/**/*.js",
 6 |     "example/**/*.ts",
 7 |     "example/**/*.js",
 8 |     ".eslintrc.js",
 9 |     "jest.config.js",
10 |     "plugins/**/*.ts"
11 |   ]
12 | }
13 | 


--------------------------------------------------------------------------------
/example/vite.config.js:
--------------------------------------------------------------------------------
 1 | // vite.config.js
 2 | import { resolve } from 'path'
 3 | import { defineConfig } from 'vite'
 4 | 
 5 | export default defineConfig({
 6 |   build: {
 7 |     rollupOptions: {
 8 |       input: {
 9 |         main: resolve(__dirname, 'index.html'),
10 |       }
11 |     }
12 |   }
13 | })


--------------------------------------------------------------------------------
/tsup.config.ts:
--------------------------------------------------------------------------------
 1 | import { Options } from 'tsup';
 2 | 
 3 | const defaultOptions: Options = {
 4 |   entry: ['src/index.ts'],
 5 |   format: ['cjs', 'esm'],
 6 |   splitting: true,
 7 |   sourcemap: true,
 8 |   // for the type maps to work, we use tsc's declaration-only command
 9 |   dts: false,
10 |   clean: true,
11 |   external: ['livekit-client'],
12 | };
13 | export default defaultOptions;
14 | 


--------------------------------------------------------------------------------
/.changeset/config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "$schema": "https://unpkg.com/@changesets/config@2.0.0/schema.json",
 3 |   "changelog": ["@livekit/changesets-changelog-github", { "repo": "livekit/track-processors-js" }],
 4 |   "commit": true,
 5 |   "fixed": [],
 6 |   "linked": [],
 7 |   "access": "public",
 8 |   "baseBranch": "main",
 9 |   "updateInternalDependencies": "patch",
10 |   "ignore": []
11 | }
12 | 


--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
 1 | {
 2 |   "singleQuote": true,
 3 |   "trailingComma": "all",
 4 |   "semi": true,
 5 |   "tabWidth": 2,
 6 |   "printWidth": 100,
 7 |   "importOrder": ["<THIRD_PARTY_MODULES>", "^[./]"],
 8 |   "importOrderSeparation": false,
 9 |   "importOrderSortSpecifiers": true,
10 |   "importOrderParserPlugins": ["typescript"],
11 |   "plugins": ["@trivago/prettier-plugin-sort-imports"]
12 | }
13 | 


--------------------------------------------------------------------------------
/src/webgl/shader-programs/vertexShader.ts:
--------------------------------------------------------------------------------
 1 | // Vertex shader source
 2 | export const vertexShaderSource = (flipY: boolean = true) => `#version 300 es
 3 |   in vec2 position;
 4 |   out vec2 texCoords;
 5 | 
 6 |   void main() {
 7 |     texCoords = (position + 1.0) / 2.0;
 8 |     texCoords.y = ${flipY ? '1.0 - texCoords.y' : 'texCoords.y'};
 9 |     gl_Position = vec4(position, 0, 1.0);
10 |   }
11 | `;
12 | 


--------------------------------------------------------------------------------
/.eslintrc.cjs:
--------------------------------------------------------------------------------
 1 | module.exports = {
 2 |   root: true,
 3 |   extends: ['plugin:import/recommended', 'airbnb-typescript/base', 'prettier'],
 4 |   parserOptions: {
 5 |     project: './tsconfig.eslint.json',
 6 |   },
 7 |   rules: {
 8 |     'import/export': 'off',
 9 |     'max-classes-per-file': 'off',
10 |     'no-param-reassign': 'off',
11 |     'no-await-in-loop': 'off',
12 |     'no-restricted-syntax': 'off',
13 |     'consistent-return': 'off',
14 |     'class-methods-use-this': 'off',
15 |     'no-underscore-dangle': 'off',
16 |     '@typescript-eslint/no-use-before-define': 'off',
17 |   },
18 | };
19 | 


--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
 1 | Copyright 2023 LiveKit, Inc.
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |    http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 


--------------------------------------------------------------------------------
/.github/workflows/release.yaml:
--------------------------------------------------------------------------------
 1 | name: Release
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - main
 7 | 
 8 | concurrency: ${{ github.workflow }}-${{ github.ref }}
 9 | 
10 | jobs:
11 |   release:
12 |     name: Release
13 |     runs-on: ubuntu-latest
14 |     steps:
15 |       - name: Checkout Repo
16 |         uses: actions/checkout@v4
17 | 
18 |       - uses: pnpm/action-setup@v2
19 |         with:
20 |           version: 8
21 | 
22 |       - name: Setup Node.js 20.x
23 |         uses: actions/setup-node@v4
24 |         with:
25 |           node-version: 20.x
26 |           cache: pnpm
27 | 
28 |       - name: Install Dependencies
29 |         run: pnpm i
30 | 
31 |       - name: Create Release Pull Request or Publish to npm
32 |         id: changesets
33 |         uses: changesets/action@v1
34 |         with:
35 |           # This expects you to have a script called release which does a build for your packages and calls changeset publish
36 |           publish: pnpm release
37 |         env:
38 |           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
39 |           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
40 | 


--------------------------------------------------------------------------------
/src/utils.ts:
--------------------------------------------------------------------------------
 1 | /* eslint-disable @typescript-eslint/naming-convention */
 2 | export const supportsOffscreenCanvas = () => typeof OffscreenCanvas !== 'undefined';
 3 | 
 4 | async function sleep(time: number) {
 5 |   return new Promise((resolve) => setTimeout(resolve, time));
 6 | }
 7 | 
 8 | export async function waitForTrackResolution(track: MediaStreamTrack) {
 9 |   const timeout = 500;
10 | 
11 |   // browsers report wrong initial resolution on iOS.
12 |   // when slightly delaying the call to .getSettings(), the correct resolution is being reported
13 |   await sleep(10);
14 | 
15 |   const started = Date.now();
16 |   while (Date.now() - started < timeout) {
17 |     const { width, height } = track.getSettings();
18 |     if (width && height) {
19 |       return { width, height };
20 |     }
21 |     await sleep(50);
22 |   }
23 |   return { width: undefined, height: undefined };
24 | }
25 | 
26 | export function createCanvas(width: number, height: number) {
27 |   if (supportsOffscreenCanvas()) {
28 |     return new OffscreenCanvas(width, height);
29 |   }
30 |   const canvas = document.createElement('canvas');
31 |   canvas.width = width;
32 |   canvas.height = height;
33 |   return canvas;
34 | }
35 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "compilerOptions": {
 3 |     "lib": ["DOM", "ES2018"],
 4 |     "target": "ES2018" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
 5 |     "module": "ES2020" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
 6 |     "rootDir": "./",
 7 |     "declarationDir": "./dist",
 8 |     "outDir": "./dist",
 9 |     "declaration": true,
10 |     "sourceMap": true,
11 |     "strict": true /* Enable all strict type-checking options. */,
12 |     "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
13 |     "skipLibCheck": true /* Skip type checking of declaration files. */,
14 |     "noUnusedLocals": true,
15 |     "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
16 |     "moduleResolution": "node",
17 |     "resolveJsonModule": true
18 |   },
19 |   "include": ["src/**/*", "plugins/**/*"],
20 |   "typedocOptions": {
21 |     "entryPoints": ["src/index.ts"],
22 |     "excludeInternal": true,
23 |     "excludePrivate": true,
24 |     "excludeProtected": true,
25 |     "excludeExternals": true,
26 |     "includeVersion": true,
27 |     "name": "LiveKit track processors",
28 |     "out": "docs",
29 |     "theme": "default"
30 |   }
31 | }
32 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "@livekit/track-processors",
 3 |   "version": "0.7.0",
 4 |   "description": "LiveKit track processors",
 5 |   "main": "dist/index.js",
 6 |   "module": "dist/index.mjs",
 7 |   "source": "src/index.ts",
 8 |   "types": "dist/src/index.d.ts",
 9 |   "repository": {
10 |     "type": "git",
11 |     "url": "https://github.com/livekit/track-processors-js.git"
12 |   },
13 |   "author": "Lukas Seiler",
14 |   "license": "Apache-2.0",
15 |   "scripts": {
16 |     "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\"",
17 |     "build-sample": "cd example && vite build",
18 |     "lint": "eslint src",
19 |     "release": "pnpm build && changeset publish",
20 |     "test": "jest",
21 |     "dev": "vite example -c vite.config.mjs --open",
22 |     "sample": "pnpm run dev"
23 |   },
24 |   "files": [
25 |     "dist",
26 |     "src"
27 |   ],
28 |   "dependencies": {
29 |     "@mediapipe/tasks-vision": "0.10.14"
30 |   },
31 |   "peerDependencies": {
32 |     "@types/dom-mediacapture-transform": "^0.1.9",
33 |     "livekit-client": "^1.12.0 || ^2.1.0"
34 |   },
35 |   "devDependencies": {
36 |     "@changesets/cli": "^2.26.2",
37 |     "@livekit/changesets-changelog-github": "^0.0.4",
38 |     "@trivago/prettier-plugin-sort-imports": "^4.2.1",
39 |     "@types/offscreencanvas": "^2019.7.3",
40 |     "@typescript-eslint/eslint-plugin": "^5.62.0",
41 |     "eslint": "8.39.0",
42 |     "eslint-config-airbnb-typescript": "17.0.0",
43 |     "eslint-config-prettier": "8.8.0",
44 |     "eslint-plugin-import": "2.27.5",
45 |     "prettier": "^2.8.8",
46 |     "tsup": "^7.2.0",
47 |     "typescript": "^5.8.3",
48 |     "vite": "^7.0.2"
49 |   },
50 |   "packageManager": "pnpm@9.15.9+sha512.68046141893c66fad01c079231128e9afb89ef87e2691d69e4d40eee228988295fd4682181bae55b58418c3a253bde65a505ec7c5f9403ece5cc3cd37dcf2531"
51 | }
52 | 


--------------------------------------------------------------------------------
/src/webgl/shader-programs/boxBlurShader.ts:
--------------------------------------------------------------------------------
 1 | import { createProgram, createShader, glsl } from '../utils';
 2 | import { vertexShaderSource } from './vertexShader';
 3 | 
 4 | export const boxBlurFragmentShader = glsl`#version 300 es
 5 | precision mediump float;
 6 | 
 7 | in vec2 texCoords;
 8 | 
 9 | uniform sampler2D u_texture;
10 | uniform vec2 u_texelSize;    // 1.0 / texture size
11 | uniform vec2 u_direction;    // (1.0, 0.0) for horizontal, (0.0, 1.0) for vertical
12 | uniform float u_radius;      // blur radius in texels
13 | 
14 | out vec4 fragColor;
15 | 
16 | void main() {
17 |     vec3 sum = vec3(0.0);
18 |     float count = 0.0;
19 | 
20 |     // Limit radius to avoid excessive loop cost
21 |     const int MAX_RADIUS = 16;
22 |     int radius = int(min(float(MAX_RADIUS), u_radius));
23 | 
24 |     for (int i = -MAX_RADIUS; i <= MAX_RADIUS; ++i) {
25 |         if (abs(i) > radius) continue;
26 | 
27 |         vec2 offset = u_direction * u_texelSize * float(i);
28 |         sum += texture(u_texture, texCoords + offset).rgb;
29 |         count += 1.0;
30 |   }
31 | 
32 |   fragColor = vec4(sum / count, 1.0);
33 | }
34 | `;
35 | 
36 | /**
37 |  * Create the box blur shader program
38 |  */
39 | export function createBoxBlurProgram(gl: WebGL2RenderingContext) {
40 |   const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
41 |   const fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, boxBlurFragmentShader);
42 | 
43 |   const program = createProgram(gl, vertexShader, fragmentShader);
44 | 
45 |   // Get attribute and uniform locations
46 |   const uniforms = {
47 |     position: gl.getAttribLocation(program, 'position'),
48 |     texture: gl.getUniformLocation(program, 'u_texture'),
49 |     texelSize: gl.getUniformLocation(program, 'u_texelSize'),
50 |     direction: gl.getUniformLocation(program, 'u_direction'),
51 |     radius: gl.getUniformLocation(program, 'u_radius'),
52 |   };
53 | 
54 |   return {
55 |     program,
56 |     vertexShader,
57 |     fragmentShader,
58 |     uniforms,
59 |   };
60 | }
61 | 


--------------------------------------------------------------------------------
/src/transformers/VideoTransformer.ts:
--------------------------------------------------------------------------------
 1 | import { createCanvas } from '../utils';
 2 | import { setupWebGL } from '../webgl/index';
 3 | import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';
 4 | 
 5 | export default abstract class VideoTransformer<Options extends Record<string, unknown>>
 6 |   implements VideoTrackTransformer<Options>
 7 | {
 8 |   transformer?: TransformStream;
 9 | 
10 |   canvas?: OffscreenCanvas | HTMLCanvasElement;
11 | 
12 |   // ctx?: OffscreenCanvasRenderingContext2D;
13 | 
14 |   inputVideo?: HTMLVideoElement;
15 | 
16 |   gl?: ReturnType<typeof setupWebGL>;
17 | 
18 |   protected isDisabled?: boolean = false;
19 | 
20 |   async init({
21 |     outputCanvas,
22 |     inputElement: inputVideo,
23 |   }: VideoTransformerInitOptions): Promise<void> {
24 |     if (!(inputVideo instanceof HTMLVideoElement)) {
25 |       throw TypeError('Video transformer needs a HTMLVideoElement as input');
26 |     }
27 | 
28 |     this.transformer = new TransformStream({
29 |       transform: (frame, controller) => this.transform(frame, controller),
30 |     });
31 |     this.canvas = outputCanvas || null;
32 |     if (outputCanvas) {
33 |       // this.ctx = this.canvas?.getContext('2d') || undefined;
34 |       this.gl = setupWebGL(
35 |         this.canvas || createCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
36 |       );
37 |     }
38 |     this.inputVideo = inputVideo;
39 |     this.isDisabled = false;
40 |   }
41 | 
42 |   async restart({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
43 |     this.canvas = outputCanvas || null;
44 |     this.gl?.cleanup();
45 |     this.gl = setupWebGL(
46 |       this.canvas || createCanvas(inputVideo.videoWidth, inputVideo.videoHeight),
47 |     );
48 | 
49 |     this.inputVideo = inputVideo;
50 |     this.isDisabled = false;
51 |   }
52 | 
53 |   async destroy() {
54 |     this.isDisabled = true;
55 |     this.canvas = undefined;
56 |     this.gl?.cleanup();
57 |     this.gl = undefined;
58 |   }
59 | 
60 |   abstract transform(
61 |     frame: VideoFrame,
62 |     controller: TransformStreamDefaultController<VideoFrame>,
63 |   ): void;
64 | 
65 |   abstract update(options: Options): void;
66 | }
67 | 


--------------------------------------------------------------------------------
/src/transformers/types.ts:
--------------------------------------------------------------------------------
 1 | export type TrackTransformerInitOptions = {
 2 |   inputElement: HTMLMediaElement;
 3 | };
 4 | 
 5 | export interface VideoTransformerInitOptions extends TrackTransformerInitOptions {
 6 |   outputCanvas: OffscreenCanvas | HTMLCanvasElement;
 7 |   inputElement: HTMLVideoElement;
 8 | }
 9 | 
10 | export interface AudioTransformerInitOptions extends TrackTransformerInitOptions {}
11 | 
12 | export interface VideoTrackTransformer<Options extends Record<string, unknown>>
13 |   extends BaseTrackTransformer<VideoTransformerInitOptions, VideoFrame, TrackTransformerDestroyOptions> {
14 |   init: (options: VideoTransformerInitOptions) => void;
15 |   destroy: (options?: TrackTransformerDestroyOptions) => void;
16 |   restart: (options: VideoTransformerInitOptions) => void;
17 |   transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
18 |   transformer?: TransformStream;
19 |   update: (options: Options) => void;
20 | }
21 | 
22 | export interface AudioTrackTransformer<Options extends Record<string, unknown>>
23 |   extends BaseTrackTransformer<AudioTransformerInitOptions, AudioData, TrackTransformerDestroyOptions> {
24 |   init: (options: AudioTransformerInitOptions) => void;
25 |   destroy: (options: TrackTransformerDestroyOptions) => void;
26 |   restart: (options: AudioTransformerInitOptions) => void;
27 |   transform: (frame: AudioData, controller: TransformStreamDefaultController) => void;
28 |   transformer?: TransformStream;
29 |   update: (options: Options) => void;
30 | }
31 | 
32 | export type TrackTransformerDestroyOptions = { willProcessorRestart: boolean };
33 | 
34 | export type TrackTransformer<Options extends Record<string, unknown>> =
35 |   | VideoTrackTransformer<Options>
36 |   | AudioTrackTransformer<Options>;
37 | 
38 | export interface BaseTrackTransformer<
39 |   InitOpts extends TrackTransformerInitOptions,
40 |   DataType extends VideoFrame | AudioData,
41 |   DestroyOpts extends TrackTransformerDestroyOptions = TrackTransformerDestroyOptions,
42 | > {
43 |   init: (options: InitOpts) => void;
44 |   destroy: (options: DestroyOpts) => void;
45 |   restart: (options: InitOpts) => void;
46 |   transform: (frame: DataType, controller: TransformStreamDefaultController) => void;
47 |   transformer?: TransformStream;
48 | }
49 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Logs
  2 | logs
  3 | *.log
  4 | npm-debug.log*
  5 | yarn-debug.log*
  6 | yarn-error.log*
  7 | lerna-debug.log*
  8 | 
  9 | # Diagnostic reports (https://nodejs.org/api/report.html)
 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 11 | 
 12 | # Runtime data
 13 | pids
 14 | *.pid
 15 | *.seed
 16 | *.pid.lock
 17 | 
 18 | # Directory for instrumented libs generated by jscoverage/JSCover
 19 | lib-cov
 20 | 
 21 | # Coverage directory used by tools like istanbul
 22 | coverage
 23 | *.lcov
 24 | 
 25 | # nyc test coverage
 26 | .nyc_output
 27 | 
 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
 29 | .grunt
 30 | 
 31 | # Bower dependency directory (https://bower.io/)
 32 | bower_components
 33 | 
 34 | # node-waf configuration
 35 | .lock-wscript
 36 | 
 37 | # Compiled binary addons (https://nodejs.org/api/addons.html)
 38 | build/Release
 39 | 
 40 | # Dependency directories
 41 | node_modules/
 42 | jspm_packages/
 43 | 
 44 | # TypeScript v1 declaration files
 45 | typings/
 46 | 
 47 | # TypeScript cache
 48 | *.tsbuildinfo
 49 | 
 50 | # Optional npm cache directory
 51 | .npm
 52 | 
 53 | # Optional eslint cache
 54 | .eslintcache
 55 | 
 56 | # Microbundle cache
 57 | .rpt2_cache/
 58 | .rts2_cache_cjs/
 59 | .rts2_cache_es/
 60 | .rts2_cache_umd/
 61 | 
 62 | # Optional REPL history
 63 | .node_repl_history
 64 | 
 65 | # Output of 'npm pack'
 66 | *.tgz
 67 | 
 68 | # Yarn Integrity file
 69 | .yarn-integrity
 70 | 
 71 | # dotenv environment variables file
 72 | .env
 73 | .env.test
 74 | 
 75 | # parcel-bundler cache (https://parceljs.org/)
 76 | .cache
 77 | 
 78 | # Next.js build output
 79 | .next
 80 | 
 81 | # Nuxt.js build / generate output
 82 | .nuxt
 83 | dist
 84 | 
 85 | # Gatsby files
 86 | .cache/
 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js
 88 | # https://nextjs.org/blog/next-9-1#public-directory-support
 89 | # public
 90 | 
 91 | # vuepress build output
 92 | .vuepress/dist
 93 | 
 94 | # Serverless directories
 95 | .serverless/
 96 | 
 97 | # FuseBox cache
 98 | .fusebox/
 99 | 
100 | # DynamoDB Local files
101 | .dynamodb/
102 | 
103 | # TernJS port file
104 | .tern-port
105 | 
106 | # macos
107 | .DS_Store
108 | 
109 | # Docs folder, generated
110 | docs/
111 | 
112 | pkg/
113 | bin/
114 | 


--------------------------------------------------------------------------------
/src/logger.ts:
--------------------------------------------------------------------------------
 1 | import { getLogger as clientGetLogger } from 'livekit-client';
 2 | 
 3 | export enum LogLevel {
 4 |   trace = 0,
 5 |   debug = 1,
 6 |   info = 2,
 7 |   warn = 3,
 8 |   error = 4,
 9 |   silent = 5,
10 | }
11 | 
12 | export enum LoggerNames {
13 |   ProcessorWrapper = 'livekit-processor-wrapper',
14 |   BackgroundProcessor = 'livekit-background-processor',
15 |   WebGl = 'livekit-track-processor-web-gl',
16 | }
17 | 
18 | type LogLevelString = keyof typeof LogLevel;
19 | 
20 | export type StructuredLogger = ReturnType<typeof clientGetLogger>;
21 | 
22 | let livekitLogger = getLogger('livekit');
23 | const livekitLoggers = Object.values(LoggerNames).map((name) => getLogger(name));
24 | 
25 | livekitLogger.setDefaultLevel(LogLevel.info);
26 | 
27 | export default livekitLogger as StructuredLogger;
28 | 
29 | /**
30 |  * @internal
31 |  */
32 | export function getLogger(name: string): StructuredLogger {
33 |   return clientGetLogger(name);
34 | }
35 | 
36 | export function setLogLevel(level: LogLevel | LogLevelString, loggerName?: LoggerNames) {
37 |   if (loggerName) {
38 |     getLogger(loggerName).setLevel(level);
39 |   } else {
40 |     for (const logger of livekitLoggers) {
41 |       logger.setLevel(level);
42 |     }
43 |   }
44 | }
45 | 
46 | export type LogExtension = (level: LogLevel, msg: string, context?: object) => void;
47 | 
48 | /**
49 |  * use this to hook into the logging function to allow sending internal livekit logs to third party services
50 |  * if set, the browser logs will lose their stacktrace information (see https://github.com/pimterry/loglevel#writing-plugins)
51 |  */
52 | export function setLogExtension(extension: LogExtension, logger?: StructuredLogger) {
53 |   const loggers = logger ? [logger] : livekitLoggers;
54 | 
55 |   loggers.forEach((logR) => {
56 |     const originalFactory = logR.methodFactory;
57 | 
58 |     logR.methodFactory = (methodName, configLevel, loggerName) => {
59 |       const rawMethod = originalFactory(methodName, configLevel, loggerName);
60 | 
61 |       const logLevel = LogLevel[methodName as LogLevelString];
62 |       const needLog = logLevel >= configLevel && logLevel < LogLevel.silent;
63 | 
64 |       return (msg, context?: [msg: string, context: object]) => {
65 |         if (context) rawMethod(msg, context);
66 |         else rawMethod(msg);
67 |         if (needLog) {
68 |           extension(logLevel, msg, context);
69 |         }
70 |       };
71 |     };
72 |     logR.setLevel(logR.getLevel());
73 |   });
74 | }
75 | 


--------------------------------------------------------------------------------
/src/webgl/shader-programs/compositeShader.ts:
--------------------------------------------------------------------------------
 1 | import { createProgram, createShader, glsl } from '../utils';
 2 | import { vertexShaderSource } from './vertexShader';
 3 | 
 4 | // Fragment shader source for compositing
 5 | export const compositeFragmentShader = glsl`#version 300 es
 6 |   precision mediump float;
 7 |   in vec2 texCoords;
 8 |   uniform sampler2D background;
 9 |   uniform bool disableBackground;
10 |   uniform sampler2D frame;
11 |   uniform sampler2D mask;
12 |   out vec4 fragColor;
13 |   
14 |   void main() {
15 |     vec4 frameTex = texture(frame, texCoords);
16 | 
17 |     if (disableBackground) {
18 |       fragColor = frameTex;
19 |     } else {
20 |       vec4 bgTex = texture(background, texCoords);
21 | 
22 |       float maskVal = texture(mask, texCoords).r;
23 | 
24 |       // Compute screen-space gradient to detect edge sharpness
25 |       float grad = length(vec2(dFdx(maskVal), dFdy(maskVal)));
26 | 
27 |       float edgeSoftness = 2.0; // higher = softer
28 | 
29 |       // Create a smooth edge around binary transition
30 |       float smoothAlpha = smoothstep(0.5 - grad * edgeSoftness, 0.5 + grad * edgeSoftness, maskVal);
31 | 
32 |       // Optional: preserve frame alpha, or override as fully opaque
33 |       vec4 blended = mix(bgTex, vec4(frameTex.rgb, 1.0), 1.0 - smoothAlpha);
34 |       fragColor = blended;
35 |     }
36 |   
37 |   }
38 | `;
39 | 
40 | /**
41 |  * Create the composite shader program
42 |  */
43 | export function createCompositeProgram(gl: WebGL2RenderingContext) {
44 |   const vertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
45 |   const compositeShader = createShader(gl, gl.FRAGMENT_SHADER, compositeFragmentShader);
46 | 
47 |   const compositeProgram = createProgram(gl, vertexShader, compositeShader);
48 | 
49 |   // Get attribute and uniform locations
50 |   const attribLocations = {
51 |     position: gl.getAttribLocation(compositeProgram, 'position'),
52 |   };
53 | 
54 |   const uniformLocations = {
55 |     mask: gl.getUniformLocation(compositeProgram, 'mask')!,
56 |     frame: gl.getUniformLocation(compositeProgram, 'frame')!,
57 |     background: gl.getUniformLocation(compositeProgram, 'background')!,
58 |     disableBackground: gl.getUniformLocation(compositeProgram, 'disableBackground')!,
59 |     stepWidth: gl.getUniformLocation(compositeProgram, 'u_stepWidth')!,
60 |   };
61 | 
62 |   return {
63 |     program: compositeProgram,
64 |     vertexShader,
65 |     fragmentShader: compositeShader,
66 |     attribLocations,
67 |     uniformLocations,
68 |   };
69 | }
70 | 


--------------------------------------------------------------------------------
/src/webgl/shader-programs/downSampler.ts:
--------------------------------------------------------------------------------
 1 | import { createProgram, createShader } from '../utils';
 2 | 
 3 | export function createDownSampler(
 4 |   gl: WebGL2RenderingContext,
 5 |   width: number,
 6 |   height: number,
 7 | ): {
 8 |   framebuffer: WebGLFramebuffer;
 9 |   texture: WebGLTexture;
10 |   program: WebGLProgram;
11 |   uniforms: any;
12 | } {
13 |   // Create texture
14 |   const texture = gl.createTexture()!;
15 |   gl.bindTexture(gl.TEXTURE_2D, texture);
16 |   gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
17 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
18 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
19 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
20 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
21 | 
22 |   // Create framebuffer
23 |   const framebuffer = gl.createFramebuffer()!;
24 |   gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
25 |   gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
26 | 
27 |   // Create shader program for copying
28 |   const vertexSource = `
29 |       attribute vec2 position;
30 |       varying vec2 v_uv;
31 |       void main() {
32 |         v_uv = (position + 1.0) * 0.5;
33 |         gl_Position = vec4(position, 0.0, 1.0);
34 |       }
35 |     `;
36 | 
37 |   const fragmentSource = `
38 |       precision mediump float;
39 |       varying vec2 v_uv;
40 |       uniform sampler2D u_texture;
41 |       void main() {
42 |         gl_FragColor = texture2D(u_texture, v_uv);
43 |       }
44 |     `;
45 | 
46 |   const vertShader = createShader(gl, gl.VERTEX_SHADER, vertexSource);
47 |   const fragShader = createShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
48 |   const program = createProgram(gl, vertShader, fragShader);
49 | 
50 |   const uniforms = {
51 |     texture: gl.getUniformLocation(program, 'u_texture'),
52 |     position: gl.getAttribLocation(program, 'position'),
53 |   };
54 | 
55 |   return {
56 |     framebuffer,
57 |     texture,
58 |     program,
59 |     uniforms,
60 |   };
61 | }
62 | 
63 | export function applyDownsampling(
64 |   gl: WebGL2RenderingContext,
65 |   inputTexture: WebGLTexture,
66 |   downSampler: {
67 |     framebuffer: WebGLFramebuffer;
68 |     texture: WebGLTexture;
69 |     program: WebGLProgram;
70 |     uniforms: any;
71 |   },
72 |   vertexBuffer: WebGLBuffer,
73 |   width: number,
74 |   height: number,
75 | ): WebGLTexture {
76 |   gl.useProgram(downSampler.program);
77 | 
78 |   gl.bindFramebuffer(gl.FRAMEBUFFER, downSampler.framebuffer);
79 |   gl.viewport(0, 0, width, height);
80 | 
81 |   gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
82 |   gl.enableVertexAttribArray(downSampler.uniforms.position);
83 |   gl.vertexAttribPointer(downSampler.uniforms.position, 2, gl.FLOAT, false, 0, 0);
84 | 
85 |   gl.activeTexture(gl.TEXTURE0);
86 |   gl.bindTexture(gl.TEXTURE_2D, inputTexture);
87 |   gl.uniform1i(downSampler.uniforms.texture, 0);
88 | 
89 |   gl.drawArrays(gl.TRIANGLES, 0, 6);
90 | 
91 |   gl.bindFramebuffer(gl.FRAMEBUFFER, null);
92 | 
93 |   return downSampler.texture;
94 | }
95 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # LiveKit track processors
 2 | 
 3 | ## Install
 4 | 
 5 | ```
 6 | npm add @livekit/track-processors
 7 | ```
 8 | 
 9 | ## Usage of prebuilt processors
10 | 
11 | ### Available processors
12 | 
13 | This package exposes the `BackgroundProcessor` pre-prepared processor pipeline, which can be used in a few ways:
14 | 
15 | - `BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 /* (optional) */ })`
16 | - `BackgroundProcessor({ mode: 'virtual-background', imagePath: "http://path.to/image.png" })`
17 | - `BackgroundProcessor({ mode: 'disabled' })`
18 | 
19 | ### Usage example
20 | 
21 | ```ts
22 | import { BackgroundProcessor, supportsBackgroundProcessors, supportsModernBackgroundProcessors } from '@livekit/track-processors';
23 | 
24 | if(!supportsBackgroundProcessors()) {
25 |   throw new Error("this browser does not support background processors")
26 | }
27 | 
28 | if(supportsModernBackgroundProcessors()) {
29 |   console.log("this browser supports modern APIs that are more performant");
30 | }
31 | 
32 | const videoTrack = await createLocalVideoTrack();
33 | const processor = BackgroundProcessor({ mode: 'background-blur' });
34 | await videoTrack.setProcessor(processor);
35 | room.localParticipant.publishTrack(videoTrack);
36 | 
37 | async function disableBackgroundBlur() {
38 |   await videoTrack.stopProcessor();
39 | }
40 | 
41 | async function updateBlurRadius(radius) {
42 |   return processor.switchTo({ mode: 'background-blur', blurRadius: radius });
43 | }
44 | ```
45 | 
46 | In a real application, it's likely you will want to only sometimes apply background effects. You
47 | could accomplish this by calling `videoTrack.setProcessor(...)` / `videoTrack.stopProcessor(...)` on
48 | demand, but these functions can sometimes result in output visual artifacts as part of the switching
49 | process, which can result in a poor user experience.
50 | 
51 | A better option which won't result in any visual artifacts while switching is to initialize the
52 | `BackgroundProcessor` in its "disabled" mode, and then later on switch to the desired mode. For
53 | example:
54 | ```ts
55 | const videoTrack = await createLocalVideoTrack();
56 | const processor = BackgroundProcessor({ mode: 'disabled' });
57 | await videoTrack.setProcessor(processor);
58 | room.localParticipant.publishTrack(videoTrack);
59 | 
60 | async function enableBlur(radius) {
61 |   await processor.switchTo({ mode: 'background-blur', blurRadius: radius });
62 | }
63 | 
64 | async function disableBlur() {
65 |   await processor.switchTo({ mode: 'disabled' });
66 | }
67 | ```
68 | 
69 | ## Developing your own processors
70 | 
71 | A track processor is instantiated with a Transformer.
72 | 
73 | ```ts
74 | // src/index.ts
75 | export const VirtualBackground = (imagePath: string) => {
76 |   const pipeline = new ProcessorWrapper(new BackgroundTransformer({ imagePath }));
77 |   return pipeline;
78 | };
79 | ```
80 | 
81 | ### Available base transformers
82 | 
83 | - BackgroundTransformer (can blur background, use a virtual background, or be put into a disabled state);
84 | 
85 | 
86 | ## Running the sample app
87 | 
88 | This repository includes a small example app built on [Vite](https://vitejs.dev/). Run it with:
89 | 
90 | ```
91 | # install pnpm: https://pnpm.io/installation
92 | pnpm install
93 | pnpm sample
94 | ```
95 | 


--------------------------------------------------------------------------------
/example/styles.css:
--------------------------------------------------------------------------------
  1 | #connect-area {
  2 |   display: grid;
  3 |   grid-template-columns: 1fr 1fr;
  4 |   grid-template-rows: min-content min-content;
  5 |   grid-auto-flow: column;
  6 |   grid-gap: 10px;
  7 |   margin-bottom: 15px;
  8 | }
  9 | 
 10 | #options-area {
 11 |   display: flex;
 12 |   flex-wrap: wrap;
 13 |   margin-left: 1.25rem;
 14 |   margin-right: 1.25rem;
 15 |   column-gap: 3rem;
 16 |   row-gap: 1rem;
 17 |   margin-bottom: 10px;
 18 | }
 19 | 
 20 | #actions-area {
 21 |   display: grid;
 22 |   grid-template-columns: fit-content(100px) auto;
 23 |   grid-gap: 1.25rem;
 24 |   margin-bottom: 15px;
 25 | }
 26 | 
 27 | #inputs-area {
 28 |   display: grid;
 29 |   grid-template-columns: repeat(3, 1fr);
 30 |   grid-gap: 1.25rem;
 31 |   margin-bottom: 10px;
 32 | }
 33 | 
 34 | #chat-input-area {
 35 |   margin-top: 1.2rem;
 36 |   display: grid;
 37 |   grid-template-columns: auto min-content;
 38 |   gap: 1.25rem;
 39 | }
 40 | 
 41 | #screenshare-area {
 42 |   position: relative;
 43 |   margin-top: 1.25rem;
 44 |   margin-bottom: 1.25rem;
 45 |   display: none;
 46 | }
 47 | 
 48 | #screenshare-area video {
 49 |   max-width: 900px;
 50 |   max-height: 900px;
 51 |   border: 3px solid rgba(0, 0, 0, 0.5);
 52 | }
 53 | 
 54 | #participants-area {
 55 |   /* display: grid;
 56 |   grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
 57 |   gap: 20px; */
 58 | }
 59 | 
 60 | #participants-area > .participant {
 61 |   width: 100%;
 62 | }
 63 | 
 64 | #participants-area > .participant::before {
 65 |   content: '';
 66 |   display: inline-block;
 67 |   width: 1px;
 68 |   height: 0;
 69 |   padding-bottom: calc(100% / (16 / 9));
 70 | }
 71 | 
 72 | #log-area {
 73 |   margin-top: 1.25rem;
 74 |   margin-bottom: 1rem;
 75 | }
 76 | 
 77 | #log {
 78 |   width: 66.6%;
 79 |   height: 100px;
 80 | }
 81 | 
 82 | .participant {
 83 |   position: relative;
 84 |   padding: 0;
 85 |   margin: 0;
 86 |   border-radius: 5px;
 87 |   border: 3px solid rgba(0, 0, 0, 0);
 88 |   overflow: hidden;
 89 | }
 90 | 
 91 | .participant video {
 92 |   position: absolute;
 93 |   left: 0;
 94 |   top: 0;
 95 |   width: 100%;
 96 |   height: 100%;
 97 |   background-color: #aaa;
 98 |   object-fit: cover;
 99 |   border-radius: 5px;
100 | }
101 | 
102 | .participant .info-bar {
103 |   position: absolute;
104 |   width: 100%;
105 |   bottom: 0;
106 |   display: grid;
107 |   color: #eee;
108 |   padding: 2px 8px 2px 8px;
109 |   background-color: rgba(0, 0, 0, 0.35);
110 |   grid-template-columns: minmax(50px, auto) 1fr minmax(50px, auto);
111 |   z-index: 5;
112 | }
113 | 
114 | .participant .size {
115 |   text-align: center;
116 | }
117 | 
118 | .participant .right {
119 |   text-align: right;
120 | }
121 | 
122 | .participant.speaking {
123 |   border: 3px solid rgba(94, 166, 190, 0.7);
124 | }
125 | 
126 | .participant .mic-off {
127 |   color: #d33;
128 |   text-align: right;
129 | }
130 | 
131 | .participant .mic-on {
132 |   text-align: right;
133 | }
134 | 
135 | .participant .connection-excellent {
136 |   color: green;
137 | }
138 | 
139 | .participant .connection-good {
140 |   color: orange;
141 | }
142 | 
143 | .participant .connection-poor {
144 |   color: red;
145 | }
146 | 
147 | .participant .volume-control {
148 |   position: absolute;
149 |   top: 4px;
150 |   right: 2px;
151 |   display: flex;
152 |   z-index: 4;
153 |   height: 100%;
154 | }
155 | 
156 | .participant .volume-control > input {
157 |   width: 16px;
158 |   height: 40%;
159 |   -webkit-appearance: slider-vertical; /* Chromium */
160 | }
161 | 
162 | .participant .volume-meter {
163 |   position: absolute;
164 |   z-index: 4;
165 | }
166 | 


--------------------------------------------------------------------------------
/src/webgl/shader-programs/blurShader.ts:
--------------------------------------------------------------------------------
  1 | import { createProgram, createShader, glsl } from '../utils';
  2 | import { vertexShaderSource } from './vertexShader';
  3 | 
  4 | // Define the blur fragment shader
  5 | export const blurFragmentShader = glsl`#version 300 es
  6 |   precision mediump float;
  7 |   in vec2 texCoords;
  8 |   uniform sampler2D u_texture;
  9 |   uniform vec2 u_texelSize;
 10 |   uniform vec2 u_direction;
 11 |   uniform float u_radius;
 12 |   out vec4 fragColor;
 13 | 
 14 |   void main() {
 15 |     float sigma = u_radius;
 16 |     float twoSigmaSq = 2.0 * sigma * sigma;
 17 |     float totalWeight = 0.0;
 18 |     vec3 result = vec3(0.0);
 19 |     const int MAX_SAMPLES = 16;
 20 |     int radius = int(min(float(MAX_SAMPLES), ceil(u_radius)));
 21 | 
 22 |     for (int i = -MAX_SAMPLES; i <= MAX_SAMPLES; ++i) {
 23 |       float offset = float(i);
 24 |       if (abs(offset) > float(radius)) continue;
 25 |       float weight = exp(-(offset * offset) / twoSigmaSq);
 26 |       vec2 sampleCoord = texCoords + u_direction * u_texelSize * offset;
 27 |       result += texture(u_texture, sampleCoord).rgb * weight;
 28 |       totalWeight += weight;
 29 |     }
 30 | 
 31 |     fragColor = vec4(result / totalWeight, 1.0);
 32 |   }
 33 | `;
 34 | 
 35 | export function createBlurProgram(gl: WebGL2RenderingContext) {
 36 |   const blurVertexShader = createShader(gl, gl.VERTEX_SHADER, vertexShaderSource());
 37 |   const blurFrag = createShader(gl, gl.FRAGMENT_SHADER, blurFragmentShader);
 38 | 
 39 |   const blurProgram = createProgram(gl, blurVertexShader, blurFrag);
 40 | 
 41 |   // Get uniform locations
 42 |   const blurUniforms = {
 43 |     position: gl.getAttribLocation(blurProgram, 'position'),
 44 |     texture: gl.getUniformLocation(blurProgram, 'u_texture'),
 45 |     texelSize: gl.getUniformLocation(blurProgram, 'u_texelSize'),
 46 |     direction: gl.getUniformLocation(blurProgram, 'u_direction'),
 47 |     radius: gl.getUniformLocation(blurProgram, 'u_radius'),
 48 |   };
 49 | 
 50 |   return {
 51 |     program: blurProgram,
 52 |     shader: blurFrag,
 53 |     vertexShader: blurVertexShader,
 54 |     uniforms: blurUniforms,
 55 |   };
 56 | }
 57 | 
 58 | export function applyBlur(
 59 |   gl: WebGL2RenderingContext,
 60 |   sourceTexture: WebGLTexture,
 61 |   width: number,
 62 |   height: number,
 63 |   blurRadius: number,
 64 |   blurProgram: WebGLProgram,
 65 |   blurUniforms: any,
 66 |   vertexBuffer: WebGLBuffer,
 67 |   processFramebuffers: WebGLFramebuffer[],
 68 |   processTextures: WebGLTexture[],
 69 | ) {
 70 |   gl.useProgram(blurProgram);
 71 | 
 72 |   // Set common attributes
 73 |   gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
 74 |   gl.vertexAttribPointer(blurUniforms.position, 2, gl.FLOAT, false, 0, 0);
 75 |   gl.enableVertexAttribArray(blurUniforms.position);
 76 | 
 77 |   const texelWidth = 1.0 / width;
 78 |   const texelHeight = 1.0 / height;
 79 | 
 80 |   // First pass - horizontal blur
 81 |   gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[0]);
 82 |   gl.viewport(0, 0, width, height);
 83 | 
 84 |   gl.activeTexture(gl.TEXTURE0);
 85 |   gl.bindTexture(gl.TEXTURE_2D, sourceTexture);
 86 |   gl.uniform1i(blurUniforms.texture, 0);
 87 |   gl.uniform2f(blurUniforms.texelSize, texelWidth, texelHeight);
 88 |   gl.uniform2f(blurUniforms.direction, 1.0, 0.0); // Horizontal
 89 |   gl.uniform1f(blurUniforms.radius, blurRadius);
 90 | 
 91 |   gl.drawArrays(gl.TRIANGLES, 0, 6);
 92 | 
 93 |   // Second pass - vertical blur
 94 |   gl.bindFramebuffer(gl.FRAMEBUFFER, processFramebuffers[1]);
 95 |   gl.viewport(0, 0, width, height);
 96 | 
 97 |   gl.activeTexture(gl.TEXTURE0);
 98 |   gl.bindTexture(gl.TEXTURE_2D, processTextures[0]);
 99 |   gl.uniform1i(blurUniforms.texture, 0);
100 |   gl.uniform2f(blurUniforms.direction, 0.0, 1.0); // Vertical
101 | 
102 |   gl.drawArrays(gl.TRIANGLES, 0, 6);
103 | 
104 |   // Reset framebuffer
105 |   gl.bindFramebuffer(gl.FRAMEBUFFER, null);
106 | 
107 |   return processTextures[1];
108 | }
109 | 


--------------------------------------------------------------------------------
/src/webgl/utils.ts:
--------------------------------------------------------------------------------
  1 | import { getLogger, LoggerNames} from '../logger';
  2 | 
  3 | const log = getLogger(LoggerNames.WebGl);
  4 | 
  5 | /**
  6 |  * Initialize a WebGL texture
  7 |  */
  8 | export function initTexture(gl: WebGL2RenderingContext, texIndex: number) {
  9 |   const texRef = gl.TEXTURE0 + texIndex;
 10 |   gl.activeTexture(texRef);
 11 |   const texture = gl.createTexture();
 12 |   gl.bindTexture(gl.TEXTURE_2D, texture);
 13 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
 14 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
 15 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
 16 |   gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
 17 |   gl.bindTexture(gl.TEXTURE_2D, texture);
 18 | 
 19 |   return texture;
 20 | }
 21 | 
 22 | export function createShader(
 23 |   gl: WebGL2RenderingContext,
 24 |   type: number,
 25 |   source: string,
 26 | ): WebGLShader {
 27 |   const shader = gl.createShader(type)!;
 28 |   gl.shaderSource(shader, source);
 29 |   gl.compileShader(shader);
 30 |   if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
 31 |     log.error('Shader compile failed:', gl.getShaderInfoLog(shader));
 32 |     gl.deleteShader(shader);
 33 |     throw new Error('Shader compile failed');
 34 |   }
 35 |   return shader;
 36 | }
 37 | 
 38 | export function createProgram(
 39 |   gl: WebGL2RenderingContext,
 40 |   vs: WebGLShader,
 41 |   fs: WebGLShader,
 42 | ): WebGLProgram {
 43 |   const program = gl.createProgram()!;
 44 |   gl.attachShader(program, vs);
 45 |   gl.attachShader(program, fs);
 46 |   gl.linkProgram(program);
 47 |   if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
 48 |     log.error('Program link failed:', gl.getProgramInfoLog(program));
 49 |     throw new Error('Program link failed');
 50 |   }
 51 |   return program;
 52 | }
 53 | 
 54 | /**
 55 |  * Create a WebGL framebuffer with the given texture as color attachment
 56 |  */
 57 | export function createFramebuffer(
 58 |   gl: WebGL2RenderingContext,
 59 |   texture: WebGLTexture,
 60 |   width: number,
 61 |   height: number,
 62 | ) {
 63 |   const framebuffer = gl.createFramebuffer();
 64 |   gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
 65 | 
 66 |   // Set the texture as the color attachment
 67 |   gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0);
 68 | 
 69 |   // Ensure texture dimensions match the provided width and height
 70 |   gl.bindTexture(gl.TEXTURE_2D, texture);
 71 |   gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
 72 | 
 73 |   // Check if framebuffer is complete
 74 |   const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
 75 |   if (status !== gl.FRAMEBUFFER_COMPLETE) {
 76 |     throw new Error('Framebuffer not complete');
 77 |   }
 78 | 
 79 |   gl.bindFramebuffer(gl.FRAMEBUFFER, null);
 80 |   return framebuffer;
 81 | }
 82 | 
 83 | /**
 84 |  * Create a vertex buffer for a full-screen quad
 85 |  */
 86 | export function createVertexBuffer(gl: WebGL2RenderingContext): WebGLBuffer | null {
 87 |   const vertexBuffer = gl.createBuffer();
 88 |   gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
 89 |   gl.bufferData(
 90 |     gl.ARRAY_BUFFER,
 91 |     new Float32Array([-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, -1]),
 92 |     gl.STATIC_DRAW,
 93 |   );
 94 |   return vertexBuffer;
 95 | }
 96 | 
 97 | /**
 98 |  * Resizes and crops an image to cover a target canvas while maintaining aspect ratio
 99 |  * @param image The source image
100 |  * @param targetWidth The target width
101 |  * @param targetHeight The target height
102 |  * @returns A cropped and resized ImageBitmap
103 |  */
104 | export async function resizeImageToCover(
105 |   image: ImageBitmap,
106 |   targetWidth: number,
107 |   targetHeight: number,
108 | ): Promise<ImageBitmap> {
109 |   // Calculate dimensions and crop for "cover" mode
110 |   const imgAspect = image.width / image.height;
111 |   const targetAspect = targetWidth / targetHeight;
112 | 
113 |   let sx = 0;
114 |   let sy = 0;
115 |   let sWidth = image.width;
116 |   let sHeight = image.height;
117 | 
118 |   // For cover mode, we need to crop some parts of the image
119 |   // to ensure it covers the canvas while maintaining aspect ratio
120 |   if (imgAspect > targetAspect) {
121 |     // Image is wider than target - crop the sides
122 |     sWidth = Math.round(image.height * targetAspect);
123 |     sx = Math.round((image.width - sWidth) / 2); // Center the crop horizontally
124 |   } else if (imgAspect < targetAspect) {
125 |     // Image is taller than target - crop the top/bottom
126 |     sHeight = Math.round(image.width / targetAspect);
127 |     sy = Math.round((image.height - sHeight) / 2); // Center the crop vertically
128 |   }
129 | 
130 |   // Create a new ImageBitmap with the cropped portion
131 |   return createImageBitmap(image, sx, sy, sWidth, sHeight, {
132 |     resizeWidth: targetWidth,
133 |     resizeHeight: targetHeight,
134 |     resizeQuality: 'medium',
135 |   });
136 | }
137 | 
138 | let emptyImageData: ImageData | undefined;
139 | 
140 | function getEmptyImageData() {
141 |   if (!emptyImageData) {
142 |     emptyImageData = new ImageData(2, 2);
143 |     emptyImageData.data[0] = 0;
144 |     emptyImageData.data[1] = 0;
145 |     emptyImageData.data[2] = 0;
146 |     emptyImageData.data[3] = 0;
147 |   }
148 | 
149 |   return emptyImageData;
150 | }
151 | 
152 | const glsl = (source: any) => source;
153 | 
154 | export { getEmptyImageData, glsl };
155 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
  1 | # @livekit/track-processors
  2 | 
  3 | ## 0.7.0
  4 | 
  5 | ### Minor Changes
  6 | 
  7 | - Faster disabling of background transformers - [#102](https://github.com/livekit/track-processors-js/pull/102) ([@holzgeist](https://github.com/holzgeist))
  8 | 
  9 | ### Patch Changes
 10 | 
 11 | - Fix a "black screen flash" which occurs sometimes when restarting the processor wrapper - [#111](https://github.com/livekit/track-processors-js/pull/111) ([@1egoman](https://github.com/1egoman))
 12 | 
 13 | - Add logic to BackgroundProcessor to allow dynamically switching modes elegantly - [#107](https://github.com/livekit/track-processors-js/pull/107) ([@1egoman](https://github.com/1egoman))
 14 | 
 15 | - Adds logging infrastructure to allow for changing of log levels - [#109](https://github.com/livekit/track-processors-js/pull/109) ([@1egoman](https://github.com/1egoman))
 16 | 
 17 | ## 0.6.1
 18 | 
 19 | ### Patch Changes
 20 | 
 21 | - cache background image rather than refetch it every time it is enabled - [#100](https://github.com/livekit/track-processors-js/pull/100) ([@1egoman](https://github.com/1egoman))
 22 | 
 23 | ## 0.6.0
 24 | 
 25 | ### Minor Changes
 26 | 
 27 | - Reduce the intensity of the gray "flash" when enabling the BackgroundProcessor - [#96](https://github.com/livekit/track-processors-js/pull/96) ([@1egoman](https://github.com/1egoman))
 28 | 
 29 | ## 0.5.8
 30 | 
 31 | ### Patch Changes
 32 | 
 33 | - Only destroy if no newer init method has been invoked - [#90](https://github.com/livekit/track-processors-js/pull/90) ([@lukasIO](https://github.com/lukasIO))
 34 | 
 35 | ## 0.5.7
 36 | 
 37 | ### Patch Changes
 38 | 
 39 | - fix(deps): include dom-mediacapture-transform types as a dependency - [#86](https://github.com/livekit/track-processors-js/pull/86) ([@CSantosM](https://github.com/CSantosM))
 40 | 
 41 | ## 0.5.6
 42 | 
 43 | ### Patch Changes
 44 | 
 45 | - Create empty ImageData helper lazily to avoid ssr build time errors - [#80](https://github.com/livekit/track-processors-js/pull/80) ([@lukasIO](https://github.com/lukasIO))
 46 | 
 47 | - Add HTMLCanvas fallback if OffscreenCanvas is not available - [#81](https://github.com/livekit/track-processors-js/pull/81) ([@lukasIO](https://github.com/lukasIO))
 48 | 
 49 | ## 0.5.5
 50 | 
 51 | ### Patch Changes
 52 | 
 53 | - Downsample background before applying blur - [#77](https://github.com/livekit/track-processors-js/pull/77) ([@lukasIO](https://github.com/lukasIO))
 54 | 
 55 | - Update mask when ready - [#76](https://github.com/livekit/track-processors-js/pull/76) ([@lukasIO](https://github.com/lukasIO))
 56 | 
 57 | ## 0.5.4
 58 | 
 59 | ### Patch Changes
 60 | 
 61 | - Smoothen mask edges - [#74](https://github.com/livekit/track-processors-js/pull/74) ([@lukasIO](https://github.com/lukasIO))
 62 | 
 63 | ## 0.5.3
 64 | 
 65 | ### Patch Changes
 66 | 
 67 | - Dowgrade mediapipe to 0.10.14 - [#72](https://github.com/livekit/track-processors-js/pull/72) ([@lukasIO](https://github.com/lukasIO))
 68 | 
 69 | ## 0.5.2
 70 | 
 71 | ### Patch Changes
 72 | 
 73 | - Fix device switching on Safari - [#70](https://github.com/livekit/track-processors-js/pull/70) ([@lukasIO](https://github.com/lukasIO))
 74 | 
 75 | ## 0.5.1
 76 | 
 77 | ### Patch Changes
 78 | 
 79 | - Adds missing object variable "assetPaths" for configurable asset paths in BackgroundProcessor. - [#68](https://github.com/livekit/track-processors-js/pull/68) ([@jonkad](https://github.com/jonkad))
 80 | 
 81 | ## 0.5.0
 82 | 
 83 | ### Minor Changes
 84 | 
 85 | - Add captureStream fallback for other browsers - [#65](https://github.com/livekit/track-processors-js/pull/65) ([@lukasIO](https://github.com/lukasIO))
 86 | 
 87 | - Use webGL for video processors - [#64](https://github.com/livekit/track-processors-js/pull/64) ([@lukasIO](https://github.com/lukasIO))
 88 | 
 89 | ### Patch Changes
 90 | 
 91 | - Expose frame processing stats via optional callback - [#63](https://github.com/livekit/track-processors-js/pull/63) ([@lukasIO](https://github.com/lukasIO))
 92 | 
 93 | ## 0.4.1
 94 | 
 95 | ### Patch Changes
 96 | 
 97 | - Use putImageData instead of creating new bitmap - [#61](https://github.com/livekit/track-processors-js/pull/61) ([@lukasIO](https://github.com/lukasIO))
 98 | 
 99 | ## 0.4.0
100 | 
101 | ### Minor Changes
102 | 
103 | - Update tasks-vision dependency and remove blurred outlines - [#59](https://github.com/livekit/track-processors-js/pull/59) ([@lukasIO](https://github.com/lukasIO))
104 | 
105 | ## 0.3.3
106 | 
107 | ### Patch Changes
108 | 
109 | - Ignore empty video frames - [#52](https://github.com/livekit/track-processors-js/pull/52) ([@lukasIO](https://github.com/lukasIO))
110 | 
111 | ## 0.3.2
112 | 
113 | ### Patch Changes
114 | 
115 | - Reload the background image when needed during processor initialization - [#42](https://github.com/livekit/track-processors-js/pull/42) ([@kyleparrott](https://github.com/kyleparrott))
116 | 
117 | ## 0.3.1
118 | 
119 | ### Patch Changes
120 | 
121 | - Allow configurable asset paths for task vision assets - [#35](https://github.com/livekit/track-processors-js/pull/35) ([@lukasIO](https://github.com/lukasIO))
122 | 
123 | ## 0.3.0
124 | 
125 | ### Minor Changes
126 | 
127 | - Replace ProcessorPipeline with ProcessorWrapper, allowing for direct transformer updates - [#32](https://github.com/livekit/track-processors-js/pull/32) ([@lukasIO](https://github.com/lukasIO))
128 | 
129 | ## 0.2.8
130 | 
131 | ### Patch Changes
132 | 
133 | - Update @mediapipe/tasks-vision - [`5be167d`](https://github.com/livekit/track-processors-js/commit/5be167d2f7b0aaf99d691009306691cfe7fa9d77) ([@lukasIO](https://github.com/lukasIO))
134 | 
135 | ## 0.2.7
136 | 
137 | ### Patch Changes
138 | 
139 | - Expose ProcessorPipeline and VideoTransformer - [#15](https://github.com/livekit/track-processors-js/pull/15) ([@lukasIO](https://github.com/lukasIO))
140 |   Update media vision SDK
141 | 
142 | ## 0.2.6
143 | 
144 | ### Patch Changes
145 | 
146 | - Publish workflow release - [#12](https://github.com/livekit/track-processors-js/pull/12) ([@lukasIO](https://github.com/lukasIO))
147 | 


--------------------------------------------------------------------------------
/example/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 |   <head>
  4 |     <title>LiveKit track processor sample</title>
  5 |     <meta charset="utf-8" />
  6 |     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
  7 | 
  8 |     <link
  9 |       rel="stylesheet"
 10 |       href="https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css"
 11 |       integrity="sha384-TX8t27EcRE3e/ihU7zmQxVncDAy5uIKz4rEkgIXeMed4M0jlfIDPvg6uqKI2xXr2"
 12 |       crossorigin="anonymous"
 13 |     />
 14 |     <link rel="stylesheet" href="styles.css" />
 15 |   </head>
 16 |   <body>
 17 |     <div class="container">
 18 |       <div class="row">
 19 |         <div class="col-md-8">
 20 |           <h2>LiveKit track processor sample</h2>
 21 |           <br />
 22 |           <div id="connect-area">
 23 |             <div>
 24 |               <b>LiveKit URL</b>
 25 |             </div>
 26 |             <div>
 27 |               <input type="text" class="form-control" id="url" value="ws://localhost:7880" />
 28 |             </div>
 29 |             <div>
 30 |               <b>Token</b>
 31 |             </div>
 32 |             <div>
 33 |               <input type="text" class="form-control" id="token" />
 34 |             </div>
 35 |           </div>
 36 | 
 37 |           <!-- actions -->
 38 |           <div id="actions-area">
 39 |             <div>
 40 |               <button
 41 |                 id="connect-button"
 42 |                 class="btn btn-primary mt-1"
 43 |                 type="button"
 44 |                 onclick="appActions.connectWithFormInput()"
 45 |               >
 46 |                 Connect
 47 |               </button>
 48 |             </div>
 49 |             <div>
 50 |               <button
 51 |                 id="toggle-audio-button"
 52 |                 class="btn btn-secondary mt-1"
 53 |                 disabled
 54 |                 type="button"
 55 |                 onclick="appActions.toggleAudio()"
 56 |               >
 57 |                 Enable Mic
 58 |               </button>
 59 |               <button
 60 |                 id="toggle-video-button"
 61 |                 class="btn btn-secondary mt-1"
 62 |                 disabled
 63 |                 type="button"
 64 |                 onclick="appActions.toggleVideo()"
 65 |               >
 66 |                 Enable Camera
 67 |               </button>
 68 | 
 69 |               <button
 70 |                 id="flip-video-button"
 71 |                 class="btn btn-secondary mt-1"
 72 |                 type="button"
 73 |                 onclick="appActions.flipVideo()"
 74 |               >
 75 |                 Flip Video
 76 |               </button>
 77 | 
 78 |               <button
 79 |                 id="disconnect-room-button"
 80 |                 class="btn btn-danger mt-1"
 81 |                 disabled
 82 |                 type="button"
 83 |                 onclick="appActions.disconnectRoom()"
 84 |               >
 85 |                 Disconnect
 86 |               </button>
 87 | 
 88 |               <button
 89 |                 id="start-audio-button"
 90 |                 class="btn btn-secondary mt-1"
 91 |                 disabled
 92 |                 type="button"
 93 |                 onclick="appActions.startAudio()"
 94 |               >
 95 |                 Start Audio
 96 |               </button>
 97 | 
 98 |               <div class="card mt-1 mb-1 mx-n2 p-2">
 99 |                 <div class="d-flex align-items-center justify-content-between">
100 |                   <button
101 |                     id="toggle-track-processor"
102 |                     class="btn btn-secondary mr-2"
103 |                     disabled
104 |                     type="button"
105 |                     onclick="appActions.toggleTrackProcessorEnabled()"
106 |                   >
107 |                     Insert Track Processor
108 |                   </button>
109 | 
110 |                   <div id="initial-mode-wrapper" style="display: none;">
111 |                     <span class="text-secondary">Initial mode:</span>
112 |                     <select class="custom-select" id="initial-mode-select" style="width: 200px;">
113 |                       <option value="disabled">Disabled</option>
114 |                       <option value="background-blur">Blur</option>
115 |                       <option value="virtual-background">Virtual Background</option>
116 |                     </select>
117 |                   </div>
118 |                 </div>
119 | 
120 |                 <div id="track-processor-modes" style="display: none;">
121 |                   <div class="btn-group mt-1">
122 |                     <button
123 |                       id="switch-to-disabled-button"
124 |                       class="btn btn-secondary"
125 |                       disabled
126 |                       type="button"
127 |                       onclick="appActions.switchBackgroundMode('disabled')"
128 |                     >
129 |                       Disabled
130 |                     </button>
131 |                     <button
132 |                       id="switch-to-background-blur-button"
133 |                       class="btn btn-secondary"
134 |                       disabled
135 |                       type="button"
136 |                       onclick="appActions.switchBackgroundMode('background-blur')"
137 |                     >
138 |                       Blur
139 |                     </button>
140 |                     <button
141 |                       id="switch-to-virtual-background-button"
142 |                       class="btn btn-secondary"
143 |                       disabled
144 |                       type="button"
145 |                       onclick="appActions.switchBackgroundMode('virtual-background')"
146 |                     >
147 |                       Virtual background
148 |                     </button>
149 |                   </div>
150 | 
151 |                   <button
152 |                     id="update-bg-button"
153 |                     class="btn btn-secondary mt-1"
154 |                     disabled
155 |                     type="button"
156 |                     onclick="appActions.updateVirtualBackgroundImage('/ali-kazal-tbw_KQE3Cbg-unsplash.jpg')"
157 |                   >
158 |                     Update Background Image
159 |                   </button>
160 |                 </div>
161 |               </div>
162 |             </div>
163 |           </div>
164 | 
165 |           <div id="inputs-area">
166 |             <div>
167 |               <select
168 |                 id="video-input"
169 |                 class="custom-select"
170 |                 onchange="appActions.handleDeviceSelected(event)"
171 |               >
172 |                 <option selected>Video Input (default)</option>
173 |               </select>
174 |             </div>
175 |             <div>
176 |               <select
177 |                 id="audio-input"
178 |                 class="custom-select"
179 |                 onchange="appActions.handleDeviceSelected(event)"
180 |               >
181 |                 <option selected>Audio Input (default)</option>
182 |               </select>
183 |             </div>
184 |             <div>
185 |               <select
186 |                 id="audio-output"
187 |                 class="custom-select"
188 |                 onchange="appActions.handleDeviceSelected(event)"
189 |               >
190 |                 <option selected>Audio Output (default)</option>
191 |               </select>
192 |             </div>
193 |             <div>
194 |               <input
195 |                 type="range"
196 |                 id="step-width"
197 |                 min="0"
198 |                 max="10"
199 |                 value="1"
200 |                 step="0.1"
201 |                 onchange="appActions.setStepWidth(event)"
202 |               />
203 |             </div>
204 |           </div>
205 |         </div>
206 |       </div>
207 | 
208 |       <div id="participants-area"></div>
209 | 
210 |       <div id="log-area">
211 |         <textarea id="log"></textarea>
212 |       </div>
213 |     </div>
214 |     <script src="./sample.ts" type="module"></script>
215 |     <!-- for icons -->
216 |     <script defer src="https://kit.fontawesome.com/7c1c4c7e9c.js" crossorigin="anonymous"></script>
217 |   </body>
218 | </html>
219 | 


--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
  1 | import ProcessorWrapper, { ProcessorWrapperOptions } from './ProcessorWrapper';
  2 | import BackgroundTransformer, {
  3 |   BackgroundOptions,
  4 |   FrameProcessingStats,
  5 |   SegmenterOptions,
  6 | } from './transformers/BackgroundTransformer';
  7 | 
  8 | export * from './transformers/types';
  9 | export { default as VideoTransformer } from './transformers/VideoTransformer';
 10 | export {
 11 |   ProcessorWrapper,
 12 |   type BackgroundOptions,
 13 |   type SegmenterOptions,
 14 |   BackgroundTransformer,
 15 |   type ProcessorWrapperOptions,
 16 | };
 17 | export * from './logger';
 18 | 
 19 | const DEFAULT_BLUR_RADIUS = 10;
 20 | 
 21 | /**
 22 |  * Determines if the current browser supports background processors
 23 |  */
 24 | export const supportsBackgroundProcessors = () =>
 25 |   BackgroundTransformer.isSupported && ProcessorWrapper.isSupported;
 26 | 
 27 | /**
 28 |  * Determines if the current browser supports modern background processors, which yield better performance
 29 |  */
 30 | export const supportsModernBackgroundProcessors = () =>
 31 |   BackgroundTransformer.isSupported && ProcessorWrapper.hasModernApiSupport;
 32 | 
 33 | type SwitchBackgroundProcessorBackgroundBlurOptions = {
 34 |   mode: 'background-blur';
 35 |   /** If unspecified, defaults to {@link DEFAULT_BLUR_RADIUS} */
 36 |   blurRadius?: number;
 37 | };
 38 | 
 39 | type SwitchBackgroundProcessorVirtualBackgroundOptions = {
 40 |   mode: 'virtual-background';
 41 |   imagePath: string;
 42 | };
 43 | 
 44 | type SwitchBackgroundProcessorDisabledOptions = {
 45 |   mode: 'disabled';
 46 | };
 47 | 
 48 | export type SwitchBackgroundProcessorOptions =
 49 | | SwitchBackgroundProcessorDisabledOptions
 50 | | SwitchBackgroundProcessorBackgroundBlurOptions
 51 | | SwitchBackgroundProcessorVirtualBackgroundOptions
 52 | 
 53 | type BackgroundProcessorCommonOptions = ProcessorWrapperOptions & {
 54 |   segmenterOptions?: SegmenterOptions;
 55 |   assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };
 56 |   onFrameProcessed?: (stats: FrameProcessingStats) => void;
 57 | };
 58 | 
 59 | type BackgroundProcessorModeOptions = BackgroundProcessorCommonOptions & SwitchBackgroundProcessorOptions;
 60 | type BackgroundProcessorLegacyOptions = BackgroundProcessorCommonOptions & {
 61 |   mode?: never;
 62 |   blurRadius?: number;
 63 |   imagePath?: string;
 64 | };
 65 | 
 66 | export type BackgroundProcessorOptions =
 67 |   | BackgroundProcessorModeOptions
 68 |   | BackgroundProcessorLegacyOptions;
 69 | 
 70 | /**
 71 |  * A {@link ProcessorWrapper} that supports applying effects to the background of a video track.
 72 |  *
 73 |  * For more info and for how to construct this object, see {@link BackgroundProcessor}.
 74 |  */
 75 | export class BackgroundProcessorWrapper extends ProcessorWrapper<BackgroundOptions, BackgroundTransformer> {
 76 |   get mode(): BackgroundProcessorModeOptions['mode'] | 'legacy' {
 77 |     const options = this.transformer.options;
 78 | 
 79 |     if (options.backgroundDisabled) {
 80 |       return 'disabled';
 81 |     }
 82 | 
 83 |     if (typeof options.imagePath === 'string' && typeof options.blurRadius === 'undefined') {
 84 |       return 'virtual-background';
 85 |     }
 86 | 
 87 |     if (typeof options.imagePath === 'undefined') {
 88 |       return 'background-blur';
 89 |     }
 90 | 
 91 |     return 'legacy';
 92 |   }
 93 | 
 94 |   async switchTo(options: SwitchBackgroundProcessorOptions) {
 95 |     switch (options.mode) {
 96 |       case 'background-blur':
 97 |         await this.updateTransformerOptions({
 98 |           imagePath: undefined,
 99 |           blurRadius: options.blurRadius ?? DEFAULT_BLUR_RADIUS,
100 |           backgroundDisabled: false,
101 |         });
102 |         break;
103 |       case 'virtual-background':
104 |         await this.updateTransformerOptions({
105 |           imagePath: options.imagePath,
106 |           blurRadius: undefined,
107 |           backgroundDisabled: false,
108 |         });
109 |         break;
110 |       case 'disabled':
111 |         await this.updateTransformerOptions({ imagePath: undefined, backgroundDisabled: true });
112 |         break;
113 |     }
114 |   }
115 | }
116 | 
117 | /**
118 |  * Instantiates a background processor that supports blurring the background of a user's local
119 |  * video or replacing the user's background with a virtual background image, and supports switching
120 |  * the active mode later on the fly to avoid visual artifacts.
121 |  *
122 |  * @example
123 |  * const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!.track as LocalVideoTrack;
124 |  * const processor = BackgroundProcessor({ mode: 'background-blur', blurRadius: 10 });
125 |  * camTrack.setProcessor(processor);
126 |  *
127 |  * // Change to background image:
128 |  * processor.switchToVirtualBackground('path/to/image.png');
129 |  * // Change back to background blur:
130 |  * processor.switchToBackgroundBlur(10);
131 |  */
132 | export const BackgroundProcessor = (
133 |   options: BackgroundProcessorOptions,
134 |   name = 'background-processor',
135 | ) => {
136 |   const isTransformerSupported = BackgroundTransformer.isSupported;
137 |   const isProcessorSupported = ProcessorWrapper.isSupported;
138 | 
139 |   if (!isTransformerSupported) {
140 |     throw new Error('Background transformer is not supported in this browser');
141 |   }
142 | 
143 |   if (!isProcessorSupported) {
144 |     throw new Error(
145 |       'Neither MediaStreamTrackProcessor nor canvas.captureStream() fallback is supported in this browser',
146 |     );
147 |   }
148 | 
149 |   // Extract transformer-specific options and processor options
150 |   let transformer, processorOpts;
151 |   switch (options.mode) {
152 |     case 'background-blur': {
153 |       const {
154 |         // eslint-disable-next-line no-unused-vars
155 |         mode,
156 |         blurRadius = DEFAULT_BLUR_RADIUS,
157 |         segmenterOptions,
158 |         assetPaths,
159 |         onFrameProcessed,
160 |         ...rest
161 |       } = options;
162 | 
163 |       processorOpts = rest;
164 |       transformer = new BackgroundTransformer({
165 |         blurRadius,
166 |         segmenterOptions,
167 |         assetPaths,
168 |         onFrameProcessed,
169 |       });
170 |       break;
171 |     }
172 | 
173 |     case 'virtual-background': {
174 |       const {
175 |         // eslint-disable-next-line no-unused-vars
176 |         mode,
177 |         imagePath,
178 |         segmenterOptions,
179 |         assetPaths,
180 |         onFrameProcessed,
181 |         ...rest
182 |       } = options;
183 | 
184 |       processorOpts = rest;
185 |       transformer = new BackgroundTransformer({
186 |         imagePath,
187 |         segmenterOptions,
188 |         assetPaths,
189 |         onFrameProcessed,
190 |       });
191 |       break;
192 |     }
193 | 
194 |     case 'disabled': {
195 |       const {
196 |         segmenterOptions,
197 |         assetPaths,
198 |         onFrameProcessed,
199 |         ...rest
200 |       } = options;
201 | 
202 |       processorOpts = rest;
203 |       transformer = new BackgroundTransformer({
204 |         segmenterOptions,
205 |         assetPaths,
206 |         onFrameProcessed,
207 |       });
208 |       break;
209 |     }
210 | 
211 |     default: {
212 |       const {
213 |         blurRadius,
214 |         imagePath,
215 |         segmenterOptions,
216 |         assetPaths,
217 |         onFrameProcessed,
218 |         ...rest
219 |       } = options;
220 | 
221 |       processorOpts = rest;
222 |       transformer = new BackgroundTransformer({
223 |         blurRadius,
224 |         imagePath,
225 |         segmenterOptions,
226 |         assetPaths,
227 |         onFrameProcessed,
228 |       });
229 |       break;
230 |     }
231 |   }
232 | 
233 |   const processor = new BackgroundProcessorWrapper(transformer, name, processorOpts);
234 | 
235 |   return processor;
236 | };
237 | 
238 | /**
239 |  * Instantiates a background processor that is configured in blur mode.
240 |  * @deprecated Use `BackgroundProcessor({ mode: 'background-blur', blurRadius: 10,  ... })` instead.
241 |  */
242 | export const BackgroundBlur = (
243 |   blurRadius: number = DEFAULT_BLUR_RADIUS,
244 |   segmenterOptions?: SegmenterOptions,
245 |   onFrameProcessed?: (stats: FrameProcessingStats) => void,
246 |   processorOptions?: ProcessorWrapperOptions,
247 | ) => {
248 |   return BackgroundProcessor(
249 |     {
250 |       blurRadius,
251 |       segmenterOptions,
252 |       onFrameProcessed,
253 |       ...processorOptions,
254 |     },
255 |     'background-blur',
256 |   );
257 | };
258 | 
259 | /**
260 |  * Instantiates a background processor that is configured in virtual background mode.
261 |  * @deprecated Use `BackgroundProcessor({ mode: 'virtual-background', imagePath: '...', ... })` instead.
262 |  */
263 | export const VirtualBackground = (
264 |   imagePath: string,
265 |   segmenterOptions?: SegmenterOptions,
266 |   onFrameProcessed?: (stats: FrameProcessingStats) => void,
267 |   processorOptions?: ProcessorWrapperOptions,
268 | ) => {
269 |   return BackgroundProcessor(
270 |     {
271 |       imagePath,
272 |       segmenterOptions,
273 |       onFrameProcessed,
274 |       ...processorOptions,
275 |     },
276 |     'virtual-background',
277 |   );
278 | };
279 | 
280 | 


--------------------------------------------------------------------------------
/src/transformers/BackgroundTransformer.ts:
--------------------------------------------------------------------------------
  1 | import * as vision from '@mediapipe/tasks-vision';
  2 | import { getLogger, LoggerNames } from '../logger';
  3 | import { dependencies } from '../../package.json';
  4 | import VideoTransformer from './VideoTransformer';
  5 | import { TrackTransformerDestroyOptions, VideoTransformerInitOptions } from './types';
  6 | 
  7 | export type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions']>;
  8 | 
  9 | export interface FrameProcessingStats {
 10 |   processingTimeMs: number;
 11 |   segmentationTimeMs: number;
 12 |   filterTimeMs: number;
 13 | }
 14 | 
 15 | export type BackgroundOptions = {
 16 |   blurRadius?: number;
 17 |   imagePath?: string;
 18 |   backgroundDisabled?: boolean;
 19 |   /** cannot be updated through the `update` method, needs a restart */
 20 |   segmenterOptions?: SegmenterOptions;
 21 |   /** cannot be updated through the `update` method, needs a restart */
 22 |   assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };
 23 |   /** called when a new frame is processed */
 24 |   onFrameProcessed?: (stats: FrameProcessingStats) => void;
 25 | };
 26 | 
 27 | export default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {
 28 |   static get isSupported() {
 29 |     return (
 30 |       typeof OffscreenCanvas !== 'undefined' &&
 31 |       typeof VideoFrame !== 'undefined' &&
 32 |       typeof createImageBitmap !== 'undefined' &&
 33 |       !!document.createElement('canvas').getContext('webgl2')
 34 |     );
 35 |   }
 36 | 
 37 |   imageSegmenter?: vision.ImageSegmenter;
 38 | 
 39 |   segmentationResults: vision.ImageSegmenterResult | undefined;
 40 | 
 41 |   backgroundImageAndPath: { imageData: ImageBitmap, path: string } | null = null;
 42 | 
 43 |   options: BackgroundOptions;
 44 | 
 45 |   segmentationTimeMs: number = 0;
 46 | 
 47 |   isFirstFrame = true;
 48 | 
 49 |   private log = getLogger(LoggerNames.ProcessorWrapper);
 50 | 
 51 |   constructor(opts: BackgroundOptions) {
 52 |     super();
 53 |     this.options = opts;
 54 |     this.update(opts);
 55 |   }
 56 | 
 57 |   async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
 58 |     // Initialize WebGL with appropriate options based on our current state
 59 | 
 60 |     await super.init({ outputCanvas, inputElement: inputVideo });
 61 | 
 62 |     const fileSet = await vision.FilesetResolver.forVisionTasks(
 63 |       this.options.assetPaths?.tasksVisionFileSet ??
 64 |         `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,
 65 |     );
 66 | 
 67 |     this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
 68 |       baseOptions: {
 69 |         modelAssetPath:
 70 |           this.options.assetPaths?.modelAssetPath ??
 71 |           'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
 72 |         delegate: 'GPU',
 73 |         ...this.options.segmenterOptions,
 74 |       },
 75 |       canvas: this.canvas,
 76 |       runningMode: 'VIDEO',
 77 |       outputCategoryMask: true,
 78 |       outputConfidenceMasks: false,
 79 |     });
 80 | 
 81 |     // Skip loading the image here if update already loaded the image below
 82 |     if (this.options?.imagePath) {
 83 |       await this.loadAndSetBackground(this.options.imagePath).catch((err) =>
 84 |         this.log.error('Error while loading processor background image: ', err),
 85 |       );
 86 |     }
 87 |     if (typeof this.options.blurRadius === 'number') {
 88 |       this.gl?.setBlurRadius(this.options.blurRadius);
 89 |     }
 90 |     this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
 91 |   }
 92 | 
 93 |   async destroy(options?: TrackTransformerDestroyOptions) {
 94 |     await super.destroy();
 95 |     await this.imageSegmenter?.close();
 96 |     this.backgroundImageAndPath = null;
 97 | 
 98 |     if (!options?.willProcessorRestart) {
 99 |       this.isFirstFrame = true;
100 |     }
101 |   }
102 | 
103 |   async loadAndSetBackground(path: string) {
104 |     if (!this.backgroundImageAndPath || this.backgroundImageAndPath?.path !== path) {
105 |       const img = new Image();
106 | 
107 |       await new Promise((resolve, reject) => {
108 |         img.crossOrigin = 'Anonymous';
109 |         img.onload = () => resolve(img);
110 |         img.onerror = (err) => reject(err);
111 |         img.src = path;
112 |       });
113 |       const imageData = await createImageBitmap(img);
114 |       this.backgroundImageAndPath = { imageData, path };
115 |     }
116 |     this.gl?.setBackgroundImage(this.backgroundImageAndPath.imageData);
117 |   }
118 | 
119 |   async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
120 |     let enqueuedFrame = false;
121 |     try {
122 |       if (!(frame instanceof VideoFrame) || frame.codedWidth === 0 || frame.codedHeight === 0) {
123 |         this.log.debug('empty frame detected, ignoring');
124 |         return;
125 |       }
126 | 
127 |       let skipProcessingFrame = this.isDisabled ?? this.options.backgroundDisabled ?? false;
128 |       if (typeof this.options.blurRadius !== 'number' && typeof this.options.imagePath !== 'string') {
129 |         skipProcessingFrame = true;
130 |       }
131 | 
132 |       if (skipProcessingFrame) {
133 |         controller.enqueue(frame);
134 |         enqueuedFrame = true;
135 |         return;
136 |       }
137 | 
138 |       const frameTimeMs = Date.now();
139 |       if (!this.canvas) {
140 |         throw TypeError('Canvas needs to be initialized first');
141 |       }
142 |       this.canvas.width = frame.displayWidth;
143 |       this.canvas.height = frame.displayHeight;
144 | 
145 |       // Render a copy of the first frame is rendered to the screen as soon as possible to act
146 |       // as a less jarring initial state than a solid color while the synchronous work below
147 |       // (segmentation + frame rendering) occurs.
148 |       //
149 |       // Ideally, these sync tasks could be offloaded to a webworker, but this is challenging
150 |       // given WebGLTextures cannot be easily passed in a `postMessage`.
151 |       if (this.isFirstFrame) {
152 |         controller.enqueue(frame.clone());
153 | 
154 |         // Wait for the frame that was enqueued above to render before doing the sync work
155 |         // below - otherwise, the sync work will take over the event loop and prevent the render
156 |         // from occurring
157 |         if (this.inputVideo) {
158 |           await new Promise((resolve) => {
159 |             this.inputVideo!.requestVideoFrameCallback((_now, e) => {
160 |               const durationUntilFrameRenderedInMs = e.expectedDisplayTime - e.presentationTime;
161 |               setTimeout(resolve, durationUntilFrameRenderedInMs);
162 |             });
163 |           });
164 |         }
165 |       }
166 |       this.isFirstFrame = false;
167 | 
168 |       const filterStartTimeMs = performance.now();
169 | 
170 |       const segmentationPromise = new Promise<void>((resolve, reject) => {
171 |         try {
172 |           let segmentationStartTimeMs = performance.now();
173 |           // NOTE: this.imageSegmenter?.segmentForVideo is synchronous, and blocks the event loop
174 |           // for tens to ~100 ms! The promise wrapper is just used to flatten out the call hierarchy.
175 |           this.imageSegmenter?.segmentForVideo(frame, segmentationStartTimeMs, (result) => {
176 |             this.segmentationTimeMs = performance.now() - segmentationStartTimeMs;
177 |             this.segmentationResults = result;
178 |             this.updateMask(result.categoryMask);
179 |             result.close();
180 |             resolve();
181 |           });
182 |         } catch (e) {
183 |           reject(e);
184 |         }
185 |       });
186 | 
187 |       // NOTE: `this.drawFrame` is synchronous, and could take tens of ms to run!
188 |       this.drawFrame(frame);
189 |       if (this.canvas && this.canvas.width > 0 && this.canvas.height > 0) {
190 |         const newFrame = new VideoFrame(this.canvas, {
191 |           timestamp: frame.timestamp || frameTimeMs,
192 |         });
193 |         controller.enqueue(newFrame);
194 |         const filterTimeMs = performance.now() - filterStartTimeMs;
195 |         const stats: FrameProcessingStats = {
196 |           processingTimeMs: this.segmentationTimeMs + filterTimeMs,
197 |           segmentationTimeMs: this.segmentationTimeMs,
198 |           filterTimeMs,
199 |         };
200 |         this.options.onFrameProcessed?.(stats);
201 |       } else {
202 |         controller.enqueue(frame);
203 |       }
204 |       await segmentationPromise;
205 |     } catch (e) {
206 |       this.log.error('Error while processing frame: ', e);
207 |     } finally {
208 |       if (!enqueuedFrame) {
209 |         frame.close();
210 |       }
211 |     }
212 |   }
213 | 
214 |   async update(opts: BackgroundOptions) {
215 |     this.options = { ...this.options, ...opts };
216 | 
217 |     this.gl?.setBlurRadius(opts.blurRadius ?? null);
218 |     if (opts.imagePath) {
219 |       await this.loadAndSetBackground(opts.imagePath);
220 |     } else {
221 |       this.gl?.setBackgroundImage(null);
222 |     }
223 |     this.gl?.setBackgroundDisabled(opts.backgroundDisabled ?? false);
224 |   }
225 | 
226 |   private async drawFrame(frame: VideoFrame) {
227 |     if (!this.gl) return;
228 |     this.gl?.renderFrame(frame);
229 |   }
230 | 
231 |   private async updateMask(mask: vision.MPMask | undefined) {
232 |     if (!mask) return;
233 |     this.gl?.updateMask(mask.getAsWebGLTexture());
234 |   }
235 | }
236 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 


--------------------------------------------------------------------------------
/src/webgl/index.ts:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * WebGL setup for the mask processor
  3 |  * potential improvements:
  4 |  * - downsample the video texture in background blur scenario before applying the (gaussian) blur for better performance
  5 |  *
  6 |  */
  7 | import { getLogger, LoggerNames} from '../logger';
  8 | import { applyBlur, createBlurProgram } from './shader-programs/blurShader';
  9 | import { createBoxBlurProgram } from './shader-programs/boxBlurShader';
 10 | import { createCompositeProgram } from './shader-programs/compositeShader';
 11 | import { applyDownsampling, createDownSampler } from './shader-programs/downSampler';
 12 | import {
 13 |   createFramebuffer,
 14 |   createVertexBuffer,
 15 |   getEmptyImageData,
 16 |   initTexture,
 17 |   resizeImageToCover,
 18 | } from './utils';
 19 | 
 20 | const log = getLogger(LoggerNames.WebGl);
 21 | 
 22 | export const setupWebGL = (canvas: OffscreenCanvas | HTMLCanvasElement) => {
 23 |   const gl = canvas.getContext('webgl2', {
 24 |     antialias: true,
 25 |     premultipliedAlpha: true,
 26 |   }) as WebGL2RenderingContext;
 27 | 
 28 |   let blurRadius: number | null = null;
 29 |   let maskBlurRadius: number | null = 8;
 30 |   const downsampleFactor = 4;
 31 | 
 32 |   if (!gl) {
 33 |     log.error('Failed to create WebGL context');
 34 |     return undefined;
 35 |   }
 36 | 
 37 |   gl.enable(gl.BLEND);
 38 |   gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA);
 39 | 
 40 |   // Create the composite program
 41 |   const composite = createCompositeProgram(gl);
 42 |   const compositeProgram = composite.program;
 43 |   const positionLocation = composite.attribLocations.position;
 44 |   const {
 45 |     mask: maskTextureLocation,
 46 |     frame: frameTextureLocation,
 47 |     background: bgTextureLocation,
 48 |     disableBackground: disableBackgroundLocation,
 49 |   } = composite.uniformLocations;
 50 | 
 51 |   // Create the blur program using the same vertex shader source
 52 |   const blur = createBlurProgram(gl);
 53 |   const blurProgram = blur.program;
 54 |   const blurUniforms = blur.uniforms;
 55 | 
 56 |   // Create the box blur program
 57 |   const boxBlur = createBoxBlurProgram(gl);
 58 |   const boxBlurProgram = boxBlur.program;
 59 |   const boxBlurUniforms = boxBlur.uniforms;
 60 | 
 61 |   const bgTexture = initTexture(gl, 0);
 62 |   const frameTexture = initTexture(gl, 1);
 63 |   const vertexBuffer = createVertexBuffer(gl);
 64 | 
 65 |   if (!vertexBuffer) {
 66 |     throw new Error('Failed to create vertex buffer');
 67 |   }
 68 | 
 69 |   // Create additional textures and framebuffers for processing
 70 |   let bgBlurTextures: WebGLTexture[] = [];
 71 |   let bgBlurFrameBuffers: WebGLFramebuffer[] = [];
 72 |   let blurredMaskTexture: WebGLTexture | null = null;
 73 | 
 74 |   // For double buffering the final mask
 75 |   let finalMaskTextures: WebGLTexture[] = [];
 76 |   let readMaskIndex = 0; // Index for renderFrame to read from
 77 |   let writeMaskIndex = 1; // Index for updateMask to write to
 78 | 
 79 |   // Create textures for background processing (blur)
 80 |   bgBlurTextures.push(initTexture(gl, 3)); // For blur pass 1
 81 |   bgBlurTextures.push(initTexture(gl, 4)); // For blur pass 2
 82 | 
 83 |   const bgBlurTextureWidth = Math.floor(canvas.width / downsampleFactor);
 84 |   const bgBlurTextureHeight = Math.floor(canvas.height / downsampleFactor);
 85 | 
 86 |   const downSampler = createDownSampler(gl, bgBlurTextureWidth, bgBlurTextureHeight);
 87 | 
 88 |   // Create framebuffers for background processing
 89 |   bgBlurFrameBuffers.push(
 90 |     createFramebuffer(gl, bgBlurTextures[0], bgBlurTextureWidth, bgBlurTextureHeight),
 91 |   );
 92 |   bgBlurFrameBuffers.push(
 93 |     createFramebuffer(gl, bgBlurTextures[1], bgBlurTextureWidth, bgBlurTextureHeight),
 94 |   );
 95 | 
 96 |   // Initialize texture for the first mask blur pass
 97 |   const tempMaskTexture = initTexture(gl, 5);
 98 |   const tempMaskFrameBuffer = createFramebuffer(gl, tempMaskTexture, canvas.width, canvas.height);
 99 | 
100 |   // Initialize two textures for double-buffering the final mask
101 |   finalMaskTextures.push(initTexture(gl, 6)); // For reading in renderFrame
102 |   finalMaskTextures.push(initTexture(gl, 7)); // For writing in updateMask
103 | 
104 |   // Create framebuffers for the final mask textures
105 |   const finalMaskFrameBuffers = [
106 |     createFramebuffer(gl, finalMaskTextures[0], canvas.width, canvas.height),
107 |     createFramebuffer(gl, finalMaskTextures[1], canvas.width, canvas.height),
108 |   ];
109 | 
110 |   let backgroundImageDisabled = false;
111 | 
112 |   // Set up uniforms for the composite shader
113 |   gl.useProgram(compositeProgram);
114 |   gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
115 |   gl.uniform1i(bgTextureLocation, 0);
116 |   gl.uniform1i(frameTextureLocation, 1);
117 |   gl.uniform1i(maskTextureLocation, 2);
118 | 
119 |   // Store custom background image
120 |   let customBackgroundImage: ImageBitmap | ImageData | null = null;
121 | 
122 |   function renderFrame(frame: VideoFrame) {
123 |     if (frame.codedWidth === 0 || finalMaskTextures.length === 0) {
124 |       return;
125 |     }
126 | 
127 |     const width = frame.displayWidth;
128 |     const height = frame.displayHeight;
129 | 
130 |     // Prepare frame texture
131 |     gl.activeTexture(gl.TEXTURE1);
132 |     gl.bindTexture(gl.TEXTURE_2D, frameTexture);
133 |     gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
134 | 
135 |     // Apply blur if enabled (and no custom background is set)
136 |     let backgroundTexture = bgTexture;
137 | 
138 |     if (blurRadius) {
139 |       const downSampledFrameTexture = applyDownsampling(
140 |         gl,
141 |         frameTexture,
142 |         downSampler,
143 |         vertexBuffer!,
144 |         bgBlurTextureWidth,
145 |         bgBlurTextureHeight,
146 |       );
147 |       backgroundTexture = applyBlur(
148 |         gl,
149 |         downSampledFrameTexture,
150 |         bgBlurTextureWidth,
151 |         bgBlurTextureHeight,
152 |         blurRadius,
153 |         blurProgram,
154 |         blurUniforms,
155 |         vertexBuffer!,
156 |         bgBlurFrameBuffers,
157 |         bgBlurTextures,
158 |       );
159 |     } else if (customBackgroundImage) {
160 |       gl.activeTexture(gl.TEXTURE0);
161 |       gl.bindTexture(gl.TEXTURE_2D, bgTexture);
162 |       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
163 |       backgroundTexture = bgTexture;
164 |     }
165 | 
166 |     // Render the final composite
167 |     gl.viewport(0, 0, width, height);
168 |     gl.clearColor(1.0, 1.0, 1.0, 1.0);
169 |     gl.clear(gl.COLOR_BUFFER_BIT);
170 | 
171 |     gl.useProgram(compositeProgram);
172 |     gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer);
173 |     gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
174 |     gl.enableVertexAttribArray(positionLocation);
175 | 
176 |     // Set background texture (either original, blurred or custom)
177 |     gl.activeTexture(gl.TEXTURE0);
178 |     gl.bindTexture(gl.TEXTURE_2D, backgroundTexture);
179 |     gl.uniform1i(bgTextureLocation, 0);
180 |     gl.uniform1i(disableBackgroundLocation, backgroundImageDisabled ? 1 : 0);
181 | 
182 |     // Set frame texture
183 |     gl.activeTexture(gl.TEXTURE1);
184 |     gl.bindTexture(gl.TEXTURE_2D, frameTexture);
185 |     gl.uniform1i(frameTextureLocation, 1);
186 | 
187 |     // Set mask texture - always read from the current read index
188 |     gl.activeTexture(gl.TEXTURE2);
189 |     gl.bindTexture(gl.TEXTURE_2D, finalMaskTextures[readMaskIndex]);
190 |     gl.uniform1i(maskTextureLocation, 2);
191 |     gl.drawArrays(gl.TRIANGLES, 0, 6);
192 |   }
193 | 
194 |   /**
195 |    * Set or update the background image
196 |    * @param image The background image to use, or null to clear
197 |    */
198 |   async function setBackgroundImage(image: ImageBitmap | null) {
199 |     // Clear existing background
200 |     customBackgroundImage = null;
201 | 
202 |     if (image) {
203 |       customBackgroundImage = getEmptyImageData();
204 |       try {
205 |         // Resize and crop the image to cover the canvas
206 |         const croppedImage = await resizeImageToCover(image, canvas.width, canvas.height);
207 | 
208 |         // Store the cropped and resized image
209 |         customBackgroundImage = croppedImage;
210 |       } catch (error) {
211 |         log.error(
212 |           'Error processing background image, falling back to black background:',
213 |           error,
214 |         );
215 |       }
216 | 
217 |       gl.activeTexture(gl.TEXTURE0);
218 |       gl.bindTexture(gl.TEXTURE_2D, bgTexture);
219 |       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, customBackgroundImage);
220 |     }
221 |   }
222 | 
223 |   function setBlurRadius(radius: number | null) {
224 |     blurRadius = radius ? Math.max(1, Math.floor(radius / downsampleFactor)) : null; // we are downsampling the blur texture, so decrease the radius here for better performance with a similar visual result
225 |     setBackgroundImage(null);
226 |   }
227 | 
228 |   function setBackgroundDisabled(disabled: boolean) {
229 |     backgroundImageDisabled = disabled;
230 |   }
231 | 
232 |   function updateMask(mask: WebGLTexture) {
233 |     // Use the existing applyBlur function to apply the first blur pass
234 |     // The second blur pass will be written to finalMaskTextures[writeMaskIndex]
235 | 
236 |     // Create temporary arrays for the single blur operation
237 |     const tempFramebuffers = [tempMaskFrameBuffer, finalMaskFrameBuffers[writeMaskIndex]];
238 | 
239 |     const tempTextures = [tempMaskTexture, finalMaskTextures[writeMaskIndex]];
240 | 
241 |     // Apply the blur using the existing function
242 |     applyBlur(
243 |       gl,
244 |       mask,
245 |       canvas.width,
246 |       canvas.height,
247 |       maskBlurRadius || 1.0,
248 |       boxBlurProgram,
249 |       boxBlurUniforms,
250 |       vertexBuffer!,
251 |       tempFramebuffers,
252 |       tempTextures,
253 |     );
254 | 
255 |     // Swap indices for the next frame
256 |     readMaskIndex = writeMaskIndex;
257 |     writeMaskIndex = 1 - writeMaskIndex;
258 |   }
259 | 
260 |   function cleanup() {
261 |     gl.deleteProgram(compositeProgram);
262 |     gl.deleteProgram(blurProgram);
263 |     gl.deleteProgram(boxBlurProgram);
264 |     gl.deleteTexture(bgTexture);
265 |     gl.deleteTexture(frameTexture);
266 |     gl.deleteTexture(tempMaskTexture);
267 |     gl.deleteFramebuffer(tempMaskFrameBuffer);
268 | 
269 |     for (const texture of bgBlurTextures) {
270 |       gl.deleteTexture(texture);
271 |     }
272 |     for (const framebuffer of bgBlurFrameBuffers) {
273 |       gl.deleteFramebuffer(framebuffer);
274 |     }
275 |     for (const texture of finalMaskTextures) {
276 |       gl.deleteTexture(texture);
277 |     }
278 |     for (const framebuffer of finalMaskFrameBuffers) {
279 |       gl.deleteFramebuffer(framebuffer);
280 |     }
281 |     gl.deleteBuffer(vertexBuffer);
282 | 
283 |     if (blurredMaskTexture) {
284 |       gl.deleteTexture(blurredMaskTexture);
285 |     }
286 | 
287 |     if (downSampler) {
288 |       gl.deleteTexture(downSampler.texture);
289 |       gl.deleteFramebuffer(downSampler.framebuffer);
290 |       gl.deleteProgram(downSampler.program);
291 |     }
292 | 
293 |     // Release any ImageBitmap resources
294 |     if (customBackgroundImage) {
295 |       if (customBackgroundImage instanceof ImageBitmap) {
296 |         customBackgroundImage.close();
297 |       }
298 |       customBackgroundImage = null;
299 |     }
300 |     bgBlurTextures = [];
301 |     bgBlurFrameBuffers = [];
302 |     finalMaskTextures = [];
303 |   }
304 | 
305 |   return { renderFrame, updateMask, setBackgroundImage, setBlurRadius, setBackgroundDisabled, cleanup };
306 | };
307 | 


--------------------------------------------------------------------------------
/src/ProcessorWrapper.ts:
--------------------------------------------------------------------------------
  1 | import { type ProcessorOptions, type Track, type TrackProcessor } from 'livekit-client';
  2 | import { TrackTransformer, TrackTransformerDestroyOptions } from './transformers';
  3 | import { createCanvas, waitForTrackResolution } from './utils';
  4 | import { LoggerNames, getLogger } from './logger';
  5 | 
  6 | type ProcessorWrapperLifecycleState = 'idle' | 'initializing' | 'running' | 'media-exhausted' | 'destroying' | 'destroyed';
  7 | 
  8 | export interface ProcessorWrapperOptions {
  9 |   /**
 10 |    * Maximum frame rate for fallback canvas.captureStream implementation
 11 |    * Default: 30
 12 |    */
 13 |   maxFps?: number;
 14 | }
 15 | 
 16 | export default class ProcessorWrapper<
 17 |   TransformerOptions extends Record<string, unknown>,
 18 |   Transformer extends TrackTransformer<TransformerOptions> = TrackTransformer<TransformerOptions>,
 19 | >
 20 |   implements TrackProcessor<Track.Kind>
 21 | {
 22 |   /**
 23 |    * Determines if the Processor is supported on the current browser
 24 |    */
 25 |   static get isSupported() {
 26 |     // Check for primary implementation support
 27 |     const hasStreamProcessor =
 28 |       typeof MediaStreamTrackGenerator !== 'undefined' &&
 29 |       typeof MediaStreamTrackProcessor !== 'undefined';
 30 | 
 31 |     // Check for fallback implementation support
 32 |     const hasFallbackSupport =
 33 |       typeof HTMLCanvasElement !== 'undefined' &&
 34 |       typeof VideoFrame !== 'undefined' &&
 35 |       'captureStream' in HTMLCanvasElement.prototype;
 36 | 
 37 |     // We can work if either implementation is available
 38 |     return hasStreamProcessor || hasFallbackSupport;
 39 |   }
 40 | 
 41 |   /**
 42 |    * Determines if modern browser APIs are supported, which yield better performance
 43 |    */
 44 |   static get hasModernApiSupport() {
 45 |     return (
 46 |       typeof MediaStreamTrackGenerator !== 'undefined' &&
 47 |       typeof MediaStreamTrackProcessor !== 'undefined'
 48 |     );
 49 |   }
 50 | 
 51 |   name: string;
 52 | 
 53 |   source?: MediaStreamVideoTrack;
 54 | 
 55 |   processor?: MediaStreamTrackProcessor<VideoFrame>;
 56 | 
 57 |   trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;
 58 | 
 59 |   canvas?: OffscreenCanvas | HTMLCanvasElement;
 60 | 
 61 |   displayCanvas?: HTMLCanvasElement;
 62 | 
 63 |   sourceDummy?: HTMLMediaElement;
 64 | 
 65 |   processedTrack?: MediaStreamTrack;
 66 | 
 67 |   transformer: Transformer;
 68 | 
 69 |   // For tracking whether we're using the stream API fallback
 70 |   private useStreamFallback = false;
 71 | 
 72 |   // For fallback rendering with canvas.captureStream()
 73 |   private capturedStream?: MediaStream;
 74 | 
 75 |   private animationFrameId?: number;
 76 | 
 77 |   private renderContext?: CanvasRenderingContext2D;
 78 | 
 79 |   private frameCallback?: (frame: VideoFrame) => void;
 80 | 
 81 |   private processingEnabled = false;
 82 | 
 83 |   // FPS control for fallback implementation
 84 |   private maxFps: number;
 85 | 
 86 |   private log = getLogger(LoggerNames.ProcessorWrapper);
 87 | 
 88 |   private lifecycleState: ProcessorWrapperLifecycleState = 'idle';
 89 | 
 90 |   constructor(
 91 |     transformer: Transformer,
 92 |     name: string,
 93 |     options: ProcessorWrapperOptions = {},
 94 |   ) {
 95 |     this.name = name;
 96 |     this.transformer = transformer;
 97 |     this.maxFps = options.maxFps ?? 30;
 98 |   }
 99 | 
100 |   private async setup(opts: ProcessorOptions<Track.Kind>) {
101 |     this.source = opts.track as MediaStreamVideoTrack;
102 | 
103 |     const { width, height } = await waitForTrackResolution(this.source);
104 |     this.sourceDummy = opts.element;
105 | 
106 |     if (!(this.sourceDummy instanceof HTMLVideoElement)) {
107 |       throw TypeError('Currently only video transformers are supported');
108 |     }
109 | 
110 |     if (this.sourceDummy instanceof HTMLVideoElement) {
111 |       this.sourceDummy.height = height ?? 300;
112 |       this.sourceDummy.width = width ?? 300;
113 |     }
114 | 
115 |     this.useStreamFallback = !ProcessorWrapper.hasModernApiSupport;
116 | 
117 |     if (this.useStreamFallback) {
118 |       // Create a visible canvas for the fallback implementation or use an existing one if provided
119 |       const existingCanvas = document.querySelector(
120 |         'canvas[data-livekit-processor="' + this.name + '"]',
121 |       ) as HTMLCanvasElement;
122 | 
123 |       if (existingCanvas) {
124 |         this.displayCanvas = existingCanvas;
125 |         this.displayCanvas.width = width ?? 300;
126 |         this.displayCanvas.height = height ?? 300;
127 |       } else {
128 |         this.displayCanvas = document.createElement('canvas');
129 |         this.displayCanvas.width = width ?? 300;
130 |         this.displayCanvas.height = height ?? 300;
131 |         this.displayCanvas.style.display = 'none';
132 |         this.displayCanvas.dataset.livekitProcessor = this.name;
133 |         document.body.appendChild(this.displayCanvas);
134 |       }
135 | 
136 |       this.renderContext = this.displayCanvas.getContext('2d')!;
137 |       this.capturedStream = this.displayCanvas.captureStream();
138 |       this.canvas = createCanvas(width ?? 300, height ?? 300);
139 |     } else {
140 |       // Use MediaStreamTrackProcessor API
141 |       this.processor = new MediaStreamTrackProcessor({ track: this.source });
142 |       this.trackGenerator = new MediaStreamTrackGenerator({
143 |         kind: 'video',
144 |         signalTarget: this.source,
145 |       });
146 |       this.canvas = createCanvas(width ?? 300, height ?? 300);
147 |     }
148 |   }
149 | 
150 |   async init(opts: ProcessorOptions<Track.Kind>): Promise<void> {
151 |     this.log.debug('Init called');
152 |     this.lifecycleState = 'initializing';
153 |     await this.setup(opts);
154 | 
155 |     if (!this.canvas) {
156 |       throw new TypeError('Expected canvas to be defined after setup');
157 |     }
158 | 
159 |     await this.transformer.init({
160 |       outputCanvas: this.canvas,
161 |       inputElement: this.sourceDummy as HTMLVideoElement,
162 |     });
163 | 
164 |     if (this.useStreamFallback) {
165 |       this.initFallbackPath();
166 |     } else {
167 |       this.initStreamProcessorPath();
168 |     }
169 |     this.lifecycleState = 'running';
170 |   }
171 | 
172 |   private initStreamProcessorPath() {
173 |     if (!this.processor || !this.trackGenerator) {
174 |       throw new TypeError(
175 |         'Expected processor and trackGenerator to be defined for stream processor path',
176 |       );
177 |     }
178 | 
179 |     const readableStream = this.processor.readable;
180 |     const pipedStream = readableStream.pipeThrough(this.transformer!.transformer!);
181 | 
182 |     pipedStream
183 |       .pipeTo(this.trackGenerator.writable)
184 |       // if stream finishes, the media to process is exhausted
185 |       .then(() => this.handleMediaExhausted())
186 |       // destroy processor if stream errors - unless it's an abort error
187 |       .catch((e) => {
188 |         if (e instanceof DOMException && e.name === 'AbortError') {
189 |           this.log.log('stream processor path aborted');
190 |         } else if (e instanceof DOMException && e.name === 'InvalidStateError' && e.message === 'Stream closed') {
191 |           this.log.log('stream processor underlying stream closed');
192 |           this.handleMediaExhausted();
193 |         } else {
194 |           this.log.error('error when trying to pipe', e);
195 |           this.destroy();
196 |         }
197 |       });
198 | 
199 |     this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;
200 |   }
201 | 
202 |   private initFallbackPath() {
203 |     if (!this.capturedStream || !this.source || !this.canvas || !this.renderContext) {
204 |       throw new TypeError('Missing required components for fallback implementation');
205 |     }
206 | 
207 |     this.processedTrack = this.capturedStream.getVideoTracks()[0];
208 |     this.processingEnabled = true;
209 | 
210 |     // Set up the frame callback for the transformer
211 |     this.frameCallback = (frame: VideoFrame) => {
212 |       if (!this.processingEnabled || !frame) {
213 |         frame.close();
214 |         return;
215 |       }
216 | 
217 |       const controller = {
218 |         enqueue: (processedFrame: VideoFrame) => {
219 |           if (this.renderContext && this.displayCanvas) {
220 |             // Draw the processed frame to the visible canvas
221 |             this.renderContext.drawImage(
222 |               processedFrame,
223 |               0,
224 |               0,
225 |               this.displayCanvas.width,
226 |               this.displayCanvas.height,
227 |             );
228 |             processedFrame.close();
229 |           }
230 |         },
231 |       } as TransformStreamDefaultController<VideoFrame>;
232 | 
233 |       try {
234 |         // Pass the frame through our transformer
235 |         // @ts-ignore - The controller expects both VideoFrame & AudioData but we're only using VideoFrame
236 |         this.transformer.transform(frame, controller);
237 |       } catch (e) {
238 |         this.log.error('Error in transform:', e);
239 |         frame.close();
240 |       }
241 |     };
242 | 
243 |     // Start the rendering loop
244 |     this.startRenderLoop();
245 |   }
246 | 
247 |   private startRenderLoop() {
248 |     if (!this.sourceDummy || !(this.sourceDummy instanceof HTMLVideoElement)) {
249 |       this.handleMediaExhausted();
250 |       return;
251 |     }
252 | 
253 |     // Store the last processed timestamp to avoid duplicate processing
254 |     let lastVideoTimestamp = -1;
255 |     let lastFrameTime = 0;
256 |     const videoElement = this.sourceDummy as HTMLVideoElement;
257 |     const minFrameInterval = 1000 / this.maxFps; // Minimum time between frames
258 | 
259 |     // Estimate the video's native frame rate
260 |     let estimatedVideoFps = this.maxFps;
261 |     let frameTimeHistory: number[] = [];
262 |     let lastVideoTimeChange = 0;
263 |     let frameCount = 0;
264 |     let lastFpsLog = 0;
265 | 
266 |     const renderLoop = () => {
267 |       if (
268 |         !this.processingEnabled ||
269 |         !this.sourceDummy ||
270 |         !(this.sourceDummy instanceof HTMLVideoElement)
271 |       ) {
272 |         this.handleMediaExhausted();
273 |         return;
274 |       }
275 | 
276 |       if (this.sourceDummy.paused) {
277 |         this.log.warn('Video is paused, trying to play');
278 |         this.sourceDummy.play();
279 |         return;
280 |       }
281 | 
282 |       // Only process a new frame if the video has actually updated
283 |       const videoTime = videoElement.currentTime;
284 |       const now = performance.now();
285 |       const timeSinceLastFrame = now - lastFrameTime;
286 | 
287 |       // Detect if video has a new frame
288 |       const hasNewFrame = videoTime !== lastVideoTimestamp;
289 | 
290 |       // Update frame rate estimation if we have a new frame
291 |       if (hasNewFrame) {
292 |         if (lastVideoTimeChange > 0) {
293 |           const timeBetweenFrames = now - lastVideoTimeChange;
294 |           frameTimeHistory.push(timeBetweenFrames);
295 | 
296 |           // Keep a rolling window of the last 10 frame times
297 |           if (frameTimeHistory.length > 10) {
298 |             frameTimeHistory.shift();
299 |           }
300 | 
301 |           // Calculate average frame interval
302 |           if (frameTimeHistory.length > 2) {
303 |             const avgFrameTime =
304 |               frameTimeHistory.reduce((sum, time) => sum + time, 0) / frameTimeHistory.length;
305 |             estimatedVideoFps = 1000 / avgFrameTime;
306 | 
307 |             // Log estimated FPS every 5 seconds in development environments
308 |             // Use a simpler check that works in browsers without process.env
309 |             const isDevelopment =
310 |               (typeof window !== 'undefined' && window.location.hostname === 'localhost') ||
311 |               window.location.hostname === '127.0.0.1';
312 | 
313 |             if (isDevelopment && now - lastFpsLog > 5000) {
314 |               this.log.debug(
315 |                 `[${this.name}] Estimated video FPS: ${estimatedVideoFps.toFixed(
316 |                   1,
317 |                 )}, Processing at: ${(frameCount / 5).toFixed(1)} FPS`,
318 |               );
319 |               frameCount = 0;
320 |               lastFpsLog = now;
321 |             }
322 |           }
323 |         }
324 |         lastVideoTimeChange = now;
325 |       }
326 | 
327 |       // Determine if we should process this frame
328 |       // We'll process if:
329 |       // 1. The video has a new frame
330 |       // 2. Enough time has passed since last frame (respecting maxFps)
331 |       const timeThresholdMet = timeSinceLastFrame >= minFrameInterval;
332 | 
333 |       if (hasNewFrame && timeThresholdMet) {
334 |         lastVideoTimestamp = videoTime;
335 |         lastFrameTime = now;
336 |         frameCount++;
337 | 
338 |         try {
339 |           // Create a VideoFrame from the video element
340 |           if (videoElement.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
341 |             const frame = new VideoFrame(videoElement);
342 |             if (this.frameCallback) {
343 |               this.frameCallback(frame);
344 |             } else {
345 |               frame.close();
346 |             }
347 |           }
348 |         } catch (e) {
349 |           this.log.error('Error in render loop:', e);
350 |         }
351 |       }
352 |       this.animationFrameId = requestAnimationFrame(renderLoop);
353 |     };
354 | 
355 |     this.animationFrameId = requestAnimationFrame(renderLoop);
356 |   }
357 | 
358 |   async restart(opts: ProcessorOptions<Track.Kind>): Promise<void> {
359 |     this.log.debug('Restart called');
360 |     await this.destroy({ willProcessorRestart: true });
361 |     await this.init(opts);
362 |   }
363 | 
364 |   async restartTransformer(...options: Parameters<(typeof this.transformer)['restart']>) {
365 |     // @ts-ignore unclear why the restart method only accepts VideoTransformerInitOptions instead of either those or AudioTransformerInitOptions
366 |     await this.transformer.restart(options[0]);
367 |   }
368 | 
369 |   async updateTransformerOptions(...options: Parameters<(typeof this.transformer)['update']>) {
370 |     await this.transformer.update(options[0]);
371 |   }
372 | 
373 |   /** Called if the media pipeline no longer can read frames to process from the source media */
374 |   private async handleMediaExhausted() {
375 |     this.log.debug('Media was exhausted from source');
376 |     if (this.lifecycleState !== 'running') {
377 |       return;
378 |     }
379 |     this.lifecycleState = 'media-exhausted'
380 |     await this.cleanup();
381 |   }
382 | 
383 |   /** Tears down the media stack logic initialized in initStreamProcessorPath / initFallbackPath */
384 |   private async cleanup() {
385 |     if (this.useStreamFallback) {
386 |       this.processingEnabled = false;
387 |       if (this.animationFrameId) {
388 |         cancelAnimationFrame(this.animationFrameId);
389 |         this.animationFrameId = undefined;
390 |       }
391 |       if (this.displayCanvas && this.displayCanvas.parentNode) {
392 |         this.displayCanvas.parentNode.removeChild(this.displayCanvas);
393 |       }
394 |       this.capturedStream?.getTracks().forEach((track) => track.stop());
395 |     } else {
396 |       // NOTE: closing writableControl below terminates the stream in initStreamProcessorPath /
397 |       // calls the .then(...) which calls this.handleMediaExhausted
398 |       await this.processor?.writableControl?.close();
399 |       this.trackGenerator?.stop();
400 |     }
401 |   }
402 | 
403 |   async destroy(transformerDestroyOptions: TrackTransformerDestroyOptions = { willProcessorRestart: false }) {
404 |     this.log.debug(`Destroy called - lifecycleState=${this.lifecycleState}, transformerDestroyOptions=${JSON.stringify(transformerDestroyOptions)}`);
405 |     switch (this.lifecycleState) {
406 |       case 'running':
407 |       case 'media-exhausted':
408 |         this.lifecycleState = 'destroying';
409 | 
410 |         await this.cleanup();
411 | 
412 |         await this.transformer.destroy(transformerDestroyOptions);
413 |         this.lifecycleState = 'destroyed';
414 |         break;
415 | 
416 |       default:
417 |         break;
418 |     }
419 |   }
420 | }
421 | 


--------------------------------------------------------------------------------
/example/sample.ts:
--------------------------------------------------------------------------------
  1 | import {
  2 |   ConnectionQuality,
  3 |   ConnectionState,
  4 |   DisconnectReason,
  5 |   LocalAudioTrack,
  6 |   LocalParticipant,
  7 |   LocalVideoTrack,
  8 |   LogLevel,
  9 |   MediaDeviceFailure,
 10 |   Participant,
 11 |   ParticipantEvent,
 12 |   RemoteParticipant,
 13 |   RemoteVideoTrack,
 14 |   Room,
 15 |   RoomConnectOptions,
 16 |   RoomEvent,
 17 |   RoomOptions,
 18 |   Track,
 19 |   TrackPublication,
 20 |   VideoCaptureOptions,
 21 |   VideoPresets,
 22 |   VideoPresets43,
 23 |   createAudioAnalyser,
 24 |   facingModeFromLocalTrack,
 25 |   setLogLevel,
 26 | } from 'livekit-client';
 27 | import { BackgroundProcessor, BackgroundProcessorOptions } from '../src';
 28 | 
 29 | const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
 30 | 
 31 | const BLUR_RADIUS = 10;
 32 | const IMAGE_PATH = '/samantha-gades-BlIhVfXbi9s-unsplash.jpg';
 33 | 
 34 | const state = {
 35 |   defaultDevices: new Map<MediaDeviceKind, string>(),
 36 |   bitrateInterval: undefined as any,
 37 |   isBackgroundProcessorEnabled: false,
 38 |   backgroundProcessor: BackgroundProcessor({ mode: 'background-blur', blurRadius: BLUR_RADIUS }),
 39 | };
 40 | let currentRoom: Room | undefined;
 41 | 
 42 | let startTime: number;
 43 | 
 44 | const searchParams = new URLSearchParams(window.location.search);
 45 | const storedUrl = searchParams.get('url') ?? 'ws://localhost:7880';
 46 | const storedToken = searchParams.get('token') ?? '';
 47 | $<HTMLInputElement>('url').value = storedUrl;
 48 | $<HTMLInputElement>('token').value = storedToken;
 49 | 
 50 | function updateSearchParams(url: string, token: string) {
 51 |   const params = new URLSearchParams({ url, token });
 52 |   window.history.replaceState(null, '', `${window.location.pathname}?${params.toString()}`);
 53 | }
 54 | 
 55 | // handles actions from the HTML
 56 | const appActions = {
 57 |   connectWithFormInput: async () => {
 58 |     const url = (<HTMLInputElement>$('url')).value;
 59 |     const token = (<HTMLInputElement>$('token')).value;
 60 | 
 61 |     setLogLevel(LogLevel.debug);
 62 |     updateSearchParams(url, token);
 63 | 
 64 |     const roomOpts: RoomOptions = {
 65 |       adaptiveStream: true,
 66 |       dynacast: true,
 67 |       publishDefaults: {
 68 |         simulcast: true,
 69 |         videoSimulcastLayers: [VideoPresets43.h120, VideoPresets43.h240],
 70 |       },
 71 |       videoCaptureDefaults: {
 72 |         resolution: VideoPresets43.h540.resolution,
 73 |       },
 74 |     };
 75 | 
 76 |     const connectOpts: RoomConnectOptions = {
 77 |       autoSubscribe: true,
 78 |     };
 79 |     await appActions.connectToRoom(url, token, roomOpts, connectOpts, true);
 80 | 
 81 |     state.bitrateInterval = setInterval(renderBitrate, 1000);
 82 |   },
 83 | 
 84 |   connectToRoom: async (
 85 |     url: string,
 86 |     token: string,
 87 |     roomOptions?: RoomOptions,
 88 |     connectOptions?: RoomConnectOptions,
 89 |     shouldPublish?: boolean,
 90 |   ): Promise<Room | undefined> => {
 91 |     const room = new Room(roomOptions);
 92 | 
 93 |     startTime = Date.now();
 94 |     await room.prepareConnection(url);
 95 |     const prewarmTime = Date.now() - startTime;
 96 |     appendLog(`prewarmed connection in ${prewarmTime}ms`);
 97 | 
 98 |     room
 99 |       .on(RoomEvent.ParticipantConnected, participantConnected)
100 |       .on(RoomEvent.ParticipantDisconnected, participantDisconnected)
101 |       .on(RoomEvent.Disconnected, handleRoomDisconnect)
102 |       .on(RoomEvent.Reconnecting, () => appendLog('Reconnecting to room'))
103 |       .on(RoomEvent.Reconnected, async () => {
104 |         appendLog(
105 |           'Successfully reconnected. server',
106 |           await room.engine.getConnectedServerAddress(),
107 |         );
108 |       })
109 |       .on(RoomEvent.LocalTrackPublished, (pub) => {
110 |         const track = pub.track as LocalAudioTrack;
111 | 
112 |         if (track instanceof LocalAudioTrack) {
113 |           const { calculateVolume } = createAudioAnalyser(track);
114 | 
115 |           setInterval(() => {
116 |             $('local-volume')?.setAttribute('value', calculateVolume().toFixed(4));
117 |           }, 200);
118 |         }
119 |         renderParticipant(room.localParticipant);
120 |         updateButtonsForPublishState();
121 |       })
122 |       .on(RoomEvent.LocalTrackUnpublished, () => {
123 |         renderParticipant(room.localParticipant);
124 |         updateButtonsForPublishState();
125 |       })
126 |       .on(RoomEvent.RoomMetadataChanged, (metadata) => {
127 |         appendLog('new metadata for room', metadata);
128 |       })
129 |       .on(RoomEvent.MediaDevicesChanged, handleDevicesChanged)
130 |       .on(RoomEvent.AudioPlaybackStatusChanged, () => {
131 |         if (room.canPlaybackAudio) {
132 |           $('start-audio-button')?.setAttribute('disabled', 'true');
133 |         } else {
134 |           $('start-audio-button')?.removeAttribute('disabled');
135 |         }
136 |       })
137 |       .on(RoomEvent.MediaDevicesError, (e: Error) => {
138 |         const failure = MediaDeviceFailure.getFailure(e);
139 |         appendLog('media device failure', failure);
140 |       })
141 |       .on(
142 |         RoomEvent.ConnectionQualityChanged,
143 |         (quality: ConnectionQuality, participant?: Participant) => {
144 |           appendLog('connection quality changed', participant?.identity, quality);
145 |         },
146 |       )
147 |       .on(RoomEvent.TrackSubscribed, (track, pub, participant) => {
148 |         appendLog('subscribed to track', pub.trackSid, participant.identity);
149 |         renderParticipant(participant);
150 |       })
151 |       .on(RoomEvent.TrackUnsubscribed, (_, pub, participant) => {
152 |         appendLog('unsubscribed from track', pub.trackSid);
153 |         renderParticipant(participant);
154 |       })
155 |       .on(RoomEvent.SignalConnected, async () => {
156 |         const signalConnectionTime = Date.now() - startTime;
157 |         appendLog(`signal connection established in ${signalConnectionTime}ms`);
158 |         // speed up publishing by starting to publish before it's fully connected
159 |         // publishing is accepted as soon as signal connection has established
160 |         if (shouldPublish) {
161 |           await room.localParticipant.enableCameraAndMicrophone();
162 |           appendLog(`tracks published in ${Date.now() - startTime}ms`);
163 |           updateButtonsForPublishState();
164 |         }
165 |       });
166 | 
167 |     try {
168 |       await room.connect(url, token, connectOptions);
169 |       const elapsed = Date.now() - startTime;
170 |       appendLog(
171 |         `successfully connected to ${room.name} in ${Math.round(elapsed)}ms`,
172 |         await room.engine.getConnectedServerAddress(),
173 |       );
174 |     } catch (error: any) {
175 |       let message: any = error;
176 |       if (error.message) {
177 |         message = error.message;
178 |       }
179 |       appendLog('could not connect:', message);
180 |       return;
181 |     }
182 |     currentRoom = room;
183 |     window.currentRoom = room;
184 |     setButtonsForState(true);
185 | 
186 |     room.remoteParticipants.forEach((participant) => {
187 |       participantConnected(participant);
188 |     });
189 |     participantConnected(room.localParticipant);
190 | 
191 |     return room;
192 |   },
193 | 
194 |   toggleAudio: async () => {
195 |     if (!currentRoom) return;
196 |     const enabled = currentRoom.localParticipant.isMicrophoneEnabled;
197 |     setButtonDisabled('toggle-audio-button', true);
198 |     if (enabled) {
199 |       appendLog('disabling audio');
200 |     } else {
201 |       appendLog('enabling audio');
202 |     }
203 |     await currentRoom.localParticipant.setMicrophoneEnabled(!enabled);
204 |     setButtonDisabled('toggle-audio-button', false);
205 |     updateButtonsForPublishState();
206 |   },
207 | 
208 |   toggleVideo: async () => {
209 |     if (!currentRoom) return;
210 |     setButtonDisabled('toggle-video-button', true);
211 |     const enabled = currentRoom.localParticipant.isCameraEnabled;
212 |     if (enabled) {
213 |       appendLog('disabling video');
214 |     } else {
215 |       appendLog('enabling video');
216 |     }
217 |     await currentRoom.localParticipant.setCameraEnabled(!enabled);
218 | 
219 |     setButtonDisabled('toggle-video-button', false);
220 |     renderParticipant(currentRoom.localParticipant);
221 | 
222 |     // update display
223 |     updateButtonsForPublishState();
224 |   },
225 | 
226 |   flipVideo: () => {
227 |     const videoPub = currentRoom?.localParticipant.getTrackPublication(Track.Source.Camera);
228 |     if (!videoPub) {
229 |       return;
230 |     }
231 |     const track = videoPub.track as LocalVideoTrack | undefined;
232 |     const facingMode = track && facingModeFromLocalTrack(track);
233 |     if (facingMode?.facingMode === 'environment') {
234 |       setButtonState('flip-video-button', 'Front Camera', false);
235 |     } else {
236 |       setButtonState('flip-video-button', 'Back Camera', false);
237 |     }
238 |     const options: VideoCaptureOptions = {
239 |       resolution: VideoPresets.h720.resolution,
240 |       facingMode: facingMode?.facingMode === 'environment' ? 'user' : 'environment',
241 |     };
242 |     videoPub.videoTrack?.restartTrack(options);
243 |   },
244 | 
245 |   toggleTrackProcessorEnabled: async () => {
246 |     if (!currentRoom) return;
247 | 
248 |     setButtonDisabled('toggle-track-processor', true);
249 | 
250 |     try {
251 |       const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
252 |         .track as LocalVideoTrack;
253 | 
254 |       if (state.isBackgroundProcessorEnabled) {
255 |         await camTrack.stopProcessor();
256 |         state.isBackgroundProcessorEnabled = false;
257 |         $("initial-mode-wrapper").style.display = 'block';
258 |       } else {
259 |         $("initial-mode-wrapper").style.display = 'none';
260 |         const initialMode = $<HTMLSelectElement>("initial-mode-select").value as BackgroundProcessorOptions['mode'];
261 | 
262 |         switch (initialMode) {
263 |           case 'disabled':
264 |             await state.backgroundProcessor.switchTo({ mode: 'disabled' });
265 |             break;
266 |           case 'virtual-background':
267 |             await state.backgroundProcessor.switchTo({ mode: 'virtual-background', imagePath: IMAGE_PATH });
268 |             break;
269 |           case 'background-blur':
270 |             await state.backgroundProcessor.switchTo({ mode: 'background-blur' });
271 |             break;
272 |         }
273 | 
274 |         state.isBackgroundProcessorEnabled = true;
275 |         await camTrack.setProcessor(state.backgroundProcessor);
276 |       }
277 |     } catch (e: any) {
278 |       appendLog(`ERROR: ${e.message}`);
279 |     } finally {
280 |       renderParticipant(currentRoom.localParticipant);
281 |       updateButtonsForPublishState();
282 |       updateTrackProcessorModeButtons();
283 |     }
284 |   },
285 | 
286 |   switchBackgroundMode: async (newMode: NonNullable<BackgroundProcessorOptions['mode']>) => {
287 |     if (!currentRoom) return;
288 | 
289 |     const controlButtonId = `switch-to-${newMode}-button`;
290 |     setButtonDisabled(controlButtonId, true);
291 | 
292 |     try {
293 |       const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
294 |         .track as LocalVideoTrack;
295 | 
296 |       switch (newMode) {
297 |         case 'disabled':
298 |           await state.backgroundProcessor.switchTo({ mode: 'disabled' });
299 |           break;
300 |         case 'virtual-background':
301 |           await state.backgroundProcessor.switchTo({ mode: 'virtual-background', imagePath: IMAGE_PATH });
302 |           break;
303 |         case 'background-blur':
304 |           await state.backgroundProcessor.switchTo({ mode: 'background-blur' });
305 |           break;
306 |       }
307 | 
308 |       if (!state.isBackgroundProcessorEnabled) {
309 |         await camTrack.setProcessor(state.backgroundProcessor);
310 |         state.isBackgroundProcessorEnabled = true;
311 |       }
312 |     } catch (e: any) {
313 |       appendLog(`ERROR: ${e.message}`);
314 |     } finally {
315 |       setButtonDisabled(controlButtonId, false);
316 |       renderParticipant(currentRoom.localParticipant);
317 |       updateButtonsForPublishState();
318 |       updateTrackProcessorModeButtons();
319 |     }
320 |   },
321 | 
322 |   updateVirtualBackgroundImage: async (imagePath: string) => {
323 |     if (!currentRoom) return;
324 | 
325 |     try {
326 |       const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
327 |         .track as LocalVideoTrack;
328 |       await state.backgroundProcessor.switchTo({ mode: 'virtual-background', imagePath });
329 |       if (!state.isBackgroundProcessorEnabled) {
330 |         await camTrack.stopProcessor();
331 |         await camTrack.setProcessor(state.backgroundProcessor);
332 |       }
333 |     } catch (e: any) {
334 |       appendLog(`ERROR: ${e.message}`);
335 |     } finally {
336 |       setButtonDisabled('switch-to-background-blur-button', false);
337 |       renderParticipant(currentRoom.localParticipant);
338 |       updateButtonsForPublishState();
339 |     }
340 |   },
341 | 
342 |   startAudio: () => {
343 |     currentRoom?.startAudio();
344 |   },
345 | 
346 |   disconnectRoom: () => {
347 |     if (currentRoom) {
348 |       currentRoom.disconnect();
349 |     }
350 |     if (state.bitrateInterval) {
351 |       clearInterval(state.bitrateInterval);
352 |     }
353 |   },
354 | 
355 |   handleDeviceSelected: async (e: Event) => {
356 |     const deviceId = (<HTMLSelectElement>e.target).value;
357 |     const elementId = (<HTMLSelectElement>e.target).id;
358 |     const kind = elementMapping[elementId];
359 |     if (!kind) {
360 |       return;
361 |     }
362 | 
363 |     state.defaultDevices.set(kind, deviceId);
364 | 
365 |     if (currentRoom) {
366 |       await currentRoom.switchActiveDevice(kind, deviceId);
367 |     }
368 |   },
369 | };
370 | 
371 | declare global {
372 |   interface Window {
373 |     currentRoom: any;
374 |     appActions: typeof appActions;
375 |   }
376 | }
377 | 
378 | window.appActions = appActions;
379 | 
380 | // --------------------------- event handlers ------------------------------- //
381 | 
382 | function participantConnected(participant: Participant) {
383 |   appendLog('participant', participant.identity, 'connected', participant.metadata);
384 |   participant
385 |     .on(ParticipantEvent.TrackMuted, (pub: TrackPublication) => {
386 |       appendLog('track was muted', pub.trackSid, participant.identity);
387 |       renderParticipant(participant);
388 |     })
389 |     .on(ParticipantEvent.TrackUnmuted, (pub: TrackPublication) => {
390 |       appendLog('track was unmuted', pub.trackSid, participant.identity);
391 |       renderParticipant(participant);
392 |     })
393 |     .on(ParticipantEvent.IsSpeakingChanged, () => {
394 |       renderParticipant(participant);
395 |     })
396 |     .on(ParticipantEvent.ConnectionQualityChanged, () => {
397 |       renderParticipant(participant);
398 |     });
399 | }
400 | 
401 | function participantDisconnected(participant: RemoteParticipant) {
402 |   appendLog('participant', participant.sid, 'disconnected');
403 | 
404 |   renderParticipant(participant, true);
405 | }
406 | 
407 | function handleRoomDisconnect(reason?: DisconnectReason) {
408 |   if (!currentRoom) return;
409 |   appendLog('disconnected from room', { reason });
410 |   setButtonsForState(false);
411 |   state.isBackgroundProcessorEnabled = false;
412 |   updateTrackProcessorModeButtons();
413 |   renderParticipant(currentRoom.localParticipant, true);
414 |   currentRoom.remoteParticipants.forEach((p) => {
415 |     renderParticipant(p, true);
416 |   });
417 | 
418 |   const container = $('participants-area');
419 |   if (container) {
420 |     container.innerHTML = '';
421 |   }
422 | 
423 |   currentRoom = undefined;
424 |   window.currentRoom = undefined;
425 | }
426 | 
427 | // -------------------------- rendering helpers ----------------------------- //
428 | 
429 | function appendLog(...args: any[]) {
430 |   const logger = $('log')!;
431 |   for (let i = 0; i < arguments.length; i += 1) {
432 |     if (typeof args[i] === 'object') {
433 |       logger.innerHTML += `${
434 |         JSON && JSON.stringify ? JSON.stringify(args[i], undefined, 2) : args[i]
435 |       } `;
436 |     } else {
437 |       logger.innerHTML += `${args[i]} `;
438 |     }
439 |   }
440 |   logger.innerHTML += '\n';
441 |   (() => {
442 |     logger.scrollTop = logger.scrollHeight;
443 |   })();
444 | }
445 | 
446 | // updates participant UI
447 | function renderParticipant(participant: Participant, remove: boolean = false) {
448 |   const container = $('participants-area');
449 |   if (!container) return;
450 |   const { identity } = participant;
451 |   let div = $(`participant-${identity}`);
452 |   if (!div && !remove) {
453 |     div = document.createElement('div');
454 |     div.id = `participant-${identity}`;
455 |     div.className = 'participant';
456 |     div.innerHTML = `
457 |       <video id="video-${identity}"></video>
458 |       <audio id="audio-${identity}"></audio>
459 |       <div class="info-bar">
460 |         <div id="name-${identity}" class="name">
461 |         </div>
462 |         <div style="text-align: center;">
463 |           <span id="codec-${identity}" class="codec">
464 |           </span>
465 |           <span id="size-${identity}" class="size">
466 |           </span>
467 |           <span id="bitrate-${identity}" class="bitrate">
468 |           </span>
469 |         </div>
470 |         <div class="right">
471 |           <span id="signal-${identity}"></span>
472 |           <span id="mic-${identity}" class="mic-on"></span>
473 |         </div>
474 |       </div>
475 |       ${
476 |         participant instanceof RemoteParticipant
477 |           ? `<div class="volume-control">
478 |         <input id="volume-${identity}" type="range" min="0" max="1" step="0.1" value="1" orient="vertical" />
479 |       </div>`
480 |           : `<progress id="local-volume" max="1" value="0" />`
481 |       }
482 | 
483 |     `;
484 |     container.appendChild(div);
485 | 
486 |     const sizeElm = $(`size-${identity}`);
487 |     const videoElm = <HTMLVideoElement>$(`video-${identity}`);
488 |     videoElm.onresize = () => {
489 |       updateVideoSize(videoElm!, sizeElm!);
490 |     };
491 |   }
492 |   const videoElm = <HTMLVideoElement>$(`video-${identity}`);
493 |   const audioELm = <HTMLAudioElement>$(`audio-${identity}`);
494 |   if (remove) {
495 |     div?.remove();
496 |     if (videoElm) {
497 |       videoElm.srcObject = null;
498 |       videoElm.src = '';
499 |     }
500 |     if (audioELm) {
501 |       audioELm.srcObject = null;
502 |       audioELm.src = '';
503 |     }
504 |     return;
505 |   }
506 | 
507 |   // update properties
508 |   $(`name-${identity}`)!.innerHTML = participant.identity;
509 |   if (participant instanceof LocalParticipant) {
510 |     $(`name-${identity}`)!.innerHTML += ' (you)';
511 |   }
512 |   const micElm = $(`mic-${identity}`)!;
513 |   const signalElm = $(`signal-${identity}`)!;
514 |   const cameraPub = participant.getTrackPublication(Track.Source.Camera);
515 |   const micPub = participant.getTrackPublication(Track.Source.Microphone);
516 |   if (participant.isSpeaking) {
517 |     div!.classList.add('speaking');
518 |   } else {
519 |     div!.classList.remove('speaking');
520 |   }
521 | 
522 |   if (participant instanceof RemoteParticipant) {
523 |     const volumeSlider = <HTMLInputElement>$(`volume-${identity}`);
524 |     volumeSlider.addEventListener('input', (ev) => {
525 |       participant.setVolume(Number.parseFloat((ev.target as HTMLInputElement).value));
526 |     });
527 |   }
528 | 
529 |   const cameraEnabled = cameraPub && cameraPub.isSubscribed && !cameraPub.isMuted;
530 |   if (cameraEnabled) {
531 |     if (participant instanceof LocalParticipant) {
532 |       // flip
533 |       videoElm.style.transform = 'scale(-1, 1)';
534 |     } else if (!cameraPub?.videoTrack?.attachedElements.includes(videoElm)) {
535 |       const renderStartTime = Date.now();
536 |       // measure time to render
537 |       videoElm.onloadeddata = () => {
538 |         const elapsed = Date.now() - renderStartTime;
539 |         let fromJoin = 0;
540 |         if (participant.joinedAt && participant.joinedAt.getTime() < startTime) {
541 |           fromJoin = Date.now() - startTime;
542 |         }
543 |         appendLog(
544 |           `RemoteVideoTrack ${cameraPub?.trackSid} (${videoElm.videoWidth}x${videoElm.videoHeight}) rendered in ${elapsed}ms`,
545 |           fromJoin > 0 ? `, ${fromJoin}ms from start` : '',
546 |         );
547 |       };
548 |     }
549 |     cameraPub?.videoTrack?.attach(videoElm);
550 |   } else {
551 |     // clear information display
552 |     $(`size-${identity}`)!.innerHTML = '';
553 |     if (cameraPub?.videoTrack) {
554 |       // detach manually whenever possible
555 |       cameraPub.videoTrack?.detach(videoElm);
556 |     } else {
557 |       videoElm.src = '';
558 |       videoElm.srcObject = null;
559 |     }
560 |   }
561 | 
562 |   const micEnabled = micPub && micPub.isSubscribed && !micPub.isMuted;
563 |   if (micEnabled) {
564 |     if (!(participant instanceof LocalParticipant)) {
565 |       // don't attach local audio
566 |       audioELm.onloadeddata = () => {
567 |         if (participant.joinedAt && participant.joinedAt.getTime() < startTime) {
568 |           const fromJoin = Date.now() - startTime;
569 |           appendLog(`RemoteAudioTrack ${micPub?.trackSid} played ${fromJoin}ms from start`);
570 |         }
571 |       };
572 |       micPub?.audioTrack?.attach(audioELm);
573 |     }
574 |     micElm.className = 'mic-on';
575 |     micElm.innerHTML = '<i class="fas fa-microphone"></i>';
576 |   } else {
577 |     micElm.className = 'mic-off';
578 |     micElm.innerHTML = '<i class="fas fa-microphone-slash"></i>';
579 |   }
580 | 
581 |   switch (participant.connectionQuality) {
582 |     case ConnectionQuality.Excellent:
583 |     case ConnectionQuality.Good:
584 |     case ConnectionQuality.Poor:
585 |       signalElm.className = `connection-${participant.connectionQuality}`;
586 |       signalElm.innerHTML = '<i class="fas fa-circle"></i>';
587 |       break;
588 |     default:
589 |       signalElm.innerHTML = '';
590 |     // do nothing
591 |   }
592 | }
593 | 
594 | function renderBitrate() {
595 |   if (!currentRoom || currentRoom.state !== ConnectionState.Connected) {
596 |     return;
597 |   }
598 |   const participants: Participant[] = [...currentRoom.remoteParticipants.values()];
599 |   participants.push(currentRoom.localParticipant);
600 | 
601 |   for (const p of participants) {
602 |     const elm = $(`bitrate-${p.identity}`);
603 |     let totalBitrate = 0;
604 |     for (const t of p.trackPublications.values()) {
605 |       if (t.track) {
606 |         totalBitrate += t.track.currentBitrate;
607 |       }
608 | 
609 |       if (t.source === Track.Source.Camera) {
610 |         if (t.videoTrack instanceof RemoteVideoTrack) {
611 |           const codecElm = $(`codec-${p.identity}`)!;
612 |           codecElm.innerHTML = t.videoTrack.getDecoderImplementation() ?? '';
613 |         }
614 |       }
615 |     }
616 |     let displayText = '';
617 |     if (totalBitrate > 0) {
618 |       displayText = `${Math.round(totalBitrate / 1024).toLocaleString()} kbps`;
619 |     }
620 |     if (elm) {
621 |       elm.innerHTML = displayText;
622 |     }
623 |   }
624 | }
625 | 
626 | function updateVideoSize(element: HTMLVideoElement, target: HTMLElement) {
627 |   target.innerHTML = `(${element.videoWidth}x${element.videoHeight})`;
628 | }
629 | 
630 | function setButtonState(
631 |   buttonId: string,
632 |   buttonText: string,
633 |   isActive: boolean,
634 |   isDisabled: boolean | undefined = undefined,
635 | ) {
636 |   const el = $(buttonId) as HTMLButtonElement;
637 |   if (!el) return;
638 |   if (isDisabled !== undefined) {
639 |     el.disabled = isDisabled;
640 |   }
641 |   el.innerHTML = buttonText;
642 |   if (isActive) {
643 |     el.classList.add('active');
644 |   } else {
645 |     el.classList.remove('active');
646 |   }
647 | }
648 | 
649 | function setButtonDisabled(buttonId: string, isDisabled: boolean) {
650 |   const el = $(buttonId) as HTMLButtonElement;
651 |   el.disabled = isDisabled;
652 | }
653 | 
654 | setTimeout(handleDevicesChanged, 100);
655 | 
656 | function setButtonsForState(connected: boolean) {
657 |   const connectedSet = [
658 |     'toggle-video-button',
659 |     'toggle-audio-button',
660 |     'disconnect-room-button',
661 |     'toggle-track-processor',
662 |     'switch-to-background-blur-button',
663 |     'switch-to-virtual-background-button',
664 |     'switch-to-disabled-button',
665 |   ];
666 |   const disconnectedSet = ['connect-button'];
667 | 
668 |   const toRemove = connected ? connectedSet : disconnectedSet;
669 |   const toAdd = connected ? disconnectedSet : connectedSet;
670 | 
671 |   toRemove.forEach((id) => $(id)?.removeAttribute('disabled'));
672 |   toAdd.forEach((id) => $(id)?.setAttribute('disabled', 'true'));
673 | 
674 |   $("initial-mode-wrapper").style.display = connected ? 'block' : 'none';
675 | }
676 | 
677 | const elementMapping: { [k: string]: MediaDeviceKind } = {
678 |   'video-input': 'videoinput',
679 |   'audio-input': 'audioinput',
680 |   'audio-output': 'audiooutput',
681 | };
682 | async function handleDevicesChanged() {
683 |   Promise.all(
684 |     Object.keys(elementMapping).map(async (id) => {
685 |       const kind = elementMapping[id];
686 |       if (!kind) {
687 |         return;
688 |       }
689 |       const devices = await Room.getLocalDevices(kind);
690 |       const element = <HTMLSelectElement>$(id);
691 |       populateSelect(element, devices, state.defaultDevices.get(kind));
692 |     }),
693 |   );
694 | }
695 | 
696 | function populateSelect(
697 |   element: HTMLSelectElement,
698 |   devices: MediaDeviceInfo[],
699 |   selectedDeviceId?: string,
700 | ) {
701 |   // clear all elements
702 |   element.innerHTML = '';
703 | 
704 |   for (const device of devices) {
705 |     const option = document.createElement('option');
706 |     option.text = device.label;
707 |     option.value = device.deviceId;
708 |     if (device.deviceId === selectedDeviceId) {
709 |       option.selected = true;
710 |     }
711 |     element.appendChild(option);
712 |   }
713 | }
714 | 
715 | function updateButtonsForPublishState() {
716 |   if (!currentRoom) {
717 |     return;
718 |   }
719 |   const lp = currentRoom.localParticipant;
720 |   // video
721 |   setButtonState(
722 |     'toggle-video-button',
723 |     `${lp.isCameraEnabled ? 'Disable' : 'Enable'} Video`,
724 |     lp.isCameraEnabled,
725 |   );
726 | 
727 |   // audio
728 |   setButtonState(
729 |     'toggle-audio-button',
730 |     `${lp.isMicrophoneEnabled ? 'Disable' : 'Enable'} Audio`,
731 |     lp.isMicrophoneEnabled,
732 |   );
733 | }
734 | 
735 | function updateTrackProcessorModeButtons() {
736 |   const toggleTrackProcessorButtonEnabled = currentRoom?.state === ConnectionState.Connected;
737 |   if (state.isBackgroundProcessorEnabled) {
738 |     setButtonState('toggle-track-processor', 'Remove Track Processor', false, !toggleTrackProcessorButtonEnabled);
739 |     $('track-processor-modes').style.display = 'block';
740 |   } else {
741 |     setButtonState('toggle-track-processor', 'Insert Track Processor', false, !toggleTrackProcessorButtonEnabled);
742 |     $('track-processor-modes').style.display = 'none';
743 |   }
744 | 
745 |   const {active: activeButtonId, inactive: inactiveButtonIds} = {
746 |     'disabled': { active: 'switch-to-disabled-button', inactive: ['switch-to-virtual-background-button', 'switch-to-background-blur-button'] },
747 |     'virtual-background': { active: 'switch-to-virtual-background-button', inactive: ['switch-to-disabled-button', 'switch-to-background-blur-button'] },
748 |     'background-blur': { active: 'switch-to-background-blur-button', inactive: ['switch-to-virtual-background-button', 'switch-to-disabled-button'] },
749 |     'legacy': { active: null, inactive: [] }, // NOTE: should be impossible, but here for thoroughness
750 |     'off': { active: null, inactive: [] },
751 |   }[state.isBackgroundProcessorEnabled ? state.backgroundProcessor.mode : 'off'];
752 | 
753 |   if (activeButtonId) {
754 |     $(activeButtonId).classList.remove('btn-secondary');
755 |     $(activeButtonId).classList.add('btn-primary');
756 |   }
757 |   for (const inactiveId of inactiveButtonIds) {
758 |     $(inactiveId).classList.remove('btn-primary');
759 |     $(inactiveId).classList.add('btn-secondary');
760 |   }
761 | 
762 |   if (state.backgroundProcessor.mode === 'virtual-background') {
763 |     setButtonDisabled('update-bg-button', false);
764 |   } else {
765 |     setButtonDisabled('update-bg-button', true);
766 |   }
767 | }
768 | 
769 | async function acquireDeviceList() {
770 |   handleDevicesChanged();
771 | }
772 | 
773 | acquireDeviceList();
774 | 


--------------------------------------------------------------------------------