├── .gitignore ├── LICENSE ├── LICENSE_COMMERCIAL ├── README.md ├── assets ├── banner.png ├── benchmarks.png ├── captions.mp4 ├── composition.png ├── custom-captions.mp4 ├── font.mp4 ├── icon.png └── reddit_story.mp4 ├── biome.json ├── index.html ├── package-lock.json ├── package.json ├── playground ├── controls.ts ├── index.css ├── main.ts ├── render.ts └── timeline.ts ├── public ├── audio.mp3 ├── dvd_logo.svg ├── harvard.MP3 ├── lenna.png ├── piano.mp3 ├── sample_aac_h264_yuv420p_1080p_30fps.mp4 ├── sample_aac_h264_yuv420p_1080p_60fps.mov ├── sample_aac_h264_yuv420p_1080p_60fps.mp4 ├── sample_ogg_vp8_yuv420p_1080p_30fps.webm ├── sample_ogg_vp8_yuv420p_1080p_60fps.webm ├── silences.mp3 └── test.html ├── publish.sh ├── samples.sh ├── src ├── clips │ ├── audio │ │ ├── audio.interfaces.ts │ │ ├── audio.spec.ts │ │ ├── audio.ts │ │ └── index.ts │ ├── clip │ │ ├── clip.deserializer.spec.ts │ │ ├── clip.desierializer.ts │ │ ├── clip.interfaces.ts │ │ ├── clip.spec.ts │ │ ├── clip.ts │ │ ├── clip.types.ts │ │ ├── clip.utils.ts │ │ └── index.ts │ ├── html │ │ ├── html.interfaces.ts │ │ ├── html.spec.ts │ │ ├── html.ts │ │ └── index.ts │ ├── image │ │ ├── image.interfaces.ts │ │ ├── image.spec.ts │ │ ├── image.ts │ │ └── index.ts │ ├── index.ts │ ├── mask │ │ ├── circleMask.ts │ │ ├── ellipseMask.ts │ │ ├── index.ts │ │ ├── mask.ts │ │ ├── mask.types.ts │ │ ├── rectangleMask.ts │ │ ├── roundRectangleMask.ts │ │ └── starMask.ts │ ├── media │ │ ├── index.ts │ │ ├── media.deserializer.ts │ │ ├── media.interfaces.ts │ │ ├── media.spec.ts │ │ ├── media.ts │ │ └── media.types.ts │ ├── mixins │ │ ├── index.ts │ │ ├── visual.animation.ts │ │ ├── visual.decorator.spec.ts │ │ ├── visual.decorator.ts │ │ ├── visual.deserializers.spec.ts │ │ ├── visual.deserializers.ts │ │ ├── visual.interfaces.ts │ │ ├── visual.spec.ts │ │ └── visual.ts │ ├── text │ │ ├── font.fixtures.ts │ │ ├── font.spec.ts │ │ ├── font.static.ts │ │ ├── font.ts │ │ ├── font.types.ts │ │ ├── index.ts │ │ ├── text.complex.deserializer.ts │ │ ├── text.complex.interfaces.ts │ │ ├── text.complex.spec.ts │ │ ├── text.complex.ts │ │ ├── text.fixtures.ts │ │ ├── text.interfaces.ts │ │ ├── text.spec.ts │ │ ├── text.ts │ │ ├── text.types.ts │ │ └── text.utils.ts │ ├── utils │ │ ├── index.spec.ts │ │ └── index.ts │ └── video │ │ ├── buffer.spec.ts │ │ ├── buffer.ts │ │ ├── decoder.spec.ts │ │ ├── decoder.ts │ │ ├── demuxer │ │ ├── ffmpeg.worker.ts │ │ ├── index.ts │ │ ├── types │ │ │ ├── avutil.ts │ │ │ ├── demuxer.ts │ │ │ ├── ffmpeg-worker-message.ts │ │ │ └── index.ts │ │ └── web-demuxer.ts │ │ ├── index.ts │ │ ├── video.decorator.ts │ │ ├── video.interfaces.ts │ │ ├── video.spec.ts │ │ ├── video.ts │ │ ├── video.types.ts │ │ ├── worker.ts │ │ ├── worker.types.ts │ │ └── worker.utils.ts ├── composition │ ├── composition.spec.ts │ ├── composition.ts │ ├── composition.types.ts │ └── index.ts ├── encoders │ ├── canvas.spec.ts │ ├── canvas.ts │ ├── encoder.spec.ts │ ├── encoder.ts │ ├── index.ts │ ├── interfaces.ts │ ├── opus │ │ ├── index.ts │ │ ├── opus.encoder.ts │ │ ├── opus.fixtures.ts │ │ ├── opus.types.ts │ │ ├── opus.utils.spec.ts │ │ └── opus.utils.ts │ ├── types.ts │ ├── utils.spec.ts │ ├── utils.ts │ ├── webassembly.audio.spec.ts │ ├── webassembly.audio.ts │ ├── webcodecs.audio.spec.ts │ ├── webcodecs.audio.ts │ ├── webcodecs.video.spec.ts │ └── webcodecs.video.ts ├── errors │ ├── base-error.ts │ ├── encoder-error.ts │ ├── export-error.ts │ ├── index.ts │ ├── io-error.ts │ ├── reference-error.ts │ └── validation-error.ts ├── fixtures │ └── index.ts ├── index.ts ├── mixins │ ├── event.spec.ts │ ├── event.ts │ ├── event.types.ts │ └── index.ts ├── models │ ├── animation-builder.spec.ts │ ├── animation-builder.ts │ ├── index.ts │ ├── keyframe.spec.ts │ ├── keyframe.ts │ ├── keyframe.types.ts │ ├── keyframe.utils.spec.ts │ ├── keyframe.utils.ts │ ├── timestamp.fixtures.ts │ ├── timestamp.spec.ts │ ├── timestamp.ts │ ├── timestamp.types.ts │ ├── timestamp.utils.spec.ts │ ├── timestamp.utils.ts │ ├── transcript.group.ts │ ├── transcript.spec.ts │ ├── transcript.ts │ ├── transcript.types.ts │ ├── transcript.utils.spec.ts │ ├── transcript.utils.ts │ └── transcript.word.ts ├── services │ ├── event.ts │ ├── index.ts │ ├── serializer.spec.ts │ ├── serializer.ts │ ├── storage-item.spec.ts │ ├── storage-item.ts │ ├── store.spec.ts │ ├── store.ts │ ├── store.types.ts │ ├── thread.spec.ts │ └── thread.ts ├── sources │ ├── audio.fixtures.ts │ ├── audio.spec.ts │ ├── audio.ts │ ├── audio.types.ts │ ├── audio.utils.ts │ ├── html.spec.ts │ ├── html.ts │ ├── html.utils.spec.ts │ ├── html.utils.ts │ ├── image.spec.ts │ ├── image.ts │ ├── index.ts │ ├── source.spec.ts │ ├── source.ts │ ├── video.spec.ts │ └── video.ts ├── test │ └── captions.ts ├── tracks │ ├── audio │ │ ├── audio.spec.ts │ │ ├── audio.ts │ │ └── index.ts │ ├── caption │ │ ├── caption.spec.ts │ │ ├── caption.ts │ │ ├── index.ts │ │ ├── preset.0.spec.ts │ │ ├── preset.1.spec.ts │ │ ├── preset.cascade.ts │ │ ├── preset.classic.ts │ │ ├── preset.deserializer.ts │ │ ├── preset.guinea.ts │ │ ├── preset.interface.ts │ │ ├── preset.solar.spec.ts │ │ ├── preset.solar.ts │ │ ├── preset.spotlight.ts │ │ ├── preset.types.ts │ │ ├── preset.verdant.spec.ts │ │ ├── preset.verdant.ts │ │ ├── preset.whisper.spec.ts │ │ └── preset.whisper.ts │ ├── html │ │ ├── html.ts │ │ └── index.ts │ ├── image │ │ ├── image.spec.ts │ │ ├── image.ts │ │ └── index.ts │ ├── index.ts │ ├── media │ │ ├── index.ts │ │ ├── media.spec.ts │ │ └── media.ts │ ├── text │ │ ├── index.ts │ │ └── text.ts │ ├── track │ │ ├── index.ts │ │ ├── track.deserializer.ts │ │ ├── track.fixtures.ts │ │ ├── track.interfaces.ts │ │ ├── track.render.spec.ts │ │ ├── track.spec.ts │ │ ├── track.strategies.spec.ts │ │ ├── track.strategies.ts │ │ ├── track.ts │ │ └── track.types.ts │ └── video │ │ ├── index.ts │ │ ├── video.spec.ts │ │ └── video.ts ├── types │ └── index.ts ├── utils │ ├── audio.spec.ts │ ├── audio.ts │ ├── browser.spec.ts │ ├── browser.ts │ ├── common.spec.ts │ ├── common.ts │ ├── index.ts │ ├── webcodecs.spec.ts │ └── webcodecs.ts └── vite-env.d.ts ├── tsconfig.json ├── vite.config.ts ├── vitest.config.ts ├── vitest.mocks.ts └── vitest.setup.ts /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | **/*.mjs 27 | **/*.env 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Diffusion Studio Non-Commercial License 2 | 3 | Version 1.0 4 | 5 | Copyright (c) 2025 Diffusion Studio Inc. 6 | All rights reserved. 7 | 8 | 1. **Grant of License** 9 | Subject to the terms of this License, Diffusion Studio Inc. ("Licensor") grants you ("Licensee") a non-exclusive, non-transferable, royalty-free license to use, modify, and distribute the Software solely for **non-commercial purposes**. 10 | 11 | 2. **Non-Commercial Use** 12 | a. This license applies to individuals and organizations that **do not monetize** or derive any **commercial benefit** from the product, service, or project that incorporates this Software. 13 | b. "Monetization" includes (but is not limited to) selling, licensing, offering paid access, receiving advertising revenue, donations exceeding operational costs, or any other form of revenue generation. 14 | 15 | 3. **Restrictions** 16 | a. **No Commercial Use**: You may not use the Software, in whole or in part, in any manner that constitutes monetization or commercial benefit, as defined in Section 2, without obtaining a Commercial License. 17 | b. **No Modification of License Terms**: The text of this license must remain unmodified and included in any distribution of the Software. 18 | c. **No Removal of Notices**: You may not remove or alter any copyright, trademark, or attribution notices in the Software. 19 | d. **Deployment Optimization**: You may compress, minify, or otherwise optimize the Software for deployment, but you may not alter its functionality beyond what is necessary for optimization. 20 | 21 | 4. **Commercial Licensing** 22 | If you or your organization wish to use the Software for **commercial purposes**, you must obtain a **Commercial License** from Diffusion Studio. 23 | Visit [https://diffusion.studio](https://diffusion.studio) for more details. 24 | 25 | 5. **Disclaimer of Warranty** 26 | THE SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 27 | 28 | 6. **Limitation of Liability** 29 | IN NO EVENT SHALL THE LICENSOR BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 30 | 31 | 7. **Termination** 32 | If you violate any terms of this License, your rights under this License will automatically terminate. 33 | 34 | For inquiries, contact: [contact (at) diffusion.studio] 35 | -------------------------------------------------------------------------------- /LICENSE_COMMERCIAL: -------------------------------------------------------------------------------- 1 | Diffusion Studio Commercial License 2 | 3 | Version 1.0 4 | 5 | This Commercial License Agreement ("Agreement") is made between you ("Licensee") and Diffusion Studio Inc. ("Licensor"). By obtaining, downloading, installing, or using the Software, Licensee agrees to be bound by the terms of this Agreement. 6 | 7 | 1. **Grant of License** 8 | Subject to payment and compliance with this Agreement, Licensor grants Licensee a non-exclusive, non-transferable, worldwide right to use, modify, and distribute the Software **for commercial purposes**. 9 | 10 | 2. **Commercial Use Definition** 11 | Commercial use includes, but is not limited to: 12 | a. Using the Software in a product, service, or project that generates revenue (directly or indirectly). 13 | b. Using the Software in internal business operations of a for-profit organization. 14 | c. Any deployment of the Software in a manner that provides financial gain. 15 | 16 | 3. **Modifications and Optimization** 17 | Licensee may modify, compress, minify, or otherwise optimize the Software **only for deployment purposes**, provided that all License notices remain intact. 18 | 19 | 4. **License Fee** 20 | The Licensee must purchase a Commercial License from Diffusion Studio. Pricing and licensing tiers are available at: 21 | [https://diffusion.studio](https://diffusion.studio) 22 | 23 | 5. **License Compliance** 24 | a. Licensee must retain the original License notice in any distribution. 25 | b. Proof of purchase must be available upon request by Licensor. 26 | c. Licensor reserves the right to audit compliance with this Agreement. 27 | 28 | 6. **Restrictions** 29 | a. **No Resale or Redistribution**: Licensee may not sell, sublicense, or otherwise transfer the Software except as integrated into a larger product or service. 30 | b. **No Unauthorized Use**: Licensee may not use the Software in violation of any applicable laws. 31 | c. **No Attribution Removal**: The License notice must be included in any final product delivered to end-users. 32 | 33 | 7. **Support & Updates** 34 | Commercial License holders are entitled to updates and support as outlined in their specific licensing tier. 35 | 36 | 8. **Termination** 37 | Licensor may terminate this Agreement if Licensee fails to comply with any of the terms herein. Upon termination, all rights granted under this Agreement will cease. 38 | 39 | 9. **Liability Disclaimer** 40 | THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY. LICENSOR IS NOT LIABLE FOR ANY DAMAGES RESULTING FROM ITS USE. 41 | 42 | 10. **Governing Law** 43 | This Agreement shall be governed by the laws of Delaware, USA. 44 | 45 | For commercial licensing inquiries, contact: [contact (at) diffusion.studio] 46 | -------------------------------------------------------------------------------- /assets/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/banner.png -------------------------------------------------------------------------------- /assets/benchmarks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/benchmarks.png -------------------------------------------------------------------------------- /assets/captions.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/captions.mp4 -------------------------------------------------------------------------------- /assets/composition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/composition.png -------------------------------------------------------------------------------- /assets/custom-captions.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/custom-captions.mp4 -------------------------------------------------------------------------------- /assets/font.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/font.mp4 -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/icon.png -------------------------------------------------------------------------------- /assets/reddit_story.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/assets/reddit_story.mp4 -------------------------------------------------------------------------------- /biome.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", 3 | "organizeImports": { 4 | "enabled": true 5 | }, 6 | "linter": { 7 | "enabled": false 8 | }, 9 | "javascript": { 10 | "formatter": { 11 | "quoteStyle": "single", 12 | "jsxQuoteStyle": "double", 13 | "trailingCommas": "all", 14 | "semicolons": "always", 15 | "lineWidth": 100 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | DS Playground 8 | 9 | 10 | 11 | 12 |
13 |
14 |
15 | 18 |
19 |
20 |
21 |
22 |
23 |
24 | 25 | 26 | 27 | 28 |
29 | 00:00 / 00:00 30 | 31 | 35 |
36 |
37 | 38 | 39 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@diffusionstudio/core", 3 | "private": false, 4 | "version": "1.6.0", 5 | "type": "module", 6 | "description": "Build bleeding edge video processing applications", 7 | "files": [ 8 | "dist/*" 9 | ], 10 | "types": "./dist/index.d.ts", 11 | "module": "./dist/ds.js", 12 | "exports": { 13 | ".": { 14 | "import": "./dist/ds.js", 15 | "types": "./dist/index.d.ts" 16 | } 17 | }, 18 | "license": "MPL-2.0", 19 | "scripts": { 20 | "dev": "vite", 21 | "build": "tsc && vite build", 22 | "preview": "vite preview", 23 | "format": "npx @biomejs/biome format --write ./src", 24 | "test": "vitest", 25 | "coverage": "vitest run --coverage", 26 | "release": "tsc && vite build && npm publish", 27 | "docs": "typedoc src/index.ts --plugin typedoc-plugin-markdown --out ./docs" 28 | }, 29 | "devDependencies": { 30 | "@biomejs/biome": "1.9.2", 31 | "@types/dom-webcodecs": "^0.1.11", 32 | "@types/node": "^22.5.5", 33 | "@types/wicg-file-system-access": "^2023.10.5", 34 | "@vitest/coverage-v8": "^2.1.1", 35 | "@vitest/web-worker": "^2.1.1", 36 | "@webgpu/types": "^0.1.46", 37 | "jsdom": "^25.0.1", 38 | "rollup-plugin-node-externals": "^7.1.3", 39 | "typedoc": "^0.26.7", 40 | "typedoc-plugin-markdown": "^4.2.8", 41 | "typescript": "^5.6.2", 42 | "user-agent-data-types": "^0.4.2", 43 | "vite": "^5.4.7", 44 | "vite-plugin-dts": "^4.2.1", 45 | "vitest": "^2.1.1", 46 | "vitest-canvas-mock": "^0.3.3" 47 | }, 48 | "dependencies": { 49 | "mp4-muxer": "^5.1.3" 50 | }, 51 | "peerDependencies": { 52 | "pixi-filters": ">=6.0.0", 53 | "pixi.js": ">=8.0.0" 54 | }, 55 | "author": "Diffusion Studio GmbH", 56 | "publishConfig": { 57 | "access": "public" 58 | }, 59 | "repository": { 60 | "type": "git", 61 | "url": "git@github.com:diffusionstudio/core.git" 62 | }, 63 | "keywords": [ 64 | "mp4", 65 | "web", 66 | "aac", 67 | "h264", 68 | "opus", 69 | "edit", 70 | "webgl", 71 | "video", 72 | "audio", 73 | "record", 74 | "canvas", 75 | "tiktok", 76 | "webgpu", 77 | "encode", 78 | "editor", 79 | "decode", 80 | "editing", 81 | "youtube", 82 | "recorder", 83 | "automate", 84 | "webcodecs", 85 | "client-side", 86 | "programmatic", 87 | "browser-based" 88 | ] 89 | } 90 | -------------------------------------------------------------------------------- /playground/controls.ts: -------------------------------------------------------------------------------- 1 | import * as core from '../src'; 2 | import { render } from './render'; 3 | 4 | export function setupControls(composition: core.Composition) { 5 | const handlePlay = () => composition.play(); 6 | const handlePause = () => composition.pause(); 7 | const handleBack = () => composition.seek(0); 8 | const handleForward = () => composition.seek(composition.duration.frames); 9 | const handleExport = () => render(composition); 10 | 11 | playButton.addEventListener('click', handlePlay); 12 | pauseButton.addEventListener('click', handlePause); 13 | backButton.addEventListener('click', handleBack); 14 | forwardButton.addEventListener('click', handleForward); 15 | exportButton.addEventListener('click', handleExport); 16 | 17 | composition.on('play', () => { 18 | playButton.style.display = 'none'; 19 | pauseButton.style.display = 'block'; 20 | }); 21 | composition.on('pause', () => { 22 | pauseButton.style.display = 'none'; 23 | playButton.style.display = 'block'; 24 | }); 25 | composition.on('currentframe', () => { 26 | time.textContent = composition.time(); 27 | }); 28 | 29 | composition.attachPlayer(player); 30 | 31 | const observer = new ResizeObserver(() => { 32 | const scale = Math.min( 33 | container.clientWidth / composition.width, 34 | container.clientHeight / composition.height 35 | ); 36 | 37 | player.style.width = `${composition.width}px`; 38 | player.style.height = `${composition.height}px`; 39 | player.style.transform = `scale(${scale})`; 40 | player.style.transformOrigin = 'center'; 41 | }); 42 | 43 | observer.observe(document.body); 44 | time.textContent = composition.time(); 45 | composition.seek(0); 46 | } 47 | 48 | const container = document.querySelector('[id="player-container"]') as HTMLDivElement; 49 | const player = document.querySelector('[id="player"]') as HTMLDivElement; 50 | const time = document.querySelector('[id="time"]') as HTMLSpanElement; 51 | const exportButton = document.querySelector('[id="export"]') as HTMLButtonElement; 52 | const playButton = document.querySelector('[data-lucide="play"]') as HTMLElement; 53 | const pauseButton = document.querySelector('[data-lucide="pause"]') as HTMLElement; 54 | const backButton = document.querySelector('[data-lucide="skip-back"]') as HTMLElement; 55 | const forwardButton = document.querySelector('[data-lucide="skip-forward"]') as HTMLElement; 56 | -------------------------------------------------------------------------------- /playground/render.ts: -------------------------------------------------------------------------------- 1 | 2 | import * as core from '../src'; 3 | 4 | let fps = 30; 5 | 6 | export async function render(composition: core.Composition) { 7 | if (loader.style.display != 'none') return; 8 | 9 | try { 10 | const encoder = new core.Encoder(composition, { debug: true, fps }); 11 | 12 | encoder.on('render', (event) => { 13 | const { progress, total } = event.detail; 14 | container.style.display = 'flex'; 15 | text.innerHTML = `${Math.round(progress * 100 / total)}%`; 16 | }) 17 | 18 | const fileHandle = await window.showSaveFilePicker({ 19 | suggestedName: `untitled_video.mp4`, 20 | types: [ 21 | { 22 | description: 'Video File', 23 | accept: { 'video/mp4': ['.mp4'] }, 24 | }, 25 | ], 26 | }); 27 | 28 | loader.style.display = 'block'; 29 | await encoder.render(fileHandle); 30 | } catch (e) { 31 | if (e instanceof DOMException) { 32 | console.log(e) 33 | // user canceled file picker 34 | } else if (e instanceof core.EncoderError) { 35 | alert(e.message); 36 | } else { 37 | alert(String(e)); 38 | } 39 | } finally { 40 | loader.style.display = 'none'; 41 | container.style.display = 'none'; 42 | } 43 | } 44 | 45 | const container = document.querySelector('[id="progress"]') as HTMLDivElement; 46 | const text = document.querySelector('[id="progress"] > h1') as HTMLHeadingElement; 47 | const loader = document.querySelector('.loader') as HTMLDivElement; 48 | const fpsButton = document.querySelector('[data-lucide="gauge"]') as HTMLElement; 49 | 50 | fpsButton.addEventListener('click', () => { 51 | const value = parseFloat( 52 | prompt("Please enter the desired frame rate", fps.toString()) ?? fps.toString() 53 | ); 54 | 55 | if (!Number.isNaN(value)) fps = value 56 | }); 57 | 58 | if (!('showSaveFilePicker' in window)) { 59 | Object.assign(window, { showSaveFilePicker: async () => undefined }); 60 | } 61 | -------------------------------------------------------------------------------- /playground/timeline.ts: -------------------------------------------------------------------------------- 1 | import * as core from '../src'; 2 | 3 | export function setupTimeline(composition: core.Composition) { 4 | composition.on('currentframe', (evt) => { 5 | const pos = evt.detail / composition.duration.frames; 6 | 7 | cursor.style.left = `${timeline.clientWidth * pos}px`; 8 | }); 9 | 10 | timeline.addEventListener('click', (evt: MouseEvent) => { 11 | const pos = evt.offsetX / timeline.clientWidth; 12 | 13 | composition.seek(composition.duration.frames * pos); 14 | }); 15 | } 16 | 17 | const timeline = document.querySelector('[id="timeline"]') as HTMLDivElement; 18 | const cursor = document.querySelector('[id="timeline"] > div') as HTMLDivElement; 19 | -------------------------------------------------------------------------------- /public/audio.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/audio.mp3 -------------------------------------------------------------------------------- /public/dvd_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /public/harvard.MP3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/harvard.MP3 -------------------------------------------------------------------------------- /public/lenna.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/lenna.png -------------------------------------------------------------------------------- /public/piano.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/piano.mp3 -------------------------------------------------------------------------------- /public/sample_aac_h264_yuv420p_1080p_30fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/sample_aac_h264_yuv420p_1080p_30fps.mp4 -------------------------------------------------------------------------------- /public/sample_aac_h264_yuv420p_1080p_60fps.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/sample_aac_h264_yuv420p_1080p_60fps.mov -------------------------------------------------------------------------------- /public/sample_aac_h264_yuv420p_1080p_60fps.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/sample_aac_h264_yuv420p_1080p_60fps.mp4 -------------------------------------------------------------------------------- /public/sample_ogg_vp8_yuv420p_1080p_30fps.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/sample_ogg_vp8_yuv420p_1080p_30fps.webm -------------------------------------------------------------------------------- /public/sample_ogg_vp8_yuv420p_1080p_60fps.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/sample_ogg_vp8_yuv420p_1080p_60fps.webm -------------------------------------------------------------------------------- /public/silences.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/diffusionstudio/core/02dbff926c892c64ee7e47d0990ef0509aa1f694/public/silences.mp3 -------------------------------------------------------------------------------- /public/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 19 | 20 | 21 |
22 | This is Html render inside a Foreign Object 23 |
24 | 25 | 26 | -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Step 1: Run Vitest for unit testing in "run" mode to avoid waiting for changes 4 | echo "Running unit tests with Vitest..." 5 | npx vitest run 6 | 7 | # Check if the tests were successful 8 | if [ $? -ne 0 ]; then 9 | echo "Tests failed. Aborting publish." 10 | exit 1 11 | fi 12 | 13 | echo "Tests passed. Proceeding with build and publish..." 14 | 15 | # Step 2: Remove the dist folder 16 | echo "Removing dist folder..." 17 | rm -rf dist 18 | 19 | # Step 3: Build the library 20 | echo "Building the library..." 21 | npm run build 22 | 23 | # Check if the build was successful 24 | if [ $? -ne 0 ]; then 25 | echo "Build failed. Aborting publish." 26 | exit 1 27 | fi 28 | 29 | echo "Build successful." 30 | 31 | # Step 4: Publish the package 32 | echo "Publishing the package..." 33 | npm publish 34 | 35 | # Check if the publish was successful 36 | if [ $? -ne 0 ]; then 37 | echo "Publish failed." 38 | exit 1 39 | fi 40 | 41 | echo "Package published successfully!" 42 | -------------------------------------------------------------------------------- /samples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define variables 4 | RESOLUTION="1080p" # or 2160p 5 | INPUT_URL="http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_${RESOLUTION}_60fps_normal.mp4" 6 | LOCAL_VIDEO="downloaded_video.mp4" 7 | START_TIME="00:07:57" 8 | DURATION="20" 9 | 10 | # Output directory 11 | OUTPUT_DIR="./outputs" 12 | mkdir -p $OUTPUT_DIR 13 | 14 | # Download the video 15 | echo "Downloading video from $INPUT_URL..." 16 | curl -o $LOCAL_VIDEO $INPUT_URL 17 | 18 | # Check if the download was successful 19 | if [ $? -ne 0 ]; then 20 | echo "Failed to download the video." 21 | exit 1 22 | fi 23 | 24 | echo "Download completed." 25 | 26 | FPS_LIST=("24" "30" "60") 27 | PIX_FMT="yuv420p" 28 | 29 | # Process video with different pixel formats 30 | for FPS in "${FPS_LIST[@]}" 31 | do 32 | OUTPUT_FILE="${OUTPUT_DIR}/sample_aac_h264_${PIX_FMT}_${RESOLUTION}_${FPS}fps.mp4" 33 | echo "Processing with fps: $FPS" 34 | ffmpeg -i $LOCAL_VIDEO -ss $START_TIME -t $DURATION -c:v libx264 -c:a aac -pix_fmt $PIX_FMT -r $FPS $OUTPUT_FILE 35 | done 36 | 37 | echo "Processing completed. Check the $OUTPUT_DIR directory for output files." 38 | -------------------------------------------------------------------------------- /src/clips/audio/audio.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { MediaClipProps } from '../media'; 9 | 10 | export interface AudioClipProps extends MediaClipProps { } 11 | -------------------------------------------------------------------------------- /src/clips/audio/audio.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { MediaClip } from '../media'; 9 | import { AudioSource } from '../../sources'; 10 | import { IOError } from '../../errors'; 11 | 12 | import type { Track } from '../../tracks'; 13 | import type { AudioClipProps } from './audio.interfaces'; 14 | 15 | export class AudioClip extends MediaClip { 16 | public readonly type = 'audio'; 17 | public declare track?: Track; 18 | public source = new AudioSource(); 19 | 20 | /** 21 | * Access to the HTML5 audio element 22 | */ 23 | public readonly element = new Audio(); 24 | 25 | public constructor(source?: File | AudioSource, props: AudioClipProps = {}) { 26 | super(); 27 | 28 | if (source instanceof AudioSource) { 29 | this.source = source; 30 | } 31 | 32 | if (source instanceof File) { 33 | this.source.from(source); 34 | } 35 | 36 | this.element.addEventListener('play', () => { 37 | this.playing = true; 38 | }); 39 | 40 | this.element.addEventListener('pause', () => { 41 | this.playing = false; 42 | }); 43 | 44 | Object.assign(this, props); 45 | } 46 | 47 | public async init(): Promise { 48 | const objectURL = await this.source.createObjectURL(); 49 | this.element.setAttribute('src', objectURL); 50 | this.element.load(); 51 | 52 | await new Promise((resolve, reject) => { 53 | this.element.oncanplay = () => { 54 | this.duration.seconds = this.element.duration; 55 | this.state = 'READY'; 56 | resolve(); 57 | } 58 | 59 | this.element.onerror = () => { 60 | this.state = 'ERROR'; 61 | 62 | const error = new IOError({ 63 | code: 'sourceNotProcessable', 64 | message: 'An error occurred while processing the input medium.', 65 | }); 66 | 67 | reject(this.element.error ?? error); 68 | } 69 | }); 70 | } 71 | 72 | public update(): void | Promise { 73 | if (this.track?.composition?.rendering) { 74 | return this.exit(); 75 | } else if (this.track?.composition?.playing && !this.playing) { 76 | this.element.play(); 77 | } else if (!this.track?.composition?.playing && this.playing) { 78 | this.element.pause(); 79 | } 80 | } 81 | 82 | public exit(): void { 83 | if (this.playing) { 84 | this.element.pause(); 85 | }; 86 | } 87 | 88 | public copy(): AudioClip { 89 | const clip = AudioClip.fromJSON(JSON.parse(JSON.stringify(this))); 90 | clip.source = this.source; 91 | 92 | return clip; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/clips/audio/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './audio'; 9 | export * from './audio.interfaces'; 10 | -------------------------------------------------------------------------------- /src/clips/clip/clip.deserializer.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from 'vitest'; 2 | import { ClipDeserializer } from './clip.desierializer'; 3 | import { 4 | AudioClip, 5 | VideoClip, 6 | HtmlClip, 7 | ImageClip, 8 | TextClip, 9 | ComplexTextClip, 10 | Clip 11 | } from '..'; 12 | import { AudioSource, HtmlSource, ImageSource, VideoSource } from '../../sources'; 13 | import type { Source } from '../../sources'; 14 | 15 | describe('ClipDeserializer', () => { 16 | it('should return correct clip based on type', () => { 17 | expect(ClipDeserializer.fromType({ type: 'video' })).toBeInstanceOf(VideoClip); 18 | expect(ClipDeserializer.fromType({ type: 'audio' })).toBeInstanceOf(AudioClip); 19 | expect(ClipDeserializer.fromType({ type: 'html' })).toBeInstanceOf(HtmlClip); 20 | expect(ClipDeserializer.fromType({ type: 'image' })).toBeInstanceOf(ImageClip); 21 | expect(ClipDeserializer.fromType({ type: 'text' })).toBeInstanceOf(TextClip); 22 | expect(ClipDeserializer.fromType({ type: 'complex_text' })).toBeInstanceOf(ComplexTextClip); 23 | expect(ClipDeserializer.fromType({ type: 'unknown' as any })).toBeInstanceOf(Clip); // Default case 24 | }); 25 | 26 | it('should return correct clip based on source', () => { 27 | // Mock instances for different source types 28 | const audioSource = new AudioSource(); 29 | const videoSource = new VideoSource(); 30 | const imageSource = new ImageSource(); 31 | const htmlSource = new HtmlSource(); 32 | 33 | const res = ClipDeserializer.fromSource(audioSource) 34 | 35 | // Ensure proper class instantiation based on source type 36 | expect(res).toBeInstanceOf(AudioClip); 37 | expect(ClipDeserializer.fromSource(videoSource)).toBeInstanceOf(VideoClip); 38 | expect(ClipDeserializer.fromSource(imageSource)).toBeInstanceOf(ImageClip); 39 | expect(ClipDeserializer.fromSource(htmlSource)).toBeInstanceOf(HtmlClip); 40 | }); 41 | 42 | it('should return undefined if source type does not match', () => { 43 | const invalidSourceMock = { type: 'unknown' } as any as Source; 44 | expect(ClipDeserializer.fromSource(invalidSourceMock)).toBeUndefined(); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /src/clips/clip/clip.desierializer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { AudioClip, Clip, HtmlClip, ComplexTextClip, ImageClip, TextClip, VideoClip } from '..'; 9 | import { AudioSource, HtmlSource, ImageSource, VideoSource } from '../../sources'; 10 | 11 | import type { ClipType } from '..'; 12 | import type { Source } from '../../sources'; 13 | 14 | export class ClipDeserializer { 15 | public static fromType(data: { type: ClipType }): Clip { 16 | switch (data.type) { 17 | case 'video': 18 | return new VideoClip(); 19 | case 'audio': 20 | return new AudioClip(); 21 | case 'html': 22 | return new HtmlClip(); 23 | case 'image': 24 | return new ImageClip(); 25 | case 'text': 26 | return new TextClip(); 27 | case 'complex_text': 28 | return new ComplexTextClip(); 29 | default: 30 | return new Clip(); 31 | } 32 | } 33 | 34 | public static fromSource(data: Source) { 35 | if (data.type == 'audio' && data instanceof AudioSource) { 36 | return new AudioClip(data); 37 | } 38 | if (data.type == 'video' && data instanceof VideoSource) { 39 | return new VideoClip(data); 40 | } 41 | if (data.type == 'image' && data instanceof ImageSource) { 42 | return new ImageClip(data); 43 | } 44 | if (data.type == 'html' && data instanceof HtmlSource) { 45 | return new HtmlClip(data); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/clips/clip/clip.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { frame } from '../../types'; 9 | import type { Timestamp } from '../../models'; 10 | 11 | export interface ClipProps { 12 | disabled?: boolean; 13 | name?: string; 14 | start?: frame | Timestamp; 15 | stop?: frame | Timestamp; 16 | } 17 | -------------------------------------------------------------------------------- /src/clips/clip/clip.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Timestamp } from '../../models'; 9 | 10 | export type ClipType = 'image' | 'audio' | 'text' | 'video' | 'base' | 'html' | 'complex_text'; 11 | 12 | export type ClipState = 'IDLE' | 'LOADING' | 'ATTACHED' | 'READY' | 'ERROR'; 13 | 14 | export type ClipEvents = { 15 | offsetBy: Timestamp; 16 | update: any; 17 | frame: number | undefined; 18 | attach: undefined; 19 | detach: undefined; 20 | load: undefined; 21 | }; 22 | -------------------------------------------------------------------------------- /src/clips/clip/clip.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Container, Filter } from "pixi.js"; 9 | import { Keyframe } from "../../models"; 10 | 11 | import type { Timestamp } from "../../models"; 12 | 13 | /** 14 | * Replace all keyframes of an object and nested objects 15 | * @param obj The object to crawl 16 | * @param time Time of replacement 17 | * @param depth Portection agains circular deps 18 | * @returns void 19 | */ 20 | export function replaceKeyframes(obj: any, time: Timestamp, depth = 0) { 21 | if ( 22 | obj instanceof Container 23 | || obj instanceof Filter 24 | || depth == 3 25 | ) return; 26 | 27 | for (const key in obj) { 28 | const value = obj[key]; 29 | 30 | if (!key) continue; 31 | 32 | if (value instanceof Keyframe) { 33 | obj[key] = value.value(time); 34 | } 35 | 36 | if (value != null && typeof value == 'object' && Object.keys(value).length) { 37 | replaceKeyframes(value, time, depth + 1); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/clips/clip/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './clip'; 9 | export * from './clip.types'; 10 | export * from './clip.desierializer'; 11 | export * from './clip.interfaces'; 12 | -------------------------------------------------------------------------------- /src/clips/html/html.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { ClipProps } from '../clip'; 9 | import type { VisualMixinProps } from '../mixins'; 10 | 11 | export interface HtmlClipProps extends ClipProps, VisualMixinProps { } 12 | -------------------------------------------------------------------------------- /src/clips/html/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './html'; 9 | export * from './html.interfaces'; 10 | -------------------------------------------------------------------------------- /src/clips/image/image.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { ClipProps } from '../clip'; 9 | import type { VisualMixinProps } from '../mixins'; 10 | 11 | export interface ImageClipProps extends ClipProps, VisualMixinProps { } 12 | -------------------------------------------------------------------------------- /src/clips/image/image.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Sprite, Texture } from 'pixi.js'; 9 | import { ImageSource } from '../../sources'; 10 | import { Clip } from '../clip'; 11 | import { VisualMixin, visualize } from '../mixins'; 12 | import { IOError } from '../../errors'; 13 | 14 | import type { Track } from '../../tracks'; 15 | import type { ImageClipProps } from './image.interfaces'; 16 | import type { Timestamp } from '../../models'; 17 | 18 | export class ImageClip extends VisualMixin(Clip) { 19 | public readonly type = 'image'; 20 | public declare track?: Track; 21 | public readonly element = new Image(); 22 | public source = new ImageSource(); 23 | 24 | /** 25 | * Access to the sprite containing the image texture 26 | */ 27 | public readonly sprite = new Sprite(); 28 | 29 | public constructor(source?: File | ImageSource, props: ImageClipProps = {}) { 30 | super(); 31 | 32 | this.view.addChild(this.sprite); 33 | 34 | if (source instanceof ImageSource) { 35 | this.source = source; 36 | } 37 | 38 | if (source instanceof File) { 39 | this.source.from(source); 40 | } 41 | 42 | Object.assign(this, props); 43 | } 44 | 45 | public async init(): Promise { 46 | this.element.setAttribute('src', await this.source.createObjectURL()); 47 | 48 | await new Promise((resolve, reject) => { 49 | this.element.onload = () => { 50 | this.sprite.texture = Texture.from(this.element); 51 | this.state = 'READY'; 52 | resolve(); 53 | } 54 | this.element.onerror = (e) => { 55 | console.error(e); 56 | this.state = 'ERROR'; 57 | reject(new IOError({ 58 | code: 'sourceNotProcessable', 59 | message: 'An error occurred while processing the input medium.', 60 | })); 61 | } 62 | }); 63 | } 64 | 65 | @visualize 66 | public update(_: Timestamp): void | Promise { } 67 | 68 | public copy(): ImageClip { 69 | const clip = ImageClip.fromJSON(JSON.parse(JSON.stringify(this))); 70 | clip.filters = this.filters; 71 | clip.source = this.source; 72 | 73 | return clip; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/clips/image/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './image'; 9 | export * from './image.interfaces'; 10 | -------------------------------------------------------------------------------- /src/clips/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './clip'; 9 | export * from './media'; 10 | export * from './mixins'; 11 | export * from './image'; 12 | export * from './text'; 13 | export * from './video'; 14 | export * from './audio'; 15 | export * from './utils'; 16 | export * from './html'; 17 | export * from './mask'; 18 | -------------------------------------------------------------------------------- /src/clips/mask/circleMask.ts: -------------------------------------------------------------------------------- 1 | import { Mask } from "./mask"; 2 | import { CircleMaskProps } from "./mask.types"; 3 | 4 | /** 5 | * A circular mask of a given radius 6 | */ 7 | export class CircleMask extends Mask { 8 | 9 | private _radius: number; 10 | 11 | public constructor(props: CircleMaskProps){ 12 | super(props); 13 | 14 | this._radius = props.radius; 15 | 16 | this.circle(this.position.x, this.position.y, this._radius) 17 | this.fill({color: '#FFF'}) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/clips/mask/ellipseMask.ts: -------------------------------------------------------------------------------- 1 | import { EllipseMaskProps } from './mask.types'; 2 | import { Mask } from './mask'; 3 | 4 | export class EllipseMask extends Mask { 5 | private _radius: { x: number; y: number }; 6 | 7 | public constructor(props: EllipseMaskProps) { 8 | super(props); 9 | 10 | this._radius = props.radius; 11 | 12 | this.ellipse( 13 | this.position.x, 14 | this.position.y, 15 | this._radius.x, 16 | this._radius.y, 17 | ); 18 | this.fill({ color: '#FFF' }); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/clips/mask/index.ts: -------------------------------------------------------------------------------- 1 | import { CircleMask } from "./circleMask"; 2 | import { EllipseMask } from "./ellipseMask"; 3 | import { RectangleMask } from "./rectangleMask"; 4 | import { RoundRectangleMask } from "./roundRectangleMask"; 5 | import { StarMask } from "./starMask"; 6 | 7 | export { CircleMask, EllipseMask, RectangleMask, RoundRectangleMask, StarMask }; 8 | 9 | -------------------------------------------------------------------------------- /src/clips/mask/mask.ts: -------------------------------------------------------------------------------- 1 | import { Graphics } from "pixi.js"; 2 | import type { MaskProps } from "./mask.types"; 3 | 4 | export class Mask extends Graphics { 5 | constructor(props: MaskProps) { 6 | super(); 7 | this.position = props.position ?? {x: 0, y: 0}; 8 | } 9 | } -------------------------------------------------------------------------------- /src/clips/mask/mask.types.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | export interface MaskProps { 4 | /** 5 | * The position of the center of the mask for circular, elliptical and star masks and 6 | * the top left corner of the mask for rectangular masks 7 | */ 8 | position?: { x: number, y: number }, 9 | } 10 | 11 | export interface CircleMaskProps extends MaskProps { 12 | radius: number, 13 | } 14 | 15 | export interface EllipseMaskProps extends MaskProps { 16 | radius: { 17 | x: number, 18 | y: number, 19 | } 20 | } 21 | 22 | export interface RectangleMaskProps extends MaskProps { 23 | rectangleWidth: number; 24 | rectangleHeight: number; 25 | } 26 | 27 | export interface RoundRectangleMaskProps extends RectangleMaskProps { 28 | borderRadius: number, 29 | } 30 | 31 | export interface StarMaskProps extends MaskProps { 32 | numberOfPoints: number, 33 | radius: number, 34 | innerRadius?: number, 35 | } -------------------------------------------------------------------------------- /src/clips/mask/rectangleMask.ts: -------------------------------------------------------------------------------- 1 | import { Mask } from './mask'; 2 | import { RectangleMaskProps } from './mask.types'; 3 | 4 | 5 | 6 | export class RectangleMask extends Mask { 7 | private _rectangleWidth: number; 8 | private _rectangleHeight: number; 9 | 10 | public constructor(props: RectangleMaskProps) { 11 | super(props); 12 | 13 | this._rectangleWidth = props.rectangleWidth; 14 | this._rectangleHeight = props.rectangleHeight; 15 | 16 | this.rect(this.position.x, this.position.y, this._rectangleWidth, this._rectangleHeight); 17 | this.fill({ color: '#FFF' }); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/clips/mask/roundRectangleMask.ts: -------------------------------------------------------------------------------- 1 | import { RoundRectangleMaskProps } from "./mask.types"; 2 | import { Mask } from "./mask"; 3 | 4 | export class RoundRectangleMask extends Mask { 5 | private _rectangleWidth: number; 6 | private _rectangleHeight: number; 7 | private _borderRadius: number; 8 | 9 | 10 | public constructor(props: RoundRectangleMaskProps) { 11 | super(props); 12 | 13 | this._rectangleWidth = props.rectangleWidth; 14 | this._rectangleHeight = props.rectangleHeight; 15 | this._borderRadius = props.borderRadius; 16 | 17 | this.roundRect(this.position.x, this.position.y, this._rectangleWidth, this._rectangleHeight, this._borderRadius) 18 | this.fill({color: '#FFF'}) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/clips/mask/starMask.ts: -------------------------------------------------------------------------------- 1 | import { StarMaskProps } from "./mask.types"; 2 | import { Mask } from "./mask"; 3 | 4 | export class StarMask extends Mask { 5 | 6 | private _numberOfPoints: number; 7 | private _radius: number; 8 | private _innerRadius: number | undefined; 9 | public constructor(props: StarMaskProps){ 10 | super(props); 11 | 12 | this._numberOfPoints = props.numberOfPoints; 13 | this._radius = props.radius; 14 | this._innerRadius = props.innerRadius; 15 | 16 | this.star(this.position.x, this.position.y, this._numberOfPoints, this._radius, this._innerRadius, this.rotation) 17 | this.fill({color: '#FFF'}) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/clips/media/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './media'; 9 | export * from './media.interfaces'; 10 | export * from './media.types'; 11 | -------------------------------------------------------------------------------- /src/clips/media/media.deserializer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Timestamp } from "../../models"; 9 | 10 | type millis = ReturnType; 11 | 12 | export class RangeDeserializer { 13 | public static fromJSON(obj: [millis, millis]): [Timestamp, Timestamp] { 14 | return [new Timestamp(obj[0]), new Timestamp(obj[1])] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/clips/media/media.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Timestamp, Transcript } from '../../models'; 9 | import type { frame } from '../../types'; 10 | import type { ClipProps } from '../clip'; 11 | 12 | export interface MediaClipProps extends ClipProps { 13 | playing?: boolean; 14 | transcript?: Transcript; 15 | offset?: frame | Timestamp; 16 | volume?: number; 17 | muted?: boolean; 18 | } 19 | -------------------------------------------------------------------------------- /src/clips/media/media.types.ts: -------------------------------------------------------------------------------- 1 | import { SilenceDetectionOptions } from "../../sources"; 2 | 3 | export type SilenceRemoveOptions = { 4 | /** 5 | * Adds padding in milliseconds after each detected non-silent segment. 6 | * This helps prevent cutting off audio too abruptly. 7 | * @default 500 8 | */ 9 | padding?: number; 10 | } & SilenceDetectionOptions; 11 | -------------------------------------------------------------------------------- /src/clips/mixins/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './visual'; 9 | export * from './visual.decorator'; 10 | export * from './visual.interfaces'; 11 | -------------------------------------------------------------------------------- /src/clips/mixins/visual.animation.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { AnimationBuilder as Builder, AnimationFunction } from '../../models/animation-builder'; 9 | 10 | export interface AnimationBuilder extends Builder { 11 | height: AnimationFunction; 12 | width: AnimationFunction; 13 | x: AnimationFunction; 14 | y: AnimationFunction; 15 | translateX: AnimationFunction; 16 | translateY: AnimationFunction; 17 | rotation: AnimationFunction; 18 | alpha: AnimationFunction; 19 | scale: AnimationFunction; 20 | } 21 | 22 | export class AnimationBuilder extends Builder { } 23 | -------------------------------------------------------------------------------- /src/clips/mixins/visual.deserializers.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Keyframe } from '../../models'; 9 | 10 | import type { Position } from '../../types'; 11 | 12 | export class Deserializer1D { 13 | public static fromJSON(json: any) { 14 | if (typeof json == 'object') { 15 | return Keyframe.fromJSON(json); 16 | } 17 | 18 | return json; 19 | } 20 | } 21 | 22 | export class Deserializer2D { 23 | public static fromJSON(json: Position) { 24 | if (typeof json.x == 'object') { 25 | json.x = Keyframe.fromJSON(json.x); 26 | } 27 | if (typeof json.y == 'object') { 28 | json.y = Keyframe.fromJSON(json.y); 29 | } 30 | 31 | return json; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/clips/mixins/visual.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Filter } from 'pixi.js'; 9 | import type { Keyframe } from '../../models'; 10 | import type { float, int, Anchor, Position, Scale, Translate2D, NumberCallback, Percent } from '../../types'; 11 | 12 | export interface VisualMixinProps { 13 | filters?: Filter | Filter[]; 14 | rotation?: number | Keyframe | NumberCallback; 15 | alpha?: number | Keyframe | NumberCallback; 16 | translate?: Translate2D; 17 | position?: Position | 'center'; 18 | scale?: Scale | float | Keyframe | NumberCallback; 19 | x?: int | Keyframe | Percent | NumberCallback; 20 | y?: int | Keyframe | Percent | NumberCallback; 21 | translateX?: int | Keyframe | NumberCallback; 22 | translateY?: int | Keyframe | NumberCallback; 23 | height?: Keyframe | Percent | int | NumberCallback; 24 | width?: Keyframe | Percent | int | NumberCallback; 25 | anchor?: Anchor | float; 26 | } 27 | -------------------------------------------------------------------------------- /src/clips/text/font.fixtures.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export const WebFonts = { 9 | 'The Bold Font': { 10 | weights: ['500'], 11 | url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/the-bold-font.ttf', 12 | }, 13 | 'Komika Axis': { 14 | weights: ['400'], 15 | url: 'https://diffusion-studio-public.s3.eu-central-1.amazonaws.com/fonts/komika-axis.ttf', 16 | }, 17 | Geologica: { 18 | weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'], 19 | url: 'https://fonts.gstatic.com/s/geologica/v1/oY1l8evIr7j9P3TN9YwNAdyjzUyDKkKdAGOJh1UlCDUIhAIdhCZOn1fLsig7jfvCCPHZckUWE1lELWNN-w.woff2', 20 | }, 21 | Figtree: { 22 | weights: ['300', '400', '500', '600', '700', '800', '900'], 23 | url: 'https://fonts.gstatic.com/s/figtree/v5/_Xms-HUzqDCFdgfMm4S9DaRvzig.woff2', 24 | }, 25 | Urbanist: { 26 | weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'], 27 | url: 'https://fonts.gstatic.com/s/urbanist/v15/L0x-DF02iFML4hGCyMqlbS1miXK2.woff2', 28 | }, 29 | Montserrat: { 30 | weights: ['100', '200', '300', '400', '500', '600', '700', '800', '900'], 31 | url: 'https://fonts.gstatic.com/s/montserrat/v26/JTUSjIg1_i6t8kCHKm459WlhyyTh89Y.woff2', 32 | }, 33 | Bangers: { 34 | weights: ['400'], 35 | url: 'https://fonts.gstatic.com/s/bangers/v20/FeVQS0BTqb0h60ACH55Q2J5hm24.woff2', 36 | }, 37 | Chewy: { 38 | weights: ['400'], 39 | url: 'https://fonts.gstatic.com/s/chewy/v18/uK_94ruUb-k-wn52KjI9OPec.woff2', 40 | }, 41 | 'Source Code Pro': { 42 | weights: ['200', '300', '400', '500', '600', '700', '800', '900'], 43 | url: 'https://fonts.gstatic.com/s/sourcecodepro/v22/HI_SiYsKILxRpg3hIP6sJ7fM7PqlPevWnsUnxg.woff2', 44 | }, 45 | } as const; 46 | -------------------------------------------------------------------------------- /src/clips/text/font.static.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export const SAFE_BROSER_FONTS = [ 9 | 'Helvetica', 10 | 'Arial', 11 | 'Arial Black', 12 | 'Verdana', 13 | 'Tahoma', 14 | 'Trebuchet MS', 15 | 'Impact', 16 | 'Gill Sans', 17 | 'Times New Roman', 18 | 'Georgia', 19 | 'Palatino', 20 | 'Baskerville', 21 | 'Andalé Mono', 22 | 'Courier', 23 | 'Lucida', 24 | 'Monaco', 25 | 'Bradley Hand', 26 | 'Brush Script MT', 27 | 'Luminari', 28 | 'Comic Sans MS', 29 | ] as const; 30 | 31 | export const FONT_WEIGHTS = { 32 | '100': 'Thin', 33 | '200': 'Extra Light', 34 | '300': 'Light', 35 | '400': 'Normal', 36 | '500': 'Medium', 37 | '600': 'Semi Bold', 38 | '700': 'Bold', 39 | '800': 'Extra Bold', 40 | '900': 'Black', 41 | } as const; 42 | -------------------------------------------------------------------------------- /src/clips/text/font.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { WebFonts } from './font.fixtures'; 9 | import type { FONT_WEIGHTS } from './font.static'; 10 | 11 | /** 12 | * Defines all available font families 13 | */ 14 | export type FontFamily = keyof typeof WebFonts | string; 15 | 16 | /** 17 | * Defines all available font weights 18 | */ 19 | export type FontWeight = keyof typeof FONT_WEIGHTS; 20 | 21 | /** 22 | * Defines the style of the font 23 | */ 24 | export type FontStyle = 'normal' | 'italic' | 'oblique'; 25 | 26 | /** 27 | * Defines all available font subsets which 28 | * limit the number of characters 29 | */ 30 | export type FontSubset = 'latin' | 'latin-ext' | 'vietnamese' | 'cyrillic' | 'cyrillic-ext'; 31 | 32 | /** 33 | * Defines the source where the font is coming from 34 | */ 35 | export type FontType = 'local' | 'web'; 36 | 37 | /** 38 | * Defines the properties that are required 39 | * to load a new font 40 | */ 41 | export type FontSource = { 42 | /** 43 | * Name of the Family 44 | * @example 'Arial' 45 | */ 46 | family: string; 47 | /** 48 | * Source of the Variant 49 | * @example url(arial.ttf) 50 | */ 51 | source: string; 52 | /** 53 | * Defines the font style 54 | * @example 'italic' 55 | */ 56 | style?: string; 57 | /** 58 | * The weight of the font 59 | * @example '400' 60 | */ 61 | weight?: string; 62 | }; 63 | 64 | /** 65 | * Defines a single font that has one or 66 | * more variants 67 | */ 68 | export type FontSources = { 69 | family: string; 70 | variants: FontSource[]; 71 | }; 72 | 73 | /** 74 | * Defines the arguments to identify 75 | * a default webfont 76 | */ 77 | export type WebfontProperties = { 78 | family: T; 79 | weight: typeof WebFonts[T]['weights'][number]; 80 | }; 81 | -------------------------------------------------------------------------------- /src/clips/text/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './font.fixtures'; 9 | export * from './font'; 10 | export * from './font.types'; 11 | export * from './text'; 12 | export * from './text.complex'; 13 | export * from './text.types'; 14 | export * from './text.interfaces'; 15 | export * from './text.complex.interfaces'; 16 | -------------------------------------------------------------------------------- /src/clips/text/text.complex.deserializer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Font } from './font'; 9 | 10 | import type * as types from './text.types'; 11 | 12 | export class StylesDeserializer { 13 | public static fromJSON(obj: types.StyleOption[]) { 14 | return obj.map(item => { 15 | if (item.font) { 16 | item.font = Font.fromJSON(item.font); 17 | } 18 | return item; 19 | }) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/clips/text/text.complex.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { TextClipProps } from './text.interfaces'; 9 | import type { Background, StyleOption, TextSegment } from './text.types'; 10 | 11 | export interface ComplexTextClipProps extends TextClipProps { 12 | segments?: TextSegment[]; 13 | background?: Background; 14 | styles?: StyleOption[]; 15 | } 16 | -------------------------------------------------------------------------------- /src/clips/text/text.complex.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { beforeEach, describe, expect, it, vi } from 'vitest'; 9 | import { ComplexTextClip } from './text.complex'; 10 | import { Font } from './font'; 11 | 12 | // Blend of different test files 13 | describe('Copying the ComplexTextClip', () => { 14 | let clip: ComplexTextClip; 15 | const fontAddFn = vi.fn(); 16 | 17 | Object.assign(document, { fonts: { add: fontAddFn } }); 18 | 19 | beforeEach(() => { 20 | clip = new ComplexTextClip('Hello World'); 21 | }); 22 | 23 | it('should transfer base properties', () => { 24 | const font = new Font({ 25 | family: 'Bangers', 26 | style: 'normal', 27 | source: 'local(banger.ttf)' 28 | }); 29 | font.loaded = true; 30 | 31 | clip.styles = [{ 32 | fillStyle: '#FF00FF', 33 | font: font, 34 | fontSize: 12, 35 | stroke: { 36 | join: 'bevel', 37 | width: 20, 38 | }, 39 | }]; 40 | 41 | clip.background = { 42 | alpha: 0.2, 43 | fill: '#00FF00' 44 | }; 45 | 46 | clip.maxWidth = 534; 47 | clip.textAlign = 'right'; 48 | clip.textBaseline = 'bottom'; 49 | 50 | const copy = clip.copy(); 51 | 52 | expect(copy).toBeInstanceOf(ComplexTextClip) 53 | expect(copy.id).not.toBe(clip.id); 54 | expect(copy.name).toBe('Hello World'); 55 | expect(copy.anchor.x).toBe(1); 56 | expect(copy.anchor.y).toBe(1); 57 | expect(copy.background?.alpha).toBe(0.2); 58 | expect(copy.background?.fill).toBe('#00FF00'); 59 | expect(copy.maxWidth).toBe(534); 60 | expect(copy.textAlign).toBe('right'); 61 | expect(copy.textBaseline).toBe('bottom'); 62 | expect(copy.styles?.length).toBe(1); 63 | expect(copy.styles?.[0].fillStyle).toBe('#FF00FF'); 64 | expect(copy.styles?.[0].font).toBeInstanceOf(Font); 65 | expect(copy.styles?.[0].font?.name).toBe('Bangers normal'); 66 | expect(copy.styles?.[0].fontSize).toBe(12); 67 | expect(copy.styles?.[0].stroke?.join).toBe('bevel'); 68 | expect(copy.styles?.[0].stroke?.width).toBe(20); 69 | }); 70 | }); 71 | -------------------------------------------------------------------------------- /src/clips/text/text.fixtures.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { TextAlign, TextBaseline } from './text.types'; 9 | 10 | export const SCALE_OFFSET = 4; 11 | 12 | export const alignToAnchor: Record = { 13 | center: 0.5, 14 | justify: 0.5, 15 | left: 0, 16 | right: 1, 17 | }; 18 | 19 | export const baselineToAnchor: Record = { 20 | alphabetic: 0, 21 | top: 0, 22 | middle: 0.5, 23 | hanging: 1, 24 | bottom: 1, 25 | ideographic: 1, 26 | }; 27 | -------------------------------------------------------------------------------- /src/clips/text/text.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { hex } from '../../types'; 9 | import type { ClipProps } from '../clip'; 10 | import type { VisualMixinProps } from '../mixins'; 11 | import type { Font } from './font'; 12 | import type { Stroke, TextAlign, TextBaseline, TextCase, TextShadow } from './text.types'; 13 | 14 | export interface TextClipProps extends ClipProps, Omit { 15 | text?: string; 16 | font?: Font; 17 | maxWidth?: number; 18 | textAlign?: TextAlign; 19 | padding?: number; 20 | textBaseline?: TextBaseline; 21 | fillStyle?: hex; 22 | stroke?: Partial; 23 | textCase?: TextCase; 24 | shadow?: Partial; 25 | fontSize?: number; 26 | leading?: number; 27 | } 28 | -------------------------------------------------------------------------------- /src/clips/text/text.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { SCALE_OFFSET } from './text.fixtures'; 9 | import type { CanvasTextMetrics } from 'pixi.js'; 10 | 11 | export function split(text: string) { 12 | const tokens = text.split(' ').map((t) => `${t} `); 13 | tokens[tokens.length - 1] = tokens[tokens.length - 1].replace(/ $/, ''); 14 | 15 | return tokens; 16 | } 17 | 18 | const handler: ProxyHandler = { 19 | get(target, property) { 20 | const value = (target as any)[property]; 21 | if (typeof value === 'number') { 22 | return value * SCALE_OFFSET; 23 | } 24 | if (Array.isArray(value) && typeof value[0] === 'number') { 25 | return value.map((v) => v * SCALE_OFFSET); 26 | } 27 | return value; 28 | }, 29 | }; 30 | 31 | export function createMetricsProxy(obj: CanvasTextMetrics): CanvasTextMetrics { 32 | return new Proxy(obj, handler); 33 | } 34 | 35 | export class TextMetricLine { 36 | public tokens: { metrics: CanvasTextMetrics; index: number }[] = []; 37 | 38 | public get width() { 39 | return this.tokens.reduce((prev, item) => prev + item.metrics.lineWidths[0], 0); 40 | } 41 | 42 | public get height() { 43 | return Math.max(...this.tokens.map((token) => token.metrics.lineHeight)); 44 | } 45 | } 46 | 47 | export class TextMetrics { 48 | public lines: TextMetricLine[] = []; 49 | 50 | public get width() { 51 | return Math.max(...this.lines.map((token) => token.width)); 52 | } 53 | 54 | public get height() { 55 | return this.lines.reduce((prev, item) => prev + item.height, 0); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/clips/utils/index.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { SUPPORTED_MIME_TYPES } from '../../fixtures'; 10 | import { parseMimeType } from '.'; 11 | 12 | describe('The Clip utils', () => { 13 | it('should be able to validate mime types', () => { 14 | for (const mimeType of Object.keys(SUPPORTED_MIME_TYPES.MIXED)) { 15 | expect(parseMimeType(mimeType)).toBe(mimeType); 16 | } 17 | 18 | const invalidTypes = ['video/x-msvideo', 'image/bmp', 'text/css']; 19 | for (const mimeType of invalidTypes) { 20 | expect(() => parseMimeType(mimeType)).toThrow(); 21 | } 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /src/clips/utils/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { SUPPORTED_MIME_TYPES } from '../../fixtures'; 9 | import * as errors from '../../errors'; 10 | 11 | import type { MimeType } from '../../types'; 12 | 13 | /** 14 | * Make sure a mimetype is valid 15 | * @param mimeType The mimetype to check 16 | * @returns A valid mimetype 17 | */ 18 | export function parseMimeType(mimeType?: string | null): MimeType { 19 | if (!Object.keys(SUPPORTED_MIME_TYPES.MIXED).includes(mimeType ?? '')) { 20 | throw new errors.ValidationError({ 21 | message: `${mimeType} is not an accepted mime type`, 22 | code: 'invalid_mimetype', 23 | }); 24 | } 25 | return mimeType as MimeType; 26 | } 27 | -------------------------------------------------------------------------------- /src/clips/video/buffer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export class FrameBuffer { 9 | public frames: Array = []; 10 | public state: 'active' | 'closed' = 'active'; 11 | 12 | public onenqueue?: () => void; 13 | public onclose?: () => void; 14 | 15 | public enqueue(data: VideoFrame) { 16 | this.frames.unshift(data); 17 | this.onenqueue?.(); 18 | } 19 | 20 | public async dequeue() { 21 | if (this.frames.length == 0 && this.state == 'active') { 22 | await this.waitFor(20e3); 23 | } 24 | 25 | if (this.frames.length == 0 && this.state == 'closed') { 26 | return; 27 | } 28 | 29 | return this.frames.pop(); 30 | } 31 | 32 | public close() { 33 | this.state = 'closed'; 34 | this.onclose?.(); 35 | } 36 | 37 | public terminate() { 38 | for (const frame of this.frames) { 39 | frame.close(); 40 | } 41 | } 42 | 43 | private async waitFor(timeout: number) { 44 | await new Promise((resolve, reject) => { 45 | const timer = setTimeout(() => { 46 | reject(`Promise timed out after ${timeout} ms`); 47 | }, timeout); 48 | 49 | this.onenqueue = () => { 50 | clearTimeout(timer); 51 | resolve(); 52 | }; 53 | 54 | this.onclose = () => { 55 | clearTimeout(timer); 56 | resolve(); 57 | }; 58 | }); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/clips/video/decoder.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect, vi, beforeEach, Mock } from 'vitest'; 9 | import { Decoder } from './decoder'; 10 | 11 | describe('Decoder', () => { 12 | let mockPostMessage: Mock 13 | let mockVideoDecoder: Mock 14 | let mockVideoFrame: any; 15 | let decoder: Decoder; 16 | 17 | beforeEach(() => { 18 | // Mock postMessage for 'self' 19 | mockPostMessage = vi.fn(); 20 | (global as any).self = { 21 | postMessage: mockPostMessage, 22 | close: vi.fn(), 23 | }; 24 | 25 | // Mock VideoDecoder 26 | mockVideoDecoder = vi.fn().mockImplementation(({ output, error }) => { 27 | return { 28 | output, 29 | error, 30 | decode: vi.fn(), 31 | close: vi.fn(), 32 | }; 33 | }); 34 | (global as any).VideoDecoder = mockVideoDecoder; 35 | 36 | // Mock VideoFrame 37 | mockVideoFrame = { 38 | timestamp: 0, 39 | duration: 1000000, // 1 second in nanoseconds 40 | close: vi.fn(), 41 | }; 42 | }); 43 | 44 | it('should initialize with correct properties', () => { 45 | const range = [0, 5] satisfies [number, number]; // 5 seconds range 46 | const fps = 30; 47 | 48 | decoder = new Decoder(range, fps); 49 | 50 | expect(decoder.video).toBeDefined(); 51 | expect(mockVideoDecoder).toHaveBeenCalled(); 52 | expect(decoder['currentTime']).toBe(range[0] * 1e6); 53 | expect(decoder['firstTimestamp']).toBe(range[0] * 1e6); 54 | expect(decoder['totalFrames']).toBe(((range[1] - range[0]) * fps) + 1); 55 | expect(decoder['fps']).toBe(fps); 56 | }); 57 | 58 | it('should post a frame and update current time and count', () => { 59 | const range = [0, 5] satisfies [number, number]; 60 | const fps = 30; 61 | 62 | decoder = new Decoder(range, fps); 63 | 64 | decoder['postFrame'](mockVideoFrame); 65 | 66 | expect(mockPostMessage).toHaveBeenCalledWith({ type: 'frame', frame: mockVideoFrame }); 67 | expect(decoder['currentTime']).toBeGreaterThan(range[0] * 1e6); // Time should increase 68 | expect(decoder['currentFrames']).toBe(1); 69 | }); 70 | 71 | it('should handle frame output within range and post frames', () => { 72 | const range = [0, 5] satisfies [number, number]; 73 | const fps = 30; 74 | 75 | decoder = new Decoder(range, fps); 76 | mockVideoFrame.timestamp = range[0] * 1e6; // Start time 77 | 78 | decoder['handleFrameOutput'](mockVideoFrame); 79 | 80 | expect(mockPostMessage).toHaveBeenCalledWith({ type: 'frame', frame: mockVideoFrame }); 81 | expect(mockVideoFrame.close).toHaveBeenCalled(); 82 | }); 83 | 84 | it('should handle errors and post error messages', () => { 85 | const range = [0, 5] satisfies [number, number]; 86 | const fps = 30; 87 | const mockError = new DOMException('Test Error'); 88 | 89 | decoder = new Decoder(range, fps); 90 | 91 | decoder['handleError'](mockError); 92 | 93 | expect(mockPostMessage).toHaveBeenCalledWith({ 94 | type: 'error', 95 | message: 'Test Error', 96 | }); 97 | expect(self.close).toHaveBeenCalled(); 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /src/clips/video/decoder.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export class Decoder { 9 | /** 10 | * Defines the VideoDecoder instance 11 | */ 12 | public readonly video: VideoDecoder; 13 | 14 | /** 15 | * Defines the current number of posted frames 16 | */ 17 | private currentFrames: number = 0; 18 | 19 | /** 20 | * Defines the total number of required frames 21 | */ 22 | private readonly totalFrames: number; // The total number of required frames 23 | 24 | /** 25 | * Defines the time up to which the video 26 | * has been decoded, in nano seconds 27 | */ 28 | private currentTime: number; 29 | 30 | /** 31 | * Defines the requested frame rate 32 | */ 33 | private readonly fps: number; 34 | 35 | /** 36 | * Defines the first requested frame timestamp 37 | * in nanoseconds 38 | */ 39 | private readonly firstTimestamp: number; 40 | 41 | /** 42 | * Create a new video decoder and send the decoded frames 43 | * from the worker to the main process 44 | * @param range The start and stop of the window to decode in seconds 45 | * @param fps The desired framerate 46 | */ 47 | public constructor(range: [number, number], fps: number) { 48 | this.currentTime = range[0] * 1e6; 49 | this.firstTimestamp = range[0] * 1e6; 50 | this.totalFrames = ((range[1] - range[0]) * fps) + 1; 51 | this.fps = fps; 52 | this.video = new VideoDecoder({ 53 | output: this.handleFrameOutput.bind(this), 54 | error: this.handleError.bind(this), 55 | }); 56 | } 57 | 58 | /** 59 | * Method to post the frame and update the current time and count 60 | */ 61 | private postFrame(frame: VideoFrame) { 62 | self.postMessage({ type: 'frame', frame }); 63 | 64 | // Update the expected time for the next frame 65 | this.currentTime += Math.floor((1 / this.fps) * 1e6); 66 | this.currentFrames += 1; 67 | }; 68 | 69 | /** 70 | * Method to handle the output of the decoder 71 | */ 72 | private handleFrameOutput(frame: VideoFrame) { 73 | const timestamp = frame.timestamp; 74 | const duration = frame.duration ?? 0; 75 | const endsAt = timestamp + duration; 76 | 77 | if (!this.isFrameInRange(timestamp)) { 78 | frame.close(); 79 | return; 80 | } 81 | 82 | while (endsAt > this.currentTime && this.currentFrames <= this.totalFrames) { 83 | this.postFrame(frame); 84 | } 85 | 86 | frame.close(); 87 | }; 88 | 89 | /** 90 | * Method to check if the frame is within the specified range 91 | */ 92 | private isFrameInRange(timestamp: number): boolean { 93 | return timestamp >= this.firstTimestamp; 94 | }; 95 | 96 | /** 97 | * Method for handling decoding errors 98 | */ 99 | private handleError(error: DOMException) { 100 | console.error('error in worker', error); 101 | self.postMessage({ 102 | type: 'error', 103 | message: error.message ?? 'An unknown worker error occurred', 104 | }); 105 | self.close(); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/ffmpeg.worker.ts: -------------------------------------------------------------------------------- 1 | /* c8 ignore start */ 2 | import type { WebAVPacket, WebAVStream } from './types'; 3 | 4 | let Module: any; // TODO: rm any 5 | 6 | self.postMessage({ 7 | type: 'FFmpegWorkerLoaded', 8 | }); 9 | 10 | self.addEventListener('message', async function (e) { 11 | const { type, data = {}, msgId } = e.data; 12 | 13 | try { 14 | if (type === 'LoadWASM') { 15 | const { wasmLoaderPath } = data || {}; 16 | 17 | const ModuleLoader = await import(/* @vite-ignore */ wasmLoaderPath); 18 | Module = await ModuleLoader.default(); 19 | } else if (type === 'GetAVStream') { 20 | const { file, streamType, streamIndex } = data; 21 | const result = Module.getAVStream(file, streamType, streamIndex); 22 | 23 | self.postMessage( 24 | { 25 | type, 26 | msgId, 27 | result, 28 | }, 29 | [result.codecpar.extradata.buffer], 30 | ); 31 | } else if (type === 'GetAVStreams') { 32 | const { file } = data; 33 | const result = Module.getAVStreams(file); 34 | 35 | self.postMessage( 36 | { 37 | type, 38 | msgId, 39 | result, 40 | }, 41 | result.map((stream: WebAVStream) => stream.codecpar.extradata.buffer), 42 | ); 43 | } else if (type === 'GetAVPacket') { 44 | const { file, time, streamType, streamIndex } = data; 45 | const result = Module.getAVPacket(file, time, streamType, streamIndex); 46 | 47 | self.postMessage( 48 | { 49 | type, 50 | msgId, 51 | result, 52 | }, 53 | [result.data.buffer], 54 | ); 55 | } else if (type === 'GetAVPackets') { 56 | const { file, time } = data; 57 | const result = Module.getAVPackets(file, time); 58 | 59 | self.postMessage( 60 | { 61 | type, 62 | msgId, 63 | result, 64 | }, 65 | result.map((packet: WebAVPacket) => packet.data.buffer), 66 | ); 67 | } else if (type === 'ReadAVPacket') { 68 | const { file, start, end, streamType, streamIndex } = data; 69 | const result = Module.readAVPacket(msgId, file, start, end, streamType, streamIndex); 70 | 71 | self.postMessage({ 72 | type, 73 | msgId, 74 | result, 75 | }); 76 | } 77 | } catch (e) { 78 | self.postMessage({ 79 | type, 80 | msgId, 81 | errMsg: e instanceof Error ? e.message : 'Unknown Error', 82 | }); 83 | } 84 | }); 85 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/index.ts: -------------------------------------------------------------------------------- 1 | export type { WebAVStream, WebAVPacket, WebAVCodecParameters } from './types'; 2 | export { AVMediaType } from './types'; 3 | export { WebDemuxer as Demuxer } from './web-demuxer'; 4 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/types/avutil.ts: -------------------------------------------------------------------------------- 1 | /* c8 ignore start */ 2 | /** 3 | * sync with ffmpeg libavutil/avutil.h 4 | */ 5 | export enum AVMediaType { 6 | AVMEDIA_TYPE_UNKNOWN = -1, ///< Usually treated as AVMEDIA_TYPE_DATA 7 | AVMEDIA_TYPE_VIDEO, 8 | AVMEDIA_TYPE_AUDIO, 9 | AVMEDIA_TYPE_DATA, ///< Opaque data information usually continuous 10 | AVMEDIA_TYPE_SUBTITLE, 11 | AVMEDIA_TYPE_ATTACHMENT, ///< Opaque data information usually sparse 12 | AVMEDIA_TYPE_NB, 13 | } 14 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/types/demuxer.ts: -------------------------------------------------------------------------------- 1 | /* c8 ignore start */ 2 | /** 3 | * sync with web-demuxer.h 4 | */ 5 | import type { AVMediaType } from './avutil'; 6 | 7 | export interface WebAVCodecParameters { 8 | codec_type: AVMediaType; 9 | codec_id: number; 10 | codec_string: string; 11 | format: number; 12 | profile: number; 13 | level: number; 14 | width: number; 15 | height: number; 16 | channels: number; 17 | sample_rate: number; 18 | extradata_size: number; 19 | extradata: Uint8Array; 20 | } 21 | 22 | export interface WebAVStream { 23 | index: number; 24 | id: number; 25 | start_time: number; 26 | duration: number; 27 | nb_frames: number; 28 | codecpar: WebAVCodecParameters; 29 | } 30 | 31 | export interface WebAVPacket { 32 | keyframe: 0 | 1; 33 | timestamp: number; 34 | duration: number; 35 | size: number; 36 | data: Uint8Array; 37 | } 38 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/types/ffmpeg-worker-message.ts: -------------------------------------------------------------------------------- 1 | /* c8 ignore start */ 2 | import type { AVMediaType } from './avutil'; 3 | 4 | export enum FFMpegWorkerMessageType { 5 | FFmpegWorkerLoaded = 'FFmpegWorkerLoaded', 6 | WASMRuntimeInitialized = 'WASMRuntimeInitialized', 7 | LoadWASM = 'LoadWASM', 8 | GetAVPacket = 'GetAVPacket', 9 | GetAVPackets = 'GetAVPackets', 10 | GetAVStream = 'GetAVStream', 11 | GetAVStreams = 'GetAVStreams', 12 | ReadAVPacket = 'ReadAVPacket', 13 | AVPacketStream = 'AVPacketStream', 14 | ReadNextAVPacket = 'ReadNextAVPacket', 15 | StopReadAVPacket = 'StopReadAVPacket', 16 | } 17 | 18 | export type FFMpegWorkerMessageData = 19 | | GetAVPacketMessageData 20 | | GetAVPacketsMessageData 21 | | GetAVStreamMessageData 22 | | GetAVStreamsMessageData 23 | | ReadAVPacketMessageData 24 | | LoadWASMMessageData; 25 | 26 | export interface GetAVStreamMessageData { 27 | file: File; 28 | streamType: AVMediaType; 29 | streamIndex: number; 30 | } 31 | 32 | export interface GetAVStreamsMessageData { 33 | file: File; 34 | } 35 | 36 | export interface GetAVPacketMessageData { 37 | file: File; 38 | time: number; 39 | streamType: AVMediaType; 40 | streamIndex: number; 41 | } 42 | 43 | export interface GetAVPacketsMessageData { 44 | file: File; 45 | time: number; 46 | } 47 | 48 | export interface ReadAVPacketMessageData { 49 | file: File; 50 | start: number; 51 | end: number; 52 | streamType: AVMediaType; 53 | streamIndex: number; 54 | } 55 | 56 | export interface LoadWASMMessageData { 57 | wasmLoaderPath: string; 58 | } 59 | 60 | export interface FFMpegWorkerMessage { 61 | type: FFMpegWorkerMessageType; 62 | data: FFMpegWorkerMessageData; 63 | } 64 | -------------------------------------------------------------------------------- /src/clips/video/demuxer/types/index.ts: -------------------------------------------------------------------------------- 1 | export * from './avutil'; 2 | export * from './ffmpeg-worker-message'; 3 | export * from './demuxer'; 4 | -------------------------------------------------------------------------------- /src/clips/video/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './video'; 9 | export * from './video.interfaces'; 10 | -------------------------------------------------------------------------------- /src/clips/video/video.decorator.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { VideoClip } from "./video"; 9 | import type { Timestamp } from "../../models"; 10 | 11 | /** 12 | * Decorator for swapping the texture depending on the 13 | * current composition state/ environement 14 | */ 15 | export function textureSwap // @ts-ignore 16 | (target: T, propertyKey: string, descriptor: PropertyDescriptor) { 17 | const originalMethod = descriptor.value; 18 | 19 | descriptor.value = function (this: T, ...args: [Timestamp]) { 20 | if (this.track?.composition?.rendering 21 | && this.sprite.texture.source.uid != this.textrues.canvas.source.uid) { 22 | this.sprite.texture = this.textrues.canvas; 23 | } 24 | 25 | if (!this.track?.composition?.rendering 26 | && this.sprite.texture.source.uid != this.textrues.html5.source.uid) { 27 | this.sprite.texture = this.textrues.html5; 28 | } 29 | 30 | return originalMethod.apply(this, args); 31 | }; 32 | 33 | return descriptor; 34 | } 35 | -------------------------------------------------------------------------------- /src/clips/video/video.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { MediaClipProps } from '../media'; 9 | import { VisualMixinProps } from '../mixins'; 10 | 11 | export interface VideoClipProps extends MediaClipProps, VisualMixinProps { } 12 | -------------------------------------------------------------------------------- /src/clips/video/video.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export interface GetWavBytesOptions { 9 | isFloat: boolean; // floating point or 16-bit integer 10 | numChannels: number; 11 | sampleRate: number; 12 | } 13 | 14 | export interface GetWavHeaderOptions extends GetWavBytesOptions { 15 | numFrames: number; 16 | } 17 | -------------------------------------------------------------------------------- /src/clips/video/worker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Demuxer } from './demuxer'; 9 | import { Decoder } from './decoder'; 10 | import { withThreadErrorHandler } from '../../services'; 11 | import { validateDecoderConfig } from './worker.utils'; 12 | 13 | import type { InitMessageData } from './worker.types'; 14 | 15 | const MAX_QUEUE = 30; 16 | 17 | async function main(event: MessageEvent) { 18 | if (event.data?.type != 'init') return; 19 | 20 | const { file, range, fps } = event.data; 21 | 22 | const demuxer = new Demuxer({ 23 | wasmLoaderPath: 24 | 'https://cdn.jsdelivr.net/npm/@diffusionstudio/ffmpeg-wasm@1.0.0/dist/ffmpeg.js', 25 | }); 26 | 27 | await demuxer.load(file); 28 | 29 | const config = await demuxer.getVideoDecoderConfig(); 30 | validateDecoderConfig(config); 31 | 32 | const decoder = new Decoder(range, fps); 33 | 34 | decoder.video.configure(config); 35 | 36 | const reader = demuxer.readAVPacket(range[0], range[1]).getReader(); 37 | 38 | reader.read().then(async function processPacket({ done, value }): Promise { 39 | if (done) { 40 | await decoder.video.flush(); 41 | self.postMessage({ type: 'done' }); 42 | self.close(); 43 | return; 44 | } 45 | 46 | const chunk = demuxer.genEncodedVideoChunk(value); 47 | 48 | if (decoder.video.decodeQueueSize > MAX_QUEUE) { 49 | await new Promise((resolve) => { 50 | decoder.video.ondequeue = () => resolve(); 51 | }); 52 | } 53 | 54 | if (chunk.timestamp <= range[1] * 1e6) { 55 | decoder.video.decode(chunk); 56 | } 57 | 58 | return reader.read().then(processPacket); 59 | }); 60 | } 61 | 62 | self.addEventListener('message', withThreadErrorHandler(main)); 63 | -------------------------------------------------------------------------------- /src/clips/video/worker.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export type InitMessageData = { 9 | type: 'init'; 10 | file: File; 11 | range: [number, number]; 12 | fps: number; 13 | }; 14 | -------------------------------------------------------------------------------- /src/clips/video/worker.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * Validates the coded and replaces it 10 | * if necessary 11 | */ 12 | export function validateDecoderConfig(config: VideoDecoderConfig) { 13 | if (config.codec == 'vp09') { 14 | config.codec = 'vp09.00.10.08'; 15 | } 16 | 17 | return config; 18 | } 19 | -------------------------------------------------------------------------------- /src/composition/composition.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { frame, hex, int } from "../types"; 9 | 10 | export type CompositionSettings = { 11 | /** 12 | * Height of the composition 13 | * 14 | * @default 1080 15 | */ 16 | height: int; 17 | /** 18 | * Width of the composition 19 | * 20 | * @default 1920 21 | */ 22 | width: int; 23 | /** 24 | * Background color of the composition 25 | * 26 | * @default #000000 27 | */ 28 | background: hex; 29 | /** 30 | * Overwrite the backend auto detection. 31 | * *While webgpu is faster than webgl 32 | * it might not be available in your 33 | * browser yet.* 34 | */ 35 | backend: 'webgpu' | 'webgl' 36 | }; 37 | 38 | /** 39 | * Defines the available image formats 40 | */ 41 | export type ScreenshotImageFormat = 'webp' | 'png' | 'jpeg'; 42 | 43 | /** 44 | * Defines the type of events emitted by the 45 | * composition 46 | */ 47 | export type CompositionEvents = { 48 | play: frame; 49 | pause: frame; 50 | init: undefined; 51 | currentframe: frame; 52 | update: any; 53 | frame: number | undefined; 54 | attach: undefined; 55 | detach: undefined; 56 | load: undefined; 57 | resize: undefined; 58 | }; 59 | 60 | export type CompositionState = 'IDLE' | 'RENDER' | 'PLAY'; 61 | -------------------------------------------------------------------------------- /src/composition/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './composition'; 9 | export * from './composition.types'; 10 | -------------------------------------------------------------------------------- /src/encoders/encoder.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { beforeEach, describe, expect, it, vi } from 'vitest'; 9 | import { Composition } from '../composition'; 10 | import { Encoder } from './encoder'; 11 | import { Clip } from '../clips'; 12 | 13 | 14 | describe('The Encoder', () => { 15 | let composition: Composition; 16 | let encoder: Encoder; 17 | 18 | const configureSpy = vi.spyOn(VideoEncoder.prototype, 'configure').mockImplementation(vi.fn()); 19 | const encodeSpy = vi.spyOn(VideoEncoder.prototype, 'encode').mockImplementation(vi.fn()); 20 | 21 | beforeEach(() => { 22 | composition = new Composition(); 23 | 24 | encoder = new Encoder(composition, { 25 | audio: false, 26 | debug: false, 27 | fps: 60, 28 | gpuBatchSize: 4, 29 | numberOfChannels: 1, 30 | resolution: 0.5, 31 | sampleRate: 8000, 32 | videoBitrate: 1e6, 33 | }); 34 | 35 | configureSpy.mockClear(); 36 | encodeSpy.mockClear(); 37 | 38 | expect(encoder.audio).toBe(false); 39 | expect(encoder.debug).toBe(false); 40 | expect(encoder.fps).toBe(60); 41 | }); 42 | 43 | it('should render the compostion', async () => { 44 | await composition.add(new Clip({ stop: 10 })); 45 | 46 | const pauseSpy = vi.spyOn(composition, 'pause').mockImplementation(async () => undefined); 47 | const seekSpy = vi.spyOn(composition, 'seek').mockImplementation(async () => undefined); 48 | 49 | await encoder.render(); 50 | 51 | expect(pauseSpy).toBeCalledTimes(1); 52 | expect(seekSpy).toBeCalledTimes(1); 53 | }); 54 | 55 | it('should not render when the composition renderer is not defined', async () => { 56 | delete composition.renderer; 57 | 58 | await expect(() => encoder.render()).rejects.toThrowError(); 59 | }); 60 | 61 | it('should debug the render process', async () => { 62 | encoder.debug = true; 63 | 64 | await composition.add(new Clip({ stop: 10 })); 65 | 66 | const logSpy = vi.spyOn(console, 'info'); 67 | 68 | await encoder.render(); 69 | 70 | expect(logSpy).toBeCalledTimes(3); 71 | }); 72 | }); 73 | -------------------------------------------------------------------------------- /src/encoders/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './encoder'; 9 | export * from './canvas'; 10 | export * from './opus'; 11 | -------------------------------------------------------------------------------- /src/encoders/interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { StreamTarget } from 'mp4-muxer'; 9 | import type { Muxer } from 'mp4-muxer'; 10 | 11 | export interface WebAudioEncoder { 12 | encode(muxer: Muxer, config: AudioEncoderConfig): Promise 13 | } 14 | 15 | export interface EncoderInit { 16 | /** 17 | * A floating point number indicating the audio context's sample rate, in samples per second. 18 | * 19 | * @default 48000 20 | */ 21 | sampleRate?: number 22 | 23 | /** 24 | * Defines the number of channels 25 | * of the composed audio 26 | * 27 | * @default 2 28 | */ 29 | numberOfChannels?: number; 30 | 31 | /** 32 | * Defines the bitrate at which the video 33 | * should be rendered at 34 | * @default 10e6 35 | */ 36 | videoBitrate?: number; 37 | 38 | /** 39 | * Defines the maximum size of the video 40 | * encoding queue, increasing this number 41 | * will put a higher pressure on the gpu. 42 | * It's restricted to a value between 1 and 100 43 | * @default 5 44 | */ 45 | gpuBatchSize?: number; 46 | /** 47 | * Defines the fps at which the composition 48 | * will be rendered 49 | * @default 30 50 | */ 51 | fps?: number; 52 | 53 | /** 54 | * Defines if the audio should be encoded 55 | */ 56 | audio?: boolean; 57 | }; 58 | 59 | export interface VideoEncoderInit extends EncoderInit { 60 | /** 61 | * Multiplier of the composition size 62 | * @example 2 // 1080p -> 4K 63 | * @default 1 // 1080p -> 1080p 64 | */ 65 | resolution?: number; 66 | /** 67 | * Defines if the performance should be logged 68 | * @default false; 69 | */ 70 | debug?: boolean; 71 | } 72 | -------------------------------------------------------------------------------- /src/encoders/opus/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './opus.encoder'; 9 | export * from './opus.types'; 10 | -------------------------------------------------------------------------------- /src/encoders/opus/opus.fixtures.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * Sample rates supported by the opus encoder 10 | */ 11 | export const SUPPORTED_RATES = [8000, 12000, 16000, 24000, 48000]; 12 | export const OPUS_WASM_PATH = 'https://cdn.jsdelivr.net/npm/@diffusionstudio/libopus-wasm@1.0.0/dist/opus.wasm'; 13 | export const OPUS_JS_PATH = 'https://cdn.jsdelivr.net/npm/@diffusionstudio/libopus-wasm@1.0.0/dist/opus.js'; 14 | -------------------------------------------------------------------------------- /src/encoders/opus/opus.types.ts: -------------------------------------------------------------------------------- 1 | export type OpusEncoderSamples = { 2 | /** 3 | * 16-bit signed integer array of interleaved audio samples 4 | */ 5 | data: Int16Array, 6 | /** 7 | * The number of frames (usually total samples / number of channels) 8 | */ 9 | numberOfFrames: number, 10 | /** 11 | * Defines the timestamp of the first frame 12 | */ 13 | timestamp?: number 14 | } 15 | 16 | export type EncodedOpusChunk = { 17 | data: Uint8Array; 18 | timestamp: number; 19 | type: 'key' | 'delta'; 20 | duration: number; 21 | } 22 | 23 | export type OpusEncoderConfig = Omit; 24 | 25 | export type EncodedOpusChunkOutputCallback = (output: EncodedOpusChunk, metadata: EncodedAudioChunkMetadata) => void; 26 | 27 | export type OpusEncoderInit = { output: EncodedOpusChunkOutputCallback, error: WebCodecsErrorCallback }; 28 | -------------------------------------------------------------------------------- /src/encoders/opus/opus.utils.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect } from 'vitest'; 9 | import { createOpusHead } from './opus.utils'; 10 | 11 | describe('createOpusHead', () => { 12 | it('should generate a correct Opus header', () => { 13 | const sampleRate = 48000; 14 | const numberOfChannels = 2; 15 | const result = createOpusHead(sampleRate, numberOfChannels); 16 | 17 | // Check that the result is a Uint8Array of length 19 18 | expect(result).toBeInstanceOf(Uint8Array); 19 | expect(result.length).toBe(19); 20 | 21 | // Check magic signature "OpusHead" 22 | expect(result[0]).toBe('O'.charCodeAt(0)); 23 | expect(result[1]).toBe('p'.charCodeAt(0)); 24 | expect(result[2]).toBe('u'.charCodeAt(0)); 25 | expect(result[3]).toBe('s'.charCodeAt(0)); 26 | expect(result[4]).toBe('H'.charCodeAt(0)); 27 | expect(result[5]).toBe('e'.charCodeAt(0)); 28 | expect(result[6]).toBe('a'.charCodeAt(0)); 29 | expect(result[7]).toBe('d'.charCodeAt(0)); 30 | 31 | // Check version is set to 1 32 | expect(result[8]).toBe(1); 33 | 34 | // Check number of channels 35 | expect(result[9]).toBe(numberOfChannels); 36 | 37 | // Check pre-skip is 0 (bytes 10 and 11) 38 | expect(result[10]).toBe(0); 39 | expect(result[11]).toBe(0); 40 | 41 | // Check sample rate is correctly encoded (bytes 12-15) 42 | expect(result[12]).toBe(sampleRate & 0xFF); 43 | expect(result[13]).toBe((sampleRate >> 8) & 0xFF); 44 | expect(result[14]).toBe((sampleRate >> 16) & 0xFF); 45 | expect(result[15]).toBe((sampleRate >> 24) & 0xFF); 46 | 47 | // Check gain is 0 (bytes 16 and 17) 48 | expect(result[16]).toBe(0); 49 | expect(result[17]).toBe(0); 50 | 51 | // Check channel mapping is 0 52 | expect(result[18]).toBe(0); 53 | }); 54 | 55 | it('should correctly encode a different sample rate and number of channels', () => { 56 | const sampleRate = 44100; 57 | const numberOfChannels = 1; 58 | const result = createOpusHead(sampleRate, numberOfChannels); 59 | 60 | // Check number of channels 61 | expect(result[9]).toBe(numberOfChannels); 62 | 63 | // Check sample rate is correctly encoded (bytes 12-15) 64 | expect(result[12]).toBe(sampleRate & 0xFF); 65 | expect(result[13]).toBe((sampleRate >> 8) & 0xFF); 66 | expect(result[14]).toBe((sampleRate >> 16) & 0xFF); 67 | expect(result[15]).toBe((sampleRate >> 24) & 0xFF); 68 | }); 69 | }); 70 | -------------------------------------------------------------------------------- /src/encoders/opus/opus.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * Utility for creating the OpusHead 10 | */ 11 | export function createOpusHead(sampleRate: number, numberOfChannels: number) { 12 | const head = new Uint8Array(19); 13 | head[0] = 'O'.charCodeAt(0); // Magic signature 'OpusHead' 14 | head[1] = 'p'.charCodeAt(0); 15 | head[2] = 'u'.charCodeAt(0); 16 | head[3] = 's'.charCodeAt(0); 17 | head[4] = 'H'.charCodeAt(0); 18 | head[5] = 'e'.charCodeAt(0); 19 | head[6] = 'a'.charCodeAt(0); 20 | head[7] = 'd'.charCodeAt(0); 21 | head[8] = 1; // Version 22 | head[9] = numberOfChannels; // Number of channels 23 | head[10] = 0; // Pre-skip (2 bytes, default is 0) 24 | head[11] = 0; 25 | head[12] = sampleRate & 0xFF; // Sample rate (4 bytes) 26 | head[13] = (sampleRate >> 8) & 0xFF; 27 | head[14] = (sampleRate >> 16) & 0xFF; 28 | head[15] = (sampleRate >> 24) & 0xFF; 29 | head[16] = 0; // Gain (2 bytes, default is 0) 30 | head[17] = 0; 31 | head[18] = 0; // Channel mapping (0 = single stream, default) 32 | return head; 33 | } 34 | -------------------------------------------------------------------------------- /src/encoders/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export type EncoderEvents = { 9 | render: { 10 | /** 11 | * Defines how many were rendered yet 12 | */ 13 | progress: number; 14 | /** 15 | * Defines the total number of frames 16 | * to be rendered 17 | */ 18 | total: number; 19 | /** 20 | * Defines the estimated remaining 21 | * render time 22 | */ 23 | remaining: Date; 24 | }; 25 | }; 26 | -------------------------------------------------------------------------------- /src/encoders/webassembly.audio.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 9 | import { AudioBufferMock } from '../../vitest.mocks'; 10 | import { Composition } from '../composition'; 11 | import { WebassemblyAudioEncoder } from './webassembly.audio'; 12 | import { Muxer, ArrayBufferTarget } from 'mp4-muxer'; 13 | import { OpusEncoder } from './opus'; 14 | 15 | const config = { 16 | codec: 'opus', 17 | numberOfChannels: 2, 18 | sampleRate: 48000, 19 | } as const; 20 | 21 | describe('WebassemblyAudioEncoder', () => { 22 | const composition = new Composition(); 23 | 24 | const bufferSpy = vi.spyOn(composition, 'audio').mockImplementation( 25 | async (numberOfChannels: number = 1, sampleRate: number = 1e6) => 26 | new AudioBufferMock({ sampleRate, numberOfChannels, length: 50 }) as AudioBuffer 27 | ); 28 | const configureSpy = vi.spyOn(OpusEncoder.prototype, 'configure').mockImplementation(vi.fn()); 29 | const encodeSpy = vi.spyOn(OpusEncoder.prototype, 'encode') 30 | .mockImplementation(function (this: OpusEncoder, ...args: any[]) { 31 | this.output({ 32 | data: new Uint8Array(10), 33 | duration: 20, 34 | timestamp: 0, 35 | type: 'key' 36 | }, { decoderConfig: config }) 37 | return vi.fn()(...args); 38 | }); 39 | 40 | const encoder = new WebassemblyAudioEncoder(composition); 41 | 42 | const muxer = new Muxer({ 43 | target: new ArrayBufferTarget(), 44 | audio: config, 45 | fastStart: 'in-memory' 46 | }); 47 | 48 | const muxSpy = vi.spyOn(muxer, 'addAudioChunkRaw'); 49 | 50 | beforeEach(() => { 51 | bufferSpy.mockClear(); 52 | configureSpy.mockClear(); 53 | encodeSpy.mockClear(); 54 | muxSpy.mockClear(); 55 | }) 56 | 57 | it('should encode the audio of the composition using the provided configuration', async () => { 58 | await encoder.encode(muxer, { ...config, sampleRate: 50_000 }); 59 | 60 | expect(bufferSpy).toHaveBeenCalledTimes(1); 61 | expect(configureSpy).toHaveBeenCalledWith(config); 62 | expect(encodeSpy.mock.calls[0][0].numberOfFrames).toBe(50); 63 | // 2 channels times 50 64 | expect(encodeSpy.mock.calls[0][0].data.length).toBe(100); 65 | 66 | expect(muxSpy).toHaveBeenCalledTimes(1); 67 | expect(muxSpy.mock.calls[0][0]).toBeInstanceOf(Uint8Array); 68 | expect(muxSpy.mock.calls[0][0].length).toBe(10); 69 | expect(muxSpy.mock.calls[0][1]).toBe('key'); 70 | expect(muxSpy.mock.calls[0][2]).toBe(0); 71 | expect(muxSpy.mock.calls[0][3]).toBe(20); 72 | }); 73 | }); 74 | -------------------------------------------------------------------------------- /src/encoders/webassembly.audio.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { bufferToI16Interleaved } from "../utils"; 9 | import { OpusEncoder } from "./opus"; 10 | 11 | import type { StreamTarget } from 'mp4-muxer'; 12 | import type { Composition } from "../composition"; 13 | import type { Muxer } from 'mp4-muxer'; 14 | import type { WebAudioEncoder } from "./interfaces"; 15 | import { toOpusSampleRate } from "./utils"; 16 | 17 | export class WebassemblyAudioEncoder implements WebAudioEncoder { 18 | private composition: Composition; 19 | 20 | public constructor(composition: Composition) { 21 | this.composition = composition; 22 | } 23 | 24 | public async encode(muxer: Muxer, config: AudioEncoderConfig): Promise { 25 | const numberOfChannels = config.numberOfChannels; 26 | const sampleRate = toOpusSampleRate(config.sampleRate); 27 | 28 | const output = await this.composition.audio(numberOfChannels, sampleRate); 29 | 30 | if (!output) return; 31 | 32 | const encoder = new OpusEncoder({ 33 | output: (chunk, meta) => { 34 | muxer.addAudioChunkRaw( 35 | chunk.data, 36 | chunk.type, 37 | chunk.timestamp, 38 | chunk.duration, 39 | meta 40 | ); 41 | }, 42 | error: console.error, 43 | }); 44 | 45 | await encoder.configure({ ...config, sampleRate }); 46 | 47 | encoder.encode({ 48 | data: bufferToI16Interleaved(output), 49 | numberOfFrames: output.length, 50 | }); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/encoders/webcodecs.audio.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 9 | import { AudioBufferMock } from '../../vitest.mocks'; 10 | import { Composition } from '../composition'; 11 | import { WebcodecsAudioEncoder } from './webcodecs.audio'; 12 | import { Muxer, ArrayBufferTarget } from 'mp4-muxer'; 13 | 14 | const config = { 15 | codec: 'opus', 16 | numberOfChannels: 2, 17 | sampleRate: 48000, 18 | } as const; 19 | 20 | describe('WebcodecsAudioEncoder', () => { 21 | const composition = new Composition(); 22 | 23 | const bufferSpy = vi.spyOn(composition, 'audio').mockImplementation( 24 | async (numberOfChannels: number = 1, sampleRate: number = 1e6) => 25 | new AudioBufferMock({ sampleRate, numberOfChannels, length: 50 }) as AudioBuffer 26 | ); 27 | const configureSpy = vi.spyOn(AudioEncoder.prototype, 'configure').mockImplementation(vi.fn()); 28 | const encodeSpy = vi.spyOn(AudioEncoder.prototype, 'encode').mockImplementation(vi.fn()); 29 | 30 | const encoder = new WebcodecsAudioEncoder(composition); 31 | 32 | const muxer = new Muxer({ 33 | target: new ArrayBufferTarget(), 34 | audio: config, 35 | fastStart: 'in-memory' 36 | }); 37 | 38 | const muxSpy = vi.spyOn(muxer, 'addAudioChunk'); 39 | 40 | beforeEach(() => { 41 | bufferSpy.mockClear(); 42 | configureSpy.mockClear(); 43 | encodeSpy.mockClear(); 44 | muxSpy.mockClear(); 45 | }) 46 | 47 | it('should encode the audio of the composition using the provided configuration', async () => { 48 | await encoder.encode(muxer, config); 49 | 50 | expect(bufferSpy).toHaveBeenCalledTimes(1); 51 | expect(configureSpy).toHaveBeenCalledWith(config); 52 | expect(encodeSpy.mock.calls[0][0]).toBeInstanceOf(AudioData); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/encoders/webcodecs.audio.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { bufferToF32Planar } from "../utils"; 9 | import { Muxer } from 'mp4-muxer'; 10 | 11 | import type { StreamTarget } from 'mp4-muxer'; 12 | import type { Composition } from "../composition"; 13 | import type { WebAudioEncoder } from "./interfaces"; 14 | 15 | export class WebcodecsAudioEncoder implements WebAudioEncoder { 16 | private composition: Composition; 17 | 18 | public constructor(composition: Composition) { 19 | this.composition = composition; 20 | } 21 | 22 | public async encode(muxer: Muxer, config: AudioEncoderConfig): Promise { 23 | const { numberOfChannels, sampleRate } = config; 24 | 25 | const output = await this.composition.audio(numberOfChannels, sampleRate); 26 | 27 | if (!output) return; 28 | 29 | const encoder = new AudioEncoder({ 30 | output: (chunk, meta) => { 31 | meta && muxer.addAudioChunk(chunk, meta); 32 | }, 33 | error: console.error, 34 | }); 35 | 36 | encoder.configure(config); 37 | 38 | const data = new AudioData({ 39 | format: 'f32-planar', 40 | sampleRate: output.sampleRate, 41 | numberOfChannels: output.numberOfChannels, 42 | numberOfFrames: output.length, 43 | timestamp: 0, 44 | data: bufferToF32Planar(output), 45 | }); 46 | 47 | encoder.encode(data); 48 | await encoder.flush(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/errors/base-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export class BaseError extends Error { 9 | public readonly message: string; 10 | public readonly code: string; 11 | public constructor({ message = '', code = '' }, ...args: any[]) { 12 | super(message, ...(args as [])); 13 | console.error(message); 14 | this.code = code; 15 | this.message = message; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/errors/encoder-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { BaseError } from './base-error'; 9 | 10 | export class EncoderError extends BaseError {} 11 | -------------------------------------------------------------------------------- /src/errors/export-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { BaseError } from './base-error'; 9 | 10 | /** 11 | * @deprecated please replace with `EncoderError` 12 | */ 13 | export class ExportError extends BaseError {} 14 | -------------------------------------------------------------------------------- /src/errors/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './io-error'; 9 | export * from './base-error'; 10 | export * from './validation-error'; 11 | export * from './export-error'; 12 | export * from './encoder-error'; 13 | export * from './reference-error'; 14 | -------------------------------------------------------------------------------- /src/errors/io-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { BaseError } from './base-error'; 9 | 10 | export class IOError extends BaseError {} 11 | -------------------------------------------------------------------------------- /src/errors/reference-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { BaseError } from './base-error'; 9 | 10 | export class ReferenceError extends BaseError { } 11 | -------------------------------------------------------------------------------- /src/errors/validation-error.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { BaseError } from './base-error'; 9 | 10 | export class ValidationError extends BaseError {} 11 | -------------------------------------------------------------------------------- /src/fixtures/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export const SUPPORTED_MIME_TYPES = { 9 | IMAGE: { 10 | 'image/jpeg': 'jpg', 11 | 'image/png': 'png', 12 | 'image/webp': 'webp', 13 | 'image/svg+xml': 'svg', 14 | }, 15 | VIDEO: { 16 | 'video/mp4': 'mp4', 17 | 'video/webm': 'webm', 18 | 'video/quicktime': 'mov', 19 | // 'video/x-msvideo': 'avi', 20 | // 'video/x-matroska': 'mkv', 21 | }, 22 | AUDIO: { 23 | 'audio/mp3': 'mp3', 24 | 'audio/mpeg': 'mp3', 25 | 'audio/aac': 'aac', 26 | 'audio/wav': 'wav', 27 | 'audio/x-wav': 'wav', 28 | }, 29 | DOCUMENT: { 30 | 'text/html': 'html', 31 | }, 32 | 33 | get MIXED() { 34 | return { 35 | ...this.IMAGE, 36 | ...this.VIDEO, 37 | ...this.AUDIO, 38 | ...this.DOCUMENT, 39 | }; 40 | }, 41 | }; 42 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './composition'; 9 | export * from './encoders'; 10 | export * from './tracks'; 11 | export * from './models'; 12 | export * from './clips'; 13 | export * from './sources'; 14 | export * from './utils'; 15 | export * from './types'; 16 | export * from './mixins'; 17 | export * from './services'; 18 | export * from './errors'; 19 | export * from './fixtures'; 20 | -------------------------------------------------------------------------------- /src/mixins/event.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Constructor } from '../types'; 9 | import type { BaseEvents, EmittedEvent } from './event.types'; 10 | 11 | export function EventEmitterMixin(Base: T) { 12 | return class EventEmitter extends Base { 13 | _handlers: { 14 | [T in keyof BaseEvents]?: { 15 | [x: string]: (event: EmittedEvent[T], any>) => void; 16 | }; 17 | } = {}; 18 | 19 | public on>( 20 | eventType: T, 21 | callback: (event: EmittedEvent[T], this>) => void, 22 | ): string { 23 | if (typeof callback != 'function') { 24 | throw new Error('The callback of an event listener needs to be a function.'); 25 | } 26 | 27 | const id = crypto.randomUUID(); 28 | 29 | if (!this._handlers[eventType]) { 30 | this._handlers[eventType] = { [id]: callback }; 31 | } else { 32 | // @ts-ignore 33 | this._handlers[eventType][id] = callback; 34 | } 35 | 36 | return id; 37 | } 38 | 39 | public off(id?: string | '*', ...ids: string[]) { 40 | if (!id) return; 41 | 42 | if (id === '*') { 43 | this._handlers = {}; 44 | return; 45 | } 46 | 47 | for (const obj of Object.values(this._handlers)) { 48 | if (id in obj) { 49 | delete obj[id]; 50 | } 51 | } 52 | 53 | for (const id of ids) { 54 | this.off(id); 55 | } 56 | } 57 | 58 | public trigger>(eventType: T, detail: BaseEvents[T]) { 59 | const event = new CustomEvent[T]>(eventType as string, { 60 | detail, 61 | }); 62 | Object.defineProperty(event, 'currentTarget', { writable: false, value: this }); 63 | 64 | for (const handler in this._handlers[eventType] ?? {}) { 65 | this._handlers[eventType]?.[handler](event); 66 | } 67 | for (const handler in this._handlers['*'] ?? {}) { 68 | this._handlers['*']?.[handler](event); 69 | } 70 | } 71 | 72 | public bubble(target: EventEmitter) { 73 | return this.on('*', (event: EmittedEvent) => { 74 | target.trigger(event.type as any, event.detail); 75 | }); 76 | } 77 | 78 | public resolve(eventType: keyof BaseEvents) { 79 | return (resolve: (value: unknown) => void, reject: (reason?: any) => void) => { 80 | this.on('error', reject); 81 | this.on(eventType, resolve); 82 | }; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/mixins/event.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export type ErrorEventDetail = Error; 9 | 10 | export type OverrideValues = Omit & Pick>; 11 | 12 | export type BaseEvents = { 13 | '*': any; 14 | error: Error; 15 | } & E; 16 | 17 | export type EmittedEvent = OverrideValues, { target: T }>; 18 | -------------------------------------------------------------------------------- /src/mixins/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './event'; 9 | -------------------------------------------------------------------------------- /src/models/animation-builder.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { ValidationError } from "../errors"; 9 | import { EasingFunction, Keyframe, Timestamp } from "../models"; 10 | 11 | export type AnimationFunction = 12 | (value: V, delay?: number, easing?: EasingFunction) => T; 13 | 14 | export interface AnimationBuilder { 15 | to(value: number, relframe: number): this; 16 | } 17 | 18 | export class AnimationBuilder { 19 | private target: any; 20 | public animation: Keyframe | undefined; 21 | 22 | constructor(target: any) { 23 | this.target = target; 24 | } 25 | 26 | init(property: string | symbol, value: number | string, delay: number = 0, easing?: EasingFunction) { 27 | if (!(property in this.target)) { 28 | throw new Error(`Property [${String(property)}] cannot be assigned`); 29 | } 30 | 31 | const input = [delay]; 32 | const ouptut = [value]; 33 | 34 | // animate from current value to next value 35 | if (typeof (this.target[property]) == typeof (value) && delay != 0) { 36 | input.unshift(0); 37 | ouptut.unshift(this.target[property]); 38 | } 39 | 40 | this.target[property] = this.animation = new Keyframe(input, ouptut, { easing }); 41 | 42 | } 43 | } 44 | 45 | export function createAnimationBuilder(builder: T) { 46 | const proxy = new Proxy(builder, { 47 | get(obj: T, prop) { 48 | if (prop == 'to') { 49 | return (value: number, relframe: number) => { 50 | if (!obj.animation) { 51 | throw new ValidationError({ 52 | code: 'undefinedKeyframe', 53 | message: "Cannot use 'to() before selecting a property" 54 | }); 55 | } 56 | 57 | const timestamp = new Timestamp(obj.animation.input.at(-1)); 58 | const absframe = timestamp.frames + relframe; 59 | 60 | // ATTENTION: The arguments are inversed here 61 | obj.animation.push(absframe, value); 62 | 63 | return proxy; 64 | } 65 | } 66 | 67 | return (value: number, delay?: number, easing?: EasingFunction) => { 68 | obj.init(prop, value, delay, easing); 69 | 70 | return proxy; 71 | }; 72 | }, 73 | }); 74 | 75 | return proxy; 76 | } 77 | -------------------------------------------------------------------------------- /src/models/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './timestamp.fixtures'; 9 | export * from './timestamp'; 10 | export * from './timestamp.utils'; 11 | export * from './transcript'; 12 | export * from './transcript.group'; 13 | export * from './transcript.word'; 14 | export * from './transcript.types'; 15 | export * from './keyframe'; 16 | export * from './keyframe.types'; 17 | -------------------------------------------------------------------------------- /src/models/keyframe.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { easingFunctions } from "./keyframe.utils"; 9 | 10 | export type EasingFunctions = typeof easingFunctions; 11 | export type EasingFunction = keyof EasingFunctions; 12 | 13 | /** 14 | * Options for configuring a Keyframe instance. 15 | */ 16 | export type KeyframeOptions = { 17 | /** 18 | * Defines the extrapolation behavior outside the input range. 19 | * - "clamp": Clamps the value to the nearest endpoint within the range. 20 | * - "extend": Allows values to extend beyond the range. 21 | * @default "clamp" 22 | */ 23 | extrapolate?: "clamp" | "extend"; 24 | 25 | /** 26 | * Specifies the type of output values. 27 | * - "number": Output values are numbers. 28 | * - "color": Output values are colors in hex format. 29 | * @default "number" 30 | */ 31 | type?: "number" | "color"; 32 | 33 | /** 34 | * An optional easing function to apply to the interpolation. 35 | * Easing functions can modify the interpolation to be non-linear. 36 | * @default "linear" 37 | */ 38 | easing?: EasingFunction; 39 | } 40 | -------------------------------------------------------------------------------- /src/models/keyframe.utils.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect } from 'vitest'; 9 | import { lerp, interpolateColor } from './keyframe.utils'; 10 | 11 | describe('lerp', () => { 12 | it('should interpolate between two numbers correctly', () => { 13 | expect(lerp(0, 10, 0.5)).toBe(5); 14 | expect(lerp(10, 20, 0.25)).toBe(12.5); 15 | expect(lerp(-10, 10, 0.75)).toBe(5); 16 | }); 17 | }); 18 | 19 | describe('interpolateColor', () => { 20 | it('should interpolate between two hex colors correctly', () => { 21 | expect(interpolateColor('#000000', '#FFFFFF', 0.5)).toBe('#808080'); 22 | expect(interpolateColor('#FF0000', '#00FF00', 0.5)).toBe('#808000'); 23 | expect(interpolateColor('#0000FF', '#FFFF00', 0.25)).toBe('#4040bf'); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /src/models/keyframe.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * Performs linear interpolation between two numbers. 10 | * @param start - The starting value. 11 | * @param end - The ending value. 12 | * @param t - The interpolation factor (between 0 and 1). 13 | * @returns The interpolated value. 14 | */ 15 | export function lerp(start: number, end: number, t: number): number { 16 | return start + (end - start) * t; 17 | } 18 | 19 | /** 20 | * Interpolates between two hex color values. 21 | * @param color1 - The starting color in hex format. 22 | * @param color2 - The ending color in hex format. 23 | * @param t - The interpolation factor (between 0 and 1). 24 | * @returns The interpolated color in hex format. 25 | */ 26 | export function interpolateColor(color1: string, color2: string, t: number): string { 27 | // Assuming colors are in the format '#RRGGBB' 28 | const c1 = Number.parseInt(color1.slice(1), 16); 29 | const c2 = Number.parseInt(color2.slice(1), 16); 30 | 31 | const r1 = (c1 >> 16) & 0xff; 32 | const g1 = (c1 >> 8) & 0xff; 33 | const b1 = c1 & 0xff; 34 | 35 | const r2 = (c2 >> 16) & 0xff; 36 | const g2 = (c2 >> 8) & 0xff; 37 | const b2 = c2 & 0xff; 38 | 39 | const r = Math.round(lerp(r1, r2, t)); 40 | const g = Math.round(lerp(g1, g2, t)); 41 | const b = Math.round(lerp(b1, b2, t)); 42 | 43 | return `#${((1 << 24) + (r << 16) + (g << 8) + b).toString(16).slice(1)}`; 44 | } 45 | 46 | export const easingFunctions = { 47 | linear: (t: number) => t, 48 | easeIn: (t: number) => t * t, 49 | easeOut: (t: number) => t * (2 - t), 50 | easeInOut: (t: number) => { 51 | if (t < 0.5) return 2 * t * t; 52 | return -1 + (4 - 2 * t) * t; 53 | }, 54 | } 55 | -------------------------------------------------------------------------------- /src/models/timestamp.fixtures.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export const FPS_DEFAULT = 30; 9 | -------------------------------------------------------------------------------- /src/models/timestamp.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { float, frame } from '../types'; 9 | 10 | export type TimestampProps = { 11 | /** 12 | * Defines the global frame rate 13 | */ 14 | fps: float; 15 | /** 16 | * Defines the duration or time 17 | */ 18 | frames: frame; 19 | }; 20 | -------------------------------------------------------------------------------- /src/models/timestamp.utils.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, it, expect } from 'vitest'; 9 | import { Timestamp } from '../models'; 10 | import { framesToMillis, framesToSeconds, secondsToFrames } from './timestamp.utils'; 11 | 12 | describe('Timestamp utils', () => { 13 | it('should be able to convert frames to milliseconds', () => { 14 | expect(framesToMillis(10)).toBe(333); 15 | expect(framesToMillis(-10)).toBe(-333); 16 | expect(framesToMillis(0)).toBe(0); 17 | expect(framesToMillis(10, 2)).toBe(5000); 18 | expect(framesToMillis(-10, 2)).toBe(-5000); 19 | expect(framesToMillis(5, 7)).toBe(714); 20 | expect(framesToMillis(8, 9)).toBe(889); 21 | expect(() => framesToMillis(8, 0)).toThrowError(); 22 | expect(() => framesToMillis(8, -1)).toThrowError(); 23 | }); 24 | 25 | it('should be able to convert frames to seconds', () => { 26 | expect(framesToSeconds(10)).toBe(0.333); 27 | expect(framesToSeconds(-10)).toBe(-0.333); 28 | expect(framesToSeconds(0)).toBe(0); 29 | expect(framesToSeconds(10, 2)).toBe(5); 30 | expect(framesToSeconds(-10, 2)).toBe(-5); 31 | expect(framesToSeconds(5, 7)).toBe(0.714); 32 | expect(framesToSeconds(8, 9)).toBe(0.889); 33 | expect(() => framesToSeconds(8, 0)).toThrowError(); 34 | expect(() => framesToSeconds(8, -1)).toThrowError(); 35 | }); 36 | 37 | it('should be able to convert seconds to frames', () => { 38 | expect(secondsToFrames(0.333)).toBe(10); 39 | expect(secondsToFrames(-0.333)).toBe(-10); 40 | expect(secondsToFrames(0)).toBe(0); 41 | expect(secondsToFrames(5, 2)).toBe(10); 42 | expect(secondsToFrames(-5, 2)).toBe(-10); 43 | expect(secondsToFrames(0.714, 7)).toBe(5); 44 | expect(secondsToFrames(0.889, 9)).toBe(8); 45 | expect(() => secondsToFrames(8, 0)).toThrowError(); 46 | expect(() => secondsToFrames(8, -1)).toThrowError(); 47 | }) 48 | 49 | it('should result in a bidirectional conversion frames - seconds - frames', () => { 50 | const ts = new Timestamp(77686.86858); 51 | const seconds = framesToSeconds(ts.frames); 52 | expect(secondsToFrames(seconds)).toBe(ts.frames); 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /src/models/timestamp.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { FPS_DEFAULT } from './timestamp.fixtures'; 9 | import type { frame } from '../types'; 10 | import { ValidationError } from '../errors'; 11 | 12 | /** 13 | * Convert seconds into frames 14 | */ 15 | export function secondsToFrames(seconds: number, fps = FPS_DEFAULT): frame { 16 | if (fps < 1) throw new ValidationError({ 17 | code: 'invalidArgument', 18 | message: 'FPS must be greater or equal to 1' 19 | }); 20 | 21 | return Math.round(seconds * fps); 22 | } 23 | 24 | /** 25 | * Convert frames into seconds 26 | */ 27 | export function framesToSeconds(frames: frame, fps = FPS_DEFAULT): number { 28 | if (fps < 1) throw new ValidationError({ 29 | code: 'invalidArgument', 30 | message: 'FPS must be greater or equal to 1' 31 | }); 32 | 33 | return Math.round((frames / fps) * 1000) / 1000; 34 | } 35 | 36 | /** 37 | * Convert frames to milliseconds 38 | */ 39 | export function framesToMillis(frames: frame, fps = FPS_DEFAULT): number { 40 | if (fps < 1) throw new ValidationError({ 41 | code: 'invalidArgument', 42 | message: 'FPS must be greater or equal to 1' 43 | }); 44 | 45 | return Math.round((frames / fps) * 1000); 46 | } 47 | -------------------------------------------------------------------------------- /src/models/transcript.group.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Timestamp } from './timestamp'; 9 | import type { Word } from './transcript.word'; 10 | 11 | export class WordGroup { 12 | public words: Word[] = []; 13 | 14 | public constructor(words?: Word[]) { 15 | if (!words) return; 16 | this.words = words; 17 | } 18 | 19 | public get duration(): Timestamp { 20 | return this.stop.subtract(this.start); 21 | } 22 | 23 | public get text(): string { 24 | return this.words.map(({ text }) => text).join(' '); 25 | } 26 | 27 | public get start(): Timestamp { 28 | return this.words.at(0)?.start ?? new Timestamp(); 29 | } 30 | 31 | public get stop(): Timestamp { 32 | return this.words.at(-1)?.stop ?? new Timestamp(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/models/transcript.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export enum Language { 9 | en = 'en', 10 | de = 'de', 11 | } 12 | 13 | export type GeneratorOptions = { 14 | /** 15 | * Iterates by word count 16 | */ 17 | count?: [number, number?]; 18 | /** 19 | * Iterates by group duration 20 | */ 21 | duration?: [number, number?]; 22 | /** 23 | * Iterates by number of characters within the group 24 | */ 25 | length?: [number, number?]; 26 | }; 27 | -------------------------------------------------------------------------------- /src/models/transcript.utils.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { formatTime, secondsToTime } from './transcript.utils'; 10 | 11 | describe('The transcript utils', () => { 12 | it('should be able convert seconds into time', () => { 13 | const time = secondsToTime(90); 14 | expect(time.seconds).toBe(30); 15 | expect(time.minutes).toBe(1); 16 | expect(time.hours).toBe(0); 17 | expect(time.milliseconds).toBe(0); 18 | }); 19 | 20 | it('should be able to format the time', () => { 21 | const time = secondsToTime(90); 22 | expect(formatTime(time)).toBe('00:01:30,000'); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /src/models/transcript.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export type Time = { 9 | hours: number; 10 | minutes: number; 11 | seconds: number; 12 | milliseconds: number; 13 | }; 14 | 15 | export function formatTime(time: Time) { 16 | return ( 17 | '' + 18 | `${time.hours.toString().padStart(2, '0')}:` + 19 | `${time.minutes.toString().padStart(2, '0')}:` + 20 | `${time.seconds.toString().padStart(2, '0')},` + 21 | time.milliseconds.toString().padStart(3, '0') 22 | ); 23 | } 24 | 25 | export function secondsToTime(seconds: number): Time { 26 | const time = new Date(1970, 0, 1); // Epoch 27 | time.setSeconds(seconds); 28 | time.setMilliseconds(Math.round((seconds % 1) * 1000)); 29 | 30 | return { 31 | hours: time.getHours(), 32 | minutes: time.getMinutes(), 33 | seconds: time.getSeconds(), 34 | milliseconds: time.getMilliseconds(), 35 | }; 36 | } 37 | -------------------------------------------------------------------------------- /src/models/transcript.word.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Timestamp } from './timestamp'; 9 | 10 | export class Word { 11 | /** 12 | * A unique identifier for the word 13 | */ 14 | public id = crypto.randomUUID(); 15 | /** 16 | * Defines the text to be displayed 17 | */ 18 | public text: string; 19 | /** 20 | * Defines the time stamp at 21 | * which the text is spoken 22 | */ 23 | public start: Timestamp; 24 | /** 25 | * Defines the time stamp at 26 | * which the text was spoken 27 | */ 28 | public stop: Timestamp; 29 | /** 30 | * Defines the confidence of 31 | * the predicition 32 | */ 33 | public confidence?: number; 34 | 35 | /** 36 | * Create a new Word object 37 | * @param text The string contents of the word 38 | * @param start Start in **milliseconds** 39 | * @param stop Stop in **milliseconds** 40 | * @param confidence Predicition confidence 41 | */ 42 | public constructor(text: string, start: number, stop: number, confidence?: number) { 43 | this.text = text; 44 | this.start = new Timestamp(start); 45 | this.stop = new Timestamp(stop); 46 | this.confidence = confidence; 47 | } 48 | 49 | /** 50 | * Defines the time between start 51 | * and stop returned as a timestamp 52 | */ 53 | public get duration(): Timestamp { 54 | return this.stop.subtract(this.start); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/services/event.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { EventEmitterMixin } from "../mixins"; 9 | 10 | export function EventEmitter() { 11 | return class extends EventEmitterMixin(class { }) { } 12 | } 13 | -------------------------------------------------------------------------------- /src/services/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './serializer'; 9 | export * from './storage-item'; 10 | export * from './store'; 11 | export * from './store.types'; 12 | export * from './thread'; 13 | export * from './event'; 14 | -------------------------------------------------------------------------------- /src/services/serializer.ts: -------------------------------------------------------------------------------- 1 | import type { Constructor } from '../types'; 2 | 3 | export class Serializer { 4 | /** 5 | * Unique identifier of the object 6 | */ 7 | public id = crypto.randomUUID(); 8 | 9 | toJSON(): any { 10 | const obj: any = {}; 11 | const properties = (this.constructor as any).__serializableProperties || []; 12 | 13 | properties.forEach(({ propertyKey, serializer }: any) => { 14 | const value = (this as any)[propertyKey]; 15 | if (serializer && value instanceof serializer) { 16 | obj[propertyKey] = value.toJSON(); 17 | } else { 18 | obj[propertyKey] = value; 19 | } 20 | }); 21 | 22 | return obj; 23 | } 24 | 25 | static fromJSON(this: new () => T, obj: K extends string ? never : K): T { 26 | const instance = new this(); 27 | const properties = (this as any).__serializableProperties || []; 28 | 29 | properties.forEach(({ propertyKey, serializer }: any) => { 30 | if ((obj as any).hasOwnProperty(propertyKey)) { 31 | if (serializer) { 32 | const nestedInstance = serializer.fromJSON((obj as any)[propertyKey]); 33 | (instance as any)[propertyKey] = nestedInstance; 34 | } else { 35 | (instance as any)[propertyKey] = (obj as any)[propertyKey]; 36 | } 37 | } 38 | }); 39 | 40 | return instance; 41 | } 42 | } 43 | 44 | export function serializable(serializer?: Omit, 'toJSON'>) { 45 | return function (target: any, propertyKey: string) { 46 | if (!target.constructor.__serializableProperties) { 47 | target.constructor.__serializableProperties = []; 48 | } 49 | target.constructor.__serializableProperties.push({ 50 | propertyKey, 51 | serializer, 52 | }); 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /src/services/storage-item.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it, afterEach } from 'vitest'; 9 | import { StorageItem, Store } from '.'; 10 | 11 | describe('The Storage Item Object', () => { 12 | const store = new Store(); 13 | 14 | it('should be able get the key and value from the constructor', () => { 15 | const item = new StorageItem(store, 'MY_KEY', 5); 16 | 17 | expect(item.key).toBe('MY_KEY'); 18 | expect(item.value).toBe(5); 19 | }); 20 | 21 | it('should update the value when a new value is set', () => { 22 | const item = new StorageItem(store, 'MY_KEY_1', 3); 23 | 24 | expect(item.value).toBe(3); 25 | 26 | item.value = 8; 27 | 28 | expect(item.value).toBe(8); 29 | expect(store.get(item.key)).toBe(8); 30 | }); 31 | 32 | afterEach(() => { 33 | localStorage.clear(); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /src/services/storage-item.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { EventEmitterMixin } from '../mixins'; 9 | import { Serializer } from '../services'; 10 | 11 | import type { Store } from '.'; 12 | 13 | type Events = { 14 | update: any; 15 | } 16 | 17 | export class StorageItem extends EventEmitterMixin(Serializer) { 18 | private _key: string; 19 | private _value: T | undefined; 20 | private _store: Store; 21 | 22 | public loaded = false; 23 | 24 | public constructor(store: Store, key: string, value: T | Promise) { 25 | super(); 26 | 27 | this._store = store; 28 | this._key = key; 29 | this.initValue(value); 30 | } 31 | 32 | public get key(): string { 33 | return this._key; 34 | } 35 | 36 | public get value(): T { 37 | return this._value!; 38 | } 39 | 40 | public set value(newValue: T) { 41 | this._value = newValue; 42 | this._store.set(this._key, newValue); 43 | this.trigger('update', undefined); 44 | } 45 | 46 | private async initValue(value: T | Promise) { 47 | if (value instanceof Promise) { 48 | this._value = await value; 49 | } else { 50 | this._value = value; 51 | } 52 | this.loaded = true; 53 | this.trigger('update', undefined); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/services/store.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { StorageItem } from './storage-item'; 9 | import type { Deserializer } from './store.types'; 10 | 11 | export class Store { 12 | public readonly storageEngine: Storage; 13 | public readonly namespace?: string; 14 | 15 | public constructor(namespace?: string, storageEngine = localStorage) { 16 | this.storageEngine = storageEngine; 17 | this.namespace = namespace; 18 | } 19 | 20 | public define(key: string, defaultValue: T, deserializer?: Deserializer): StorageItem { 21 | const storedValue = this.get(key); 22 | 23 | if (storedValue === null) { 24 | this.set(key, defaultValue); 25 | return new StorageItem(this, key, defaultValue); 26 | } 27 | if (deserializer && storedValue != undefined) { 28 | return new StorageItem(this, key, deserializer(storedValue)); 29 | } 30 | 31 | return new StorageItem(this, key, storedValue); 32 | } 33 | 34 | public set(key: string, value: T): void { 35 | this.storageEngine.setItem( 36 | this.getStorageId(key), 37 | JSON.stringify({ 38 | value: value, 39 | }), 40 | ); 41 | } 42 | 43 | public get(key: string): T | null { 44 | const item = this.storageEngine.getItem(this.getStorageId(key)); 45 | 46 | if (item) { 47 | return JSON.parse(item).value as T; 48 | } 49 | 50 | return null; 51 | } 52 | 53 | public remove(key: string): void { 54 | this.storageEngine.removeItem(this.getStorageId(key)); 55 | } 56 | 57 | private getStorageId(key: string): string { 58 | if (this.namespace) { 59 | return `${this.namespace}.${key}`; 60 | } 61 | return key; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/services/store.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export type Deserializer = (data: any) => Promise | T; 9 | -------------------------------------------------------------------------------- /src/services/thread.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Constructor } from '../types'; 9 | 10 | type TreadResponse = 11 | | { 12 | result: R; 13 | error: undefined; 14 | } 15 | | { 16 | result: undefined; 17 | error: string; 18 | }; 19 | 20 | type EventListener = (event: MessageEvent['data']) => void; 21 | 22 | export class Thread { 23 | public worker: Worker; 24 | 25 | public constructor(Worker: Constructor) { 26 | this.worker = new Worker(); 27 | this.worker.onerror = console.error; 28 | } 29 | 30 | public async run(payload?: Arg, listner?: EventListener): Promise> { 31 | this.worker.postMessage({ type: 'init', ...(payload ?? {}) }); 32 | 33 | return await new Promise((resolve, reject) => { 34 | this.worker.addEventListener('message', (event) => { 35 | listner?.(event.data); 36 | 37 | if (event.data.type == 'result') { 38 | event.data.type = undefined; 39 | resolve(event.data); 40 | } 41 | 42 | if (event.data.type == 'error') { 43 | reject(event.data.message); 44 | } 45 | }); 46 | }) 47 | .then((data) => { 48 | return { result: data as Result, error: undefined }; 49 | }) 50 | .catch((error: string) => { 51 | return { result: undefined, error }; 52 | }) 53 | .finally(() => { 54 | this.worker.terminate(); 55 | }); 56 | } 57 | } 58 | 59 | export function withThreadErrorHandler(main: (event: MessageEvent) => Promise) { 60 | return async (event: MessageEvent) => { 61 | try { 62 | await main(event); 63 | } catch (e: any) { 64 | self.postMessage({ 65 | type: 'error', 66 | message: e?.message ?? 'An unkown worker error occured', 67 | }); 68 | } 69 | }; 70 | } 71 | -------------------------------------------------------------------------------- /src/sources/audio.fixtures.ts: -------------------------------------------------------------------------------- 1 | export const MIN_SAMPLE_RATE = 3000; 2 | -------------------------------------------------------------------------------- /src/sources/audio.types.ts: -------------------------------------------------------------------------------- 1 | import type { Timestamp } from '../models'; 2 | 3 | /** 4 | * Fast sampler options. 5 | */ 6 | export type FastSamplerOptions = { 7 | /** 8 | * The number of samples to return. 9 | */ 10 | length?: number; 11 | /** 12 | * The start time in **milliseconds** relative to the beginning of the clip. 13 | */ 14 | start?: Timestamp | number; 15 | /** 16 | * The stop time in **milliseconds** relative to the beginning of the clip. 17 | */ 18 | stop?: Timestamp | number; 19 | /** 20 | * Whether to use a logarithmic scale. 21 | */ 22 | logarithmic?: boolean; 23 | }; 24 | 25 | export type SilenceDetectionOptions = { 26 | /** 27 | * If the RMS is below the threshold, the frame is considered silent. 28 | * @default 0.02 29 | */ 30 | threshold?: number; 31 | /** 32 | * This parameter affects how accurately the algorithm captures short silences. 33 | * @default 1024 34 | */ 35 | hopSize?: number; 36 | /** 37 | * Setting a minimum duration in **milliseconds** for a silence period helps avoid detecting brief gaps between sounds as silences. 38 | * @default 500 39 | */ 40 | minDuration?: number; 41 | }; 42 | 43 | export type AudioSlice = { 44 | start: Timestamp; 45 | stop: Timestamp; 46 | }; 47 | -------------------------------------------------------------------------------- /src/sources/audio.utils.ts: -------------------------------------------------------------------------------- 1 | import { Timestamp } from '../models'; 2 | 3 | import type { AudioSlice, SilenceDetectionOptions } from './audio.types'; 4 | 5 | 6 | /** 7 | * Detect silences in an audio buffer 8 | * @param audioBuffer - The web audio buffer. 9 | * @param options - Options for silence detection 10 | */ 11 | export function detectSilences( 12 | audioBuffer: AudioBuffer, 13 | options: SilenceDetectionOptions = {} 14 | ): AudioSlice[] { 15 | const { threshold = 0.02, hopSize = 1024, minDuration = 500 } = options; 16 | 17 | const slices: AudioSlice[] = []; 18 | const channel = audioBuffer.getChannelData(0); 19 | const sampleRate = audioBuffer.sampleRate; 20 | 21 | // Convert minDuration from milliseconds to samples 22 | const minSamples = Math.floor((minDuration / 1000) * sampleRate); 23 | 24 | let silenceStart: number | null = null; 25 | let consecutiveSilentSamples = 0; 26 | 27 | // Process audio in frames 28 | for (let i = 0; i < channel.length; i += hopSize) { 29 | // Calculate RMS for current frame 30 | let rms = 0; 31 | const frameEnd = Math.min(i + hopSize, channel.length); 32 | 33 | for (let j = i; j < frameEnd; j++) { 34 | rms += channel[j] * channel[j]; 35 | } 36 | rms = Math.sqrt(rms / (frameEnd - i)); 37 | 38 | // Check if frame is silent 39 | if (rms < threshold) { 40 | consecutiveSilentSamples += hopSize; 41 | if (silenceStart === null) { 42 | silenceStart = i; 43 | } 44 | } else { 45 | // If we had a silence of sufficient duration, add it to slices 46 | if (silenceStart !== null && consecutiveSilentSamples >= minSamples) { 47 | slices.push({ 48 | start: Timestamp.fromSeconds(silenceStart / sampleRate), 49 | stop: Timestamp.fromSeconds(i / sampleRate) 50 | }); 51 | } 52 | silenceStart = null; 53 | consecutiveSilentSamples = 0; 54 | } 55 | } 56 | 57 | // Handle silence at the end of audio 58 | if (silenceStart !== null && consecutiveSilentSamples >= minSamples) { 59 | slices.push({ 60 | start: Timestamp.fromSeconds(silenceStart / sampleRate), 61 | stop: Timestamp.fromSeconds(channel.length / sampleRate) 62 | }); 63 | } 64 | 65 | return slices; 66 | } 67 | -------------------------------------------------------------------------------- /src/sources/html.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Source } from './source'; 9 | import { documentToSvgImageUrl } from './html.utils'; 10 | 11 | import type { ClipType } from '../clips'; 12 | 13 | export class HtmlSource extends Source { 14 | public readonly type: ClipType = 'html'; 15 | /** 16 | * Access to the iframe that is required 17 | * for extracting the html's dimensions 18 | */ 19 | public readonly iframe: HTMLIFrameElement; 20 | 21 | public constructor() { 22 | super(); 23 | 24 | const iframe = document.createElement('iframe'); 25 | 26 | iframe.style.position = "absolute"; 27 | iframe.style.width = "0"; 28 | iframe.style.height = "0"; 29 | iframe.style.border = "0"; 30 | iframe.style.visibility = 'hidden'; 31 | 32 | document.body.appendChild(iframe); 33 | 34 | this.iframe = iframe; 35 | } 36 | 37 | /** 38 | * Access to the html document as loaded 39 | * within the iframe. Can be manipulated with 40 | * javascript 41 | */ 42 | public get document(): Document | undefined { 43 | return this.iframe.contentWindow?.document; 44 | } 45 | 46 | public async createObjectURL(): Promise { 47 | if (!this.file && this.state == 'LOADING') { 48 | await new Promise(this.resolve('load')); 49 | } 50 | 51 | if (this.objectURL) return this.objectURL; 52 | 53 | this.objectURL = documentToSvgImageUrl(this.document); 54 | 55 | return this.objectURL; 56 | } 57 | 58 | protected async loadUrl(url: string | URL | Request, init?: RequestInit) { 59 | await super.loadUrl(url, init); 60 | 61 | this.iframe.setAttribute('src', URL.createObjectURL(this.file!)); 62 | 63 | await new Promise((resolve, reject) => { 64 | this.iframe.onload = () => resolve(); 65 | this.iframe.onerror = (e) => reject(e); 66 | }); 67 | } 68 | 69 | protected async loadFile(file: File) { 70 | await super.loadFile(file); 71 | 72 | this.iframe.setAttribute('src', URL.createObjectURL(this.file!)); 73 | 74 | await new Promise((resolve, reject) => { 75 | this.iframe.onload = () => resolve(); 76 | this.iframe.onerror = (e) => reject(e); 77 | }); 78 | } 79 | 80 | /** 81 | * Update the object url using the current 82 | * contents of the iframe document 83 | */ 84 | public update() { 85 | // url has not been created yet 86 | if (!this.objectURL) return; 87 | 88 | this.objectURL = documentToSvgImageUrl(this.document); 89 | } 90 | 91 | public async thumbnail(): Promise { 92 | const image = new Image(); 93 | image.src = await this.createObjectURL(); 94 | image.className = 'object-contain w-full aspect-video h-auto'; 95 | return image; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/sources/html.utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | const emptySvg = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg'%3E%3C/svg%3E"; 9 | 10 | // Function to escape special characters 11 | export function utf8ToBase64(str: string) { 12 | const utf8Bytes = new TextEncoder().encode(str); 13 | let binary = ''; 14 | const len = utf8Bytes.byteLength; 15 | for (let i = 0; i < len; i++) { 16 | binary += String.fromCharCode(utf8Bytes[i]); 17 | } 18 | return btoa(binary); 19 | } 20 | 21 | export function documentToSvgImageUrl(doc?: Document) { 22 | if (!doc || !doc.body) return emptySvg; 23 | 24 | const width = doc.body.scrollWidth; 25 | const height = doc.body.scrollHeight; 26 | 27 | const document = doc.cloneNode(true) as Document; 28 | const style = document.getElementsByTagName('style').item(0); 29 | const body = document.getElementsByTagName('body').item(0); 30 | body?.setAttribute('xmlns', 'http://www.w3.org/1999/xhtml'); 31 | 32 | if (!body) return emptySvg; 33 | const xmlSerializer = new XMLSerializer(); 34 | 35 | const styleString = style ? xmlSerializer.serializeToString(style) : ''; 36 | const bodyString = xmlSerializer.serializeToString(body); 37 | 38 | const svgString = ` 39 | 40 | body { padding: 0; } 41 | ${styleString} 42 | 43 | ${bodyString} 44 | 45 | `; 46 | 47 | return 'data:image/svg+xml;base64,' + utf8ToBase64(svgString); 48 | } 49 | 50 | export async function fontToBas64Url(url: string): Promise { 51 | const response = await fetch(url); 52 | const blob = await response.blob(); 53 | 54 | const base64 = await new Promise((resolve) => { 55 | const reader = new FileReader(); 56 | reader.onloadend = () => resolve(reader.result); 57 | reader.readAsDataURL(blob); 58 | }); 59 | 60 | let format = 'woff2'; 61 | if (url?.endsWith('woff')) format = 'woff'; 62 | if (url?.endsWith('ttf')) format = 'truetype'; 63 | 64 | return `url(${base64}) format('${format}')`; 65 | } 66 | -------------------------------------------------------------------------------- /src/sources/image.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { ImageSource } from './image'; 10 | 11 | describe('The Image Source Object', () => { 12 | it('should retrun a video as thumbnail', async () => { 13 | const file = new File([], 'file.png', { type: 'image/png' }); 14 | const source = new ImageSource(); 15 | 16 | await source.from(file); 17 | 18 | const thumbnail = await source.thumbnail(); 19 | 20 | expect(thumbnail).toBeInstanceOf(Image); 21 | }); 22 | 23 | it('should accept custom metadata', async () => { 24 | const metadata = { a: 1, b: 2 }; 25 | const source = new ImageSource(); 26 | source.metadata = metadata; 27 | expect(source.metadata).toEqual(metadata); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/sources/image.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Source } from './source'; 9 | import type { ClipType } from '../clips'; 10 | 11 | export class ImageSource extends Source { 12 | public readonly type: ClipType = 'image'; 13 | 14 | public async thumbnail(): Promise { 15 | const image = new Image(); 16 | image.src = await this.createObjectURL(); 17 | image.className = 'object-cover w-full aspect-video h-auto'; 18 | return image; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/sources/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './source'; 9 | export * from './html'; 10 | export * from './audio'; 11 | export * from './audio.types'; 12 | export * from './image'; 13 | export * from './video'; 14 | -------------------------------------------------------------------------------- /src/sources/video.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { AudioSource } from './'; 9 | import { parseMimeType } from '../clips'; 10 | import { IOError, ValidationError } from '../errors'; 11 | 12 | import type { ClipType } from '../clips'; 13 | 14 | export class VideoSource extends AudioSource { 15 | public readonly type: ClipType = 'video'; 16 | private downloadInProgress = true; 17 | 18 | protected async loadUrl(url: string | URL | Request, init?: RequestInit | undefined) { 19 | const res = await fetch(url, init); 20 | 21 | if (!res?.ok) throw new IOError({ 22 | code: 'unexpectedIOError', 23 | message: 'An unexpected error occurred while fetching the file', 24 | }); 25 | 26 | this.name = url.toString().split('/').at(-1) ?? ''; 27 | this.external = true; 28 | this.externalURL = url; 29 | this.objectURL = String(url); 30 | this.mimeType = parseMimeType(res.headers.get('Content-type')); 31 | 32 | this.getBlob(res); 33 | } 34 | 35 | public async getFile(): Promise { 36 | if (!this.file && this.downloadInProgress) { 37 | await new Promise(this.resolve('load')); 38 | } 39 | 40 | if (!this.file) { 41 | throw new ValidationError({ 42 | code: 'fileNotAccessible', 43 | message: "The desired file cannot be accessed", 44 | }); 45 | } 46 | 47 | return this.file; 48 | } 49 | 50 | public async thumbnail(): Promise { 51 | const video = document.createElement('video'); 52 | video.className = 'object-cover w-full aspect-video h-auto'; 53 | video.controls = false; 54 | 55 | video.addEventListener('loadedmetadata', () => { 56 | this.duration.seconds = video.duration; 57 | this.trigger('update', undefined); 58 | }); 59 | 60 | video.addEventListener('mousemove', (evt: MouseEvent) => { 61 | const clip = evt.currentTarget as HTMLVideoElement | null; 62 | const rect = clip?.getBoundingClientRect(); 63 | const x = evt.clientX - (rect?.left ?? 0); 64 | const duration = clip?.duration; 65 | 66 | if (duration && rect && rect.width > 0) { 67 | clip.currentTime = Math.round(duration * (x / rect.width)); 68 | } 69 | }); 70 | 71 | video.src = await this.createObjectURL(); 72 | return video; 73 | } 74 | 75 | private async getBlob(response: Response) { 76 | try { 77 | this.downloadInProgress = true; 78 | const blob = await response.blob(); 79 | 80 | this.file = new File([blob], this.name, { type: blob.type }); 81 | this.trigger('load', undefined); 82 | } catch (e) { 83 | this.state = 'ERROR'; 84 | this.trigger('error', new Error(String(e))); 85 | } finally { 86 | this.downloadInProgress = false; 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/tracks/audio/audio.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { AudioTrack } from './audio'; 10 | 11 | describe('The Audio Track Object', () => { 12 | it('should have a certain intitial state', () => { 13 | const track = new AudioTrack(); 14 | expect(track.type).toBe('audio'); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/tracks/audio/audio.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { AudioClip } from '../../clips'; 9 | import { MediaTrack } from '../media'; 10 | 11 | export class AudioTrack extends MediaTrack { 12 | public readonly type = 'audio'; 13 | } 14 | -------------------------------------------------------------------------------- /src/tracks/audio/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './audio'; 9 | -------------------------------------------------------------------------------- /src/tracks/caption/caption.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { beforeEach, describe, expect, it, vi } from 'vitest'; 9 | import { Composition } from '../../composition'; 10 | import { Font, MediaClip } from '../../clips'; 11 | import { CaptionTrack } from './caption'; 12 | import { Transcript, Word, WordGroup } from '../../models'; 13 | 14 | describe('The Caption Track Object', () => { 15 | vi.spyOn(Font.prototype, 'load').mockImplementation(async () => new Font()); 16 | 17 | let composition: Composition; 18 | let track: CaptionTrack; 19 | let media: MediaClip; 20 | 21 | beforeEach(() => { 22 | composition = new Composition(); 23 | track = composition.createTrack('caption'); 24 | media = new MediaClip().set({ 25 | transcript: new Transcript([ 26 | new WordGroup([ 27 | new Word('Lorem', 0, 1e3), 28 | new Word('Ipsum', 2e3, 3e3), 29 | new Word('is', 4e3, 5e3), 30 | new Word('simply', 6e3, 7e3), 31 | new Word('dummy', 8e3, 9e3), 32 | new Word('text', 10e3, 11e3), 33 | new Word('of', 12e3, 13e3), 34 | new Word('the', 14e3, 15e3), 35 | new Word('printing', 16e3, 17e3), 36 | new Word('and', 18e3, 19e3), 37 | new Word('typesetting', 20e3, 21e3), 38 | new Word('industry', 22e3, 23e3), 39 | ]), 40 | new WordGroup([ 41 | new Word('Lorem', 24e3, 25e3), 42 | new Word('Ipsum', 26e3, 27e3), 43 | new Word('has', 28e3, 29e3), 44 | new Word('been', 30e3, 31e3), 45 | new Word('the', 32e3, 33e3), 46 | new Word("industry's", 34e3, 35e3), 47 | ]), 48 | ]), 49 | }); 50 | track.from(media); 51 | expect(track.clip?.transcript).toBeDefined(); 52 | expect(track.clip?.id).toBe(media.id); 53 | }); 54 | 55 | it('should have a certain intitial state', () => { 56 | expect(track.type).toBe('caption'); 57 | }); 58 | 59 | it('should generate captions', async () => { 60 | expect(track.clips.length).toBe(0); 61 | await track.generate(); 62 | expect(track.clips.length).not.toBe(0); 63 | }); 64 | 65 | it('should update the offset when the media keyframes change', async () => { 66 | await track.generate(); 67 | expect(track.clips.at(0)?.start.seconds).toBe(0); 68 | 69 | media.offsetBy(10); 70 | expect(track.clips.at(0)?.start.frames).toBe(10); 71 | 72 | media.offsetBy(-5); 73 | expect(track.clips.at(0)?.start.frames).toBe(5); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /src/tracks/caption/caption.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Track } from '../track'; 9 | import { ClassicCaptionPreset } from './preset.classic'; 10 | 11 | import type { MediaClip, TextClip } from '../../clips'; 12 | import type { CaptionPresetStrategy } from './preset.interface'; 13 | 14 | export class CaptionTrack extends Track { 15 | /** 16 | * Defines the media clip that will be 17 | * used for creating the captions 18 | */ 19 | public clip?: MediaClip; 20 | 21 | public readonly type = 'caption'; 22 | 23 | /** 24 | * The currently active captioning strategy 25 | */ 26 | public preset: CaptionPresetStrategy = new ClassicCaptionPreset(); 27 | 28 | /** 29 | * Defines the media resource from which the 30 | * captions will be created. It must contain 31 | * a `Transcript` 32 | */ 33 | public from(value: MediaClip | undefined): this { 34 | this.clip = value; 35 | 36 | this.clip?.on('offsetBy', (evt) => this.offsetBy(evt.detail)); 37 | 38 | return this; 39 | } 40 | 41 | /** 42 | * If a transcript has been added to the resource 43 | * you can generate captions with this function 44 | * @param strategy The caption strategy to use 45 | * @default ClassicCaptionPreset 46 | */ 47 | public async generate(strategy?: CaptionPresetStrategy | (new () => CaptionPresetStrategy)): Promise { 48 | let preset = this.preset; 49 | 50 | if (typeof strategy == 'object') { 51 | preset = strategy; 52 | } else if (strategy) { 53 | preset = new strategy(); 54 | } 55 | 56 | this.clips = []; 57 | this.trigger('update', undefined); 58 | this.preset = preset; 59 | await preset.applyTo(this); 60 | this.trigger('update', undefined); 61 | return this; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/tracks/caption/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './caption'; 9 | export * from './preset.classic'; 10 | export * from './preset.spotlight'; 11 | export * from './preset.guinea'; 12 | export * from './preset.cascade'; 13 | export * from './preset.deserializer'; 14 | export * from './preset.interface'; 15 | export * from './preset.solar'; 16 | export * from './preset.whisper'; 17 | export * from './preset.verdant'; 18 | export * from './preset.types'; 19 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.cascade.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Timestamp } from '../../models'; 9 | import { Serializer, serializable } from '../../services'; 10 | import { Font, TextClip } from '../../clips'; 11 | import { ValidationError } from '../../errors'; 12 | 13 | import type { GeneratorOptions } from '../../models'; 14 | import type { DefaultCaptionPresetConfig } from './preset.types'; 15 | import type { CaptionTrack } from './caption'; 16 | import type { CaptionPresetStrategy } from './preset.interface'; 17 | import type { Position } from '../../types'; 18 | 19 | export class CascadeCaptionPreset extends Serializer implements CaptionPresetStrategy { 20 | @serializable() 21 | public generatorOptions: GeneratorOptions; 22 | 23 | @serializable() 24 | public readonly type = 'CASCADE'; 25 | 26 | @serializable() 27 | public position: Position; 28 | 29 | public constructor(config: Partial = {}) { 30 | super(); 31 | 32 | this.generatorOptions = config.generatorOptions ?? { duration: [1.4] }; 33 | this.position = config.position ?? { x: '12%', y: '44%' }; 34 | } 35 | 36 | public async applyTo(track: CaptionTrack): Promise { 37 | if (!track.clip?.transcript || !track.composition?.width) { 38 | throw new ValidationError({ 39 | code: 'referenceError', 40 | message: 'Captions need to be applied with a defined transcript and composition', 41 | }); 42 | } 43 | 44 | const offset = track.clip?.offset ?? new Timestamp(); 45 | const font = await Font.fromFamily({ family: 'Geologica', weight: '400' }).load(); 46 | 47 | // add captions 48 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 49 | for (let i = 0; i < sequence.words.length; i++) { 50 | const getText = () => { 51 | if (sequence.words.length == 1) return sequence.text; 52 | 53 | const words = sequence.words.map((word) => word.text); 54 | return words.slice(0, i + 1).join(' '); 55 | } 56 | 57 | await track.add( 58 | new TextClip({ 59 | text: getText(), 60 | textAlign: 'left', 61 | textBaseline: 'top', 62 | fillStyle: '#FFFFFF', 63 | fontSize: 16, 64 | font, 65 | maxWidth: track.composition.width * 0.7, 66 | stroke: { 67 | color: '#000000', 68 | width: 4, 69 | join: 'round', 70 | }, 71 | shadow: { 72 | color: '#000000', 73 | blur: 8, 74 | alpha: 0.4, 75 | angle: Math.PI / 4, 76 | distance: 2, 77 | }, 78 | position: this.position, 79 | stop: sequence.words[i].stop.add(offset), 80 | start: sequence.words[i].start.add(offset), 81 | }) 82 | ); 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.classic.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { serializable, Serializer } from '../../services'; 9 | import { Keyframe, Timestamp } from '../../models'; 10 | import { Font, TextClip } from '../../clips'; 11 | import { ValidationError } from '../../errors'; 12 | 13 | import type { CaptionPresetType, DefaultCaptionPresetConfig } from './preset.types'; 14 | import type { GeneratorOptions } from '../../models'; 15 | import type { CaptionTrack } from './caption'; 16 | import type { CaptionPresetStrategy } from './preset.interface'; 17 | import type { Position } from '../../types'; 18 | 19 | export class ClassicCaptionPreset extends Serializer implements CaptionPresetStrategy { 20 | @serializable() 21 | public generatorOptions: GeneratorOptions; 22 | 23 | @serializable() 24 | public readonly type: CaptionPresetType = 'CLASSIC'; 25 | 26 | @serializable() 27 | public position: Position; 28 | 29 | public constructor(config: Partial = {}) { 30 | super(); 31 | 32 | this.generatorOptions = config.generatorOptions ?? { duration: [0.2] }; 33 | this.position = config.position ?? { x: '50%', y: '50%' }; 34 | } 35 | 36 | public async applyTo(track: CaptionTrack): Promise { 37 | if (!track.clip?.transcript || !track.composition?.width) { 38 | throw new ValidationError({ 39 | code: 'referenceError', 40 | message: 'Captions need to be applied with a defined transcript and composition', 41 | }); 42 | } 43 | 44 | const offset = track.clip?.offset ?? new Timestamp(); 45 | const font = await Font.fromFamily({ family: 'Figtree', weight: '700' }).load(); 46 | 47 | // add captions 48 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 49 | await track.add( 50 | new TextClip({ 51 | text: sequence.words.map((word) => word.text).join(' '), 52 | textAlign: 'center', 53 | textBaseline: 'middle', 54 | fontSize: 21, 55 | fillStyle: '#FFFFFF', 56 | font, 57 | stroke: { 58 | color: '#000000', 59 | width: 4, 60 | join: 'round', 61 | }, 62 | maxWidth: track.composition.width * 0.85, 63 | shadow: { 64 | color: '#000000', 65 | blur: 0, 66 | distance: 1.1, 67 | angle: Math.PI * 0.40, 68 | alpha: 1, 69 | }, 70 | position: this.position, 71 | stop: sequence.stop.add(offset), 72 | start: sequence.start.add(offset), 73 | scale: new Keyframe([0, 8], [0.96, 1], { easing: 'easeOut' }), 74 | alpha: new Keyframe([0, 4], [0, 1], { easing: 'easeOut' }), 75 | }) 76 | ); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.deserializer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { 9 | SpotlightCaptionPreset, 10 | CascadeCaptionPreset, 11 | GuineaCaptionPreset, 12 | ClassicCaptionPreset, 13 | SolarCaptionPreset, 14 | WhisperCaptionPreset, 15 | } from '.'; 16 | 17 | import type { CaptionPresetStrategy } from './preset.interface'; 18 | import type { CaptionPresetType } from './preset.types'; 19 | 20 | export class CaptionPresetDeserializer { 21 | public static fromJSON(data: K extends string ? never : K): CaptionPresetStrategy { 22 | switch (data.type) { 23 | case 'SPOTLIGHT': 24 | return SpotlightCaptionPreset.fromJSON(data); 25 | case 'CASCADE': 26 | return CascadeCaptionPreset.fromJSON(data); 27 | case 'GUINEA': 28 | return GuineaCaptionPreset.fromJSON(data); 29 | case 'SOLAR': 30 | return SolarCaptionPreset.fromJSON(data); 31 | case 'WHISPER': 32 | return WhisperCaptionPreset.fromJSON(data); 33 | default: 34 | return ClassicCaptionPreset.fromJSON(data); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.interface.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { CaptionPresetType } from './preset.types'; 9 | import type { CaptionTrack } from './caption'; 10 | 11 | export interface CaptionPresetStrategy { 12 | /** 13 | * Defines the type of strategy 14 | */ 15 | type: CaptionPresetType; 16 | /** 17 | * This function applies the settings to the track 18 | */ 19 | applyTo(track: CaptionTrack): Promise; 20 | } 21 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.solar.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it, vi, beforeEach } from 'vitest'; 9 | import { Composition } from '../../composition'; 10 | import { MediaClip, TextClip } from '../../clips'; 11 | import { Transcript, Word, WordGroup } from '../../models'; 12 | import { CaptionTrack } from './caption'; 13 | import { SolarCaptionPreset } from './preset.solar'; 14 | import { GlowFilter } from 'pixi-filters'; 15 | 16 | describe('The SolarCaptionPreset', () => { 17 | const mockFn = vi.fn(); 18 | Object.assign(document, { fonts: { add: mockFn } }); 19 | 20 | let composition: Composition; 21 | let track: CaptionTrack; 22 | 23 | beforeEach(() => { 24 | composition = new Composition(); 25 | track = composition.createTrack('caption'); 26 | }); 27 | 28 | it('should apply complex clips to the track', async () => { 29 | await track 30 | .from(new MediaClip({ transcript })) 31 | .generate(SolarCaptionPreset); 32 | 33 | expect(track.clips.length).toBe(13); 34 | expect(track.clips[0]).toBeInstanceOf(TextClip); 35 | expect(track.clips[0].start.frames).toBe(0); 36 | expect(track.clips[0].text).toBe('Lorem'); 37 | expect(track.clips[0].filters).toBeInstanceOf(GlowFilter); 38 | }); 39 | 40 | it('should not apply clips if the transcript or composition is not devined', async () => { 41 | await expect(() => track 42 | .from(new MediaClip()) 43 | .generate(SolarCaptionPreset)).rejects.toThrowError(); 44 | 45 | await expect(() => new CaptionTrack() 46 | .from(new MediaClip({ transcript })) 47 | .generate(SolarCaptionPreset)).rejects.toThrowError(); 48 | }); 49 | }); 50 | 51 | const transcript = new Transcript([ 52 | new WordGroup([ 53 | new Word('Lorem', 0, 1e3), 54 | new Word('Ipsum', 2e3, 3e3), 55 | new Word('is', 4e3, 5e3), 56 | new Word('simply', 6e3, 7e3), 57 | new Word('dummy', 8e3, 9e3), 58 | new Word('text', 10e3, 11e3), 59 | new Word('of', 12e3, 13e3), 60 | new Word('the', 14e3, 15e3), 61 | new Word('printing', 16e3, 17e3), 62 | new Word('and', 18e3, 19e3), 63 | new Word('typesetting', 20e3, 21e3), 64 | new Word('industry', 22e3, 23e3), 65 | ]), 66 | new WordGroup([ 67 | new Word('Lorem', 24e3, 25e3), 68 | ]), 69 | ]); 70 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.solar.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { GlowFilter } from 'pixi-filters'; 9 | import { Keyframe, Timestamp } from '../../models'; 10 | import { Serializer, serializable } from '../../services'; 11 | import { Font, TextClip } from '../../clips'; 12 | import { ValidationError } from '../../errors'; 13 | 14 | import type { CaptionPresetType, DefaultCaptionPresetConfig } from './preset.types'; 15 | import type { CaptionTrack } from './caption'; 16 | import type { GeneratorOptions } from '../../models'; 17 | import type { CaptionPresetStrategy } from './preset.interface'; 18 | import type { Position } from '../../types'; 19 | 20 | export class SolarCaptionPreset extends Serializer implements CaptionPresetStrategy { 21 | @serializable() 22 | public generatorOptions: GeneratorOptions; 23 | 24 | @serializable() 25 | public readonly type: CaptionPresetType = 'SOLAR'; 26 | 27 | @serializable() 28 | public position: Position; 29 | 30 | public constructor(config: Partial = {}) { 31 | super(); 32 | 33 | this.generatorOptions = config.generatorOptions ?? { duration: [0.2] }; 34 | this.position = config.position ?? { x: '50%', y: '50%' }; 35 | } 36 | 37 | public async applyTo(track: CaptionTrack): Promise { 38 | if (!track.clip?.transcript || !track.composition?.width) { 39 | throw new ValidationError({ 40 | code: 'referenceError', 41 | message: 'Captions need to be applied with a defined transcript and composition', 42 | }); 43 | } 44 | 45 | const font = await Font.fromFamily({ family: 'Urbanist', weight: '800' }).load(); 46 | const offset = track.clip?.offset ?? new Timestamp(); 47 | const filters = new GlowFilter({ 48 | color: '#fffe41', 49 | alpha: 0.25, 50 | distance: 90, 51 | quality: 0.05, 52 | }); 53 | 54 | // add captions 55 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 56 | await track.add( 57 | new TextClip({ 58 | text: sequence.words.map((word) => word.text).join(' '), 59 | textAlign: 'center', 60 | textBaseline: 'middle', 61 | fontSize: 19, 62 | fillStyle: '#fffe41', 63 | font, 64 | maxWidth: track.composition.width * 0.85, 65 | textCase: 'upper', 66 | shadow: { 67 | color: '#ab7a00', 68 | blur: 0, 69 | distance: 2.1, 70 | angle: Math.PI / 2.5, 71 | alpha: 1, 72 | }, 73 | position: this.position, 74 | stop: sequence.stop.add(offset), 75 | start: sequence.start.add(offset), 76 | scale: new Keyframe([0, 8], [0.96, 1], { easing: 'easeOut' }), 77 | alpha: new Keyframe([0, 4], [0, 1], { easing: 'easeOut' }), 78 | filters, 79 | }) 80 | ); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.spotlight.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Font, ComplexTextClip } from '../../clips'; 9 | import { serializable, Serializer } from '../../services'; 10 | import { Timestamp } from '../../models'; 11 | import { ValidationError } from '../../errors'; 12 | 13 | import type { SingleColorCaptionPresetConfig } from './preset.types'; 14 | import type { CaptionPresetStrategy } from './preset.interface'; 15 | import type { GeneratorOptions } from '../../models'; 16 | import type { CaptionTrack } from './caption'; 17 | import type { hex, Position } from '../../types'; 18 | 19 | export class SpotlightCaptionPreset extends Serializer implements CaptionPresetStrategy { 20 | @serializable() 21 | public generatorOptions: GeneratorOptions; 22 | 23 | @serializable() 24 | public readonly type = 'SPOTLIGHT'; 25 | 26 | @serializable() 27 | public color: hex; 28 | 29 | @serializable() 30 | public position: Position; 31 | 32 | public constructor(config: Partial = {}) { 33 | super(); 34 | 35 | this.generatorOptions = config.generatorOptions ?? { duration: [0.2] }; 36 | this.color = config.color ?? '#00FF4C'; 37 | this.position = config.position ?? { x: '50%', y: '50%' }; 38 | } 39 | 40 | public async applyTo(track: CaptionTrack): Promise { 41 | if (!track.clip?.transcript || !track.composition?.width) { 42 | throw new ValidationError({ 43 | code: 'referenceError', 44 | message: 'Captions need to be applied with a defined transcript and composition', 45 | }); 46 | } 47 | 48 | const offset = track.clip?.offset ?? new Timestamp(); 49 | const font = await Font.fromFamily({ family: 'The Bold Font', weight: '500' }).load(); 50 | 51 | // add captions 52 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 53 | for (let i = 0; i < sequence.words.length; i++) { 54 | const tokens = sequence.words.map((s) => s.text); 55 | 56 | await track.add( 57 | new ComplexTextClip({ 58 | text: tokens.join(' '), 59 | textAlign: 'center', 60 | textBaseline: 'middle', 61 | fillStyle: '#FFFFFF', 62 | fontSize: 22, 63 | maxWidth: track.composition.width * 0.8, 64 | font, 65 | stroke: { 66 | width: 5, 67 | color: '#000000', 68 | }, 69 | shadow: { 70 | color: '#000000', 71 | blur: 12, 72 | alpha: 0.7, 73 | angle: Math.PI / 4, 74 | distance: 2, 75 | }, 76 | position: this.position, 77 | styles: [{ 78 | fillStyle: this.color, 79 | }], 80 | 81 | segments: 82 | sequence.words.length > 1 83 | ? [ 84 | { 85 | index: 0, 86 | start: tokens.slice(0, i).join(' ').length, 87 | stop: tokens.slice(0, i + 1).join(' ').length, 88 | }, 89 | ] 90 | : undefined, 91 | stop: sequence.words[i].stop.add(offset), 92 | start: sequence.words[i].start.add(offset), 93 | }) 94 | ); 95 | } 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { GeneratorOptions } from '../../models'; 9 | import type { hex, Position } from '../../types'; 10 | 11 | export type CaptionPresetType = 12 | | 'CLASSIC' 13 | | 'SPOTLIGHT' 14 | | 'CASCADE' 15 | | 'GUINEA' 16 | | 'SOLAR' 17 | | 'WHISPER' 18 | | 'VERDANT' 19 | | string; 20 | 21 | export type DefaultCaptionPresetConfig = { 22 | generatorOptions: GeneratorOptions; 23 | position: Position; 24 | }; 25 | 26 | export type SingleColorCaptionPresetConfig = { 27 | color: hex; 28 | } & DefaultCaptionPresetConfig; 29 | 30 | export type MultiColorCaptionPresetConfig = { 31 | colors: hex[] | undefined; 32 | } & DefaultCaptionPresetConfig; 33 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.verdant.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it, vi, beforeEach } from 'vitest'; 9 | import { Composition } from '../../composition'; 10 | import { ComplexTextClip, MediaClip } from '../../clips'; 11 | import { Transcript, Word, WordGroup } from '../../models'; 12 | import { CaptionTrack } from './caption'; 13 | import { VerdantCaptionPreset } from './preset.verdant'; 14 | 15 | describe('The VerdantCaptionPreset', () => { 16 | const mockFn = vi.fn(); 17 | Object.assign(document, { fonts: { add: mockFn } }); 18 | 19 | let composition: Composition; 20 | let track: CaptionTrack; 21 | 22 | beforeEach(() => { 23 | composition = new Composition(); 24 | track = composition.createTrack('caption'); 25 | }); 26 | 27 | it('should apply complex clips to the track', async () => { 28 | await track 29 | .from(new MediaClip({ transcript })) 30 | .generate(VerdantCaptionPreset); 31 | 32 | expect(track.clips.length).toBe(13); 33 | expect(track.clips[0]).toBeInstanceOf(ComplexTextClip); 34 | expect(track.clips[0].start.frames).toBe(0); 35 | expect(track.clips[0].text).toBe('Lorem'); 36 | }); 37 | 38 | it('should not apply clips if the transcript or composition is not devined', async () => { 39 | await expect(() => track 40 | .from(new MediaClip()) 41 | .generate(VerdantCaptionPreset)).rejects.toThrowError(); 42 | 43 | await expect(() => new CaptionTrack() 44 | .from(new MediaClip({ transcript })) 45 | .generate(VerdantCaptionPreset)).rejects.toThrowError(); 46 | }); 47 | }); 48 | 49 | const transcript = new Transcript([ 50 | new WordGroup([ 51 | new Word('Lorem', 0, 1e3), 52 | new Word('Ipsum', 2e3, 3e3), 53 | new Word('is', 4e3, 5e3), 54 | new Word('simply', 6e3, 7e3), 55 | new Word('dummy', 8e3, 9e3), 56 | new Word('text', 10e3, 11e3), 57 | new Word('of', 12e3, 13e3), 58 | new Word('the', 14e3, 15e3), 59 | new Word('printing', 16e3, 17e3), 60 | new Word('and', 18e3, 19e3), 61 | new Word('typesetting', 20e3, 21e3), 62 | new Word('industry', 22e3, 23e3), 63 | ]), 64 | new WordGroup([ 65 | new Word('Lorem', 24e3, 25e3), 66 | ]), 67 | ]); 68 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.verdant.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Timestamp } from '../../models'; 9 | import { Serializer, serializable } from '../../services'; 10 | import { ComplexTextClip, Font } from '../../clips'; 11 | import { ValidationError } from '../../errors'; 12 | 13 | import type { SingleColorCaptionPresetConfig, CaptionPresetType } from './preset.types'; 14 | import type { CaptionPresetStrategy } from './preset.interface'; 15 | import type { GeneratorOptions } from '../../models'; 16 | import type { CaptionTrack } from './caption'; 17 | import type { hex, Position } from '../../types'; 18 | 19 | export class VerdantCaptionPreset extends Serializer implements CaptionPresetStrategy { 20 | @serializable() 21 | readonly type: CaptionPresetType = 'VERDANT'; 22 | 23 | @serializable() 24 | public generatorOptions: GeneratorOptions; 25 | 26 | @serializable() 27 | public color: hex; 28 | 29 | @serializable() 30 | public position: Position; 31 | 32 | public constructor(config: Partial = {}) { 33 | super(); 34 | 35 | this.generatorOptions = config.generatorOptions ?? { duration: [1] }; 36 | this.color = config.color ?? '#69E34C'; 37 | this.position = config.position ?? { x: '50%', y: '50%' }; 38 | } 39 | 40 | public async applyTo(track: CaptionTrack): Promise { 41 | if (!track.clip?.transcript || !track.composition?.width) { 42 | throw new ValidationError({ 43 | code: 'referenceError', 44 | message: 'Captions need to be applied with a defined transcript and composition', 45 | }); 46 | } 47 | 48 | const offset = track.clip?.offset ?? new Timestamp(); 49 | const font = await Font.fromFamily({ family: 'Montserrat', weight: '800' }).load(); 50 | 51 | // add captions 52 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 53 | for (let i = 0; i < sequence.words.length; i++) { 54 | const tokens = sequence.words.map((s) => s.text); 55 | 56 | await track.add( 57 | new ComplexTextClip({ 58 | text: tokens.join(' '), 59 | textAlign: 'center', 60 | textBaseline: 'middle', 61 | fontSize: 15, 62 | fillStyle: '#FFFFFF', 63 | shadow: { 64 | color: '#000000', 65 | blur: 4, 66 | alpha: 0.7, 67 | angle: Math.PI / 4, 68 | distance: 2, 69 | }, 70 | stroke: { 71 | width: 3, 72 | color: '#000000', 73 | }, 74 | maxWidth: track.composition.width * 0.5, 75 | leading: 1.1, 76 | font, 77 | textCase: 'upper', 78 | styles: [{ 79 | fillStyle: this.color, 80 | fontSize: 19, 81 | }], 82 | position: this.position, 83 | stop: sequence.words[i].stop.add(offset), 84 | start: sequence.words[i].start.add(offset), 85 | segments: [{ 86 | index: 0, 87 | start: tokens.slice(0, i).join(' ').length, 88 | stop: tokens.slice(0, i + 1).join(' ').length, 89 | }], 90 | }) 91 | ); 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.whisper.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it, vi, beforeEach } from 'vitest'; 9 | import { Composition } from '../../composition'; 10 | import { ComplexTextClip, MediaClip } from '../../clips'; 11 | import { Transcript, Word, WordGroup } from '../../models'; 12 | import { CaptionTrack } from './caption'; 13 | import { WhisperCaptionPreset } from './preset.whisper'; 14 | 15 | describe('The WhisperCaptionPreset', () => { 16 | const mockFn = vi.fn(); 17 | Object.assign(document, { fonts: { add: mockFn } }); 18 | 19 | let composition: Composition; 20 | let track: CaptionTrack; 21 | 22 | beforeEach(() => { 23 | composition = new Composition(); 24 | track = composition.createTrack('caption'); 25 | }); 26 | 27 | it('should apply complex clips to the track', async () => { 28 | await track 29 | .from(new MediaClip({ transcript })) 30 | .generate(WhisperCaptionPreset); 31 | 32 | expect(track.clips.length).toBe(13); 33 | expect(track.clips[0]).toBeInstanceOf(ComplexTextClip); 34 | expect(track.clips[0].start.frames).toBe(0); 35 | expect(track.clips[0].text).toBe('Lorem Ipsum is simply'); 36 | }); 37 | 38 | it('should not apply clips if the transcript or composition is not devined', async () => { 39 | await expect(() => track 40 | .from(new MediaClip()) 41 | .generate(WhisperCaptionPreset)).rejects.toThrowError(); 42 | 43 | await expect(() => new CaptionTrack() 44 | .from(new MediaClip({ transcript })) 45 | .generate(WhisperCaptionPreset)).rejects.toThrowError(); 46 | }); 47 | }); 48 | 49 | const transcript = new Transcript([ 50 | new WordGroup([ 51 | new Word('Lorem', 0, 1e3), 52 | new Word('Ipsum', 2e3, 3e3), 53 | new Word('is', 4e3, 5e3), 54 | new Word('simply', 6e3, 7e3), 55 | new Word('dummy', 8e3, 9e3), 56 | new Word('text', 10e3, 11e3), 57 | new Word('of', 12e3, 13e3), 58 | new Word('the', 14e3, 15e3), 59 | new Word('printing', 16e3, 17e3), 60 | new Word('and', 18e3, 19e3), 61 | new Word('typesetting', 20e3, 21e3), 62 | new Word('industry', 22e3, 23e3), 63 | ]), 64 | new WordGroup([ 65 | new Word('Lorem', 24e3, 25e3), 66 | ]), 67 | ]); 68 | -------------------------------------------------------------------------------- /src/tracks/caption/preset.whisper.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Font, ComplexTextClip } from '../../clips'; 9 | import { Timestamp } from '../../models'; 10 | import { serializable, Serializer } from '../../services'; 11 | import { ValidationError } from '../../errors'; 12 | 13 | import type { SingleColorCaptionPresetConfig } from './preset.types'; 14 | import type { CaptionTrack } from './caption'; 15 | import type { CaptionPresetStrategy } from './preset.interface'; 16 | import type { GeneratorOptions } from '../../models'; 17 | import type { hex, Position } from '../../types'; 18 | 19 | export class WhisperCaptionPreset extends Serializer implements CaptionPresetStrategy { 20 | @serializable() 21 | public generatorOptions: GeneratorOptions; 22 | 23 | @serializable() 24 | public readonly type = 'WHISPER'; 25 | 26 | @serializable() 27 | public color: hex; 28 | 29 | @serializable() 30 | public position: Position; 31 | 32 | public constructor(config: Partial = {}) { 33 | super(); 34 | 35 | this.generatorOptions = config.generatorOptions ?? { length: [20] }; 36 | this.color = config.color ?? '#8c8c8c'; 37 | this.position = config.position ?? { x: '50%', y: '50%' }; 38 | } 39 | 40 | public async applyTo(track: CaptionTrack): Promise { 41 | if (!track.clip?.transcript || !track.composition?.width) { 42 | throw new ValidationError({ 43 | code: 'referenceError', 44 | message: 'Captions need to be applied with a defined transcript and composition', 45 | }); 46 | } 47 | 48 | const offset = track.clip?.offset ?? new Timestamp(); 49 | const font = await Font.fromFamily({ family: 'Montserrat', weight: '300' }).load(); 50 | 51 | // add captions 52 | for (const sequence of track.clip.transcript.iter(this.generatorOptions)) { 53 | for (let i = 0; i < sequence.words.length; i++) { 54 | const splits = sequence.words.map((s) => s.text); 55 | 56 | await track.add( 57 | new ComplexTextClip({ 58 | text: splits.join(' '), 59 | textAlign: 'center', 60 | textBaseline: 'middle', 61 | fillStyle: '#FFFFFF', 62 | fontSize: 13, 63 | background: { 64 | alpha: 0.3, 65 | padding: { 66 | x: 50, 67 | y: 30 68 | } 69 | }, 70 | maxWidth: track.composition.width * 0.8, 71 | font, 72 | position: this.position, 73 | styles: [{ 74 | fillStyle: this.color, 75 | }], 76 | stop: sequence.words[i].stop.add(offset), 77 | start: sequence.words[i].start.add(offset), 78 | segments: 79 | splits.length > 1 80 | ? [ 81 | { 82 | index: 0, 83 | start: splits.slice(0, i + 1).join(' ').length, 84 | }, 85 | ] 86 | : undefined, 87 | }) 88 | ); 89 | } 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/tracks/html/html.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Track } from '../track'; 9 | 10 | import type { HtmlClip } from '../../clips'; 11 | 12 | export class HtmlTrack extends Track { 13 | public readonly type = 'html'; 14 | } 15 | -------------------------------------------------------------------------------- /src/tracks/html/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './html'; 9 | -------------------------------------------------------------------------------- /src/tracks/image/image.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { ImageTrack } from './image'; 10 | 11 | describe('The Image Track Object', () => { 12 | it('should have a certain intitial state', () => { 13 | const track = new ImageTrack(); 14 | expect(track.type).toBe('image'); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/tracks/image/image.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Track } from '../track'; 9 | 10 | import type { ImageClip } from '../../clips'; 11 | 12 | export class ImageTrack extends Track { 13 | public readonly type = 'image'; 14 | } 15 | -------------------------------------------------------------------------------- /src/tracks/image/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './image'; 9 | -------------------------------------------------------------------------------- /src/tracks/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './track'; 9 | export * from './media'; 10 | export * from './video'; 11 | export * from './image'; 12 | export * from './audio'; 13 | export * from './text'; 14 | export * from './html'; 15 | export * from './caption'; 16 | -------------------------------------------------------------------------------- /src/tracks/media/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './media'; 9 | -------------------------------------------------------------------------------- /src/tracks/media/media.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { beforeEach, describe, expect, it, vi } from 'vitest'; 9 | import { Composition } from '../../composition'; 10 | import { MediaClip } from '../../clips'; 11 | import { Timestamp } from '../../models'; 12 | import { MediaTrack } from './media'; 13 | 14 | describe('The Media Track Object', () => { 15 | let comp: Composition; 16 | let track: MediaTrack; 17 | const updateMock = vi.fn(); 18 | 19 | beforeEach(() => { 20 | // frame and seconds are the same 21 | comp = new Composition(); 22 | track = comp.shiftTrack(new MediaTrack()); 23 | track.on('update', updateMock); 24 | }); 25 | 26 | it('should propagate a seek call', async () => { 27 | const clip = new MediaClip(); 28 | clip.element = new Audio(); 29 | clip.duration.seconds = 60; 30 | clip.state = 'READY'; 31 | await track.add(clip); 32 | expect(track.clips.length).toBe(1); 33 | const seekSpy = vi.spyOn(clip, 'seek').mockImplementation(async (_) => {}); 34 | track.seek(Timestamp.fromFrames(5)); 35 | expect(seekSpy).toBeCalledTimes(1); 36 | }); 37 | 38 | it('should be be able to add a media clip with offset', async () => { 39 | const clip = new MediaClip(); 40 | clip.duration.frames = 30; 41 | 42 | expect(clip.duration.frames).toBe(30); 43 | expect(clip.start.frames).toBe(0); 44 | expect(clip.offset.frames).toBe(0); 45 | expect(clip.stop.frames).toBe(30); 46 | 47 | clip.state = 'READY'; 48 | await track.add(clip.offsetBy(60)); 49 | 50 | expect(track.clips.at(0)?.start.frames).toBe(60); 51 | expect(track.clips.at(0)?.stop.frames).toBe(90); 52 | expect(track.clips.at(0)?.offset.frames).toBe(60); 53 | expect(track.clips.at(0)?.duration.frames).toBe(30); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /src/tracks/media/media.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Track } from '../track'; 9 | import { Timestamp } from '../../models'; 10 | 11 | import type { MediaClip, SilenceRemoveOptions } from '../../clips'; 12 | 13 | export class MediaTrack extends Track { 14 | public clips: Clip[] = []; 15 | public async seek(time: Timestamp): Promise { 16 | for (const clip of this.clips) { 17 | await clip.seek(time); 18 | } 19 | } 20 | 21 | /** 22 | * Remove silences from all clips in the track 23 | * 24 | * @param options - Options for silence detection 25 | */ 26 | public async removeSilences(options: SilenceRemoveOptions = {}) { 27 | const clips: MediaClip[] = []; 28 | 29 | for (const clip of this.clips.map((clip) => clip.detach())) { 30 | clips.push(...(await clip.removeSilences(options))); 31 | } 32 | 33 | for (const clip of clips) { 34 | await this.add(clip); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/tracks/text/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './text'; 9 | -------------------------------------------------------------------------------- /src/tracks/text/text.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { TextClip } from '../../clips'; 9 | import { Track } from '../track'; 10 | 11 | export class TextTrack extends Track { 12 | public readonly type = 'text'; 13 | } 14 | -------------------------------------------------------------------------------- /src/tracks/track/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './track.types'; 9 | export * from './track'; 10 | export * from './track.deserializer'; 11 | -------------------------------------------------------------------------------- /src/tracks/track/track.deserializer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { Track, VideoTrack, AudioTrack, HtmlTrack, TextTrack, ImageTrack, CaptionTrack, TrackMap } from '..'; 9 | 10 | export class TrackDeserializer { 11 | public static fromType(data: { type: T }): TrackMap[T] { 12 | switch (data.type) { 13 | case 'video': 14 | return new VideoTrack() as TrackMap[T]; 15 | case 'audio': 16 | return new AudioTrack() as TrackMap[T]; 17 | case 'html': 18 | return new HtmlTrack() as TrackMap[T]; 19 | case 'image': 20 | return new ImageTrack() as TrackMap[T]; 21 | case 'text': 22 | return new TextTrack() as TrackMap[T]; 23 | case 'complex_text': 24 | return new TextTrack() as TrackMap[T]; 25 | case 'caption': 26 | return new CaptionTrack() as TrackMap[T]; 27 | default: 28 | return new Track() as TrackMap[T]; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/tracks/track/track.fixtures.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export const insertModes = ['DEFAULT', 'STACK'] as const; 9 | -------------------------------------------------------------------------------- /src/tracks/track/track.interfaces.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { InsertMode } from './track.types'; 9 | import type { Clip } from '../../clips'; 10 | import type { Track } from './track'; 11 | import type { Timestamp } from '../../models'; 12 | 13 | export interface InsertStrategy { 14 | readonly mode: T; 15 | add(clip: Clip, track: Track, index?: number): void; 16 | update(clip: Clip, track: Track): void; 17 | offset(time: Timestamp, track: Track): void; 18 | } 19 | -------------------------------------------------------------------------------- /src/tracks/track/track.types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import type { Clip } from '../../clips'; 9 | import type { insertModes } from './track.fixtures'; 10 | import type { Track, VideoTrack, AudioTrack, HtmlTrack, TextTrack, ImageTrack, CaptionTrack } from '..'; 11 | 12 | export type TrackMap = { 13 | video: VideoTrack; 14 | audio: AudioTrack; 15 | html: HtmlTrack; 16 | image: ImageTrack; 17 | text: TextTrack; 18 | complex_text: TextTrack; 19 | caption: CaptionTrack; 20 | base: Track; 21 | }; 22 | 23 | export type TrackType = keyof TrackMap; 24 | export type TrackInsertMethod = 'STACK' | 'TIMED'; 25 | /** 26 | * Defines where the track should be inserted 27 | */ 28 | export type TrackLayer = 'top' | 'bottom' | number; 29 | 30 | export type InsertMode = (typeof insertModes)[number]; 31 | -------------------------------------------------------------------------------- /src/tracks/video/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './video'; 9 | -------------------------------------------------------------------------------- /src/tracks/video/video.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { describe, expect, it } from 'vitest'; 9 | import { VideoTrack } from './video'; 10 | 11 | describe('The Video Track Object', () => { 12 | it('should have a certain intitial state', () => { 13 | const track = new VideoTrack(); 14 | expect(track.type).toBe('video'); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /src/tracks/video/video.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { MediaTrack } from '../media'; 9 | import { Timestamp } from '../../models'; 10 | 11 | import type { VideoClip } from '../../clips'; 12 | 13 | export class VideoTrack extends MediaTrack { 14 | public readonly type = 'video'; 15 | 16 | public async seek(time: Timestamp): Promise { 17 | if (this.composition?.rendering) { 18 | // ensures that 'enter' method will be called again 19 | this.view.removeChildren(); 20 | } else { 21 | super.seek(time); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/utils/browser.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | import { beforeEach, describe, expect, it, vi } from 'vitest'; 9 | import { downloadObject, showFileDialog } from './browser'; 10 | 11 | describe('The browser utils', () => { 12 | const a = document.createElement('a'); 13 | const input = document.createElement('input'); 14 | 15 | const clickSpy = vi.spyOn(a, 'click'); 16 | const removeSpy = vi.spyOn(a, 'remove'); 17 | const createSpy = vi.spyOn(document, 'createElement').mockImplementation((tag: string) => { 18 | expect(tag).toBeTypeOf('string'); 19 | 20 | if (tag == 'a') { 21 | return a; 22 | } else { 23 | return input; 24 | } 25 | }); 26 | 27 | beforeEach(() => { 28 | clickSpy.mockClear(); 29 | removeSpy.mockClear(); 30 | createSpy.mockClear(); 31 | }) 32 | 33 | it('should download an object from an url (downloadObject)', async () => { 34 | downloadObject('https://myurl.com/example.mp4'); 35 | expect(a.download).toBe('untitled'); 36 | expect(a.href).toBe('https://myurl.com/example.mp4'); 37 | 38 | expect(createSpy).toBeCalledTimes(1); 39 | expect(clickSpy).toBeCalledTimes(1); 40 | expect(removeSpy).toBeCalledTimes(1); 41 | }); 42 | 43 | it('should download an a blob with custom name (downloadObject)', async () => { 44 | downloadObject(new Blob(), 'temp.mp4'); 45 | expect(a.download).toBe('temp.mp4'); 46 | expect(a.href).toBe('blob:chrome://new-tab-page/3dc0f2b7-7773-4cd4-a397-2e43b1bba7cd'); 47 | 48 | expect(createSpy).toBeCalledTimes(1); 49 | expect(clickSpy).toBeCalledTimes(1); 50 | expect(removeSpy).toBeCalledTimes(1); 51 | }); 52 | 53 | it('should download base 64 encoded image data (downloadObject)', async () => { 54 | const img = "data:image/svg+xml;base64,"; 55 | downloadObject(img, 'temp.mp4'); 56 | expect(a.download).toBe('temp.svg'); 57 | expect(a.href).toBe('blob:chrome://new-tab-page/3dc0f2b7-7773-4cd4-a397-2e43b1bba7cd'); 58 | 59 | expect(createSpy).toBeCalledTimes(1); 60 | expect(clickSpy).toBeCalledTimes(1); 61 | expect(removeSpy).toBeCalledTimes(1); 62 | }); 63 | 64 | it('should show a save dialog (showFileDialog)', async () => { 65 | const clickSpy = vi.spyOn(input, 'click'); 66 | 67 | const promise = showFileDialog('video/mp4', false); 68 | input.dispatchEvent(new Event('change')); 69 | await promise; 70 | 71 | expect(input.type).toBe('file'); 72 | expect(input.accept).toBe('video/mp4'); 73 | expect(input.multiple).toBe(false); 74 | 75 | expect(createSpy).toBeCalledTimes(1); 76 | expect(clickSpy).toBeCalledTimes(1); 77 | }); 78 | }); 79 | -------------------------------------------------------------------------------- /src/utils/browser.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * This utility creates an anchor tag and clicks on it 10 | * @param source Blob url or base64 encoded svg 11 | * @param name File name suggestion 12 | */ 13 | export async function downloadObject( 14 | source: string | Blob, 15 | name: string = 'untitled', 16 | ): Promise { 17 | const a = document.createElement('a'); 18 | document.head.appendChild(a); 19 | a.download = name; 20 | 21 | if (typeof source == 'string' && source.startsWith('data:image/svg+xml;base64,')) { 22 | // Step 1: Extract the Base64 string from the src attribute 23 | const base64 = source.split(',')[1]; 24 | 25 | // Step 2: Create a Blob object from the Base64 string 26 | const byteCharacters = atob(base64); 27 | const byteNumbers = new Array(byteCharacters.length); 28 | for (let i = 0; i < byteCharacters.length; i++) { 29 | byteNumbers[i] = byteCharacters.charCodeAt(i); 30 | } 31 | const byteArray = new Uint8Array(byteNumbers); 32 | const blob = new Blob([byteArray], { type: 'image/svg+xml' }); 33 | 34 | // Step 3: Set the href attribute to the Blob URL 35 | a.href = URL.createObjectURL(blob); 36 | a.download = name.split('.')[0] + '.svg'; 37 | } else if (typeof source == 'string') { 38 | a.href = source; 39 | } else { 40 | a.href = URL.createObjectURL(source); 41 | } 42 | 43 | a.click(); 44 | a.remove(); 45 | } 46 | 47 | /** 48 | * This utility creates a file input element and clicks on it 49 | * @param accept comma separated mime types 50 | * @example audio/mp3, video/mp4 51 | * @param multiple enable multiselection 52 | * @default true 53 | */ 54 | export async function showFileDialog(accept: string, multiple = true): Promise { 55 | return new Promise((resolve) => { 56 | // setup input 57 | const input = document.createElement('input'); 58 | input.type = 'file'; 59 | input.accept = accept; 60 | input.multiple = multiple; 61 | // listen for changes 62 | input.onchange = (fileEvent: Event) => { 63 | const file = Array.from((fileEvent.target)?.files ?? []); 64 | resolve(file); 65 | }; 66 | input.click(); 67 | }); 68 | } 69 | -------------------------------------------------------------------------------- /src/utils/common.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | /** 9 | * Convert an alpha value to hex 10 | * @param alpha A value between 0 and 1 11 | * @returns Alpha as 2 digit hex 12 | * @example FF 13 | */ 14 | export function toHex(alpha: number): string { 15 | return Math.floor(alpha * 255) 16 | .toString(16) 17 | .padStart(2, '0') 18 | .toUpperCase(); 19 | } 20 | 21 | /** 22 | * Group an array of objects by the specified key 23 | */ 24 | export function groupBy(arr: T[], key: K) { 25 | return arr.reduce( 26 | (accumulator, val) => { 27 | const groupedKey = val[key]; 28 | if (!accumulator[groupedKey]) { 29 | accumulator[groupedKey] = []; 30 | } 31 | accumulator[groupedKey].push(val); 32 | return accumulator; 33 | }, // @ts-ignore 34 | {} as Record, 35 | ); 36 | } 37 | 38 | /** 39 | * Split an array at the specified position 40 | */ 41 | export function splitAt(list: any[] | string, index: number): T { 42 | return [list.slice(0, index), list.slice(index)].filter((i) => i.length > 0) as T; 43 | } 44 | 45 | /** 46 | * Generate a random value between two numbers 47 | */ 48 | export function randInt(min: number, max: number | undefined): number { 49 | if (!max) return min; 50 | // min and max included 51 | return Math.floor(Math.random() * (max - min + 1) + min); 52 | } 53 | 54 | /** 55 | * setTimeout async/await replacement 56 | */ 57 | export async function sleep(ms: number): Promise { 58 | if (ms <= 0) return; 59 | await new Promise((resolve) => setTimeout(resolve, ms)); 60 | } 61 | 62 | /** 63 | * clip assert replacement for the browser 64 | * @example assert(true == false) 65 | */ 66 | export function assert(condition: any) { 67 | if (!condition) { 68 | throw 'Assertion failed!'; 69 | } 70 | } 71 | 72 | /** 73 | * Limit the number of times a function can be called 74 | * per interval, timeout is in milliseconds 75 | */ 76 | export function debounce(func: Function, timeout = 300) { 77 | let timer: any; 78 | return (...args: any[]) => { 79 | clearTimeout(timer); 80 | timer = setTimeout(() => { 81 | func.apply(func, args); 82 | }, timeout); 83 | }; 84 | } 85 | 86 | /** 87 | * Move an element inside the provided array 88 | */ 89 | export function arraymove(arr: any[], fromIndex: number, toIndex: number) { 90 | if (toIndex < 0) toIndex = 0; 91 | const element = arr[fromIndex]; 92 | arr.splice(fromIndex, 1); 93 | arr.splice(toIndex, 0, element); 94 | } 95 | 96 | /** 97 | * Short unique id (not as secure as uuid 4 though) 98 | */ 99 | export function uid() { 100 | return crypto.randomUUID().split('-').at(0); 101 | } 102 | 103 | /** 104 | * Check whether a given value is a class 105 | */ 106 | export function isClass(value: any) { 107 | if (typeof value !== 'function') { 108 | return false; // Not a function 109 | } 110 | 111 | // Check if it's a class 112 | const isClass = /^class\s/.test(Function.prototype.toString.call(value)); 113 | return isClass; 114 | } 115 | 116 | export function capitalize(str: string): string { 117 | return str.charAt(0).toUpperCase() + str.slice(1); 118 | } 119 | -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2024 The Diffusion Studio Authors 3 | * 4 | * This Source Code Form is subject to the terms of the Mozilla 5 | * Public License, v. 2.0 that can be found in the LICENSE file. 6 | */ 7 | 8 | export * from './common'; 9 | export * from './audio'; 10 | export * from './webcodecs'; 11 | export * from './browser'; 12 | -------------------------------------------------------------------------------- /src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | /// 4 | 5 | declare global { 6 | interface Window { 7 | queryLocalFonts(query?: Record): FontData[]; 8 | chrome: any; 9 | } 10 | interface VideoEncoder { 11 | ondequeue(): void; 12 | } 13 | interface VideoDecoder { 14 | ondequeue(): void; 15 | } 16 | interface FileSystemHandle { 17 | remove(): Promise; 18 | } 19 | } 20 | 21 | export type {}; 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "lib": [ 5 | "DOM", 6 | "DOM.Iterable", 7 | "ESNext", 8 | "webworker" 9 | ], 10 | "experimentalDecorators": true, 11 | "useDefineForClassFields": true, 12 | "module": "ESNext", 13 | "skipLibCheck": true, 14 | "allowJs": true, 15 | "types": [ 16 | "@webgpu/types", 17 | "@types/wicg-file-system-access", 18 | "@types/dom-webcodecs" 19 | ], 20 | /* Bundler mode */ 21 | "moduleResolution": "bundler", 22 | "allowImportingTsExtensions": true, 23 | "resolveJsonModule": true, 24 | "isolatedModules": true, 25 | "moduleDetection": "force", 26 | "noEmit": true, 27 | /* Linting */ 28 | "strict": true, 29 | "noUnusedLocals": true, 30 | "noUnusedParameters": true, 31 | "noFallthroughCasesInSwitch": true, 32 | }, 33 | "include": ["src"] 34 | } -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite'; 2 | import dts from 'vite-plugin-dts'; 3 | import { nodeExternals } from 'rollup-plugin-node-externals' 4 | import path from 'node:path'; 5 | 6 | // https://vitejs.dev/config/ 7 | export default defineConfig(({ command }) => ({ 8 | publicDir: command == 'build' ? false : 'public', 9 | build: { 10 | lib: { 11 | entry: path.resolve(__dirname, 'src/index.ts'), 12 | name: 'DiffusionStudio', 13 | formats: ['es'], 14 | fileName: 'ds' 15 | }, 16 | target: 'esnext', 17 | }, 18 | plugins: [ 19 | dts({ 20 | exclude: ['**/*.spec.ts', '**/tests/*'], 21 | rollupTypes: true, 22 | include: ['src'] 23 | }), 24 | nodeExternals(), 25 | ], 26 | })); 27 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config'; 2 | 3 | export default defineConfig({ 4 | test: { 5 | setupFiles: ['./vitest.setup.ts', '@vitest/web-worker'], 6 | environment: 'jsdom', 7 | root: './src', 8 | globals: true, 9 | server: { 10 | deps: { 11 | inline: ['vitest-canvas-mock'], 12 | }, 13 | }, 14 | environmentOptions: { 15 | jsdom: { 16 | resources: 'usable', 17 | }, 18 | }, 19 | coverage: { 20 | reporter: ['text'], 21 | }, 22 | } 23 | }); 24 | -------------------------------------------------------------------------------- /vitest.setup.ts: -------------------------------------------------------------------------------- 1 | // resolve circular deps 2 | import './src/index'; 3 | import 'vitest-canvas-mock'; 4 | import { vi } from 'vitest'; 5 | import * as mocks from './vitest.mocks'; 6 | 7 | Object.assign(navigator, { 8 | storage: { 9 | getDirectory: async () => mocks.opfs 10 | } 11 | }); 12 | Object.assign(globalThis, { URL: mocks.URLMock }); 13 | vi.stubGlobal('FontFace', mocks.FontFaceMock); 14 | vi.stubGlobal('File', mocks.FileMock); 15 | vi.stubGlobal('FileSystemFileHandle', mocks.FileSystemFileHandleMock); 16 | vi.stubGlobal('FileSystemWritableFileStream', mocks.FileSystemWritableFileStreamMock); 17 | Object.assign(globalThis, { queryLocalFonts: mocks.queryLocalFonts }); 18 | Object.assign(globalThis, { 19 | AudioEncoder: mocks.AudioEncoderMock, 20 | AudioData: mocks.AudioDataMock, 21 | VideoEncoder: mocks.VideoEncoderMock, 22 | VideoFrame: mocks.VideoFrameMock 23 | }); 24 | vi.mock('pixi.js', async (importOriginal) => { 25 | class Renderer { 26 | private _canvas = document.createElement('canvas'); 27 | 28 | height: undefined | number; 29 | width: undefined | number; 30 | 31 | async init(args: any) { 32 | this.height = args.height; 33 | this.width = args.width; 34 | } 35 | 36 | get canvas() { 37 | return this._canvas; 38 | } 39 | 40 | resize(width: number, height: number) { 41 | this._canvas.width = width; 42 | this._canvas.height = height; 43 | } 44 | 45 | render = vi.fn(); 46 | } 47 | 48 | return { 49 | ...await importOriginal(), 50 | 51 | autoDetectRenderer: async (args: any) => { 52 | const renderer = new Renderer() 53 | await renderer.init(args); 54 | return renderer; 55 | }, 56 | WebGPURenderer: Renderer, 57 | WebGLRenderer: Renderer, 58 | CanvasRenderer: Renderer, 59 | } 60 | }); 61 | 62 | export const fetchMock = vi.fn(); 63 | fetchMock.mockResolvedValue(mocks.defaultFetchMockReturnValue); 64 | 65 | Object.assign(globalThis, { fetch: fetchMock }); 66 | --------------------------------------------------------------------------------