├── .gitignore ├── .npmignore ├── Makefile ├── README.md ├── package-lock.json ├── package.json ├── rollup.config.mjs ├── samples ├── audio-decoder-flac │ ├── audio-decoder-flac.js │ └── index.html ├── audio-decoder-opus │ ├── audio-decoder-opus.js │ └── index.html ├── audio-encoder-flac │ ├── audio-encoder-flac.js │ └── index.html ├── audio-encoder-opus │ ├── audio-encoder-opus.js │ └── index.html ├── sample1.flac ├── sample1.ogg ├── sample1.opus ├── sample2.webm ├── util.js ├── video-decoder-vp8-opus │ ├── index.html │ └── video-decoder-vp8-opus.js ├── video-encoder-vp8 │ ├── index.html │ └── video-encoder-vp8.js ├── webcam-encoder │ ├── index.html │ └── webcam-encoder.js ├── webcam-round-trip │ ├── index.html │ └── webcam-round-trip.js └── worker-util.js ├── src ├── audio-data.ts ├── audio-decoder.ts ├── audio-encoder.ts ├── avloader.ts ├── config.ts ├── encoded-audio-chunk.ts ├── encoded-video-chunk.ts ├── event-target.ts ├── main.ts ├── misc.ts ├── rendering.ts ├── video-decoder.ts ├── video-encoder.ts └── video-frame.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | dist/ 3 | node_modules/ 4 | src/*.js 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | src/*.js 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: dist/libavjs-webcodecs-polyfill.min.js 2 | 3 | dist/libavjs-webcodecs-polyfill.min.js: src/*.ts node_modules/.bin/tsc 4 | npm run build 5 | 6 | node_modules/.bin/tsc: 7 | npm install 8 | 9 | clean: 10 | npm run clean 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LibAVJS-WebCodecs-Polyfill 2 | 3 | This is a polyfill for the [WebCodecs API](https://w3c.github.io/webcodecs/). 4 | 5 | No, really. 6 | 7 | It supports the `VideoEncoder`, `AudioEncoder`, `VideoDecoder`, and 8 | `AudioDecoder` classes, `VideoFrame`-specific versions of 9 | `CanvasRenderingContext2D.drawImage` and `createImageBitmap`, and all the 10 | classes and interfaces required by these. There are no plans to implement image 11 | formats, only video and audio. 12 | 13 | It implements WebCodecs through 14 | [libav.js](https://github.com/Yahweasel/libav.js/), which is a port of 15 | [FFmpeg](https://ffmpeg.org/)'s library interface to WebAssembly and asm.js. 16 | 17 | To use it, simply include libav.js then this library, and then call and `await 18 | LibAVWebCodecs.load()`. `load` takes an optional `options` parameter, which is 19 | an object: 20 | 21 | ``` 22 | options: { 23 | /* Polyfill: If the WebCodecs API is not provided by the browser in the 24 | * global object, link it to this library */ 25 | polyfill?: boolean, 26 | 27 | /* Options to pass to LibAV.LibAV while constructing a LibAV instance */ 28 | libavOptions?: any 29 | } 30 | ``` 31 | 32 | Use it either by the WebCodecs API specification (if you used `polyfill`), or 33 | as a [ponyfill](https://ponyfill.com), with the API under the global 34 | `LibAVWebCodecs` object. 35 | 36 | If you don't bring your own libav.js, LibAVJS-WebCodecs-Polyfill will load its 37 | own. If you load LibAVJS-WebCodecs-Polyfill in the browser context (and not a 38 | worker thread), it is highly recommended that you do *not* use this option, 39 | because libav.js is designed to use Web Workers, and Web Workers cannot be 40 | loaded from a different origin. This will hurt both performance and 41 | responsiveness. That is, it is recommended that *either* you load libav.js 42 | yourself, *or* you use LibAVJS-WebCodecs-Polyfill in a Worker thread (or both!). 43 | 44 | For rendering, it is highly recommended that you use 45 | `LibAVWebCodecs.createImageBitmap` and draw the result on a canvas, rather than 46 | `LibAVWebCodecs.canvasDrawImage`, which is synchronous. 47 | `LibAVWebCodecs.createImageBitmap` only accepts the `resizeWidth` and 48 | `resizeHeight` options, so only the overload 49 | `LibAVWebCodecs.createImageBitmap(frame, options)` is supported, with `options` 50 | optional. 51 | 52 | If you need the synchronous API, use `LibAVWebCodecs.canvasDrawImage(ctx, 53 | ...)`. The first argument is the context, and the remaining arguments are as in 54 | `CanvasRenderingContext2D.drawImage`. It is safe to use `canvasDrawImage` with 55 | any image type, not just a `VideoFrame`; it will fall through to the original 56 | `drawImage` as needed. If you used the `polyfill` option while loading 57 | LibAVJS-WebCodecs-Polyfill, then `drawImage` itself will also support 58 | `VideoFrame`s. 59 | 60 | 61 | ## Interaction with Browser WebCodecs 62 | 63 | You can use LibAVJS-WebCodecs-Polyfill along with a browser implementation of 64 | WebCodecs, but you cannot mix and match raw data objects from each (e.g., 65 | `VideoFrame`s from a browser implementation of WebCodecs cannot be used in 66 | LibAV-WebCodecs-Polyfill and vice-versa). 67 | 68 | To make this practical, `LibAVWebCodecs.getXY(config)` (where `X` = `Video` or 69 | `Audio` and `Y` = `Encoder` or `Decoder`) are implemented, and will return a 70 | promise for an object with, e.g. `VideoEncoder`, `EncodedVideoChunk`, and 71 | `VideoFrame` set to either WebCodecs' or LibAVJS-WebCodecs-Polyfill's version. 72 | The promise is rejected if the configuration is unsupported. 73 | 74 | In addition, you can convert between the two using functions provided by the 75 | polyfill. If you have a polyfill AudioData `ad`, you can use `ad.toNative()` to 76 | convert it to a browser WebCodecs AudioData, and if you have a browser WebCodecs 77 | AudioData `ad`, you can use `LibAVWebCodecs.AudioData.fromNative(ad)`. 78 | Similarly, you can convert VideoFrames with `vf.toNative()` or 79 | `LibAVWebCodecs.VideoFrame.fromNative(vf)`. 80 | 81 | Converting involves extra copying, so is best avoided when possible. But, 82 | sometimes it's not possible. 83 | 84 | 85 | ## Compatibility 86 | 87 | LibAVJS-WebCodecs-Polyfill should be up to date with the 2024-02-29 working 88 | draft of the WebCodecs specification: 89 | https://www.w3.org/TR/2024/WD-webcodecs-20240229/ 90 | 91 | Video support in LibAVJS-WebCodecs-Polyfill requires libav.js 5.1.6 or later. 92 | Audio support should work with libav.js 4.8.6 or later, but is of course usually 93 | tested only with the latest version. 94 | 95 | Depending on the libav.js variant used, LibAVJS-WebCodecs-Polyfill supports the 96 | audio codecs FLAC (`"flac"`), Opus (`"opus"`), and Vorbis (`"vorbis"`), and the 97 | video codecs AV1 (`"av01"`), VP9 (`"vp09"`), and VP8 (`"vp8"`). The 98 | `webm-vp9` variant, which LibAVJS-WebCodecs-Polyfill uses if no libav.js is 99 | loaded, supports FLAC, Opus, VP8, and VP9. 100 | 101 | FFmpeg supports many codecs, and it's generally easy to add new codecs to 102 | libav.js and LibAVJS-WebCodecs-Polyfill. However, there are no plans to add any 103 | codecs by the Misanthropic Patent Extortion Gang (MPEG), so all useful codecs 104 | in the WebCodecs codec registry are supported. 105 | 106 | LibAVJS-WebCodecs-Polyfill also supports bypassing the codec registry entirely 107 | and using any codec FFmpeg is capable of, by using the `LibAVJSCodec` interface 108 | (see `src/libav.ts`) instead of a string for the codec. For instance, 109 | `VideoEncoder` can be configured to use H.263+ like so: 110 | 111 | ``` 112 | const enc = new LibAVJS.VideoEncoder(...); 113 | enc.configure({ 114 | codec: {libavjs: { 115 | codec: "h263p", 116 | ctx: { 117 | pix_fmt: 0, 118 | width: settings.width, 119 | height: settings.height, 120 | framerate_num: settings.frameRate, 121 | framerate_den: 1 122 | } 123 | }}, 124 | ... 125 | }); 126 | ``` 127 | 128 | This is useful because VP8, even in realtime mode, is really too slow to 129 | encode/decode in software in WebAssembly on many modern systems, but a simpler 130 | codec like H.263+ works in software nearly anywhere. 131 | 132 | 133 | ## Limitations 134 | 135 | The `createImageBitmap` polyfill is quite limited in the arguments it accepts. 136 | 137 | libav.js is surprisingly fast for what it is, but it ain't fast. All audio 138 | codecs work fine, but video struggles. This is why support for codecs outside 139 | the codec registry was added. 140 | 141 | `VideoFrame` is fairly incomplete. In particular, nothing to do with color 142 | spaces is actually implemented, and nor is cropping. The initialization of 143 | frames from canvas sources has many caveats in the spec, and none in 144 | LibAVJS-WebCodecs-Polyfill, and as a consequence, `timestamp` is always a 145 | mandatory field of `VideoFrameInit`. 146 | 147 | `VideoEncoder` assumes that `VideoFrame`s passed to it are fairly sane (i.e., 148 | the planes are lain out in the obvious way). 149 | 150 | Certain events are supposed to eagerly halt the event queue, but 151 | LibAVJS-WebCodecs-Polyfill always lets the event queue finish. 152 | 153 | The framerate reported to video codecs is the nearest whole number to the input 154 | framerate. This should usually only affect bitrate and latency calculations, as 155 | each frame is individually timestamped. 156 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "libavjs-webcodecs-polyfill", 3 | "version": "0.5.3", 4 | "description": "A WebCodecs polyfill (ponyfill, really), using libav.js", 5 | "main": "dist/libavjs-webcodecs-polyfill.js", 6 | "types": "src/main.ts", 7 | "exports": { 8 | "import": "./dist/libavjs-webcodecs-polyfill.mjs", 9 | "default": "./dist/libavjs-webcodecs-polyfill.js", 10 | "types": "./src/main.ts" 11 | }, 12 | "scripts": { 13 | "build": "tsc && rollup -c", 14 | "clean": "rm -rf dist/", 15 | "test": "echo \"Error: no test specified\" && exit 1" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/ennuicastr/libavjs-webcodecs-polyfill.git" 20 | }, 21 | "keywords": [ 22 | "webcodecs", 23 | "ffmpeg", 24 | "audio", 25 | "video", 26 | "encoding", 27 | "decoding" 28 | ], 29 | "author": "Yahweasel", 30 | "license": "0BSD", 31 | "bugs": { 32 | "url": "https://github.com/ennuicastr/libavjs-webcodecs-polyfill/issues" 33 | }, 34 | "homepage": "https://github.com/ennuicastr/libavjs-webcodecs-polyfill#readme", 35 | "dependencies": { 36 | "@libav.js/types": "^6.5.7", 37 | "@ungap/global-this": "^0.4.4" 38 | }, 39 | "devDependencies": { 40 | "@libav.js/variant-webm-vp9": "=6.5.7", 41 | "@rollup/plugin-commonjs": "^25.0.7", 42 | "@rollup/plugin-node-resolve": "^15.2.3", 43 | "@rollup/plugin-terser": "^0.4.4", 44 | "rollup": "^4.12.0", 45 | "typescript": "^5.2.2" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /rollup.config.mjs: -------------------------------------------------------------------------------- 1 | import nodeResolve from "@rollup/plugin-node-resolve"; 2 | import commonjs from "@rollup/plugin-commonjs"; 3 | import terser from "@rollup/plugin-terser"; 4 | 5 | export default { 6 | input: "src/main.js", 7 | output: [ 8 | { 9 | file: "dist/libavjs-webcodecs-polyfill.js", 10 | format: "umd", 11 | name: "LibAVWebCodecs" 12 | }, { 13 | file: "dist/libavjs-webcodecs-polyfill.min.js", 14 | format: "umd", 15 | name: "LibAVWebCodecs" 16 | }, { 17 | file: "dist/libavjs-webcodecs-polyfill.mjs", 18 | format: "es" 19 | }, { 20 | file: "dist/libavjs-webcodecs-polyfill.min.mjs", 21 | format: "es", 22 | plugins: [terser()] 23 | } 24 | ], 25 | context: "this", 26 | plugins: [nodeResolve(), commonjs()] 27 | }; 28 | -------------------------------------------------------------------------------- /samples/audio-decoder-flac/audio-decoder-flac.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[stream], allPackets] = 32 | await sampleDemux("../sample1.flac", "flac"); 33 | const packets = allPackets[stream.index]; 34 | 35 | const init = { 36 | codec: "flac", 37 | sampleRate: 48000, 38 | numberOfChannels: 2, 39 | description: stream.extradata 40 | }; 41 | 42 | const a = await decodeAudio( 43 | init, packets, stream, LibAVWebCodecs.AudioDecoder, 44 | LibAVWebCodecs.EncodedAudioChunk); 45 | let b = null; 46 | if (typeof AudioDecoder !== "undefined") { 47 | try { 48 | b = await decodeAudio( 49 | init, packets, stream, AudioDecoder, EncodedAudioChunk); 50 | } catch (ex) { 51 | console.error(ex); 52 | } 53 | } 54 | 55 | postMessage({a, b}); 56 | })(); 57 | -------------------------------------------------------------------------------- /samples/audio-decoder-flac/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Audio decoder (FLAC) 30 | 31 | 32 | 33 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /samples/audio-decoder-opus/audio-decoder-opus.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[stream], allPackets] = 32 | await sampleDemux("../sample1.opus", "opus"); 33 | const packets = allPackets[stream.index]; 34 | 35 | const init = { 36 | codec: "opus", 37 | sampleRate: 48000, 38 | numberOfChannels: 2 39 | }; 40 | 41 | const a = await decodeAudio( 42 | init, packets, stream, LibAVWebCodecs.AudioDecoder, 43 | LibAVWebCodecs.EncodedAudioChunk); 44 | let b = null; 45 | if (typeof AudioDecoder !== "undefined") { 46 | try { 47 | b = await decodeAudio( 48 | init, packets, stream, AudioDecoder, EncodedAudioChunk); 49 | } catch (ex) { 50 | console.error(ex); 51 | } 52 | } 53 | 54 | postMessage({a, b}); 55 | })(); 56 | -------------------------------------------------------------------------------- /samples/audio-decoder-opus/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Audio decoder (Opus) 30 | 31 | 32 | 33 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /samples/audio-encoder-flac/audio-encoder-flac.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[stream], allPackets] = 32 | await sampleDemux("../sample1.flac", "flac"); 33 | const packets = allPackets[stream.index]; 34 | 35 | const init = { 36 | codec: "flac", 37 | sampleRate: 48000, 38 | numberOfChannels: 2, 39 | description: stream.extradata 40 | }; 41 | 42 | // First decode it 43 | const frames = await decodeAudio( 44 | init, packets, stream, LibAVWebCodecs.AudioDecoder, 45 | LibAVWebCodecs.EncodedAudioChunk, {noextract: true}); 46 | 47 | // Then encode it as FLAC 48 | async function encode(AudioEncoder, AudioData) { 49 | const packets = []; 50 | let extradata = null; 51 | const encoder = new AudioEncoder({ 52 | output: (packet, metadata) => { 53 | packets.push(packet); 54 | if (!extradata && metadata && metadata.decoderConfig && metadata.decoderConfig.description) { 55 | const desc = metadata.decoderConfig.description; 56 | extradata = new Uint8Array(desc.buffer || desc); 57 | } 58 | }, 59 | error: x => alert(x) 60 | }); 61 | encoder.configure({ 62 | codec: "flac", 63 | sampleRate: 48000, 64 | numberOfChannels: 2 65 | }); 66 | 67 | /* NOTE: This direct-copy (_libavGetData) is here only because built-in 68 | * WebCodecs can't use our AudioData. Do not use it in production code. */ 69 | for (const frame of frames) { 70 | encoder.encode(new AudioData({ 71 | format: frame.format, 72 | sampleRate: frame.sampleRate, 73 | numberOfFrames: frame.numberOfFrames, 74 | numberOfChannels: frame.numberOfChannels, 75 | timestamp: frame.timestamp, 76 | data: frame._libavGetData() 77 | })); 78 | } 79 | 80 | await encoder.flush(); 81 | encoder.close(); 82 | 83 | const flac = await sampleMux("tmp.flac", "flac", packets, extradata); 84 | return flac; 85 | } 86 | 87 | const a = await encode(LibAVWebCodecs.AudioEncoder, LibAVWebCodecs.AudioData); 88 | let b = null; 89 | if (typeof AudioEncoder !== "undefined") { 90 | try { 91 | b = await encode(AudioEncoder, AudioData); 92 | } catch (ex) { console.error(ex); } 93 | } 94 | postMessage({a, b}); 95 | })(); 96 | -------------------------------------------------------------------------------- /samples/audio-encoder-flac/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Audio encoder (FLAC) 30 | 31 | 32 | 33 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /samples/audio-encoder-opus/audio-encoder-opus.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[stream], allPackets] = 32 | await sampleDemux("../sample1.flac", "flac"); 33 | const packets = allPackets[stream.index]; 34 | 35 | const init = { 36 | codec: "flac", 37 | sampleRate: 48000, 38 | numberOfChannels: 2, 39 | description: stream.extradata 40 | }; 41 | 42 | // First decode it 43 | const frames = await decodeAudio( 44 | init, packets, stream, LibAVWebCodecs.AudioDecoder, 45 | LibAVWebCodecs.EncodedAudioChunk, {noextract: true}); 46 | 47 | // Then encode it as Opus 48 | async function encode(AudioEncoder, AudioData) { 49 | const packets = []; 50 | let extradata = null; 51 | const encoder = new AudioEncoder({ 52 | output: (packet, metadata) => { 53 | packets.push(packet); 54 | if (!extradata && metadata && metadata.decoderConfig && metadata.decoderConfig.description) { 55 | const desc = metadata.decoderConfig.description; 56 | extradata = new Uint8Array(desc.buffer || desc); 57 | } 58 | }, 59 | error: x => alert(x) 60 | }); 61 | encoder.configure({ 62 | codec: "opus", 63 | sampleRate: 48000, 64 | numberOfChannels: 2, 65 | bitrate: 128000 66 | }); 67 | 68 | /* NOTE: This direct-copy (_libavGetData) is here only because built-in 69 | * WebCodecs can't use our AudioData. Do not use it in production code. */ 70 | for (const frame of frames) { 71 | encoder.encode(new AudioData({ 72 | format: frame.format, 73 | sampleRate: frame.sampleRate, 74 | numberOfFrames: frame.numberOfFrames, 75 | numberOfChannels: frame.numberOfChannels, 76 | timestamp: frame.timestamp, 77 | data: frame._libavGetData() 78 | })); 79 | } 80 | 81 | await encoder.flush(); 82 | encoder.close(); 83 | 84 | const opus = await sampleMux("tmp.webm", "libopus", packets, extradata); 85 | return opus; 86 | } 87 | 88 | const a = await encode(LibAVWebCodecs.AudioEncoder, LibAVWebCodecs.AudioData); 89 | let b = null; 90 | if (typeof AudioEncoder !== "undefined") { 91 | try { 92 | b = await encode(AudioEncoder, AudioData); 93 | } catch (ex) { console.error(ex); } 94 | } 95 | postMessage({a, b}); 96 | })(); 97 | -------------------------------------------------------------------------------- /samples/audio-encoder-opus/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Audio encoder (Opus) 30 | 31 | 32 | 33 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /samples/sample1.flac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ennuicastr/libavjs-webcodecs-polyfill/0c01f854f543057c63e28a5dca2c3a3e098531af/samples/sample1.flac -------------------------------------------------------------------------------- /samples/sample1.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ennuicastr/libavjs-webcodecs-polyfill/0c01f854f543057c63e28a5dca2c3a3e098531af/samples/sample1.ogg -------------------------------------------------------------------------------- /samples/sample1.opus: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ennuicastr/libavjs-webcodecs-polyfill/0c01f854f543057c63e28a5dca2c3a3e098531af/samples/sample1.opus -------------------------------------------------------------------------------- /samples/sample2.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ennuicastr/libavjs-webcodecs-polyfill/0c01f854f543057c63e28a5dca2c3a3e098531af/samples/sample2.webm -------------------------------------------------------------------------------- /samples/util.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | async function sampleCompareAudio(a, b) { 27 | // Quick concat 28 | let blob = new Blob(a); 29 | a = new Float32Array(await blob.arrayBuffer()); 30 | blob = new Blob(b); 31 | b = new Float32Array(await blob.arrayBuffer()); 32 | 33 | let diff = Array.from(a).map((x, idx) => Math.abs(x - b[idx])).reduce((x, y) => x + y); 34 | const div = document.createElement("div"); 35 | div.innerText = `Difference: ${diff}`; 36 | document.body.appendChild(div); 37 | } 38 | 39 | async function sampleOutputAudio(a) { 40 | // Quick concat 41 | const blob = new Blob(a); 42 | a = new Float32Array(await blob.arrayBuffer()); 43 | 44 | const canvas = document.createElement("canvas"); 45 | canvas.style.display = "block"; 46 | const w = canvas.width = 1024; 47 | const h = canvas.height = 64; 48 | document.body.appendChild(canvas); 49 | const ctx = canvas.getContext("2d"); 50 | 51 | for (let x = 0; x < w; x++) { 52 | const idx = Math.floor((x / w) * a.length); 53 | const y = h - (h * Math.abs(a[idx])); 54 | ctx.fillStyle = "#fff"; 55 | ctx.fillRect(x, 0, 1, y); 56 | ctx.fillStyle = "#0f0"; 57 | ctx.fillRect(x, y, 1, h - y); 58 | } 59 | } 60 | 61 | function sampleOutputVideo(v, fps) { 62 | const canvas = document.createElement("canvas"); 63 | canvas.style.display = "block"; 64 | const w = canvas.width = v[0].codedWidth; 65 | const h = canvas.height = v[0].codedHeight; 66 | document.body.appendChild(canvas); 67 | const ctx = canvas.getContext("2d"); 68 | 69 | let idx = 0; 70 | const interval = setInterval(async () => { 71 | const image = await LibAVWebCodecs.createImageBitmap(v[idx++]); 72 | ctx.drawImage(image, 0, 0); 73 | 74 | if (idx >= v.length) 75 | idx = 0; 76 | }, Math.round(1000 / fps)) 77 | } 78 | -------------------------------------------------------------------------------- /samples/video-decoder-vp8-opus/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Video decoder (VP8) 30 | 31 | 32 | 33 | 34 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /samples/video-decoder-vp8-opus/video-decoder-vp8-opus.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[videoStream, audioStream], allPackets] = 32 | await sampleDemux("../sample2.webm", "webm"); 33 | const videoPackets = allPackets[videoStream.index]; 34 | const audioPackets = allPackets[audioStream.index]; 35 | 36 | const audioInit = { 37 | codec: "opus", 38 | sampleRate: 48000, 39 | numberOfChannels: 1 40 | }; 41 | 42 | async function decodeVideo(VideoDecoder, EncodedVideoChunk) { 43 | // Feed them into the decoder 44 | const frames = []; 45 | const decoder = new VideoDecoder({ 46 | output: frame => frames.push(frame), 47 | error: x => alert(x) 48 | }); 49 | decoder.configure({ 50 | codec: "vp8", 51 | codedWidth: 1920, 52 | codedHeight: 1080 53 | }); 54 | for (const packet of videoPackets) { 55 | let pts = packet.ptshi * 0x100000000 + packet.pts; 56 | if (pts < 0) 57 | pts = 0; 58 | const ts = Math.round( 59 | pts * videoStream.time_base_num / videoStream.time_base_den * 60 | 1000000); 61 | decoder.decode(new EncodedVideoChunk({ 62 | type: (packet.flags & 1) ? "key" : "delta", 63 | timestamp: ts, 64 | data: packet.data 65 | })); 66 | } 67 | 68 | // Wait for it to finish 69 | await decoder.flush(); 70 | decoder.close(); 71 | 72 | return frames; 73 | } 74 | 75 | const a = await decodeAudio( 76 | audioInit, audioPackets, audioStream, LibAVWebCodecs.AudioDecoder, 77 | LibAVWebCodecs.EncodedAudioChunk); 78 | let b = null; 79 | if (typeof AudioDecoder !== "undefined") { 80 | try { 81 | b = await decodeAudio( 82 | audioInit, audioPackets, audioStream, AudioDecoder, 83 | EncodedAudioChunk); 84 | } catch (ex) { 85 | console.error(ex); 86 | } 87 | } 88 | const va = await decodeVideo( 89 | LibAVWebCodecs.VideoDecoder, LibAVWebCodecs.EncodedVideoChunk); 90 | let vb = null; 91 | if (typeof VideoDecoder !== "undefined") { 92 | try { 93 | vb = await decodeVideo(VideoDecoder, EncodedVideoChunk); 94 | } catch (ex) { 95 | console.error(ex); 96 | } 97 | } 98 | 99 | postMessage({a, b, va, vb}); 100 | })(); 101 | -------------------------------------------------------------------------------- /samples/video-encoder-vp8/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Video encoder (VP8) 30 | 31 | 32 | 33 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /samples/video-encoder-vp8/video-encoder-vp8.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | importScripts("../worker-util.js"); 27 | 28 | (async function() { 29 | await LibAVWebCodecs.load(); 30 | 31 | const [[videoStream, audioStream], allPackets] = 32 | await sampleDemux("../sample2.webm", "webm"); 33 | const packets = allPackets[videoStream.index]; 34 | 35 | async function decodeVideo(VideoDecoder, EncodedVideoChunk) { 36 | // Feed them into the decoder 37 | const frames = []; 38 | const decoder = new VideoDecoder({ 39 | output: frame => frames.push(frame), 40 | error: x => console.error 41 | }); 42 | decoder.configure({ 43 | codec: "vp8", 44 | codedWidth: 1920, 45 | codedHeight: 1080 46 | }); 47 | for (const packet of packets) { 48 | let pts = packet.ptshi * 0x100000000 + packet.pts; 49 | if (pts < 0) 50 | pts = 0; 51 | const ts = Math.round( 52 | pts * videoStream.time_base_num / videoStream.time_base_den * 53 | 1000000); 54 | decoder.decode(new EncodedVideoChunk({ 55 | type: (packet.flags & 1) ? "key" : "delta", 56 | timestamp: ts, 57 | data: packet.data 58 | })); 59 | } 60 | 61 | // Wait for it to finish 62 | await decoder.flush(); 63 | decoder.close(); 64 | 65 | return frames; 66 | } 67 | 68 | // First decode it 69 | let preDecode = performance.now(); 70 | const frames = await decodeVideo( 71 | LibAVWebCodecs.VideoDecoder, LibAVWebCodecs.EncodedVideoChunk); 72 | let postDecode = performance.now(); 73 | 74 | // Then encode it as VP8 75 | async function encode(VideoEncoder, VideoFrame) { 76 | const packets = []; 77 | const encoder = new VideoEncoder({ 78 | output: packet => packets.push(packet), 79 | error: x => { throw new Error(x); } 80 | }); 81 | encoder.configure({ 82 | codec: "vp8", 83 | width: 1920, 84 | height: 1080, 85 | framerate: 25, 86 | latencyMode: "realtime" 87 | }); 88 | 89 | /* NOTE: This direct-copy (_libavGetData) is here only because built-in 90 | * WebCodecs can't use our VideoData. Do not use it in production code. */ 91 | for (const frame of frames) { 92 | encoder.encode(new VideoFrame(frame._libavGetData(), { 93 | layout: frame._libavGetLayout(), 94 | format: frame.format, 95 | codedWidth: frame.codedWidth, 96 | codedHeight: frame.codedHeight, 97 | timestamp: frame.timestamp 98 | })); 99 | } 100 | 101 | await encoder.flush(); 102 | encoder.close(); 103 | 104 | return await sampleMux("tmp.webm", "libvpx", packets); 105 | } 106 | 107 | let preEncode = performance.now(); 108 | const a = await encode(LibAVWebCodecs.VideoEncoder, LibAVWebCodecs.VideoFrame); 109 | let postEncode = performance.now(); 110 | let b = null; 111 | if (typeof VideoEncoder !== "undefined") { 112 | try { 113 | b = await encode(VideoEncoder, VideoFrame); 114 | } catch (ex) { console.error(ex); } 115 | } 116 | let postEncode2 = performance.now(); 117 | 118 | postMessage({ 119 | a, b, 120 | report: "Decode time: " + ~~(postDecode - preDecode) + 121 | "ms. Encode time: " + ~~(postEncode - preEncode) + 122 | "ms. Encode time (browser implementation): " + 123 | ~~(postEncode2 - postEncode) + "ms." 124 | }); 125 | })(); 126 | -------------------------------------------------------------------------------- /samples/webcam-encoder/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Live webcam encoder 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /samples/webcam-encoder/webcam-encoder.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | (async function() { 27 | await LibAVWebCodecs.load(); 28 | 29 | // Create the elements we need 30 | const videoEl = document.createElement("video"); 31 | videoEl.style.display = "none"; 32 | document.body.appendChild(videoEl); 33 | 34 | const btn = document.createElement("button"); 35 | btn.style.display = "block"; 36 | btn.innerText = "Start"; 37 | btn.onclick = () => { go(LibAVWebCodecs); }; 38 | document.body.appendChild(btn); 39 | 40 | async function go(WebCodecs) { 41 | // Get our input stream 42 | const mediaStream = await navigator.mediaDevices.getUserMedia({video: true}); 43 | videoEl.srcObject = mediaStream; 44 | await videoEl.play(); 45 | const settings = mediaStream.getVideoTracks()[0].getSettings(); 46 | 47 | // Make a stop button 48 | let stop = false; 49 | btn.innerText = "Stop"; 50 | btn.onclick = () => { 51 | stop = true; 52 | btn.innerText = "..."; 53 | btn.onclick = () => {}; 54 | }; 55 | 56 | // Make our encoder 57 | const packets = []; 58 | const encoder = new WebCodecs.VideoEncoder({ 59 | output: packet => packets.push(packet), 60 | error: x => alert(x) 61 | }); 62 | encoder.configure({ 63 | codec: "vp8", 64 | width: settings.width, 65 | height: settings.height, 66 | framerate: settings.frameRate, 67 | latencyMode: "realtime" 68 | }); 69 | 70 | // Encode 71 | const startTime = performance.now(); 72 | await new Promise(function(res, rej) { 73 | const interval = setInterval(async function() { 74 | // Maybe stop 75 | if (stop) { 76 | clearInterval(interval); 77 | res(); 78 | return; 79 | } 80 | 81 | // Maybe skip this frame 82 | if (encoder.encodeQueueSize) { 83 | console.log("WARNING: Skipping frame!"); 84 | return; 85 | } 86 | 87 | // Make the frame 88 | const frame = new WebCodecs.VideoFrame(videoEl, { 89 | timestamp: (performance.now() - startTime) * 1000 90 | }); 91 | 92 | // And enqueue it 93 | encoder.encode(frame); 94 | }, Math.round(1000 / settings.frameRate)); 95 | }); 96 | 97 | // Wait for encoding to finish 98 | await encoder.flush(); 99 | encoder.close(); 100 | await videoEl.pause(); 101 | 102 | // And display it 103 | const out = await sampleMux("tmp.webm", "libvpx", packets); 104 | const video = document.createElement("video"); 105 | video.style.display = "block"; 106 | video.src = URL.createObjectURL(new Blob([out])); 107 | video.controls = true; 108 | document.body.appendChild(video); 109 | 110 | btn.innerText = "Start"; 111 | btn.onclick = () => { go(LibAVWebCodecs); }; 112 | } 113 | 114 | // Make a no-polyfill option 115 | if (typeof VideoEncoder !== "undefined") { 116 | const noPoly = document.createElement("button"); 117 | noPoly.style.display = "block"; 118 | noPoly.innerText = "Switch to browser encoder"; 119 | noPoly.onclick = () => { 120 | if (btn.innerText === "Start") { 121 | btn.innerText = "Start (no polyfill)"; 122 | btn.onclick = () => { go(window); }; 123 | } 124 | }; 125 | document.body.appendChild(noPoly); 126 | } 127 | })(); 128 | -------------------------------------------------------------------------------- /samples/webcam-round-trip/index.html: -------------------------------------------------------------------------------- 1 | 2 | 26 | 27 | 28 | 29 | LibAVJS WebCodecs Polyfill Example: Webcam round trip 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /samples/webcam-round-trip/webcam-round-trip.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | (async function() { 27 | await LibAVWebCodecs.load(); 28 | 29 | const canvas = document.createElement("canvas"); 30 | canvas.width = 640; 31 | canvas.height = 360; 32 | document.body.appendChild(canvas); 33 | const ctx = canvas.getContext("2d"); 34 | 35 | const videoEl = document.createElement("video"); 36 | videoEl.style.display = "none"; 37 | document.body.appendChild(videoEl); 38 | 39 | // With polyfill 40 | const btn = document.createElement("button"); 41 | btn.style.display = "block"; 42 | btn.innerText = "Start"; 43 | btn.onclick = () => { go(LibAVWebCodecs); }; 44 | document.body.appendChild(btn); 45 | 46 | // Without polyfill 47 | let noPoly = null; 48 | if (typeof VideoEncoder !== "undefined") { 49 | noPoly = document.createElement("button"); 50 | noPoly.style.display = "block"; 51 | noPoly.innerText = "Start (no polyfill)"; 52 | noPoly.onclick = () => { go(window); }; 53 | document.body.appendChild(noPoly); 54 | } 55 | 56 | async function go(WebCodecs) { 57 | btn.style.display = "none"; 58 | if (noPoly) noPoly.style.display = "none"; 59 | 60 | // Get our input stream 61 | const mediaStream = await navigator.mediaDevices.getUserMedia({video: true}); 62 | videoEl.srcObject = mediaStream; 63 | await videoEl.play(); 64 | const settings = mediaStream.getVideoTracks()[0].getSettings(); 65 | 66 | // Make our encoder 67 | const encoder = new WebCodecs.VideoEncoder({ 68 | output: encoderOutput, 69 | error: x => alert(x) 70 | }); 71 | encoder.configure({ 72 | codec: "vp8", 73 | width: settings.width, 74 | height: settings.height, 75 | framerate: settings.frameRate, 76 | latencyMode: "realtime" 77 | }); 78 | 79 | // Make our decoder 80 | const decoder = new WebCodecs.VideoDecoder({ 81 | output: decoderOutput, 82 | error: x => alert(x) 83 | }); 84 | decoder.configure({ 85 | codec: "vp8" 86 | }); 87 | 88 | function encoderOutput(data) { 89 | if (decoder.decodeQueueSize) { 90 | console.log("WARNING: Skipping decoding frame"); 91 | return; 92 | } 93 | decoder.decode(data); 94 | } 95 | 96 | async function decoderOutput(frame) { 97 | const image = await LibAVWebCodecs.createImageBitmap(frame, 98 | {resizeWidth: 640, resizeHeight: 360}); 99 | ctx.drawImage(image, 0, 0); 100 | image.close(); 101 | } 102 | 103 | // And encode 104 | const startTime = performance.now(); 105 | setInterval(async () => { 106 | if (encoder.encodeQueueSize) { 107 | console.log("WARNING: Skipping encoding frame!"); 108 | return; 109 | } 110 | 111 | // Make the frame 112 | const frame = new WebCodecs.VideoFrame(videoEl, { 113 | timestamp: (performance.now() - startTime) * 1000 114 | }); 115 | 116 | // And enqueue it 117 | encoder.encode(frame); 118 | }, Math.round(1000 / settings.frameRate)); 119 | } 120 | })(); 121 | -------------------------------------------------------------------------------- /samples/worker-util.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This (un)license applies only to this sample code, and not to 3 | * libavjs-webcodecs-polyfill as a whole: 4 | * 5 | * This is free and unencumbered software released into the public domain. 6 | * 7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute 8 | * this software, either in source code form or as a compiled binary, for any 9 | * purpose, commercial or non-commercial, and by any means. 10 | * 11 | * In jurisdictions that recognize copyright laws, the author or authors of 12 | * this software dedicate any and all copyright interest in the software to the 13 | * public domain. We make this dedication for the benefit of the public at 14 | * large and to the detriment of our heirs and successors. We intend this 15 | * dedication to be an overt act of relinquishment in perpetuity of all present 16 | * and future rights to this software under copyright law. 17 | * 18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | if (typeof importScripts !== "undefined") { 27 | const url = new URL(self.location.href); 28 | if (typeof LibAV === "undefined") { 29 | console.log("NOTE: worker-util.js will load libav.js assuming it's installed via node. If it's not, this demo will not work."); 30 | 31 | LibAV = { 32 | base: "../../node_modules/@libav.js/variant-webm-vp9/dist" 33 | }; 34 | importScripts(LibAV.base + "/libav-6.0.7.0.2-webm-vp9.js"); 35 | } 36 | importScripts("../../dist/libavjs-webcodecs-polyfill.js"); 37 | } 38 | 39 | async function sampleDemux(filename, suffix) { 40 | const inF = new Uint8Array(await (await fetch(filename)).arrayBuffer()); 41 | const libav = await LibAV.LibAV({noworker: true}); 42 | await libav.writeFile("tmp." + suffix, inF); 43 | const [fmt_ctx, streams] = await libav.ff_init_demuxer_file("tmp." + suffix); 44 | for (const stream of streams) { 45 | const extradataPtr = await libav.AVCodecParameters_extradata(stream.codecpar); 46 | if (extradataPtr) { 47 | const len = await libav.AVCodecParameters_extradata_size(stream.codecpar); 48 | stream.extradata = await libav.copyout_u8(extradataPtr, len); 49 | } 50 | } 51 | const pkt = await libav.av_packet_alloc(); 52 | const [, packets] = await libav.ff_read_frame_multi(fmt_ctx, pkt); 53 | libav.terminate(); 54 | return [streams, packets]; 55 | } 56 | 57 | async function sampleMux(filename, codec, packets, extradata) { 58 | const libavPackets = []; 59 | for (const packet of packets) { 60 | const ab = new ArrayBuffer(packet.byteLength); 61 | packet.copyTo(ab); 62 | const pts = LibAV.f64toi64(Math.floor(packet.timestamp / 1000)); 63 | libavPackets.push({ 64 | data: new Uint8Array(ab), 65 | pts: pts[0], ptshi: pts[1], 66 | dts: pts[0], dtshi: pts[1], 67 | flags: (packet.type === "key") ? 1 : 0 68 | }); 69 | } 70 | 71 | const libav = await LibAV.LibAV({noworker: true}); 72 | 73 | /* Decode a little bit (and use extradata) just to make sure everything 74 | * necessary for a header is in place */ 75 | let [, c, pkt, frame] = await libav.ff_init_decoder(codec); 76 | await libav.AVCodecContext_time_base_s(c, 1, 1000); 77 | await libav.ff_decode_multi(c, pkt, frame, [libavPackets[0]]); 78 | if (extradata) { 79 | const extradataPtr = await libav.malloc(extradata.length); 80 | await libav.copyin_u8(extradataPtr, extradata); 81 | await libav.AVCodecContext_extradata_s(c, extradataPtr); 82 | await libav.AVCodecContext_extradata_size_s(c, extradata.length); 83 | } 84 | 85 | // Now mux it 86 | const [oc, , pb] = await libav.ff_init_muxer( 87 | {filename, open: true}, [[c, 1, 1000]]); 88 | await libav.avformat_write_header(oc, 0); 89 | await libav.ff_write_multi(oc, pkt, libavPackets); 90 | await libav.av_write_trailer(oc); 91 | await libav.ff_free_muxer(oc, pb); 92 | const ret = await libav.readFile(filename); 93 | libav.terminate(); 94 | return ret; 95 | } 96 | 97 | async function decodeAudio( 98 | init, packets, stream, AudioDecoder, EncodedAudioChunk, opts = {} 99 | ) { 100 | // Feed them into the decoder 101 | const frames = []; 102 | const decoder = new AudioDecoder({ 103 | output: frame => frames.push(frame), 104 | error: console.error 105 | }); 106 | decoder.configure(init); 107 | for (const packet of packets) { 108 | let pts = packet.ptshi * 0x100000000 + packet.pts; 109 | if (pts < 0) 110 | pts = 0; 111 | const ts = Math.round( 112 | pts * stream.time_base_num / stream.time_base_den * 113 | 1000000); 114 | decoder.decode(new EncodedAudioChunk({ 115 | type: "key", 116 | timestamp: ts, 117 | data: packet.data 118 | })); 119 | } 120 | 121 | // Wait for it to finish 122 | await decoder.flush(); 123 | decoder.close(); 124 | 125 | // And output 126 | if (opts.noextract) 127 | return frames; 128 | const out = []; 129 | const copyOpts = { 130 | planeIndex: 0, 131 | format: "f32-planar" 132 | }; 133 | for (const frame of frames) { 134 | const ab = new ArrayBuffer(frame.allocationSize(copyOpts)); 135 | frame.copyTo(ab, copyOpts); 136 | out.push(new Float32Array(ab)); 137 | } 138 | 139 | return out; 140 | } 141 | -------------------------------------------------------------------------------- /src/audio-data.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | // General type for audio typed arrays 21 | type AudioTypedArray = Uint8Array | Int16Array | Int32Array | Float32Array; 22 | 23 | import "@ungap/global-this"; 24 | 25 | export class AudioData { 26 | constructor(init: AudioDataInit) { 27 | // 1. If init is not a valid AudioDataInit, throw a TypeError. 28 | AudioData._checkValidAudioDataInit(init); 29 | 30 | /* 2. If init.transfer contains more than one reference to the same 31 | * ArrayBuffer, then throw a DataCloneError DOMException. */ 32 | // 3. For each transferable in init.transfer: 33 | // 1. If [[Detached]] internal slot is true, then throw a DataCloneError DOMException. 34 | // (Not worth doing in polyfill) 35 | 36 | // 4. Let frame be a new AudioData object, initialized as follows: 37 | { 38 | 39 | // 1. Assign false to [[Detached]]. 40 | // (not doable in polyfill) 41 | 42 | // 2. Assign init.format to [[format]]. 43 | this.format = init.format; 44 | 45 | // 3. Assign init.sampleRate to [[sample rate]]. 46 | this.sampleRate = init.sampleRate; 47 | 48 | // 4. Assign init.numberOfFrames to [[number of frames]]. 49 | this.numberOfFrames = init.numberOfFrames; 50 | 51 | // 5. Assign init.numberOfChannels to [[number of channels]]. 52 | this.numberOfChannels = init.numberOfChannels; 53 | 54 | // 6. Assign init.timestamp to [[timestamp]]. 55 | this.timestamp = init.timestamp; 56 | 57 | /* 7. If init.transfer contains an ArrayBuffer referenced by 58 | * init.data the User Agent MAY choose to: */ 59 | let transfer = false; 60 | if (init.transfer) { 61 | 62 | // 1. Let resource be a new media resource referencing sample data in data. 63 | let inBuffer: ArrayBuffer; 64 | if (( init.data).buffer) 65 | inBuffer = ( init.data).buffer; 66 | else 67 | inBuffer = init.data; 68 | 69 | let t: ArrayBuffer[]; 70 | if (init.transfer instanceof Array) 71 | t = init.transfer; 72 | else 73 | t = Array.from(init.transfer); 74 | for (const b of t) { 75 | if (b === inBuffer) { 76 | transfer = true; 77 | break; 78 | } 79 | } 80 | } 81 | 82 | // 8. Otherwise: 83 | // 1. Let resource be a media resource containing a copy of init.data. 84 | 85 | // 9. Let resourceReference be a reference to resource. 86 | let inData: BufferSource, byteOffset = 0; 87 | if (transfer) { 88 | inData = init.data; 89 | byteOffset = ( init.data).byteOffset || 0; 90 | } else { 91 | inData = ( init.data).slice(0); 92 | } 93 | const resourceReference = audioView( 94 | init.format, ( inData).buffer || inData, byteOffset 95 | ); 96 | 97 | // 10. Assign resourceReference to [[resource reference]]. 98 | this._data = resourceReference; 99 | } 100 | 101 | // 5. For each transferable in init.transfer: 102 | // 1. Perform DetachArrayBuffer on transferable 103 | // (Already done by transferring) 104 | 105 | // 6. Return frame. 106 | 107 | // Duration not calculated in spec? 108 | this.duration = init.numberOfFrames / init.sampleRate * 1000000; 109 | } 110 | 111 | readonly format: AudioSampleFormat; 112 | readonly sampleRate: number; 113 | readonly numberOfFrames: number; 114 | readonly numberOfChannels: number; 115 | readonly duration: number; // microseconds 116 | readonly timestamp: number; // microseconds 117 | 118 | private _data: AudioTypedArray; 119 | 120 | /** 121 | * Convert a polyfill AudioData to a native AudioData. 122 | * @param opts Conversion options 123 | */ 124 | toNative(opts: { 125 | /** 126 | * Transfer the data, closing this AudioData. 127 | */ 128 | transfer?: boolean 129 | } = {}) { 130 | const ret = new ( globalThis).AudioData({ 131 | data: this._data, 132 | format: this.format, 133 | sampleRate: this.sampleRate, 134 | numberOfFrames: this.numberOfFrames, 135 | numberOfChannels: this.numberOfChannels, 136 | timestamp: this.timestamp, 137 | transfer: opts.transfer ? [this._data.buffer] : [] 138 | }); 139 | if (opts.transfer) 140 | this.close(); 141 | return ret; 142 | } 143 | 144 | /** 145 | * Convert a native AudioData to a polyfill AudioData. WARNING: Inefficient, 146 | * as the data cannot be transferred out. 147 | * @param from AudioData to copy in 148 | */ 149 | static fromNative(from: any /* native AudioData */) { 150 | const ad: AudioData = from; 151 | const isInterleaved_ = isInterleaved(ad.format); 152 | const planes = isInterleaved_ ? 1 : ad.numberOfChannels; 153 | const sizePerPlane = ad.allocationSize({ 154 | format: ad.format, 155 | planeIndex: 0 156 | }); 157 | const data = new Uint8Array(sizePerPlane * planes); 158 | for (let p = 0; p < planes; p++) { 159 | ad.copyTo(data.subarray(p * sizePerPlane), { 160 | format: ad.format, 161 | planeIndex: p 162 | }); 163 | } 164 | return new AudioData({ 165 | data, 166 | format: ad.format, 167 | sampleRate: ad.sampleRate, 168 | numberOfFrames: ad.numberOfFrames, 169 | numberOfChannels: ad.numberOfChannels, 170 | timestamp: ad.timestamp, 171 | transfer: [data.buffer] 172 | }); 173 | } 174 | 175 | // Internal 176 | _libavGetData() { return this._data; } 177 | 178 | private static _checkValidAudioDataInit(init: AudioDataInit) { 179 | // 1. If sampleRate less than or equal to 0, return false. 180 | if (init.sampleRate <= 0) 181 | throw new TypeError(`Invalid sample rate ${init.sampleRate}`); 182 | 183 | // 2. If numberOfFrames = 0, return false. 184 | if (init.numberOfFrames <= 0) 185 | throw new TypeError(`Invalid number of frames ${init.numberOfFrames}`); 186 | 187 | // 3. If numberOfChannels = 0, return false. 188 | if (init.numberOfChannels <= 0) 189 | throw new TypeError(`Invalid number of channels ${init.numberOfChannels}`); 190 | 191 | // 4. Verify data has enough data by running the following steps: 192 | { 193 | 194 | // 1. Let totalSamples be the product of multiplying numberOfFrames by numberOfChannels. 195 | const totalSamples = init.numberOfFrames * init.numberOfChannels; 196 | 197 | // 2. Let bytesPerSample be the number of bytes per sample, as defined by the format. 198 | const bytesPerSample_ = bytesPerSample(init.format); 199 | 200 | // 3. Let totalSize be the product of multiplying bytesPerSample with totalSamples. 201 | const totalSize = bytesPerSample_ * totalSamples; 202 | 203 | // 4. Let dataSize be the size in bytes of data. 204 | const dataSize = init.data.byteLength; 205 | 206 | // 5. If dataSize is less than totalSize, return false. 207 | if (dataSize < totalSize) 208 | throw new TypeError(`This audio data must be at least ${totalSize} bytes`); 209 | } 210 | 211 | // 5. Return true. 212 | } 213 | 214 | allocationSize(options: AudioDataCopyToOptions): number { 215 | // 1. If [[Detached]] is true, throw an InvalidStateError DOMException. 216 | if (this._data === null) 217 | throw new DOMException("Detached", "InvalidStateError"); 218 | 219 | /* 2. Let copyElementCount be the result of running the Compute Copy 220 | * Element Count algorithm with options. */ 221 | const copyElementCount = this._computeCopyElementCount(options); 222 | 223 | // 3. Let destFormat be the value of [[format]]. 224 | let destFormat = this.format; 225 | 226 | // 4. If options.format exists, assign options.format to destFormat. 227 | if (options.format) 228 | destFormat = options.format; 229 | 230 | /* 5. Let bytesPerSample be the number of bytes per sample, as defined 231 | * by the destFormat. */ 232 | const bytesPerSample_ = bytesPerSample(destFormat); 233 | 234 | /* 6. Return the product of multiplying bytesPerSample by 235 | * copyElementCount. */ 236 | return bytesPerSample_ * copyElementCount; 237 | } 238 | 239 | private _computeCopyElementCount(options: AudioDataCopyToOptions): number { 240 | // 1. Let destFormat be the value of [[format]]. 241 | let destFormat = this.format; 242 | 243 | // 2. If options.format exists, assign options.format to destFormat. 244 | if (options.format) 245 | destFormat = options.format; 246 | 247 | /* 3. If destFormat describes an interleaved AudioSampleFormat and 248 | * options.planeIndex is greater than 0, throw a RangeError. */ 249 | const isInterleaved_ = isInterleaved(destFormat); 250 | if (isInterleaved_) { 251 | if (options.planeIndex > 0) 252 | throw new RangeError("Invalid plane"); 253 | } 254 | 255 | /* 4. Otherwise, if destFormat describes a planar AudioSampleFormat and 256 | * if options.planeIndex is greater or equal to [[number of channels]], 257 | * throw a RangeError. */ 258 | else if (options.planeIndex >= this.numberOfChannels) 259 | throw new RangeError("Invalid plane"); 260 | 261 | /* 5. If [[format]] does not equal destFormat and the User Agent does 262 | * not support the requested AudioSampleFormat conversion, throw a 263 | * NotSupportedError DOMException. Conversion to f32-planar MUST always 264 | * be supported. */ 265 | if (this.format !== destFormat && 266 | destFormat !== "f32-planar") 267 | throw new DOMException("Only conversion to f32-planar is supported", "NotSupportedError"); 268 | 269 | /* 6. Let frameCount be the number of frames in the plane identified by 270 | * options.planeIndex. */ 271 | const frameCount = this.numberOfFrames; // All planes have the same number of frames 272 | 273 | /* 7. If options.frameOffset is greater than or equal to frameCount, 274 | * throw a RangeError. */ 275 | const frameOffset = options.frameOffset || 0; 276 | if (frameOffset >= frameCount) 277 | throw new RangeError("Frame offset out of range"); 278 | 279 | /* 8. Let copyFrameCount be the difference of subtracting 280 | * options.frameOffset from frameCount. */ 281 | let copyFrameCount = frameCount - frameOffset; 282 | 283 | // 9. If options.frameCount exists: 284 | if (typeof options.frameCount === "number") { 285 | /* 1. If options.frameCount is greater than copyFrameCount, throw a 286 | * RangeError. */ 287 | if (options.frameCount >= copyFrameCount) 288 | throw new RangeError("Frame count out of range"); 289 | 290 | // 2. Otherwise, assign options.frameCount to copyFrameCount. 291 | copyFrameCount = options.frameCount; 292 | } 293 | 294 | // 10. Let elementCount be copyFrameCount. 295 | let elementCount = copyFrameCount; 296 | 297 | /* 11. If destFormat describes an interleaved AudioSampleFormat, 298 | * mutliply elementCount by [[number of channels]] */ 299 | if (isInterleaved_) 300 | elementCount *= this.numberOfChannels; 301 | 302 | // 12. return elementCount. 303 | return elementCount; 304 | } 305 | 306 | copyTo(destination: BufferSource, options: AudioDataCopyToOptions): void { 307 | // 1. If [[Detached]] is true, throw an InvalidStateError DOMException. 308 | if (this._data === null) 309 | throw new DOMException("Detached", "InvalidStateError"); 310 | 311 | /* 2. Let copyElementCount be the result of running the Compute Copy 312 | * Element Count algorithm with options. */ 313 | const copyElementCount = this._computeCopyElementCount(options); 314 | 315 | // 3. Let destFormat be the value of [[format]]. 316 | let destFormat = this.format; 317 | 318 | // 4. If options.format exists, assign options.format to destFormat. 319 | if (options.format) 320 | destFormat = options.format; 321 | 322 | /* 5. Let bytesPerSample be the number of bytes per sample, as defined 323 | * by the destFormat. */ 324 | const bytesPerSample_ = bytesPerSample(destFormat); 325 | 326 | /* 6. If the product of multiplying bytesPerSample by copyElementCount 327 | * is greater than destination.byteLength, throw a RangeError. */ 328 | if (bytesPerSample_ * copyElementCount > destination.byteLength) 329 | throw new RangeError("Buffer too small"); 330 | 331 | /* 7. Let resource be the media resource referenced by [[resource 332 | * reference]]. */ 333 | const resource = this._data; 334 | 335 | /* 8. Let planeFrames be the region of resource corresponding to 336 | * options.planeIndex. */ 337 | const planeFrames = resource.subarray( 338 | options.planeIndex * this.numberOfFrames); 339 | 340 | const frameOffset = options.frameOffset || 0; 341 | const numberOfChannels = this.numberOfChannels; 342 | 343 | /* 9. Copy elements of planeFrames into destination, starting with the 344 | * frame positioned at options.frameOffset and stopping after 345 | * copyElementCount samples have been copied. If destFormat does not 346 | * equal [[format]], convert elements to the destFormat 347 | * AudioSampleFormat while making the copy. */ 348 | if (this.format === destFormat) { 349 | const dest = audioView(destFormat, 350 | ( destination).buffer || destination, 351 | ( destination).byteOffset || 0); 352 | 353 | if (isInterleaved(destFormat)) { 354 | dest.set(planeFrames.subarray( 355 | frameOffset * numberOfChannels, 356 | frameOffset * numberOfChannels + copyElementCount 357 | )); 358 | } else { 359 | dest.set(planeFrames.subarray( 360 | frameOffset, frameOffset + copyElementCount 361 | )); 362 | } 363 | 364 | } else { 365 | // Actual conversion necessary. Always to f32-planar. 366 | const out = audioView(destFormat, 367 | ( destination).buffer || destination, 368 | ( destination).byteOffset || 0); 369 | 370 | // First work out the conversion 371 | let sub = 0; 372 | let div = 1; 373 | switch (this.format) { 374 | case "u8": 375 | case "u8-planar": 376 | sub = 0x80; 377 | div = 0x80; 378 | break; 379 | 380 | case "s16": 381 | case "s16-planar": 382 | div = 0x8000; 383 | break; 384 | 385 | case "s32": 386 | case "s32-planar": 387 | div = 0x80000000; 388 | break; 389 | } 390 | 391 | // Then do it 392 | if (isInterleaved(this.format)) { 393 | for (let i = options.planeIndex + frameOffset * numberOfChannels, o = 0; 394 | o < copyElementCount; 395 | i += numberOfChannels, o++) 396 | out[o] = (planeFrames[i] - sub) / div; 397 | 398 | } else { 399 | for (let i = frameOffset, o = 0; 400 | o < copyElementCount; 401 | i++, o++) 402 | out[o] = (planeFrames[i] - sub) / div; 403 | } 404 | 405 | } 406 | } 407 | 408 | clone(): AudioData { 409 | // 1. If [[Detached]] is true, throw an InvalidStateError DOMException. 410 | if (this._data === null) 411 | throw new DOMException("Detached", "InvalidStateError"); 412 | 413 | /* 2. Return the result of running the Clone AudioData algorithm with 414 | * this. */ 415 | return new AudioData({ 416 | format: this.format, 417 | sampleRate: this.sampleRate, 418 | numberOfFrames: this.numberOfFrames, 419 | numberOfChannels: this.numberOfChannels, 420 | timestamp: this.timestamp, 421 | data: this._data 422 | }); 423 | } 424 | 425 | close(): void { 426 | this._data = null; 427 | } 428 | } 429 | 430 | export interface AudioDataInit { 431 | format: AudioSampleFormat; 432 | sampleRate: number; 433 | numberOfFrames: number; 434 | numberOfChannels: number; 435 | timestamp: number; 436 | data: BufferSource; 437 | transfer?: ArrayLike; 438 | } 439 | 440 | export type AudioSampleFormat = 441 | "u8" | 442 | "s16" | 443 | "s32" | 444 | "f32" | 445 | "u8-planar" | 446 | "s16-planar" | 447 | "s32-planar" | 448 | "f32-planar"; 449 | 450 | export interface AudioDataCopyToOptions { 451 | planeIndex: number; 452 | frameOffset?: number; 453 | frameCount?: number; 454 | format: AudioSampleFormat; 455 | } 456 | 457 | 458 | /** 459 | * Construct the appropriate type of ArrayBufferView for the given sample 460 | * format and buffer. 461 | * @param format Sample format 462 | * @param buffer ArrayBuffer (NOT view) 463 | * @param byteOffset Offset into the buffer 464 | */ 465 | function audioView( 466 | format: AudioSampleFormat, buffer: ArrayBuffer, byteOffset: number 467 | ): AudioTypedArray { 468 | switch (format) { 469 | case "u8": 470 | case "u8-planar": 471 | return new Uint8Array(buffer, byteOffset); 472 | 473 | case "s16": 474 | case "s16-planar": 475 | return new Int16Array(buffer, byteOffset); 476 | 477 | case "s32": 478 | case "s32-planar": 479 | return new Int32Array(buffer, byteOffset); 480 | 481 | case "f32": 482 | case "f32-planar": 483 | return new Float32Array(buffer, byteOffset); 484 | 485 | default: 486 | throw new TypeError("Invalid AudioSampleFormat"); 487 | } 488 | } 489 | 490 | /** 491 | * Number of bytes per sample of this format. 492 | * @param format Sample format 493 | */ 494 | function bytesPerSample(format: AudioSampleFormat): number { 495 | switch (format) { 496 | case "u8": 497 | case "u8-planar": 498 | return 1; 499 | 500 | case "s16": 501 | case "s16-planar": 502 | return 2; 503 | 504 | case "s32": 505 | case "s32-planar": 506 | case "f32": 507 | case "f32-planar": 508 | return 4; 509 | 510 | default: 511 | throw new TypeError("Invalid AudioSampleFormat"); 512 | } 513 | } 514 | 515 | /** 516 | * Is this format interleaved? 517 | * @param format Sample format 518 | */ 519 | export function isInterleaved(format: AudioSampleFormat) { 520 | switch (format) { 521 | case "u8": 522 | case "s16": 523 | case "s32": 524 | case "f32": 525 | return true; 526 | 527 | case "u8-planar": 528 | case "s16-planar": 529 | case "s32-planar": 530 | case "f32-planar": 531 | return false; 532 | 533 | default: 534 | throw new TypeError("Invalid AudioSampleFormat"); 535 | } 536 | } 537 | -------------------------------------------------------------------------------- /src/audio-decoder.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as ad from "./audio-data"; 21 | import * as eac from "./encoded-audio-chunk"; 22 | import * as et from "./event-target"; 23 | import * as libavs from "./avloader"; 24 | import * as misc from "./misc"; 25 | 26 | import type * as LibAVJS from "@libav.js/types"; 27 | 28 | export class AudioDecoder extends et.DequeueEventTarget { 29 | constructor(init: AudioDecoderInit) { 30 | super(); 31 | 32 | // 1. Let d be a new AudioDecoder object. 33 | 34 | // 2. Assign a new queue to [[control message queue]]. 35 | this._p = Promise.all([]); 36 | 37 | // 3. Assign false to [[message queue blocked]]. 38 | // (unused in polyfill) 39 | 40 | // 4. Assign null to [[codec implementation]]. 41 | this._libav = null; 42 | this._codec = this._c = this._pkt = this._frame = 0; 43 | 44 | // 5. Assign the result of starting a new parallel queue to [[codec work queue]]. 45 | // (shared with control message queue) 46 | 47 | // 6. Assign false to [[codec saturated]]. 48 | // (codec is never saturated) 49 | 50 | // 7. Assign init.output to [[output callback]]. 51 | this._output = init.output; 52 | 53 | // 8. Assign init.error to [[error callback]]. 54 | this._error = init.error; 55 | 56 | // 9. Assign true to [[key chunk required]]. 57 | // (implicit part of the underlying codec) 58 | 59 | // 10. Assign "unconfigured" to [[state]] 60 | this.state = "unconfigured"; 61 | 62 | // 11. Assign 0 to [[decodeQueueSize]]. 63 | this.decodeQueueSize = 0; 64 | 65 | // 12. Assign a new list to [[pending flush promises]]. 66 | // (shared with control message queue) 67 | 68 | // 13. Assign false to [[dequeue event scheduled]]. 69 | // (shared with control message queue) 70 | 71 | // 14. Return d. 72 | } 73 | 74 | /* NOTE: These should technically be readonly, but I'm implementing them as 75 | * plain fields, so they're writable */ 76 | state: misc.CodecState; 77 | decodeQueueSize: number; 78 | 79 | private _output: AudioDataOutputCallback; 80 | private _error: misc.WebCodecsErrorCallback; 81 | 82 | // Event queue 83 | private _p: Promise; 84 | 85 | // LibAV state 86 | private _libav: LibAVJS.LibAV | null; 87 | private _codec: number; 88 | private _c: number; 89 | private _pkt: number; 90 | private _frame: number; 91 | 92 | configure(config: AudioDecoderConfig): void { 93 | // 1. If config is not a valid AudioDecoderConfig, throw a TypeError. 94 | // NOTE: We don't support sophisticated codec string parsing (yet) 95 | 96 | // 2. If [[state]] is “closed”, throw an InvalidStateError DOMException. 97 | if (this.state === "closed") 98 | throw new DOMException("Decoder is closed", "InvalidStateError"); 99 | 100 | // Free any internal state 101 | if (this._libav) 102 | this._p = this._p.then(() => this._free()); 103 | 104 | // 3. Set [[state]] to "configured". 105 | this.state = "configured"; 106 | 107 | // 4. Set [[key chunk required]] to true. 108 | // (implicit part of underlying codecs) 109 | 110 | // 5. Queue a control message to configure the decoder with config. 111 | this._p = this._p.then(async () => { 112 | /* 1. Let supported be the result of running the Check 113 | * Configuration Support algorithm with config. */ 114 | let udesc: Uint8Array | undefined = void 0; 115 | if (config.description) { 116 | if (ArrayBuffer.isView(config.description)) { 117 | const descView = config.description as ArrayBufferView; 118 | udesc = new Uint8Array(descView.buffer, descView.byteOffset, descView.byteLength); 119 | } else { 120 | const descBuf = config.description as ArrayBuffer; 121 | udesc = new Uint8Array(descBuf); 122 | } 123 | } 124 | const supported = libavs.decoder(config.codec, config); 125 | 126 | /* 2. If supported is false, queue a task to run the Close 127 | * AudioDecoder algorithm with NotSupportedError and abort these 128 | * steps. */ 129 | if (!supported) { 130 | this._closeAudioDecoder(new DOMException("Unsupported codec", "NotSupportedError")); 131 | return; 132 | } 133 | 134 | /* 3. If needed, assign [[codec implementation]] with an 135 | * implementation supporting config. */ 136 | const libav = this._libav = await libavs.get(); 137 | const codecpara = await libav.avcodec_parameters_alloc(); 138 | const ps = [ 139 | libav.AVCodecParameters_channels_s(codecpara, config.numberOfChannels), 140 | libav.AVCodecParameters_sample_rate_s(codecpara, config.sampleRate), 141 | libav.AVCodecParameters_codec_type_s(codecpara, 1 /* AVMEDIA_TYPE_AUDIO */) 142 | ]; 143 | let extraDataPtr = 0; 144 | if (!udesc) { 145 | ps.push(libav.AVCodecParameters_extradata_s(codecpara, 0)); 146 | ps.push(libav.AVCodecParameters_extradata_size_s(codecpara, 0)); 147 | } else { 148 | ps.push(libav.AVCodecParameters_extradata_size_s(codecpara, udesc.byteLength)); 149 | extraDataPtr = await libav.calloc(udesc.byteLength + 64 /* AV_INPUT_BUFFER_PADDING_SIZE */, 1); 150 | ps.push(libav.copyin_u8(extraDataPtr, udesc)); 151 | ps.push(libav.AVCodecParameters_extradata_s(codecpara, extraDataPtr)) 152 | } 153 | await Promise.all(ps); 154 | 155 | // 4. Configure [[codec implementation]] with config. 156 | [this._codec, this._c, this._pkt, this._frame] = 157 | await libav.ff_init_decoder(supported.codec, codecpara); 158 | const fps = [ 159 | libav.AVCodecContext_time_base_s(this._c, 1, 1000), 160 | libav.avcodec_parameters_free_js(codecpara) 161 | ]; 162 | if (extraDataPtr) fps.push(libav.free(extraDataPtr)); 163 | await Promise.all(fps); 164 | 165 | // 5. queue a task to run the following steps: 166 | // 1. Assign false to [[message queue blocked]]. 167 | // 2. Queue a task to Process the control message queue. 168 | // (shared queue) 169 | 170 | }).catch(this._error); 171 | } 172 | 173 | // Our own algorithm, close libav 174 | private async _free() { 175 | if (this._c) { 176 | await this._libav!.ff_free_decoder(this._c, this._pkt, this._frame); 177 | this._codec = this._c = this._pkt = this._frame = 0; 178 | } 179 | if (this._libav) { 180 | libavs.free(this._libav); 181 | this._libav = null; 182 | } 183 | } 184 | 185 | private _closeAudioDecoder(exception: DOMException) { 186 | // 1. Run the Reset AudioDecoder algorithm with exception. 187 | this._resetAudioDecoder(exception); 188 | 189 | // 2. Set [[state]] to "closed". 190 | this.state = "closed"; 191 | 192 | /* 3. Clear [[codec implementation]] and release associated system 193 | * resources. */ 194 | this._p = this._p.then(() => this._free()); 195 | 196 | /* 4. If exception is not an AbortError DOMException, queue a task on 197 | * the control thread event loop to invoke the [[error callback]] with 198 | * exception. */ 199 | if (exception.name !== "AbortError") 200 | this._p = this._p.then(() => { this._error(exception); }); 201 | } 202 | 203 | private _resetAudioDecoder(exception: DOMException) { 204 | // 1. If [[state]] is "closed", throw an InvalidStateError. 205 | if (this.state === "closed") 206 | throw new DOMException("Decoder closed", "InvalidStateError"); 207 | 208 | // 2. Set [[state]] to "unconfigured". 209 | this.state = "unconfigured"; 210 | 211 | // ... really, we're just going to free it now 212 | this._p = this._p.then(() => this._free()); 213 | } 214 | 215 | decode(chunk: eac.EncodedAudioChunk) { 216 | // 1. If [[state]] is not "configured", throw an InvalidStateError. 217 | if (this.state !== "configured") 218 | throw new DOMException("Unconfigured", "InvalidStateError"); 219 | 220 | // 2. If [[key chunk required]] is true: 221 | // 1. If chunk.[[type]] is not key, throw a DataError. 222 | /* 2. Implementers SHOULD inspect the chunk’s [[internal data]] to 223 | * verify that it is truly a key chunk. If a mismatch is detected, 224 | * throw a DataError. */ 225 | // 3. Otherwise, assign false to [[key chunk required]]. 226 | // (handled within the codec) 227 | 228 | // 3. Increment [[decodeQueueSize]]. 229 | this.decodeQueueSize++; 230 | 231 | // 4. Queue a control message to decode the chunk. 232 | this._p = this._p.then(async () => { 233 | const libav = this._libav!; 234 | const c = this._c; 235 | const pkt = this._pkt; 236 | const frame = this._frame; 237 | 238 | let decodedOutputs: LibAVJS.Frame[] | null = null; 239 | 240 | // (1. and 2. relate to saturation) 241 | 242 | // 3. Decrement [[decodeQueueSize]] and run the Schedule Dequeue Event algorithm. 243 | this.decodeQueueSize--; 244 | this.dispatchEvent(new CustomEvent("dequeue")); 245 | 246 | // 1. Attempt to use [[codec implementation]] to decode the chunk. 247 | try { 248 | // Convert to a libav packet 249 | const ptsFull = Math.floor(chunk.timestamp / 1000); 250 | const [pts, ptshi] = libav.f64toi64(ptsFull); 251 | const packet: LibAVJS.Packet = { 252 | data: chunk._libavGetData(), 253 | pts, 254 | ptshi, 255 | dts: pts, 256 | dtshi: ptshi 257 | }; 258 | if (chunk.duration) { 259 | packet.duration = Math.floor(chunk.duration / 1000); 260 | packet.durationhi = 0; 261 | } 262 | 263 | decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [packet]); 264 | 265 | /* 2. If decoding results in an error, queue a task to run the Close 266 | * AudioDecoder algorithm with EncodingError and return. */ 267 | } catch (ex) { 268 | this._p = this._p.then(() => { 269 | this._closeAudioDecoder( ex); 270 | }); 271 | return; 272 | } 273 | 274 | /* 3. If [[codec saturated]] equals true and 275 | * [[codec implementation]] is no longer saturated, queue a task 276 | * to perform the following steps: */ 277 | // 1. Assign false to [[codec saturated]]. 278 | // 2. Process the control message queue. 279 | // (no saturation) 280 | 281 | /* 4. Let decoded outputs be a list of decoded audio data outputs 282 | * emitted by [[codec implementation]]. */ 283 | 284 | /* 5. If decoded outputs is not empty, queue a task to run the 285 | * Output AudioData algorithm with decoded outputs. */ 286 | if (decodedOutputs) 287 | this._outputAudioData(decodedOutputs); 288 | 289 | }).catch(this._error); 290 | } 291 | 292 | private _outputAudioData(outputs: LibAVJS.Frame[]) { 293 | const libav = this._libav!; 294 | 295 | for (const frame of outputs) { 296 | // 1. format 297 | let format: ad.AudioSampleFormat; 298 | let planar = false; 299 | switch (frame.format) { 300 | case libav.AV_SAMPLE_FMT_U8: 301 | format = "u8"; 302 | break; 303 | 304 | case libav.AV_SAMPLE_FMT_S16: 305 | format = "s16"; 306 | break; 307 | 308 | case libav.AV_SAMPLE_FMT_S32: 309 | format = "s32"; 310 | break; 311 | 312 | case libav.AV_SAMPLE_FMT_FLT: 313 | format = "f32"; 314 | break; 315 | 316 | case libav.AV_SAMPLE_FMT_U8P: 317 | format = "u8"; 318 | planar = true; 319 | break; 320 | 321 | case libav.AV_SAMPLE_FMT_S16P: 322 | format = "s16"; 323 | planar = true; 324 | break; 325 | 326 | case libav.AV_SAMPLE_FMT_S32P: 327 | format = "s32"; 328 | planar = true; 329 | break; 330 | 331 | case libav.AV_SAMPLE_FMT_FLTP: 332 | format = "f32"; 333 | planar = true; 334 | break; 335 | 336 | default: 337 | throw new DOMException("Unsupported libav format!", "EncodingError") 338 | } 339 | 340 | // 2. sampleRate 341 | const sampleRate = frame.sample_rate!; 342 | 343 | // 3. numberOfFrames 344 | const numberOfFrames = frame.nb_samples!; 345 | 346 | // 4. numberOfChannels 347 | const numberOfChannels = frame.channels!; 348 | 349 | // 5. timestamp 350 | const timestamp = libav.i64tof64(frame.pts!, frame.ptshi!) * 1000; 351 | 352 | // 6. data 353 | let raw: any; 354 | if (planar) { 355 | let ct = 0; 356 | for (let i = 0; i < frame.data.length; i++) 357 | ct += frame.data[i].length; 358 | raw = new (frame.data[0].constructor)(ct); 359 | ct = 0; 360 | for (let i = 0; i < frame.data.length; i++) { 361 | const part = frame.data[i]; 362 | raw.set(part, ct); 363 | ct += part.length; 364 | } 365 | } else { 366 | raw = frame.data; 367 | } 368 | 369 | const data = new ad.AudioData({ 370 | format, sampleRate, numberOfFrames, numberOfChannels, 371 | timestamp, data: raw 372 | }); 373 | 374 | this._output(data); 375 | } 376 | } 377 | 378 | flush(): Promise { 379 | /* 1. If [[state]] is not "configured", return a promise rejected with 380 | * InvalidStateError DOMException. */ 381 | if (this.state !== "configured") 382 | throw new DOMException("Invalid state", "InvalidStateError"); 383 | 384 | // 2. Set [[key chunk required]] to true. 385 | // (part of the codec) 386 | 387 | // 3. Let promise be a new Promise. 388 | // 4. Append promise to [[pending flush promises]]. 389 | // 5. Queue a control message to flush the codec with promise. 390 | // 6. Process the control message queue. 391 | // 7. Return promise. 392 | const ret = this._p.then(async () => { 393 | 394 | // 1. Signal [[codec implementation]] to emit all internal pending outputs. 395 | if (!this._c) 396 | return; 397 | 398 | // Make sure any last data is flushed 399 | const libav = this._libav!; 400 | const c = this._c; 401 | const pkt = this._pkt; 402 | const frame = this._frame; 403 | 404 | let decodedOutputs: LibAVJS.Frame[] | null = null; 405 | 406 | try { 407 | decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [], true); 408 | } catch (ex) { 409 | this._p = this._p.then(() => { 410 | this._closeAudioDecoder( ex); 411 | }); 412 | } 413 | 414 | /* 2. Let decoded outputs be a list of decoded audio data outputs 415 | * emitted by [[codec implementation]]. */ 416 | 417 | // 3. Queue a task to perform these steps: 418 | { 419 | 420 | /* 1. If decoded outputs is not empty, run the Output AudioData 421 | * algorithm with decoded outputs. */ 422 | if (decodedOutputs) 423 | this._outputAudioData(decodedOutputs); 424 | 425 | // 2. Remove promise from [[pending flush promises]]. 426 | 427 | // 3. Resolve promise. 428 | } 429 | 430 | }); 431 | this._p = ret; 432 | return ret; 433 | } 434 | 435 | reset(): void { 436 | this._resetAudioDecoder(new DOMException("Reset", "AbortError")); 437 | } 438 | 439 | close(): void { 440 | this._closeAudioDecoder(new DOMException("Close", "AbortError")); 441 | } 442 | 443 | static async isConfigSupported( 444 | config: AudioDecoderConfig 445 | ): Promise { 446 | const dec = libavs.decoder(config.codec, config); 447 | let supported = false; 448 | if (dec) { 449 | const libav = await libavs.get(); 450 | try { 451 | const [, c, pkt, frame] = await libav.ff_init_decoder(dec.codec); 452 | await libav.ff_free_decoder(c, pkt, frame); 453 | supported = true; 454 | } catch (ex) {} 455 | await libavs.free(libav); 456 | } 457 | return { 458 | supported, 459 | config: misc.cloneConfig( 460 | config, 461 | ["codec", "sampleRate", "numberOfChannels"] 462 | ) 463 | }; 464 | } 465 | } 466 | 467 | export interface AudioDecoderInit { 468 | output: AudioDataOutputCallback; 469 | error: misc.WebCodecsErrorCallback; 470 | } 471 | 472 | export type AudioDataOutputCallback = (output: ad.AudioData) => void; 473 | 474 | export interface AudioDecoderConfig { 475 | codec: string | {libavjs: libavs.LibAVJSCodec}; 476 | sampleRate: number; 477 | numberOfChannels: number; 478 | description?: BufferSource; 479 | } 480 | 481 | export interface AudioDecoderSupport { 482 | supported: boolean; 483 | config: AudioDecoderConfig; 484 | } 485 | -------------------------------------------------------------------------------- /src/avloader.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import type * as LibAVJS from "@libav.js/types"; 21 | declare let LibAV: LibAVJS.LibAVWrapper; 22 | 23 | // Wrapper function to use 24 | export let LibAVWrapper: LibAVJS.LibAVWrapper | null = null; 25 | 26 | // Currently available libav instances 27 | const libavs: LibAVJS.LibAV[] = []; 28 | 29 | // Options required to create a LibAV instance 30 | let libavOptions: any = {}; 31 | 32 | /** 33 | * Supported decoders. 34 | */ 35 | export let decoders: string[] | null = null; 36 | 37 | /** 38 | * Supported encoders. 39 | */ 40 | export let encoders: string[] | null = null; 41 | 42 | /** 43 | * libav.js-specific codec request, used to bypass the codec registry and use 44 | * anything your implementation of libav.js supports. 45 | */ 46 | export interface LibAVJSCodec { 47 | codec: string, 48 | ctx?: LibAVJS.AVCodecContextProps, 49 | options?: Record 50 | } 51 | 52 | /** 53 | * Set the libav wrapper to use. 54 | */ 55 | export function setLibAV(to: LibAVJS.LibAVWrapper) { 56 | LibAVWrapper = to; 57 | } 58 | 59 | /** 60 | * Set the libav loading options. 61 | */ 62 | export function setLibAVOptions(to: any) { 63 | libavOptions = to; 64 | } 65 | 66 | /** 67 | * Get a libav instance. 68 | */ 69 | export async function get(): Promise { 70 | if (libavs.length) 71 | return libavs.shift()!; 72 | return await LibAVWrapper!.LibAV(libavOptions); 73 | } 74 | 75 | /** 76 | * Free a libav instance for later reuse. 77 | */ 78 | export function free(libav: LibAVJS.LibAV) { 79 | libavs.push(libav); 80 | } 81 | 82 | /** 83 | * Get the list of encoders/decoders supported by libav (which are also 84 | * supported by this polyfill) 85 | * @param encoders Check for encoders instead of decoders 86 | */ 87 | async function codecs(encoders: boolean): Promise { 88 | const libav = await get(); 89 | const ret: string[] = []; 90 | 91 | for (const [avname, codec] of [ 92 | ["flac", "flac"], 93 | ["libopus", "opus"], 94 | ["libvorbis", "vorbis"], 95 | ["libaom-av1", "av01"], 96 | ["libvpx-vp9", "vp09"], 97 | ["libvpx", "vp8"] 98 | ]) { 99 | if (encoders) { 100 | if (await libav.avcodec_find_encoder_by_name(avname)) 101 | ret.push(codec); 102 | } else { 103 | if (await libav.avcodec_find_decoder_by_name(avname)) 104 | ret.push(codec); 105 | } 106 | } 107 | 108 | free(libav); 109 | return ret; 110 | } 111 | 112 | /** 113 | * Load the lists of supported decoders and encoders. 114 | */ 115 | export async function load() { 116 | LibAVWrapper = LibAVWrapper || LibAV; 117 | decoders = await codecs(false); 118 | encoders = await codecs(true); 119 | } 120 | 121 | /** 122 | * Convert a decoder from the codec registry (or libav.js-specific parameters) 123 | * to libav.js. Returns null if unsupported. 124 | */ 125 | export function decoder( 126 | codec: string | {libavjs: LibAVJSCodec}, config: any 127 | ): LibAVJSCodec | null { 128 | if (typeof codec === "string") { 129 | codec = codec.replace(/\..*/, ""); 130 | 131 | let outCodec: string = codec; 132 | switch (codec) { 133 | // Audio 134 | case "flac": 135 | if (typeof config.description === "undefined") { 136 | // description is required per spec, but one can argue, if this limitation makes sense 137 | return null; 138 | } 139 | break; 140 | 141 | case "opus": 142 | if (typeof config.description !== "undefined") { 143 | // ogg bitstream is not supported by the current implementation 144 | return null; 145 | } 146 | outCodec = "libopus"; 147 | break; 148 | 149 | case "vorbis": 150 | if (typeof config.description === "undefined") { 151 | // description is required per spec, but one can argue, if this limitation makes sense 152 | return null; 153 | } 154 | outCodec = "libvorbis"; 155 | break; 156 | 157 | // Video 158 | case "av01": 159 | outCodec = "libaom-av1"; 160 | break; 161 | 162 | case "vp09": 163 | outCodec = "libvpx-vp9"; 164 | break; 165 | 166 | case "vp8": 167 | outCodec = "libvpx"; 168 | break; 169 | 170 | // Unsupported 171 | case "mp3": 172 | case "mp4a": 173 | case "ulaw": 174 | case "alaw": 175 | case "avc1": 176 | case "avc3": 177 | case "hev1": 178 | case "hvc1": 179 | return null; 180 | 181 | // Unrecognized 182 | default: 183 | throw new TypeError("Unrecognized codec"); 184 | } 185 | 186 | // Check whether we actually support this codec 187 | if (!(decoders!.indexOf(codec) >= 0)) 188 | return null; 189 | 190 | return {codec: outCodec}; 191 | 192 | } else { 193 | return codec.libavjs; 194 | 195 | } 196 | } 197 | 198 | /** 199 | * Convert an encoder from the codec registry (or libav.js-specific parameters) 200 | * to libav.js. Returns null if unsupported. 201 | */ 202 | export function encoder( 203 | codec: string | {libavjs: LibAVJSCodec}, config: any 204 | ): LibAVJSCodec | null { 205 | if (typeof codec === "string") { 206 | const codecParts = codec.split("."); 207 | codec = codecParts[0]; 208 | 209 | let outCodec: string = codec; 210 | const ctx: LibAVJS.AVCodecContextProps = {}; 211 | const options: Record = {}; 212 | let video = false; 213 | switch (codec) { 214 | // Audio 215 | case "flac": 216 | ctx.sample_fmt = 2 /* S32 */; 217 | ctx.bit_rate = 0; 218 | 219 | if (typeof config.flac === "object" && 220 | config.flac !== null) { 221 | const flac: any = config.flac; 222 | // FIXME: Check block size 223 | if (typeof flac.blockSize === "number") 224 | ctx.frame_size = flac.blockSize; 225 | if (typeof flac.compressLevel === "number") { 226 | // Not supported 227 | return null; 228 | } 229 | } 230 | break; 231 | 232 | case "opus": 233 | outCodec = "libopus"; 234 | ctx.sample_fmt = 3 /* FLT */; 235 | ctx.sample_rate = 48000; 236 | 237 | if (typeof config.opus === "object" && 238 | config.opus !== null) { 239 | const opus: any = config.opus; 240 | // FIXME: Check frame duration 241 | if (typeof opus.frameDuration === "number") 242 | options.frame_duration = "" + (opus.frameDuration / 1000); 243 | if (typeof opus.complexity !== "undefined") { 244 | // We don't support the complexity option 245 | return null; 246 | } 247 | if (typeof opus.packetlossperc === "number") { 248 | if (opus.packetlossperc < 0 || opus.packetlossperc > 100) 249 | return null; 250 | options.packet_loss = "" + opus.packetlossperc; 251 | } 252 | if (typeof opus.useinbandfec === "boolean") 253 | options.fec = opus.useinbandfec?"1":"0"; 254 | if (typeof opus.usedtx === "boolean") { 255 | // We don't support the usedtx option 256 | return null; 257 | } 258 | if (typeof opus.format === "string") { 259 | // ogg bitstream is not supported 260 | if (opus.format !== "opus") return null; 261 | } 262 | } 263 | break; 264 | 265 | case "vorbis": 266 | outCodec = "libvorbis"; 267 | ctx.sample_fmt = 8 /* FLTP */; 268 | break; 269 | 270 | // Video 271 | case "av01": 272 | video = true; 273 | outCodec = "libaom-av1"; 274 | 275 | if (config.latencyMode === "realtime") { 276 | options.usage = "realtime"; 277 | options["cpu-used"] = "8"; 278 | } 279 | 280 | // Check for advanced options 281 | if (!av1Advanced(codecParts, ctx)) 282 | return null; 283 | 284 | break; 285 | 286 | case "vp09": 287 | video = true; 288 | outCodec = "libvpx-vp9"; 289 | 290 | if (config.latencyMode === "realtime") { 291 | options.quality = "realtime"; 292 | options["cpu-used"] = "8"; 293 | } 294 | 295 | // Check for advanced options 296 | if (!vp9Advanced(codecParts, ctx)) 297 | return null; 298 | 299 | break; 300 | 301 | case "vp8": 302 | video = true; 303 | outCodec = "libvpx"; 304 | 305 | if (config.latencyMode === "realtime") { 306 | options.quality = "realtime"; 307 | options["cpu-used"] = "8"; 308 | } 309 | break; 310 | 311 | // Unsupported 312 | case "mp3": 313 | case "mp4a": 314 | case "ulaw": 315 | case "alaw": 316 | case "avc1": 317 | return null; 318 | 319 | // Unrecognized 320 | default: 321 | throw new TypeError("Unrecognized codec"); 322 | } 323 | 324 | // Check whether we actually support this codec 325 | if (!(encoders!.indexOf(codec) >= 0)) 326 | return null; 327 | 328 | if (video) { 329 | if (typeof ctx.pix_fmt !== "number") 330 | ctx.pix_fmt = 0 /* YUV420P */; 331 | const width = ctx.width = config.width; 332 | const height = ctx.height = config.height; 333 | 334 | if (config.framerate) { 335 | /* FIXME: We need this as a rational, not a floating point, and 336 | * this is obviously not the right way to do it */ 337 | ctx.framerate_num = Math.round(config.framerate); 338 | ctx.framerate_den = 1; 339 | } 340 | 341 | // Check for non-square pixels 342 | const dWidth = config.displayWidth || config.width; 343 | const dHeight = config.displayHeight || config.height; 344 | if (dWidth !== width || dHeight !== height) { 345 | ctx.sample_aspect_ratio_num = dWidth * height; 346 | ctx.sample_aspect_ratio_den = dHeight * width; 347 | } 348 | 349 | } else { 350 | if (!ctx.sample_rate) 351 | ctx.sample_rate = config.sampleRate || 48000; 352 | if (config.numberOfChannels) { 353 | const n = config.numberOfChannels; 354 | ctx.channel_layout = (n === 1) ? 4 : ((1<= 0 && profile <= 2) 383 | ctx.profile = profile; 384 | else 385 | throw new TypeError("Invalid AV1 profile"); 386 | } 387 | 388 | if (codecParts[2]) { 389 | const level = +codecParts[2]; 390 | if (level >= 0 && level <= 23) 391 | ctx.level = level; 392 | else 393 | throw new TypeError("Invalid AV1 level"); 394 | } 395 | 396 | if (codecParts[3]) { 397 | switch (codecParts[3]) { 398 | case "M": 399 | // Default 400 | break; 401 | 402 | case "H": 403 | if (ctx.level && ctx.level >= 8) { 404 | // Valid but unsupported 405 | return false; 406 | } else { 407 | throw new TypeError("The AV1 high tier is only available for level 4.0 and up"); 408 | } 409 | break; 410 | 411 | default: 412 | throw new TypeError("Invalid AV1 tier"); 413 | } 414 | } 415 | 416 | if (codecParts[4]) { 417 | const depth = +codecParts[3]; 418 | if (depth === 10 || depth === 12) { 419 | // Valid but unsupported 420 | return false; 421 | } else if (depth !== 8) { 422 | throw new TypeError("Invalid AV1 bit depth"); 423 | } 424 | } 425 | 426 | if (codecParts[5]) { 427 | // Monochrome 428 | switch (codecParts[5]) { 429 | case "0": 430 | // Default 431 | break; 432 | 433 | case "1": 434 | // Valid but unsupported 435 | return false; 436 | 437 | default: 438 | throw new TypeError("Invalid AV1 monochrome flag"); 439 | } 440 | } 441 | 442 | if (codecParts[6]) { 443 | // Subsampling mode 444 | switch (codecParts[6]) { 445 | case "000": // YUV444 446 | ctx.pix_fmt = 5 /* YUV444P */; 447 | break; 448 | 449 | case "100": // YUV422 450 | ctx.pix_fmt = 4 /* YUV422P */; 451 | break; 452 | 453 | case "110": // YUV420P (default) 454 | ctx.pix_fmt = 0 /* YUV420P */; 455 | break; 456 | 457 | case "111": // Monochrome 458 | return false; 459 | 460 | default: 461 | throw new TypeError("Invalid AV1 subsampling mode"); 462 | } 463 | } 464 | 465 | /* The remaining values have to do with color formats, which we don't 466 | * support correctly anyway */ 467 | return true; 468 | } 469 | 470 | /** 471 | * Handler for advanced options for VP9. 472 | * @param codecParts .-separated parts of the codec string. 473 | * @param ctx Context to populate with advanced options. 474 | */ 475 | function vp9Advanced(codecParts: string[], ctx: LibAVJS.AVCodecContextProps) { 476 | if (codecParts[1]) { 477 | const profile = +codecParts[1]; 478 | if (profile >= 0 && profile <= 3) 479 | ctx.profile = profile; 480 | else 481 | throw new TypeError("Invalid VP9 profile"); 482 | } 483 | 484 | if (codecParts[2]) { 485 | const level = [+codecParts[2][0], +codecParts[2][1]]; 486 | if (level[0] >= 1 && level[0] <= 4) { 487 | if (level[1] >= 0 && level[1] <= 1) { 488 | // OK 489 | } else { 490 | throw new TypeError("Invalid VP9 level"); 491 | } 492 | } else if (level[0] >= 5 && level[0] <= 6) { 493 | if (level[1] >= 0 && level[1] <= 2) { 494 | // OK 495 | } else { 496 | throw new TypeError("Invalid VP9 level"); 497 | } 498 | } else { 499 | throw new TypeError("Invalid VP9 level"); 500 | } 501 | ctx.level = +codecParts[2]; 502 | } 503 | 504 | if (codecParts[3]) { 505 | const depth = +codecParts[3]; 506 | if (depth === 10 || depth === 12) { 507 | // Valid but unsupported 508 | return false; 509 | } else if (depth !== 8) { 510 | throw new TypeError("Invalid VP9 bit depth"); 511 | } 512 | } 513 | 514 | if (codecParts[4]) { 515 | const chromaMode = +codecParts[4]; 516 | switch (chromaMode) { 517 | case 0: 518 | case 1: 519 | // FIXME: These are subtly different YUV420P modes, but we treat them the same 520 | ctx.pix_fmt = 0 /* YUV420P */; 521 | break; 522 | 523 | case 2: // YUV422 524 | ctx.pix_fmt = 4 /* YUV422P */; 525 | break; 526 | 527 | case 3: // YUV444 528 | ctx.pix_fmt = 5 /* YUV444P */; 529 | break; 530 | 531 | default: 532 | throw new TypeError("Invalid VP9 chroma subsampling format"); 533 | } 534 | } 535 | 536 | /* The remaining values have to do with color formats, which we don't 537 | * support correctly anyway */ 538 | return true; 539 | } 540 | -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as eac from "./encoded-audio-chunk"; 21 | import * as ad from "./audio-data"; 22 | import * as adec from "./audio-decoder"; 23 | import * as aenc from "./audio-encoder"; 24 | 25 | import * as evc from "./encoded-video-chunk"; 26 | import * as vf from "./video-frame"; 27 | import * as vdec from "./video-decoder"; 28 | import * as venc from "./video-encoder"; 29 | import '@ungap/global-this'; 30 | 31 | /** 32 | * An AudioDecoder environment. 33 | */ 34 | export interface AudioDecoderEnvironment { 35 | AudioDecoder: typeof adec.AudioDecoder, 36 | EncodedAudioChunk: typeof eac.EncodedAudioChunk, 37 | AudioData: typeof ad.AudioData 38 | } 39 | 40 | /** 41 | * A VideoDecoder environment. 42 | */ 43 | export interface VideoDecoderEnvironment { 44 | VideoDecoder: typeof vdec.VideoDecoder, 45 | EncodedVideoChunk: typeof evc.EncodedVideoChunk, 46 | VideoFrame: typeof vf.VideoFrame 47 | } 48 | 49 | /** 50 | * An AudioEncoder environment. 51 | */ 52 | export interface AudioEncoderEnvironment { 53 | AudioEncoder: typeof aenc.AudioEncoder, 54 | EncodedAudioChunk: typeof eac.EncodedAudioChunk, 55 | AudioData: typeof ad.AudioData 56 | } 57 | 58 | /** 59 | * A VideoEncoder environment. 60 | */ 61 | export interface VideoEncoderEnvironment { 62 | VideoEncoder: typeof venc.VideoEncoder, 63 | EncodedVideoChunk: typeof evc.EncodedVideoChunk, 64 | VideoFrame: typeof vf.VideoFrame 65 | } 66 | 67 | /** 68 | * Error thrown to indicate a configuration is unsupported. 69 | */ 70 | export class UnsupportedException extends Error { 71 | constructor() { 72 | super("The requested configuration is not supported"); 73 | } 74 | } 75 | 76 | /** 77 | * Get an AudioDecoder environment that supports this configuration. Throws an 78 | * UnsupportedException if no environment supports the configuration. 79 | * @param config Audio decoder configuration 80 | */ 81 | export async function getAudioDecoder( 82 | config: adec.AudioDecoderConfig 83 | ): Promise { 84 | try { 85 | if (typeof ( globalThis).AudioDecoder !== "undefined" && 86 | (await ( globalThis).AudioDecoder.isConfigSupported(config)).supported) { 87 | return { 88 | AudioDecoder: ( globalThis).AudioDecoder, 89 | EncodedAudioChunk: ( globalThis).EncodedAudioChunk, 90 | AudioData: ( globalThis).AudioData 91 | }; 92 | } 93 | } catch (ex) {} 94 | 95 | if ((await adec.AudioDecoder.isConfigSupported(config)).supported) { 96 | return { 97 | AudioDecoder: adec.AudioDecoder, 98 | EncodedAudioChunk: eac.EncodedAudioChunk, 99 | AudioData: ad.AudioData 100 | }; 101 | } 102 | 103 | throw new UnsupportedException(); 104 | } 105 | 106 | /** 107 | * Get an VideoDecoder environment that supports this configuration. Throws an 108 | * UnsupportedException if no environment supports the configuration. 109 | * @param config Video decoder configuration 110 | */ 111 | export async function getVideoDecoder( 112 | config: vdec.VideoDecoderConfig 113 | ): Promise { 114 | try { 115 | if (typeof ( globalThis).VideoDecoder !== "undefined" && 116 | (await ( globalThis).VideoDecoder.isConfigSupported(config)).supported) { 117 | return { 118 | VideoDecoder: ( globalThis).VideoDecoder, 119 | EncodedVideoChunk: ( globalThis).EncodedVideoChunk, 120 | VideoFrame: ( globalThis).VideoFrame 121 | }; 122 | } 123 | } catch (ex) {} 124 | 125 | if ((await vdec.VideoDecoder.isConfigSupported(config)).supported) { 126 | return { 127 | VideoDecoder: vdec.VideoDecoder, 128 | EncodedVideoChunk: evc.EncodedVideoChunk, 129 | VideoFrame: vf.VideoFrame 130 | }; 131 | } 132 | 133 | throw new UnsupportedException(); 134 | } 135 | 136 | /** 137 | * Get an AudioEncoder environment that supports this configuration. Throws an 138 | * UnsupportedException if no environment supports the configuration. 139 | * @param config Audio encoder configuration 140 | */ 141 | export async function getAudioEncoder( 142 | config: aenc.AudioEncoderConfig 143 | ): Promise { 144 | try { 145 | if (typeof ( globalThis).AudioEncoder !== "undefined" && 146 | (await ( globalThis).AudioEncoder.isConfigSupported(config)).supported) { 147 | return { 148 | AudioEncoder: ( globalThis).AudioEncoder, 149 | EncodedAudioChunk: ( globalThis).EncodedAudioChunk, 150 | AudioData: ( globalThis).AudioData 151 | }; 152 | } 153 | } catch (ex) {} 154 | 155 | if ((await aenc.AudioEncoder.isConfigSupported(config)).supported) { 156 | return { 157 | AudioEncoder: aenc.AudioEncoder, 158 | EncodedAudioChunk: eac.EncodedAudioChunk, 159 | AudioData: ad.AudioData 160 | }; 161 | } 162 | 163 | throw new UnsupportedException(); 164 | } 165 | 166 | /** 167 | * Get an VideoEncoder environment that supports this configuration. Throws an 168 | * UnsupportedException if no environment supports the configuration. 169 | * @param config Video encoder configuration 170 | */ 171 | export async function getVideoEncoder( 172 | config: venc.VideoEncoderConfig 173 | ): Promise { 174 | try { 175 | if (typeof ( globalThis).VideoEncoder !== "undefined" && 176 | (await ( globalThis).VideoEncoder.isConfigSupported(config)).supported) { 177 | return { 178 | VideoEncoder: ( globalThis).VideoEncoder, 179 | EncodedVideoChunk: ( globalThis).EncodedVideoChunk, 180 | VideoFrame: ( globalThis).VideoFrame 181 | }; 182 | } 183 | } catch (ex) {} 184 | 185 | if ((await venc.VideoEncoder.isConfigSupported(config)).supported) { 186 | return { 187 | VideoEncoder: venc.VideoEncoder, 188 | EncodedVideoChunk: evc.EncodedVideoChunk, 189 | VideoFrame: vf.VideoFrame 190 | }; 191 | } 192 | 193 | throw new UnsupportedException(); 194 | } 195 | -------------------------------------------------------------------------------- /src/encoded-audio-chunk.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | export class EncodedAudioChunk { 21 | constructor(init: EncodedAudioChunkInit) { 22 | /* 1. If init.transfer contains more than one reference to the same 23 | * ArrayBuffer, then throw a DataCloneError DOMException. */ 24 | // 2. For each transferable in init.transfer: 25 | /* 1. If [[Detached]] internal slot is true, then throw a 26 | * DataCloneError DOMException. */ 27 | // (not worth checking in a polyfill) 28 | 29 | /* 3. Let chunk be a new EncodedAudioChunk object, initialized as 30 | * follows */ 31 | { 32 | 33 | // 1. Assign init.type to [[type]]. 34 | this.type = init.type; 35 | 36 | // 2. Assign init.timestamp to [[timestamp]]. 37 | this.timestamp = init.timestamp; 38 | 39 | /* 3. If init.duration exists, assign it to [[duration]], or assign 40 | * null otherwise. */ 41 | if (typeof init.duration === "number") 42 | this.duration = init.duration; 43 | else 44 | this.duration = null; 45 | 46 | // 4. Assign init.data.byteLength to [[byte length]]; 47 | this.byteLength = init.data.byteLength; 48 | 49 | /* 5. If init.transfer contains an ArrayBuffer referenced by 50 | * init.data the User Agent MAY choose to: */ 51 | let transfer = false; 52 | if (init.transfer) { 53 | 54 | /* 1. Let resource be a new media resource referencing sample 55 | * data in init.data. */ 56 | let inBuffer: ArrayBuffer; 57 | if (( init.data).buffer) 58 | inBuffer = ( init.data).buffer; 59 | else 60 | inBuffer = init.data; 61 | 62 | let t: ArrayBuffer[]; 63 | if (init.transfer instanceof Array) 64 | t = init.transfer; 65 | else 66 | t = Array.from(init.transfer); 67 | for (const b of t) { 68 | if (b === inBuffer) { 69 | transfer = true; 70 | break; 71 | } 72 | } 73 | } 74 | 75 | // 6. Otherwise: 76 | // 1. Assign a copy of init.data to [[internal data]]. 77 | 78 | const data = new Uint8Array( 79 | ( init.data).buffer || init.data, 80 | ( init.data).byteOffset || 0, 81 | ( init.data).BYTES_PER_ELEMENT 82 | ? (( init.data).BYTES_PER_ELEMENT * ( init.data).length) 83 | : init.data.byteLength 84 | ); 85 | if (transfer) 86 | this._data = data; 87 | else 88 | this._data = data.slice(0); 89 | } 90 | 91 | // 4. For each transferable in init.transfer: 92 | // 1. Perform DetachArrayBuffer on transferable 93 | // (already done by transferring) 94 | 95 | // 5. Return chunk. 96 | } 97 | 98 | readonly type: EncodedAudioChunkType; 99 | readonly timestamp: number; // microseconds 100 | readonly duration: number | null; // microseconds 101 | readonly byteLength: number; 102 | 103 | private _data: Uint8Array; 104 | 105 | // Internal 106 | _libavGetData() { return this._data; } 107 | 108 | copyTo(destination: BufferSource) { 109 | (new Uint8Array( 110 | ( destination).buffer || destination, 111 | ( destination).byteOffset || 0 112 | )).set(this._data); 113 | } 114 | } 115 | 116 | export interface EncodedAudioChunkInit { 117 | type: EncodedAudioChunkType; 118 | timestamp: number; // microseconds 119 | duration?: number; // microseconds 120 | data: BufferSource; 121 | transfer?: ArrayLike; 122 | } 123 | 124 | export type EncodedAudioChunkType = 125 | "key" | 126 | "delta"; 127 | -------------------------------------------------------------------------------- /src/encoded-video-chunk.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as eac from "./encoded-audio-chunk"; 21 | 22 | export type EncodedVideoChunk = eac.EncodedAudioChunk; 23 | export const EncodedVideoChunk = eac.EncodedAudioChunk; 24 | export type EncodedVideoChunkInit = eac.EncodedAudioChunkInit; 25 | export type EncodedVideoChunkType = eac.EncodedAudioChunkType; 26 | -------------------------------------------------------------------------------- /src/event-target.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | /* Unfortunately, browsers don't let us extend EventTarget. So, we implement an 21 | * EventTarget interface with a “has-a” relationship instead of an “is-a” 22 | * relationship. We have an event target, and expose its event functions as our 23 | * own. */ 24 | 25 | export class HasAEventTarget implements EventTarget { 26 | constructor() { 27 | const ev = this._eventer = new EventTarget(); 28 | this.addEventListener = ev.addEventListener.bind(ev); 29 | this.removeEventListener = ev.removeEventListener.bind(ev); 30 | this.dispatchEvent = ev.dispatchEvent.bind(ev); 31 | } 32 | 33 | public addEventListener: typeof EventTarget.prototype.addEventListener; 34 | public removeEventListener: typeof EventTarget.prototype.removeEventListener; 35 | public dispatchEvent: typeof EventTarget.prototype.dispatchEvent; 36 | 37 | private _eventer: EventTarget; 38 | } 39 | 40 | export class DequeueEventTarget extends HasAEventTarget { 41 | constructor() { 42 | super(); 43 | this.addEventListener("dequeue", ev => { 44 | if (this.ondequeue) 45 | this.ondequeue(ev); 46 | }); 47 | } 48 | 49 | public ondequeue?: (ev: Event) => unknown; 50 | } 51 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as eac from "./encoded-audio-chunk"; 21 | import * as ad from "./audio-data"; 22 | import * as adec from "./audio-decoder"; 23 | import * as aenc from "./audio-encoder"; 24 | 25 | import * as evc from "./encoded-video-chunk"; 26 | import * as vf from "./video-frame"; 27 | import * as vdec from "./video-decoder"; 28 | import * as venc from "./video-encoder"; 29 | 30 | import * as rendering from "./rendering"; 31 | 32 | import * as config from "./config"; 33 | import * as libav from "./avloader"; 34 | import * as misc from "./misc"; 35 | 36 | import type * as LibAVJS from "@libav.js/types"; 37 | import '@ungap/global-this'; 38 | 39 | declare let importScripts: any; 40 | 41 | /** 42 | * Load LibAV-WebCodecs-Polyfill. 43 | */ 44 | export async function load(options: { 45 | polyfill?: boolean, 46 | LibAV?: LibAVJS.LibAVWrapper, 47 | libavOptions?: any 48 | } = {}) { 49 | // Set up libavOptions 50 | let libavOptions: any = {}; 51 | if (options.libavOptions) 52 | Object.assign(libavOptions, options.libavOptions); 53 | 54 | // Maybe load libav 55 | if (!options.LibAV && typeof ( globalThis).LibAV === "undefined") { 56 | await new Promise((res, rej) => { 57 | // Can't load workers from another origin 58 | libavOptions.noworker = true; 59 | 60 | // Load libav 61 | const libavBase = "https://cdn.jsdelivr.net/npm/@libav.js/variant-webm-vp9@6.5.7/dist"; 62 | ( globalThis).LibAV = {base: libavBase}; 63 | const libavVar = "libav-6.0.7.0.2-webm-vp9.js"; 64 | if (typeof importScripts !== "undefined") { 65 | importScripts(`${libavBase}/${libavVar}`); 66 | res(void 0); 67 | } else { 68 | const scr = document.createElement("script"); 69 | scr.src = `${libavBase}/${libavVar}`; 70 | scr.onload = res; 71 | scr.onerror = rej; 72 | document.body.appendChild(scr); 73 | } 74 | }); 75 | } 76 | 77 | // And load the libav handler 78 | if (options.LibAV) 79 | libav.setLibAV(options.LibAV); 80 | libav.setLibAVOptions(libavOptions); 81 | await libav.load(); 82 | 83 | if (options.polyfill) { 84 | for (const exp of <[string, any][]> [ 85 | ["EncodedAudioChunk", eac.EncodedAudioChunk], 86 | ["AudioData", ad.AudioData], 87 | ["AudioDecoder", adec.AudioDecoder], 88 | ["AudioEncoder", aenc.AudioEncoder], 89 | ["EncodedVideoChunk", evc.EncodedVideoChunk], 90 | ["VideoFrame", vf.VideoFrame], 91 | ["VideoDecoder", vdec.VideoDecoder], 92 | ["VideoEncoder", venc.VideoEncoder] 93 | ]) { 94 | if (!( globalThis)[exp[0]]) 95 | ( globalThis)[exp[0]] = exp[1]; 96 | } 97 | } 98 | 99 | await rendering.load(libavOptions, !!options.polyfill); 100 | } 101 | 102 | // EncodedAudioChunk 103 | export type EncodedAudioChunk = eac.EncodedAudioChunk; 104 | export const EncodedAudioChunk = eac.EncodedAudioChunk; 105 | export type EncodedAudioChunkInit = eac.EncodedAudioChunkInit; 106 | export type EncodedAudioChunkType = eac.EncodedAudioChunkType; 107 | 108 | // AudioData 109 | export type AudioData = ad.AudioData; 110 | export const AudioData = ad.AudioData; 111 | export type AudioDataInit = ad.AudioDataInit; 112 | export type AudioSampleFormat = ad.AudioSampleFormat; 113 | export type AudioDataCopyToOptions = ad.AudioDataCopyToOptions; 114 | 115 | // AudioDecoder 116 | export type AudioDecoder = adec.AudioDecoder; 117 | export const AudioDecoder = adec.AudioDecoder; 118 | export type AudioDecoderInit = adec.AudioDecoderInit; 119 | export type AudioDataOutputCallback = adec.AudioDataOutputCallback; 120 | export type AudioDecoderConfig = adec.AudioDecoderConfig; 121 | export type AudioDecoderSupport = adec.AudioDecoderSupport; 122 | 123 | // AudioEncoder 124 | export type AudioEncoder = aenc.AudioEncoder; 125 | export const AudioEncoder = aenc.AudioEncoder; 126 | export type AudioEncoderInit = aenc.AudioEncoderInit; 127 | export type EncodedAudioChunkOutputCallback = aenc.EncodedAudioChunkOutputCallback; 128 | export type AudioEncoderConfig = aenc.AudioEncoderConfig; 129 | export type AudioEncoderSupport = aenc.AudioEncoderSupport; 130 | 131 | // EncodedVideoChunk 132 | export type EncodedVideoChunk = evc.EncodedVideoChunk; 133 | export const EncodedVideoChunk = evc.EncodedVideoChunk; 134 | export type EncodedVideoChunkInit = evc.EncodedVideoChunkInit; 135 | export type EncodedVideoChunkType = evc.EncodedVideoChunkType; 136 | 137 | // VideoFrame 138 | export type VideoFrame = vf.VideoFrame; 139 | export const VideoFrame = vf.VideoFrame; 140 | export type VideoFrameInit = vf.VideoFrameInit; 141 | export type VideoFrameBufferInit = vf.VideoFrameBufferInit; 142 | export type VideoPixelFormat = vf.VideoPixelFormat; 143 | export type PlaneLayout = vf.PlaneLayout; 144 | export type VideoFrameCopyToOptions = vf.VideoFrameCopyToOptions; 145 | 146 | // VideoDecoder 147 | export type VideoDecoder = vdec.VideoDecoder; 148 | export const VideoDecoder = vdec.VideoDecoder; 149 | export type VideoDecoderInit = vdec.VideoDecoderInit; 150 | export type VideoFrameOutputCallback = vdec.VideoFrameOutputCallback; 151 | export type VideoDecoderConfig = vdec.VideoDecoderConfig; 152 | export type VideoDecoderSupport = vdec.VideoDecoderSupport; 153 | 154 | // VideoEncoder 155 | export type VideoEncoder = venc.VideoEncoder; 156 | export const VideoEncoder = venc.VideoEncoder; 157 | export type VideoEncoderInit = venc.VideoEncoderInit; 158 | export type EncodedVideoChunkOutputCallback = venc.EncodedVideoChunkOutputCallback; 159 | export type VideoEncoderConfig = venc.VideoEncoderConfig; 160 | export type VideoEncoderEncodeOptions = venc.VideoEncoderEncodeOptions; 161 | export type LatencyMode = venc.LatencyMode; 162 | export type VideoEncoderSupport = venc.VideoEncoderSupport; 163 | 164 | // Rendering 165 | export const canvasDrawImage = rendering.canvasDrawImage; 166 | export const createImageBitmap = rendering.createImageBitmap; 167 | 168 | // Misc 169 | export type CodecState = misc.CodecState; 170 | export type WebCodecsErrorcallback = misc.WebCodecsErrorCallback; 171 | 172 | // Configurations/environments 173 | export type AudioDecoderEnvironment = config.AudioDecoderEnvironment; 174 | export type VideoDecoderEnvironment = config.VideoDecoderEnvironment; 175 | export type AudioEncoderEnvironment = config.AudioEncoderEnvironment; 176 | export type VideoEncoderEnvironment = config.VideoEncoderEnvironment; 177 | export type UnsupportedException = config.UnsupportedException; 178 | export const UnsupportedException = config.UnsupportedException; 179 | export const getAudioDecoder = config.getAudioDecoder; 180 | export const getVideoDecoder = config.getVideoDecoder; 181 | export const getAudioEncoder = config.getAudioEncoder; 182 | export const getVideoEncoder = config.getVideoEncoder; 183 | -------------------------------------------------------------------------------- /src/misc.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as LibAVJS from "@libav.js/types"; 21 | 22 | export type CodecState = 23 | "unconfigured" | 24 | "configured" | 25 | "closed"; 26 | 27 | export type WebCodecsErrorCallback = (error: DOMException) => void; 28 | 29 | /** 30 | * Clone this configuration. Just copies over the supported/recognized fields. 31 | */ 32 | export function cloneConfig(config: any, fields: string[]): any { 33 | const ret: any = {}; 34 | for (const field of fields) { 35 | if (field in config) 36 | ret[field] = config[field]; 37 | } 38 | return ret; 39 | } 40 | -------------------------------------------------------------------------------- /src/rendering.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as libav from "./avloader"; 21 | import * as vf from "./video-frame"; 22 | import '@ungap/global-this'; 23 | 24 | import type * as LibAVJS from "@libav.js/types"; 25 | 26 | // A non-threaded libav.js instance for scaling. 27 | let scalerSync: (LibAVJS.LibAV & LibAVJS.LibAVSync) | null = null; 28 | 29 | // A synchronous libav.js instance for scaling. 30 | let scalerAsync: LibAVJS.LibAV | null = null; 31 | 32 | // The original drawImage 33 | let origDrawImage: any = null; 34 | 35 | // The original drawImage Offscreen 36 | let origDrawImageOffscreen: any = null; 37 | 38 | // The original createImageBitmap 39 | let origCreateImageBitmap: any = null; 40 | 41 | /** 42 | * Load rendering capability. 43 | * @param libavOptions Options to use while loading libav 44 | * @param polyfill Set to polyfill CanvasRenderingContext2D.drawImage 45 | */ 46 | export async function load(libavOptions: any, polyfill: boolean) { 47 | // Get our scalers 48 | if ("importScripts" in globalThis) { 49 | // Make sure the worker code doesn't run 50 | ( libav.LibAVWrapper).nolibavworker = true; 51 | } 52 | scalerSync = await libav.LibAVWrapper!.LibAV({ 53 | ...libavOptions, 54 | noworker: true, 55 | yesthreads: false 56 | }); 57 | scalerAsync = await libav.LibAVWrapper!.LibAV(libavOptions); 58 | 59 | // Polyfill drawImage 60 | if ('CanvasRenderingContext2D' in globalThis) { 61 | origDrawImage = CanvasRenderingContext2D.prototype.drawImage; 62 | if (polyfill) 63 | ( CanvasRenderingContext2D.prototype).drawImage = drawImagePolyfill; 64 | } 65 | if ('OffscreenCanvasRenderingContext2D' in globalThis) { 66 | origDrawImageOffscreen = OffscreenCanvasRenderingContext2D.prototype.drawImage; 67 | if (polyfill) 68 | ( OffscreenCanvasRenderingContext2D.prototype).drawImage = drawImagePolyfillOffscreen; 69 | } 70 | 71 | // Polyfill createImageBitmap 72 | origCreateImageBitmap = globalThis.createImageBitmap; 73 | if (polyfill) 74 | ( globalThis).createImageBitmap = createImageBitmap; 75 | } 76 | 77 | /** 78 | * Draw this video frame on this canvas, synchronously. 79 | * @param ctx CanvasRenderingContext2D to draw on 80 | * @param image VideoFrame (or anything else) to draw 81 | * @param sx Source X position OR destination X position 82 | * @param sy Source Y position OR destination Y position 83 | * @param sWidth Source width OR destination width 84 | * @param sHeight Source height OR destination height 85 | * @param dx Destination X position 86 | * @param dy Destination Y position 87 | * @param dWidth Destination width 88 | * @param dHeight Destination height 89 | */ 90 | export function canvasDrawImage( 91 | ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, 92 | image: vf.VideoFrame, ax: number, ay: number, sWidth?: number, 93 | sHeight?: number, dx?: number, dy?: number, dWidth?: number, 94 | dHeight?: number 95 | ): void { 96 | if (!(( image)._data)) { 97 | // Just use the original 98 | return origDrawImage.apply(ctx, Array.prototype.slice.call(arguments, 1)); 99 | } 100 | 101 | let sx: number | undefined; 102 | let sy: number | undefined; 103 | 104 | // Normalize the arguments 105 | if (typeof sWidth === "undefined") { 106 | // dx, dy 107 | dx = ax; 108 | dy = ay; 109 | 110 | } else if (typeof dx === "undefined") { 111 | // dx, dy, dWidth, dHeight 112 | dx = ax; 113 | dy = ay; 114 | dWidth = sWidth; 115 | dHeight = sHeight; 116 | sx = void 0; 117 | sy = void 0; 118 | sWidth = void 0; 119 | sHeight = void 0; 120 | 121 | } else { 122 | sx = ax; 123 | sy = ay; 124 | 125 | } 126 | 127 | if (typeof dWidth === "undefined") { 128 | dWidth = image.displayWidth; 129 | dHeight = image.displayHeight; 130 | } 131 | 132 | // Convert the format to libav.js 133 | const format = vf.wcFormatToLibAVFormat(scalerSync!, image.format); 134 | 135 | // Convert the frame synchronously 136 | const sctx = scalerSync!.sws_getContext_sync( 137 | image.visibleRect.width, image.visibleRect.height, format, 138 | dWidth, dHeight!, scalerSync!.AV_PIX_FMT_RGBA, 139 | 2, 0, 0, 0 140 | ); 141 | const inFrame = scalerSync!.av_frame_alloc_sync(); 142 | const outFrame = scalerSync!.av_frame_alloc_sync(); 143 | 144 | let rawU8: Uint8Array; 145 | let layout: vf.PlaneLayout[]; 146 | if (image._libavGetData) { 147 | rawU8 = image._libavGetData(); 148 | layout = image._libavGetLayout(); 149 | } else { 150 | // Just have to hope this is a polyfill VideoFrame copied weirdly! 151 | rawU8 = ( image)._data; 152 | layout = ( image)._layout; 153 | } 154 | 155 | // Copy it in 156 | scalerSync!.ff_copyin_frame_sync(inFrame, { 157 | data: rawU8, 158 | layout, 159 | format, 160 | width: image.codedWidth, 161 | height: image.codedHeight, 162 | crop: { 163 | left: image.visibleRect.left, 164 | right: image.visibleRect.right, 165 | top: image.visibleRect.top, 166 | bottom: image.visibleRect.bottom 167 | } 168 | }); 169 | 170 | // Rescale 171 | scalerSync!.sws_scale_frame_sync(sctx, outFrame, inFrame); 172 | 173 | // Get the data back out again 174 | const frameData = scalerSync!.ff_copyout_frame_video_imagedata_sync(outFrame); 175 | 176 | // Finally, draw it 177 | ctx.putImageData(frameData, dx, dy!); 178 | 179 | // And clean up 180 | scalerSync!.av_frame_free_js_sync(outFrame); 181 | scalerSync!.av_frame_free_js_sync(inFrame); 182 | scalerSync!.sws_freeContext_sync(sctx); 183 | } 184 | 185 | /** 186 | * Polyfill version of canvasDrawImage. 187 | */ 188 | function drawImagePolyfill( 189 | this: CanvasRenderingContext2D, 190 | image: vf.VideoFrame, sx: number, sy: number, sWidth?: number, 191 | sHeight?: number, dx?: number, dy?: number, dWidth?: number, 192 | dHeight?: number 193 | ) { 194 | if (image instanceof vf.VideoFrame) { 195 | return canvasDrawImage( 196 | this, image, sx, sy, sWidth, sHeight, dx, dy, dWidth, dHeight 197 | ); 198 | } 199 | return origDrawImage.apply(this, arguments); 200 | } 201 | 202 | /** 203 | * Polyfill version of offscreenCanvasDrawImage. 204 | */ 205 | function drawImagePolyfillOffscreen( 206 | this: OffscreenCanvasRenderingContext2D, 207 | image: vf.VideoFrame, sx: number, sy: number, sWidth?: number, 208 | sHeight?: number, dx?: number, dy?: number, dWidth?: number, 209 | dHeight?: number 210 | ) { 211 | if (image instanceof vf.VideoFrame) { 212 | return canvasDrawImage( 213 | this, image, sx, sy, sWidth, sHeight, dx, dy, dWidth, dHeight 214 | ); 215 | } 216 | return origDrawImageOffscreen.apply(this, arguments); 217 | } 218 | 219 | /** 220 | * Create an ImageBitmap from this drawable, asynchronously. NOTE: 221 | * Sub-rectangles are not implemented for VideoFrames, so only options is 222 | * available, and there, only scaling is available. 223 | * @param image VideoFrame (or anything else) to draw 224 | * @param options Other options 225 | */ 226 | export function createImageBitmap( 227 | image: vf.VideoFrame, opts: { 228 | resizeWidth?: number, 229 | resizeHeight?: number 230 | } = {} 231 | ): Promise { 232 | if (!(( image)._data)) { 233 | // Just use the original 234 | return origCreateImageBitmap.apply(globalThis, arguments); 235 | } 236 | 237 | // Convert the format to libav.js 238 | const format = vf.wcFormatToLibAVFormat(scalerAsync!, image.format); 239 | 240 | // Normalize arguments 241 | const dWidth =(typeof opts.resizeWidth === "number") 242 | ? opts.resizeWidth : image.displayWidth; 243 | const dHeight =(typeof opts.resizeHeight === "number") 244 | ? opts.resizeHeight : image.displayHeight; 245 | 246 | // Convert the frame 247 | return (async () => { 248 | const [sctx, inFrame, outFrame] = await Promise.all([ 249 | scalerAsync!.sws_getContext( 250 | image.visibleRect.width, image.visibleRect.height, format, 251 | dWidth, dHeight, scalerAsync!.AV_PIX_FMT_RGBA, 2, 0, 0, 0 252 | ), 253 | scalerAsync!.av_frame_alloc(), 254 | scalerAsync!.av_frame_alloc() 255 | ]); 256 | 257 | // Convert the data 258 | let rawU8: Uint8Array; 259 | let layout: vf.PlaneLayout[] | undefined = void 0; 260 | if (image._libavGetData) { 261 | rawU8 = image._libavGetData(); 262 | layout = image._libavGetLayout(); 263 | } else if (( image)._data) { 264 | // Assume a VideoFrame weirdly serialized 265 | rawU8 = ( image)._data; 266 | layout = ( image)._layout; 267 | } else { 268 | rawU8 = new Uint8Array(image.allocationSize()); 269 | await image.copyTo(rawU8); 270 | } 271 | 272 | // Copy it in 273 | await scalerAsync!.ff_copyin_frame(inFrame, { 274 | data: rawU8, 275 | layout, 276 | format, 277 | width: image.codedWidth, 278 | height: image.codedHeight, 279 | crop: { 280 | left: image.visibleRect.left, 281 | right: image.visibleRect.right, 282 | top: image.visibleRect.top, 283 | bottom: image.visibleRect.bottom 284 | } 285 | }), 286 | 287 | // Rescale 288 | await scalerAsync!.sws_scale_frame(sctx, outFrame, inFrame); 289 | 290 | // Get the data back out again 291 | const frameData = 292 | await scalerAsync!.ff_copyout_frame_video_imagedata(outFrame); 293 | 294 | // And clean up 295 | await Promise.all([ 296 | scalerAsync!.av_frame_free_js(outFrame), 297 | scalerAsync!.av_frame_free_js(inFrame), 298 | scalerAsync!.sws_freeContext(sctx) 299 | ]); 300 | 301 | // Make the ImageBitmap 302 | return await origCreateImageBitmap(frameData); 303 | })(); 304 | } 305 | -------------------------------------------------------------------------------- /src/video-decoder.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as evc from "./encoded-video-chunk"; 21 | import * as et from "./event-target"; 22 | import * as libavs from "./avloader"; 23 | import * as misc from "./misc"; 24 | import * as vf from "./video-frame"; 25 | 26 | import * as LibAVJS from "@libav.js/types"; 27 | 28 | export class VideoDecoder extends et.DequeueEventTarget { 29 | constructor(init: VideoDecoderInit) { 30 | super(); 31 | 32 | // 1. Let d be a new VideoDecoder object. 33 | 34 | // 2. Assign a new queue to [[control message queue]]. 35 | this._p = Promise.all([]); 36 | 37 | // 3. Assign false to [[message queue blocked]]. 38 | // (unneeded in polyfill) 39 | 40 | // 4. Assign null to [[codec implementation]]. 41 | this._libav = null; 42 | this._codec = this._c = this._pkt = this._frame = 0; 43 | 44 | /* 5. Assign the result of starting a new parallel queue to 45 | * [[codec work queue]]. */ 46 | // (shared queue) 47 | 48 | // 6. Assign false to [[codec saturated]]. 49 | // (saturation not needed) 50 | 51 | // 7. Assign init.output to [[output callback]]. 52 | this._output = init.output; 53 | 54 | // 8. Assign init.error to [[error callback]]. 55 | this._error = init.error; 56 | 57 | // 9. Assign null to [[active decoder config]]. 58 | // (part of codec) 59 | 60 | // 10. Assign true to [[key chunk required]]. 61 | // (part of codec) 62 | 63 | // 11. Assign "unconfigured" to [[state]] 64 | this.state = "unconfigured"; 65 | 66 | // 12. Assign 0 to [[decodeQueueSize]]. 67 | this.decodeQueueSize = 0; 68 | 69 | // 13. Assign a new list to [[pending flush promises]]. 70 | // (shared queue) 71 | 72 | // 14. Assign false to [[dequeue event scheduled]]. 73 | // (not needed in polyfill) 74 | 75 | // 15. Return d. 76 | } 77 | 78 | /* NOTE: These should technically be readonly, but I'm implementing them as 79 | * plain fields, so they're writable */ 80 | state: misc.CodecState; 81 | decodeQueueSize: number; 82 | 83 | private _output: VideoFrameOutputCallback; 84 | private _error: misc.WebCodecsErrorCallback; 85 | 86 | // Event queue 87 | private _p: Promise; 88 | 89 | // LibAV state 90 | private _libav: LibAVJS.LibAV | null; 91 | private _codec: number; 92 | private _c: number; 93 | private _pkt: number; 94 | private _frame: number; 95 | 96 | configure(config: VideoDecoderConfig): void { 97 | // 1. If config is not a valid VideoDecoderConfig, throw a TypeError. 98 | // NOTE: We don't support sophisticated codec string parsing (yet) 99 | 100 | // 2. If [[state]] is “closed”, throw an InvalidStateError DOMException. 101 | if (this.state === "closed") 102 | throw new DOMException("Decoder is closed", "InvalidStateError"); 103 | 104 | // Free any internal state 105 | if (this._libav) 106 | this._p = this._p.then(() => this._free()); 107 | 108 | // 3. Set [[state]] to "configured". 109 | this.state = "configured"; 110 | 111 | // 4. Set [[key chunk required]] to true. 112 | // (part of the codec) 113 | 114 | // 5. Queue a control message to configure the decoder with config. 115 | this._p = this._p.then(async () => { 116 | /* 1. Let supported be the result of running the Check 117 | * Configuration Support algorithm with config. */ 118 | const supported = libavs.decoder(config.codec, config); 119 | 120 | /* 2. If supported is false, queue a task to run the Close 121 | * VideoDecoder algorithm with NotSupportedError and abort these 122 | * steps. */ 123 | if (!supported) { 124 | this._closeVideoDecoder(new DOMException("Unsupported codec", "NotSupportedError")); 125 | return; 126 | } 127 | 128 | /* 3. If needed, assign [[codec implementation]] with an 129 | * implementation supporting config. */ 130 | // 4. Configure [[codec implementation]] with config. 131 | const libav = this._libav = await libavs.get(); 132 | 133 | // Initialize 134 | [this._codec, this._c, this._pkt, this._frame] = 135 | await libav.ff_init_decoder(supported.codec); 136 | await libav.AVCodecContext_time_base_s(this._c, 1, 1000); 137 | 138 | // 5. queue a task to run the following steps: 139 | // 1. Assign false to [[message queue blocked]]. 140 | // 2. Queue a task to Process the control message queue. 141 | 142 | }).catch(this._error); 143 | } 144 | 145 | // Our own algorithm, close libav 146 | private async _free() { 147 | if (this._c) { 148 | await this._libav!.ff_free_decoder(this._c, this._pkt, this._frame); 149 | this._codec = this._c = this._pkt = this._frame = 0; 150 | } 151 | if (this._libav) { 152 | libavs.free(this._libav); 153 | this._libav = null; 154 | } 155 | } 156 | 157 | private _closeVideoDecoder(exception: DOMException) { 158 | // 1. Run the Reset VideoDecoder algorithm with exception. 159 | this._resetVideoDecoder(exception); 160 | 161 | // 2. Set [[state]] to "closed". 162 | this.state = "closed"; 163 | 164 | /* 3. Clear [[codec implementation]] and release associated system 165 | * resources. */ 166 | this._p = this._p.then(() => this._free()); 167 | 168 | /* 4. If exception is not an AbortError DOMException, invoke the 169 | * [[error callback]] with exception. */ 170 | if (exception.name !== "AbortError") 171 | this._p = this._p.then(() => { this._error(exception); }); 172 | } 173 | 174 | private _resetVideoDecoder(exception: DOMException) { 175 | // 1. If [[state]] is "closed", throw an InvalidStateError. 176 | if (this.state === "closed") 177 | throw new DOMException("Decoder closed", "InvalidStateError"); 178 | 179 | // 2. Set [[state]] to "unconfigured". 180 | this.state = "unconfigured"; 181 | 182 | // ... really, we're just going to free it now 183 | this._p = this._p.then(() => this._free()); 184 | } 185 | 186 | decode(chunk: evc.EncodedVideoChunk): void { 187 | const self = this; 188 | 189 | // 1. If [[state]] is not "configured", throw an InvalidStateError. 190 | if (this.state !== "configured") 191 | throw new DOMException("Unconfigured", "InvalidStateError"); 192 | 193 | // 2. If [[key chunk required]] is true: 194 | // 1. If chunk.[[type]] is not key, throw a DataError. 195 | /* 2. Implementers SHOULD inspect the chunk’s [[internal data]] to 196 | * verify that it is truly a key chunk. If a mismatch is detected, 197 | * throw a DataError. */ 198 | // 3. Otherwise, assign false to [[key chunk required]]. 199 | 200 | // 3. Increment [[decodeQueueSize]]. 201 | this.decodeQueueSize++; 202 | 203 | // 4. Queue a control message to decode the chunk. 204 | this._p = this._p.then(async function() { 205 | const libav = self._libav!; 206 | const c = self._c; 207 | const pkt = self._pkt; 208 | const frame = self._frame; 209 | 210 | let decodedOutputs: LibAVJS.Frame[] | null = null; 211 | 212 | /* 3. Decrement [[decodeQueueSize]] and run the Schedule Dequeue 213 | * Event algorithm. */ 214 | self.decodeQueueSize--; 215 | self.dispatchEvent(new CustomEvent("dequeue")); 216 | 217 | // 1. Attempt to use [[codec implementation]] to decode the chunk. 218 | try { 219 | // Convert to a libav packet 220 | const ptsFull = Math.floor(chunk.timestamp / 1000); 221 | const [pts, ptshi] = libav.f64toi64(ptsFull); 222 | const packet: LibAVJS.Packet = { 223 | data: chunk._libavGetData(), 224 | pts, 225 | ptshi, 226 | dts: pts, 227 | dtshi: ptshi 228 | }; 229 | if (chunk.duration) { 230 | packet.duration = Math.floor(chunk.duration / 1000); 231 | packet.durationhi = 0; 232 | } 233 | 234 | decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [packet]); 235 | 236 | /* 2. If decoding results in an error, queue a task on the control 237 | * thread event loop to run the Close VideoDecoder algorithm with 238 | * EncodingError. */ 239 | } catch (ex) { 240 | self._p = self._p.then(() => { 241 | self._closeVideoDecoder( ex); 242 | }); 243 | } 244 | 245 | 246 | /* 3. If [[codec saturated]] equals true and 247 | * [[codec implementation]] is no longer saturated, queue a task 248 | * to perform the following steps: */ 249 | // 1. Assign false to [[codec saturated]]. 250 | // 2. Process the control message queue. 251 | // (unneeded) 252 | 253 | /* 4. Let decoded outputs be a list of decoded video data outputs 254 | * emitted by [[codec implementation]] in presentation order. */ 255 | 256 | /* 5. If decoded outputs is not empty, queue a task to run the 257 | * Output VideoFrame algorithm with decoded outputs. */ 258 | if (decodedOutputs) 259 | self._outputVideoFrames(decodedOutputs); 260 | 261 | }).catch(this._error); 262 | } 263 | 264 | private _outputVideoFrames(frames: LibAVJS.Frame[]) { 265 | const libav = this._libav!; 266 | 267 | for (const frame of frames) { 268 | // 1. format 269 | let format: vf.VideoPixelFormat; 270 | switch (frame.format) { 271 | case libav.AV_PIX_FMT_YUV420P: format = "I420"; break; 272 | case 0x3E: /* AV_PIX_FMT_YUV420P10 */ format = "I420P10"; break; 273 | case 0x7B: /* AV_PIX_FMT_YUV420P12 */ format = "I420P12"; break; 274 | case libav.AV_PIX_FMT_YUVA420P: format = "I420A"; break; 275 | case 0x57: /* AV_PIX_FMT_YUVA420P10 */ format = "I420AP10"; break; 276 | case libav.AV_PIX_FMT_YUV422P: format = "I422"; break; 277 | case 0x40: /* AV_PIX_FMT_YUV422P10 */ format = "I422P10"; break; 278 | case 0x7F: /* AV_PIX_FMT_YUV422P12 */ format = "I422P12"; break; 279 | case 0x4E: /* AV_PIX_FMT_YUVA422P */ format = "I422A"; break; 280 | case 0x59: /* AV_PIX_FMT_YUVA422P10 */ format = "I422AP10"; break; 281 | case 0xBA: /* AV_PIX_FMT_YUVA422P12 */ format = "I422AP12"; break; 282 | case libav.AV_PIX_FMT_YUV444P: format = "I444"; break; 283 | case 0x44: /* AV_PIX_FMT_YUV444P10 */ format = "I444P10"; break; 284 | case 0x83: /* AV_PIX_FMT_YUV444P12 */ format = "I444P12"; break; 285 | case 0x4F: /* AV_PIX_FMT_YUVA444P */ format = "I444A"; break; 286 | case 0x5B: /* AV_PIX_FMT_YUVA444P10 */ format = "I444AP10"; break; 287 | case 0xBC: /* AV_PIX_FMT_YUVA444P12 */ format = "I444AP12"; break; 288 | case libav.AV_PIX_FMT_NV12: format = "NV12"; break; 289 | case libav.AV_PIX_FMT_RGBA: format = "RGBA"; break; 290 | case 0x77: /* AV_PIX_FMT_RGB0 */ format = "RGBX"; break; 291 | case libav.AV_PIX_FMT_BGRA: format = "BGRA"; break; 292 | case 0x79: /* AV_PIX_FMT_BGR0 */ format = "BGRX"; break; 293 | 294 | default: 295 | throw new DOMException("Unsupported libav format!", "EncodingError") 296 | } 297 | 298 | // 2. width and height 299 | const codedWidth = frame.width!; 300 | const codedHeight = frame.height!; 301 | 302 | // 3. cropping 303 | let visibleRect: DOMRect; 304 | if (frame.crop) { 305 | visibleRect = new DOMRect( 306 | frame.crop.left, frame.crop.top, 307 | codedWidth - frame.crop.left - frame.crop.right, 308 | codedHeight - frame.crop.top - frame.crop.bottom 309 | ); 310 | } else { 311 | visibleRect = new DOMRect(0, 0, codedWidth, codedHeight); 312 | } 313 | 314 | // Check for non-square pixels 315 | let displayWidth = codedWidth; 316 | let displayHeight = codedHeight; 317 | if (frame.sample_aspect_ratio && frame.sample_aspect_ratio[0]) { 318 | const sar = frame.sample_aspect_ratio; 319 | if (sar[0] > sar[1]) 320 | displayWidth = ~~(codedWidth * sar[0] / sar[1]); 321 | else 322 | displayHeight = ~~(codedHeight * sar[1] / sar[0]); 323 | } 324 | 325 | // 3. timestamp 326 | const timestamp = libav.i64tof64(frame.pts!, frame.ptshi!) * 1000; 327 | 328 | const data = new vf.VideoFrame(frame.data, { 329 | layout: frame.layout, 330 | format, codedWidth, codedHeight, visibleRect, displayWidth, displayHeight, 331 | timestamp 332 | }); 333 | 334 | this._output(data); 335 | } 336 | } 337 | 338 | flush(): Promise { 339 | /* 1. If [[state]] is not "configured", return a promise rejected with 340 | * InvalidStateError DOMException. */ 341 | if (this.state !== "configured") 342 | throw new DOMException("Invalid state", "InvalidStateError"); 343 | 344 | // 2. Set [[key chunk required]] to true. 345 | // (handled by codec) 346 | 347 | // 3. Let promise be a new Promise. 348 | // 4. Append promise to [[pending flush promises]]. 349 | // 5. Queue a control message to flush the codec with promise. 350 | // 6. Process the control message queue. 351 | const ret = this._p.then(async () => { 352 | /* 1. Signal [[codec implementation]] to emit all internal pending 353 | * outputs. */ 354 | if (!this._c) 355 | return; 356 | 357 | // Make sure any last data is flushed 358 | const libav = this._libav!; 359 | const c = this._c; 360 | const pkt = this._pkt; 361 | const frame = this._frame; 362 | 363 | let decodedOutputs: LibAVJS.Frame[] | null = null; 364 | 365 | try { 366 | decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [], true); 367 | } catch (ex) { 368 | this._p = this._p.then(() => { 369 | this._closeVideoDecoder( ex); 370 | }); 371 | } 372 | 373 | /* 2. Let decoded outputs be a list of decoded video data outputs 374 | * emitted by [[codec implementation]]. */ 375 | // 3. Queue a task to perform these steps: 376 | { 377 | /* 1. If decoded outputs is not empty, run the Output VideoFrame 378 | * algorithm with decoded outputs. */ 379 | if (decodedOutputs) 380 | this._outputVideoFrames(decodedOutputs); 381 | 382 | // 2. Remove promise from [[pending flush promises]]. 383 | // 3. Resolve promise. 384 | } 385 | 386 | }); 387 | this._p = ret; 388 | 389 | // 7. Return promise. 390 | return ret; 391 | } 392 | 393 | reset(): void { 394 | this._resetVideoDecoder(new DOMException("Reset", "AbortError")); 395 | } 396 | 397 | close(): void { 398 | this._closeVideoDecoder(new DOMException("Close", "AbortError")); 399 | } 400 | 401 | static async isConfigSupported( 402 | config: VideoDecoderConfig 403 | ): Promise { 404 | const dec = libavs.decoder(config.codec, config); 405 | let supported = false; 406 | if (dec) { 407 | const libav = await libavs.get(); 408 | try { 409 | const [, c, pkt, frame] = await libav.ff_init_decoder(dec.codec); 410 | await libav.ff_free_decoder(c, pkt, frame); 411 | supported = true; 412 | } catch (ex) {} 413 | await libavs.free(libav); 414 | } 415 | 416 | return { 417 | supported, 418 | config: misc.cloneConfig( 419 | config, 420 | ["codec", "codedWidth", "codedHeight"] 421 | ) 422 | }; 423 | } 424 | } 425 | 426 | export interface VideoDecoderInit { 427 | output: VideoFrameOutputCallback; 428 | error: misc.WebCodecsErrorCallback; 429 | } 430 | 431 | export type VideoFrameOutputCallback = (output: vf.VideoFrame) => void; 432 | 433 | export interface VideoDecoderConfig { 434 | codec: string | {libavjs: libavs.LibAVJSCodec}; 435 | description?: BufferSource; 436 | codedWidth?: number; 437 | codedHeight?: number; 438 | displayAspectWidth?: number; 439 | displayAspectHeight?: number; 440 | colorSpace?: vf.VideoColorSpaceInit; 441 | hardwareAcceleration?: string; // Ignored 442 | optimizeForLatency?: boolean; 443 | } 444 | 445 | export interface VideoDecoderSupport { 446 | supported: boolean; 447 | config: VideoDecoderConfig; 448 | } 449 | -------------------------------------------------------------------------------- /src/video-encoder.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the libav.js WebCodecs Polyfill implementation. The 3 | * interface implemented is derived from the W3C standard. No attribution is 4 | * required when using this library. 5 | * 6 | * Copyright (c) 2021-2024 Yahweasel 7 | * 8 | * Permission to use, copy, modify, and/or distribute this software for any 9 | * purpose with or without fee is hereby granted. 10 | * 11 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 14 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 16 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 17 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 | */ 19 | 20 | import * as evc from "./encoded-video-chunk"; 21 | import * as et from "./event-target"; 22 | import * as libavs from "./avloader"; 23 | import * as misc from "./misc"; 24 | import * as vd from "./video-decoder"; 25 | import * as vf from "./video-frame"; 26 | 27 | import type * as LibAVJS from "@libav.js/types"; 28 | 29 | export class VideoEncoder extends et.DequeueEventTarget { 30 | constructor(init: VideoEncoderInit) { 31 | super(); 32 | 33 | // 1. Let e be a new VideoEncoder object. 34 | 35 | // 2. Assign a new queue to [[control message queue]]. 36 | this._p = Promise.all([]); 37 | 38 | // 3. Assign false to [[message queue blocked]]. 39 | // (unneeded in polyfill) 40 | 41 | // 4. Assign null to [[codec implementation]]. 42 | this._libav = null; 43 | this._codec = this._c = this._frame = this._pkt = 0; 44 | 45 | /* 5. Assign the result of starting a new parallel queue to 46 | * [[codec work queue]]. */ 47 | // (shared queue) 48 | 49 | // 6. Assign false to [[codec saturated]]. 50 | // (saturation unneeded) 51 | 52 | // 7. Assign init.output to [[output callback]]. 53 | this._output = init.output; 54 | 55 | // 8. Assign init.error to [[error callback]]. 56 | this._error = init.error; 57 | 58 | // 9. Assign null to [[active encoder config]]. 59 | // (part of codec) 60 | 61 | // 10. Assign null to [[active output config]]. 62 | this._metadata = null; 63 | 64 | // 11. Assign "unconfigured" to [[state]] 65 | this.state = "unconfigured"; 66 | 67 | // 12. Assign 0 to [[encodeQueueSize]]. 68 | this.encodeQueueSize = 0; 69 | 70 | // 13. Assign a new list to [[pending flush promises]]. 71 | // (shared queue) 72 | 73 | // 14. Assign false to [[dequeue event scheduled]]. 74 | // (shared queue) 75 | 76 | // 15. Return e. 77 | } 78 | 79 | /* NOTE: These should technically be readonly, but I'm implementing them as 80 | * plain fields, so they're writable */ 81 | state: misc.CodecState; 82 | encodeQueueSize: number; 83 | 84 | private _output: EncodedVideoChunkOutputCallback; 85 | private _error: misc.WebCodecsErrorCallback; 86 | 87 | // Event queue 88 | private _p: Promise; 89 | 90 | // LibAV state 91 | private _libav: LibAVJS.LibAV | null; 92 | private _codec: number; 93 | private _c: number; 94 | private _frame: number; 95 | private _pkt: number; 96 | private _extradataSet: boolean = false; 97 | private _extradata: Uint8Array | null = null; 98 | private _metadata: EncodedVideoChunkMetadata | null; 99 | 100 | // Software scaler state, if used 101 | private _sws?: number; 102 | private _swsFrame?: number; 103 | private _swsIn?: SWScaleState; 104 | private _swsOut?: SWScaleState; 105 | 106 | // If our output uses non-square pixels, that information 107 | private _nonSquarePixels: boolean = false; 108 | private _sar_num: number = 1; 109 | private _sar_den: number = 1; 110 | 111 | configure(config: VideoEncoderConfig): void { 112 | // 1. If config is not a valid VideoEncoderConfig, throw a TypeError. 113 | // NOTE: We don't support sophisticated codec string parsing (yet) 114 | 115 | // 2. If [[state]] is "closed", throw an InvalidStateError. 116 | if (this.state === "closed") 117 | throw new DOMException("Encoder is closed", "InvalidStateError"); 118 | 119 | // Free any internal state 120 | if (this._libav) 121 | this._p = this._p.then(() => this._free()); 122 | 123 | // 3. Set [[state]] to "configured". 124 | this.state = "configured"; 125 | 126 | // 4. Queue a control message to configure the encoder using config. 127 | this._p = this._p.then(async () => { 128 | /* 1. Let supported be the result of running the Check 129 | * Configuration Support algorithm with config. */ 130 | const supported = libavs.encoder(config.codec, config); 131 | 132 | /* 2. If supported is false, queue a task to run the Close 133 | * VideoEncoder algorithm with NotSupportedError and abort these 134 | * steps. */ 135 | if (!supported) { 136 | this._closeVideoEncoder(new DOMException("Unsupported codec", "NotSupportedError")); 137 | return; 138 | } 139 | 140 | /* 3. If needed, assign [[codec implementation]] with an 141 | * implementation supporting config. */ 142 | // 4. Configure [[codec implementation]] with config. 143 | const libav = this._libav = await libavs.get(); 144 | this._metadata = { 145 | decoderConfig: { 146 | codec: supported.codec 147 | } 148 | }; 149 | 150 | // And initialize 151 | [this._codec, this._c, this._frame, this._pkt] = 152 | await libav.ff_init_encoder(supported.codec, supported); 153 | this._extradataSet = false; 154 | this._extradata = null; 155 | await libav.AVCodecContext_time_base_s(this._c, 1, 1000); 156 | 157 | const width = config.width; 158 | const height = config.height; 159 | 160 | this._sws = 0; 161 | this._swsFrame = 0; 162 | this._swsOut = { 163 | width, height, 164 | format: supported.ctx!.pix_fmt! 165 | }; 166 | 167 | // Check for non-square pixels 168 | const dWidth = config.displayWidth || width; 169 | const dHeight = config.displayHeight || height; 170 | if (dWidth !== width || dHeight !== height) { 171 | this._nonSquarePixels = true; 172 | this._sar_num = dWidth * height; 173 | this._sar_den = dHeight * width; 174 | } else { 175 | this._nonSquarePixels = false; 176 | } 177 | 178 | // 5. queue a task to run the following steps: 179 | // 1. Assign false to [[message queue blocked]]. 180 | // 2. Queue a task to Process the control message queue. 181 | 182 | }).catch(this._error); 183 | } 184 | 185 | // Our own algorithm, close libav 186 | private async _free() { 187 | if (this._sws) { 188 | await this._libav!.av_frame_free_js(this._swsFrame!); 189 | await this._libav!.sws_freeContext(this._sws); 190 | this._sws = this._swsFrame = 0; 191 | this._swsIn = this._swsOut = void 0; 192 | } 193 | if (this._c) { 194 | await this._libav!.ff_free_encoder(this._c, this._frame, this._pkt); 195 | this._codec = this._c = this._frame = this._pkt = 0; 196 | } 197 | if (this._libav) { 198 | libavs.free(this._libav); 199 | this._libav = null; 200 | } 201 | } 202 | 203 | private _closeVideoEncoder(exception: DOMException) { 204 | // 1. Run the Reset VideoEncoder algorithm with exception. 205 | this._resetVideoEncoder(exception); 206 | 207 | // 2. Set [[state]] to "closed". 208 | this.state = "closed"; 209 | 210 | /* 3. Clear [[codec implementation]] and release associated system 211 | * resources. */ 212 | this._p = this._p.then(() => this._free()); 213 | 214 | /* 4. If exception is not an AbortError DOMException, invoke the 215 | * [[error callback]] with exception. */ 216 | if (exception.name !== "AbortError") 217 | this._p = this._p.then(() => { this._error(exception); }); 218 | } 219 | 220 | private _resetVideoEncoder(exception: DOMException) { 221 | // 1. If [[state]] is "closed", throw an InvalidStateError. 222 | if (this.state === "closed") 223 | throw new DOMException("Encoder closed", "InvalidStateError"); 224 | 225 | // 2. Set [[state]] to "unconfigured". 226 | this.state = "unconfigured"; 227 | 228 | // ... really, we're just going to free it now 229 | this._p = this._p.then(() => this._free()); 230 | } 231 | 232 | encode(frame: vf.VideoFrame, options: VideoEncoderEncodeOptions = {}) { 233 | /* 1. If the value of frame’s [[Detached]] internal slot is true, throw 234 | * a TypeError. */ 235 | if (frame._libavGetData() === null) 236 | throw new TypeError("Detached"); 237 | 238 | // 2. If [[state]] is not "configured", throw an InvalidStateError. 239 | if (this.state !== "configured") 240 | throw new DOMException("Unconfigured", "InvalidStateError"); 241 | 242 | /* 3. Let frameClone hold the result of running the Clone VideoFrame 243 | * algorithm with frame. */ 244 | const frameClone = frame.clone(); 245 | 246 | // 4. Increment [[encodeQueueSize]]. 247 | this.encodeQueueSize++; 248 | 249 | // 5. Queue a control message to encode frameClone. 250 | this._p = this._p.then(async () => { 251 | const libav = this._libav!; 252 | const c = this._c; 253 | const pkt = this._pkt; 254 | const framePtr = this._frame; 255 | const swsOut = this._swsOut!; 256 | 257 | let encodedOutputs: LibAVJS.Packet[] | null = null; 258 | 259 | /* 3. Decrement [[encodeQueueSize]] and run the Schedule Dequeue 260 | * Event algorithm. */ 261 | this.encodeQueueSize--; 262 | this.dispatchEvent(new CustomEvent("dequeue")); 263 | 264 | /* 1. Attempt to use [[codec implementation]] to encode frameClone 265 | * according to options. */ 266 | try { 267 | 268 | // Convert the format 269 | const format = vf.wcFormatToLibAVFormat(libav, frameClone.format); 270 | 271 | // Convert the data 272 | const rawU8 = frameClone._libavGetData(); 273 | const layout = frameClone._libavGetLayout(); 274 | 275 | // Convert the timestamp 276 | const ptsFull = Math.floor(frameClone.timestamp / 1000); 277 | const [pts, ptshi] = libav.f64toi64(ptsFull); 278 | 279 | // Make the frame 280 | const frame: LibAVJS.Frame = { 281 | data: rawU8, layout, 282 | format, pts, ptshi, 283 | width: frameClone.codedWidth, 284 | height: frameClone.codedHeight, 285 | crop: { 286 | left: frameClone.visibleRect.left, 287 | right: frameClone.visibleRect.right, 288 | top: frameClone.visibleRect.top, 289 | bottom: frameClone.visibleRect.bottom 290 | }, 291 | key_frame: options.keyFrame ? 1 : 0, 292 | pict_type: options.keyFrame ? 1 : 0 293 | }; 294 | 295 | // Possibly scale 296 | if (frame.width !== swsOut.width || 297 | frame.height !== swsOut.height || 298 | frame.format !== swsOut.format) { 299 | if (frameClone._nonSquarePixels) { 300 | frame.sample_aspect_ratio = [ 301 | frameClone._sar_num, 302 | frameClone._sar_den 303 | ]; 304 | } 305 | 306 | // Need a scaler 307 | let sws = this._sws, swsIn = this._swsIn!, 308 | swsFrame = this._swsFrame!; 309 | if (!sws || 310 | frame.width !== swsIn.width || 311 | frame.height !== swsIn.height || 312 | frame.format !== swsIn.format) { 313 | // Need to allocate the scaler 314 | if (sws) 315 | await libav.sws_freeContext(sws); 316 | swsIn = { 317 | width: frame.width!, 318 | height: frame.height!, 319 | format: frame.format 320 | }; 321 | sws = await libav.sws_getContext( 322 | swsIn.width, swsIn.height, swsIn.format, 323 | swsOut.width, swsOut.height, swsOut.format, 324 | 2, 0, 0, 0); 325 | this._sws = sws; 326 | this._swsIn = swsIn; 327 | 328 | // Maybe need a frame 329 | if (!swsFrame) 330 | this._swsFrame = swsFrame = await libav.av_frame_alloc() 331 | } 332 | 333 | // Scale and encode the frame 334 | const [, swsRes, , , , , , encRes] = 335 | await Promise.all([ 336 | libav.ff_copyin_frame(framePtr, frame), 337 | libav.sws_scale_frame(sws, swsFrame, framePtr), 338 | this._nonSquarePixels ? 339 | libav.AVFrame_sample_aspect_ratio_s(swsFrame, 340 | this._sar_num, this._sar_den) : 341 | null, 342 | libav.AVFrame_pts_s(swsFrame, pts), 343 | libav.AVFrame_ptshi_s(swsFrame, ptshi), 344 | libav.AVFrame_key_frame_s(swsFrame, options.keyFrame ? 1 : 0), 345 | libav.AVFrame_pict_type_s(swsFrame, options.keyFrame ? 1 : 0), 346 | libav.avcodec_send_frame(c, swsFrame) 347 | ]); 348 | if (swsRes < 0 || encRes < 0) 349 | throw new Error("Encoding failed!"); 350 | encodedOutputs = []; 351 | while (true) { 352 | const recv = 353 | await libav.avcodec_receive_packet(c, pkt); 354 | if (recv === -libav.EAGAIN) 355 | break; 356 | else if (recv < 0) 357 | throw new Error("Encoding failed!"); 358 | encodedOutputs.push( 359 | await libav.ff_copyout_packet(pkt)); 360 | } 361 | 362 | } else { 363 | if (this._nonSquarePixels) { 364 | frame.sample_aspect_ratio = [ 365 | this._sar_num, 366 | this._sar_den 367 | ]; 368 | } 369 | 370 | // Encode directly 371 | encodedOutputs = 372 | await libav.ff_encode_multi(c, framePtr, pkt, [frame]); 373 | 374 | } 375 | 376 | if (encodedOutputs.length && !this._extradataSet) 377 | await this._getExtradata(); 378 | 379 | /* 2. If encoding results in an error, queue a task to run the 380 | * Close VideoEncoder algorithm with EncodingError and return. */ 381 | } catch (ex) { 382 | this._p = this._p.then(() => { 383 | this._closeVideoEncoder( ex); 384 | }); 385 | return; 386 | } 387 | 388 | /* 3. If [[codec saturated]] equals true and 389 | * [[codec implementation]] is no longer saturated, queue a task 390 | * to perform the following steps: */ 391 | // 1. Assign false to [[codec saturated]]. 392 | // 2. Process the control message queue. 393 | // (unneeded in polyfill) 394 | 395 | /* 4. Let encoded outputs be a list of encoded video data outputs 396 | * emitted by [[codec implementation]]. */ 397 | 398 | /* 5. If encoded outputs is not empty, queue a task to run the 399 | * Output EncodedVideoChunks algorithm with encoded outputs. */ 400 | if (encodedOutputs) 401 | this._outputEncodedVideoChunks(encodedOutputs); 402 | 403 | }).catch(this._error); 404 | } 405 | 406 | // Internal: Get extradata 407 | private async _getExtradata() { 408 | const libav = this._libav!; 409 | const c = this._c; 410 | const extradata = await libav.AVCodecContext_extradata(c); 411 | const extradata_size = await libav.AVCodecContext_extradata_size(c); 412 | if (extradata && extradata_size) { 413 | this._metadata!.decoderConfig!.description = this._extradata = 414 | await libav.copyout_u8(extradata, extradata_size); 415 | } 416 | this._extradataSet = true; 417 | } 418 | 419 | private _outputEncodedVideoChunks(packets: LibAVJS.Packet[]) { 420 | const libav = this._libav!; 421 | 422 | for (const packet of packets) { 423 | // 1. type 424 | const type: evc.EncodedVideoChunkType = 425 | (packet.flags! & 1) ? "key" : "delta"; 426 | 427 | // 2. timestamp 428 | const timestamp = libav.i64tof64(packet.pts!, packet.ptshi!) * 1000; 429 | 430 | const chunk = new evc.EncodedVideoChunk({ 431 | type: type, timestamp, 432 | data: packet.data 433 | }); 434 | 435 | if (this._extradataSet) 436 | this._output(chunk, this._metadata || void 0); 437 | else 438 | this._output(chunk); 439 | } 440 | } 441 | 442 | flush(): Promise { 443 | /* 1. If [[state]] is not "configured", return a promise rejected with 444 | * InvalidStateError DOMException. */ 445 | if (this.state !== "configured") 446 | throw new DOMException("Invalid state", "InvalidStateError"); 447 | 448 | // 2. Let promise be a new Promise. 449 | // 3. Append promise to [[pending flush promises]]. 450 | // 4. Queue a control message to flush the codec with promise. 451 | // 5. Process the control message queue. 452 | const ret = this._p.then(async () => { 453 | /* 1. Signal [[codec implementation]] to emit all internal pending 454 | * outputs. */ 455 | if (!this._c) 456 | return; 457 | 458 | // Make sure any last data is flushed 459 | const libav = this._libav!; 460 | const c = this._c; 461 | const frame = this._frame; 462 | const pkt = this._pkt; 463 | 464 | let encodedOutputs: LibAVJS.Packet[] | null = null; 465 | 466 | try { 467 | encodedOutputs = 468 | await libav.ff_encode_multi(c, frame, pkt, [], true); 469 | if (!this._extradataSet) 470 | await this._getExtradata(); 471 | } catch (ex) { 472 | this._p = this._p.then(() => { 473 | this._closeVideoEncoder( ex); 474 | }); 475 | } 476 | 477 | /* 2. Let encoded outputs be a list of encoded video data outputs 478 | * emitted by [[codec implementation]]. */ 479 | 480 | // 3. Queue a task to perform these steps: 481 | { 482 | /* 1. If encoded outputs is not empty, run the Output 483 | * EncodedVideoChunks algorithm with encoded outputs. */ 484 | if (encodedOutputs) 485 | this._outputEncodedVideoChunks(encodedOutputs); 486 | 487 | // 2. Remove promise from [[pending flush promises]]. 488 | // 3. Resolve promise. 489 | } 490 | 491 | }); 492 | this._p = ret; 493 | 494 | // 6. Return promise. 495 | return ret; 496 | } 497 | 498 | reset(): void { 499 | this._resetVideoEncoder(new DOMException("Reset", "AbortError")); 500 | } 501 | 502 | close(): void { 503 | this._closeVideoEncoder(new DOMException("Close", "AbortError")); 504 | } 505 | 506 | static async isConfigSupported( 507 | config: VideoEncoderConfig 508 | ): Promise { 509 | const enc = libavs.encoder(config.codec, config); 510 | let supported = false; 511 | if (enc) { 512 | const libav = await libavs.get(); 513 | try { 514 | const [, c, frame, pkt] = 515 | await libav.ff_init_encoder(enc.codec, enc); 516 | await libav.ff_free_encoder(c, frame, pkt); 517 | supported = true; 518 | } catch (ex) {} 519 | await libavs.free(libav); 520 | } 521 | 522 | return { 523 | supported, 524 | config: misc.cloneConfig( 525 | config, 526 | ["codec", "width", "height", "bitrate", "framerate", "latencyMode"] 527 | ) 528 | }; 529 | } 530 | }; 531 | 532 | export interface VideoEncoderInit { 533 | output: EncodedVideoChunkOutputCallback; 534 | error: misc.WebCodecsErrorCallback; 535 | } 536 | 537 | export type EncodedVideoChunkOutputCallback = 538 | (chunk: evc.EncodedVideoChunk, 539 | metadata?: EncodedVideoChunkMetadata) => void; 540 | 541 | export interface EncodedVideoChunkMetadata { 542 | decoderConfig?: vd.VideoDecoderConfig; 543 | svc?: any; // Unused 544 | alphaSideData?: any; // Unused 545 | } 546 | 547 | export interface VideoEncoderConfig { 548 | codec: string | {libavjs: libavs.LibAVJSCodec}; 549 | width: number; 550 | height: number; 551 | displayWidth?: number; 552 | displayHeight?: number; 553 | bitrate?: number; 554 | framerate?: number; 555 | hardwareAcceleration?: string; // Ignored, of course 556 | alpha?: string; // Ignored 557 | scalabilityMode?: string; // Ignored 558 | bitrateMode?: string; // Ignored 559 | latencyMode?: LatencyMode; 560 | } 561 | 562 | export interface VideoEncoderEncodeOptions { 563 | keyFrame?: boolean; 564 | } 565 | 566 | export type LatencyMode = 567 | "quality" | 568 | "realtime"; 569 | 570 | export interface VideoEncoderSupport { 571 | supported: boolean; 572 | config: VideoEncoderConfig; 573 | } 574 | 575 | // Used internally 576 | interface SWScaleState { 577 | width: number; 578 | height: number; 579 | format: number; 580 | } 581 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "strict": true, 4 | "target": "es6", 5 | "module": "es6", 6 | "moduleResolution": "node", 7 | "lib": ["es2020", "dom"] 8 | } 9 | } 10 | --------------------------------------------------------------------------------