├── .npmignore
├── .gitignore
├── samples
├── sample1.opus
├── decoder
│ ├── decoder.js
│ └── index.html
├── util.js
├── transcoder
│ ├── index.html
│ └── transcoder.js
└── worker-util.js
├── tsconfig.json
├── Makefile
├── rollup.config.mjs
├── src
├── build.js
├── bridge.ts
├── wctola.ts
├── latowc.ts
├── mux.ts
└── demux.ts
├── README.md
├── package.json
├── demo
├── demo.html
└── demo.js
└── docs
└── API.md
/.npmignore:
--------------------------------------------------------------------------------
1 | .*.swp
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | dist/
3 | src/*.js
4 |
--------------------------------------------------------------------------------
/samples/sample1.opus:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Yahweasel/libavjs-webcodecs-bridge/HEAD/samples/sample1.opus
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es6",
4 | "module": "es6",
5 | "moduleResolution": "node",
6 | "strict": true,
7 | "lib": ["es2020", "dom"]
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | all: dist/libavjs-webcodecs-bridge.min.js
2 |
3 | dist/libavjs-webcodecs-bridge.min.js: src/*.ts node_modules/.bin/tsc
4 | npm run build
5 |
6 | node_modules/.bin/tsc:
7 | npm install
8 |
9 | clean:
10 | npm run clean
11 |
--------------------------------------------------------------------------------
/rollup.config.mjs:
--------------------------------------------------------------------------------
1 | import nodeResolve from "@rollup/plugin-node-resolve";
2 | import commonjs from "@rollup/plugin-commonjs";
3 | import terser from "@rollup/plugin-terser";
4 |
5 | export default {
6 | input: "src/bridge.js",
7 | output: [
8 | {
9 | file: "dist/libavjs-webcodecs-bridge.js",
10 | format: "umd",
11 | name: "LibAVWebCodecsBridge"
12 | }, {
13 | file: "dist/libavjs-webcodecs-bridge.min.js",
14 | format: "umd",
15 | name: "LibAVWebCodecsBridge"
16 | }, {
17 | file: "dist/libavjs-webcodecs-bridge.mjs",
18 | format: "es"
19 | }, {
20 | file: "dist/libavjs-webcodecs-bridge.min.mjs",
21 | format: "es",
22 | plugins: [terser()]
23 | }
24 | ],
25 | context: "this",
26 | plugins: [nodeResolve(), commonjs()]
27 | };
28 |
--------------------------------------------------------------------------------
/src/build.js:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env node
2 | const fs = require("fs");
3 |
4 | const browserify = require("browserify");
5 | const browserPackFlat = require("browser-pack-flat");
6 | const tsify = require("tsify");
7 |
8 | let noImplicitAny = false;
9 | let main = "src/bridge.ts";
10 | let standalone = "LibAVWebCodecsBridge";
11 | for (let ai = 2; ai < process.argv.length; ai++) {
12 | const arg = process.argv[ai];
13 | if (arg === "-n" || arg === "--no-implicit-any")
14 | noImplicitAny = true;
15 | else if (arg === "-s" || arg === "--standalone")
16 | standalone = process.argv[++ai];
17 | else if (arg[0] !== "-")
18 | main = arg;
19 | else {
20 | console.error(`Unrecognized argument ${arg}`);
21 | process.exit(1);
22 | }
23 | }
24 |
25 | let b = browserify({standalone})
26 | .add(main)
27 | .plugin(tsify, { noImplicitAny, files: [] })
28 | .plugin(browserPackFlat)
29 | .bundle()
30 | .on("error", function (error) { console.error(error.toString()); });
31 | b.pipe(process.stdout);
32 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | libavjs-webcodecs-bridge is a bridge to help you use [libav.js](https://github.com/Yahweasel/libav.js/) and [WebCodecs](https://github.com/w3c/webcodecs) (or
2 | [libavjs-webcodecs-polyfill](https://github.com/ennuicastr/libavjs-webcodecs-polyfill)) together.
3 |
4 | WebCodecs does not come with demuxers or muxers. libav.js has those as well as
5 | encoders and decoders, but if you have WebCodecs available, you probably should
6 | use them for en/decoding instead of libav.js. That means it's common to demux
7 | with libav.js then decode with WebCodecs, or encode with WebCodecs then mux with
8 | libav.js. But, they don't speak the same language, so to speak.
9 |
10 | This bridge bridges the gap. It includes conversions from the various libav.js
11 | types to the equivalent WebCodecs types, and vice-versa.
12 |
13 | This project is by the same author as libav.js and libavjs-webcodecs-polyfill.
14 | You do not need libavjs-webcodecs-polyfill to use libavjs-webcodecs-bridge or
15 | vice versa; they have related but orthogonal purposes. For type reasons, this
16 | repository depends on both, but even if you bundle libavjs-webcodecs-bridge,
17 | neither will be included.
18 |
19 | libavjs-webcodecs-bridge's API is documented in [API.md](docs/API.md). The demo
20 | in the `demo` directory is a demonstration of start-to-finish transcoding, and
21 | there are some samples in the `samples` directory as well.
22 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "libavjs-webcodecs-bridge",
3 | "version": "0.3.2",
4 | "description": "A bridge between libav.js and WebCodecs, to allow easier decoding of files demuxed by libav.js",
5 | "main": "dist/libavjs-webcodecs-bridge.js",
6 | "types": "src/bridge.ts",
7 | "exports": {
8 | "import": "./dist/libavjs-webcodecs-bridge.mjs",
9 | "default": "./dist/libavjs-webcodecs-bridge.js",
10 | "types": "./src/bridge.ts"
11 | },
12 | "scripts": {
13 | "build": "tsc && rollup -c",
14 | "clean": "rm -f dist/",
15 | "test": "echo \"Error: no test specified\" && exit 1"
16 | },
17 | "repository": {
18 | "type": "git",
19 | "url": "git+https://github.com/Yahweasel/libavjs-webcodecs-bridge.git"
20 | },
21 | "keywords": [
22 | "webcodecs",
23 | "demuxing",
24 | "decoding"
25 | ],
26 | "author": "Yahweasel",
27 | "license": "ISC",
28 | "bugs": {
29 | "url": "https://github.com/Yahweasel/libavjs-webcodecs-bridge/issues"
30 | },
31 | "homepage": "https://github.com/Yahweasel/libavjs-webcodecs-bridge#readme",
32 | "devDependencies": {
33 | "@libav.js/variant-webcodecs": "=6.4.7",
34 | "@rollup/plugin-commonjs": "^25.0.7",
35 | "@rollup/plugin-node-resolve": "^15.2.3",
36 | "@rollup/plugin-terser": "^0.4.4",
37 | "rollup": "^4.12.0",
38 | "typescript": "^5.1.6"
39 | },
40 | "dependencies": {
41 | "libavjs-webcodecs-polyfill": "^0.5.3"
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/bridge.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * This file is part of the libav.js WebCodecs Bridge implementation.
3 | *
4 | * Copyright (c) 2023 Yahweasel
5 | *
6 | * Permission to use, copy, modify, and/or distribute this software for any
7 | * purpose with or without fee is hereby granted, provided that the above
8 | * copyright notice and this permission notice appear in all copies.
9 | *
10 | * THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
13 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
15 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
16 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 | */
18 |
19 | /*
20 | * This is the main entry point and simply exposes the interfaces provided by
21 | * other components.
22 | */
23 |
24 | import * as demux from "./demux";
25 | import * as mux from "./mux";
26 | import * as wctola from "./wctola";
27 | import * as latowc from "./latowc";
28 |
29 | export const audioStreamToConfig = demux.audioStreamToConfig;
30 | export const videoStreamToConfig = demux.videoStreamToConfig;
31 | export const packetToEncodedAudioChunk = demux.packetToEncodedAudioChunk;
32 | export const packetToEncodedVideoChunk = demux.packetToEncodedVideoChunk;
33 |
34 | export const configToAudioStream = mux.configToAudioStream;
35 | export const configToVideoStream = mux.configToVideoStream;
36 | export const encodedAudioChunkToPacket = mux.encodedAudioChunkToPacket;
37 | export const encodedVideoChunkToPacket = mux.encodedVideoChunkToPacket;
38 |
39 | export const videoFrameToLAFrame = wctola.videoFrameToLAFrame;
40 | export const audioDataToLAFrame = wctola.audioDataToLAFrame;
41 |
42 | export const laFrameToVideoFrame = latowc.laFrameToVideoFrame;
43 | export const laFrameToAudioData = latowc.laFrameToAudioData;
44 |
--------------------------------------------------------------------------------
/samples/decoder/decoder.js:
--------------------------------------------------------------------------------
1 | /*
2 | * This (un)license applies only to this sample code, and not to
3 | * libavjs-webcodecs-bridge as a whole:
4 | *
5 | * This is free and unencumbered software released into the public domain.
6 | *
7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute
8 | * this software, either in source code form or as a compiled binary, for any
9 | * purpose, commercial or non-commercial, and by any means.
10 | *
11 | * In jurisdictions that recognize copyright laws, the author or authors of
12 | * this software dedicate any and all copyright interest in the software to the
13 | * public domain. We make this dedication for the benefit of the public at
14 | * large and to the detriment of our heirs and successors. We intend this
15 | * dedication to be an overt act of relinquishment in perpetuity of all present
16 | * and future rights to this software under copyright law.
17 | *
18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 | */
25 |
26 | importScripts("../worker-util.js");
27 |
28 | onmessage = async ev => {
29 | const file = ev.data;
30 | console.error(file);
31 | let streams, configs, allPackets;
32 |
33 | try {
34 | [streams, configs, allPackets] =
35 | await sampleDemux(file);
36 | } catch (ex) {
37 | console.error(ex);
38 | return;
39 | }
40 |
41 | for (let idx = 0; idx < streams.length; idx++) {
42 | const stream = streams[idx];
43 | const config = configs[idx];
44 | if (!config)
45 | continue;
46 | const packets = allPackets[stream.index];
47 |
48 | try {
49 | if (stream.codec_type === 0 /* video */) {
50 | await decodeVideo(config, packets, stream);
51 | } else if (stream.codec_type === 1 /* audio */) {
52 | await decodeAudio(config, packets, stream);
53 | }
54 | } catch (ex) {
55 | console.error(ex);
56 | }
57 | }
58 |
59 | postMessage({c: "done"});
60 | };
61 |
--------------------------------------------------------------------------------
/demo/demo.html:
--------------------------------------------------------------------------------
1 |
2 |
26 |
27 |
28 |
29 | LibAVJS WebCodecs Streaming Transcode Example
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
43 |
44 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
--------------------------------------------------------------------------------
/samples/util.js:
--------------------------------------------------------------------------------
1 | /*
2 | * This (un)license applies only to this sample code, and not to
3 | * libavjs-webcodecs-bridge as a whole:
4 | *
5 | * This is free and unencumbered software released into the public domain.
6 | *
7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute
8 | * this software, either in source code form or as a compiled binary, for any
9 | * purpose, commercial or non-commercial, and by any means.
10 | *
11 | * In jurisdictions that recognize copyright laws, the author or authors of
12 | * this software dedicate any and all copyright interest in the software to the
13 | * public domain. We make this dedication for the benefit of the public at
14 | * large and to the detriment of our heirs and successors. We intend this
15 | * dedication to be an overt act of relinquishment in perpetuity of all present
16 | * and future rights to this software under copyright law.
17 | *
18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 | */
25 |
26 | function sampleFileInput(id, func) {
27 | const box = document.getElementById(id);
28 | box.onchange = function() {
29 | const file = box.files[0];
30 | if (!file)
31 | return;
32 | func(file, box);
33 | };
34 | }
35 |
36 | async function sampleOutputAudio(a) {
37 | // Quick concat
38 | const blob = new Blob(a);
39 | a = new Float32Array(await blob.arrayBuffer());
40 |
41 | const canvas = document.createElement("canvas");
42 | canvas.style.display = "block";
43 | const w = canvas.width = 1024;
44 | const h = canvas.height = 64;
45 | document.body.appendChild(canvas);
46 | const ctx = canvas.getContext("2d");
47 |
48 | for (let x = 0; x < w; x++) {
49 | const idx = Math.floor((x / w) * a.length);
50 | const y = h - (h * Math.abs(a[idx]));
51 | ctx.fillStyle = "#fff";
52 | ctx.fillRect(x, 0, 1, y);
53 | ctx.fillStyle = "#0f0";
54 | ctx.fillRect(x, y, 1, h - y);
55 | }
56 | }
57 |
58 | function sampleOutputVideo(v, fps) {
59 | const canvas = document.createElement("canvas");
60 | canvas.style.display = "block";
61 | const w = canvas.width = v[0].width;
62 | const h = canvas.height = v[0].height;
63 | document.body.appendChild(canvas);
64 | const ctx = canvas.getContext("2d");
65 |
66 | let idx = 0;
67 | const interval = setInterval(async () => {
68 | const image = v[idx++];
69 | ctx.drawImage(image, 0, 0);
70 |
71 | if (idx >= v.length)
72 | idx = 0;
73 | }, Math.round(1000 / fps))
74 | }
75 |
--------------------------------------------------------------------------------
/samples/transcoder/index.html:
--------------------------------------------------------------------------------
1 |
2 |
26 |
27 |
28 |
29 | LibAVJS WebCodecs Bridge Example: Transcoder
30 |
31 |
32 |
NOTE: This sample just demonstrates how to get and pass configurations. It is not efficient, and will freeze or fail on large files, as it does not stream data, but demuxes and muxes everything in one shot.
NOTE: This sample just demonstrates how to get and pass configurations. It is not efficient, and will freeze or fail on large files, as it does not stream data, but demuxes everything in one shot.
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
81 |
82 |
83 |
--------------------------------------------------------------------------------
/samples/worker-util.js:
--------------------------------------------------------------------------------
1 | /*
2 | * This (un)license applies only to this sample code, and not to
3 | * libavjs-webcodecs-bridge as a whole:
4 | *
5 | * This is free and unencumbered software released into the public domain.
6 | *
7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute
8 | * this software, either in source code form or as a compiled binary, for any
9 | * purpose, commercial or non-commercial, and by any means.
10 | *
11 | * In jurisdictions that recognize copyright laws, the author or authors of
12 | * this software dedicate any and all copyright interest in the software to the
13 | * public domain. We make this dedication for the benefit of the public at
14 | * large and to the detriment of our heirs and successors. We intend this
15 | * dedication to be an overt act of relinquishment in perpetuity of all present
16 | * and future rights to this software under copyright law.
17 | *
18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 | */
25 |
26 | if (typeof importScripts !== "undefined") {
27 | LibAV = {base: "https://cdn.jsdelivr.net/npm/@libav.js/variant-webcodecs@5.1.6/dist"};
28 | importScripts(LibAV.base + "/libav-6.0.7.0.2-webcodecs.js");
29 | importScripts("../../dist/libavjs-webcodecs-bridge.js");
30 | }
31 |
32 | async function sampleDemux(file, readOpts) {
33 | /* NOTE: noworker is not mandatory (this is in a worker, so it's fine)! */
34 | const libav = await LibAV.LibAV({noworker: true});
35 | await libav.mkreadaheadfile("input", file);
36 |
37 | const [fmt_ctx, streams] = await libav.ff_init_demuxer_file("input");
38 |
39 | const configs = await Promise.all(streams.map(stream => {
40 | if (stream.codec_type === libav.AVMEDIA_TYPE_AUDIO)
41 | return LibAVWebCodecsBridge.audioStreamToConfig(libav, stream);
42 | else if (stream.codec_type === libav.AVMEDIA_TYPE_VIDEO)
43 | return LibAVWebCodecsBridge.videoStreamToConfig(libav, stream);
44 | else
45 | return null;
46 | }));
47 |
48 | const pkt = await libav.av_packet_alloc();
49 | const [, packets] = await libav.ff_read_multi(fmt_ctx, pkt, null, readOpts);
50 |
51 | libav.terminate();
52 |
53 | return [streams, configs, packets];
54 | }
55 |
56 | async function sampleMux(filename, codec, packets, extradata) {
57 | const libavPackets = [];
58 | for (const packet of packets) {
59 | const ab = new ArrayBuffer(packet.byteLength);
60 | packet.copyTo(ab);
61 | const pts = ~~(packet.timestamp / 1000);
62 | libavPackets.push({
63 | data: new Uint8Array(ab),
64 | pts, ptshi: 0,
65 | dts: pts, dtshi: 0,
66 | flags: (packet.type === "key") ? 1 : 0
67 | });
68 | }
69 |
70 | const libav = await LibAV.LibAV({noworker: true});
71 |
72 | /* Decode a little bit (and use extradata) just to make sure everything
73 | * necessary for a header is in place */
74 | let [, c, pkt, frame] = await libav.ff_init_decoder(codec);
75 | await libav.AVCodecContext_time_base_s(c, 1, 1000);
76 | await libav.ff_decode_multi(c, pkt, frame, [libavPackets[0]]);
77 | if (extradata) {
78 | const extradataPtr = await libav.malloc(extradata.length);
79 | await libav.copyin_u8(extradataPtr, extradata);
80 | await libav.AVCodecContext_extradata_s(c, extradataPtr);
81 | await libav.AVCodecContext_extradata_size_s(c, extradata.length);
82 | }
83 |
84 | // Now mux it
85 | const [oc, , pb] = await libav.ff_init_muxer(
86 | {filename, open: true}, [[c, 1, 1000]]);
87 | await libav.avformat_write_header(oc, 0);
88 | await libav.ff_write_multi(oc, pkt, libavPackets);
89 | await libav.av_write_trailer(oc);
90 | await libav.ff_free_muxer(oc, pb);
91 | const ret = await libav.readFile(filename);
92 | libav.terminate();
93 | return ret;
94 | }
95 |
96 | async function decodeAudio(init, packets, stream) {
97 | // Feed them into the decoder
98 | const decoder = new AudioDecoder({
99 | output: frame => {
100 | const copyOpts = {
101 | planeIndex: 0,
102 | format: "f32-planar"
103 | };
104 | const ab = new ArrayBuffer(frame.allocationSize(copyOpts));
105 | frame.copyTo(ab, copyOpts);
106 | postMessage({c: "frame", idx: stream.index, a: true, frame: ab}, [ab]);
107 | frame.close();
108 | },
109 | error: x => console.error
110 | });
111 | decoder.configure(init);
112 | for (const packet of packets) {
113 | const eac = LibAVWebCodecsBridge.packetToEncodedAudioChunk(packet, stream);
114 | decoder.decode(eac);
115 | }
116 |
117 | // Wait for it to finish
118 | await decoder.flush();
119 | decoder.close();
120 | }
121 |
122 | async function decodeVideo(init, packets, stream) {
123 | // Feed them into the decoder
124 | let frameP = Promise.all([]);
125 | const decoder = new VideoDecoder({
126 | output: frame => {
127 | frameP = frameP.then(async function() {
128 | const ib = await createImageBitmap(frame);
129 | postMessage({c: "frame", idx: stream.index, v: true, frame: ib}, [ib]);
130 | frame.close();
131 | }).catch(console.error);
132 | },
133 | error: x => console.error
134 | });
135 | decoder.configure(init);
136 |
137 | for (const packet of packets.slice(0, 128)) {
138 | const evc = LibAVWebCodecsBridge.packetToEncodedVideoChunk(packet, stream);
139 | decoder.decode(evc);
140 | }
141 |
142 | // Wait for it to finish
143 | await decoder.flush();
144 | await frameP;
145 | decoder.close();
146 | }
147 |
--------------------------------------------------------------------------------
/src/wctola.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * This file is part of the libav.js WebCodecs Bridge implementation.
3 | *
4 | * Copyright (c) 2024 Yahweasel and contributors
5 | *
6 | * Permission to use, copy, modify, and/or distribute this software for any
7 | * purpose with or without fee is hereby granted, provided that the above
8 | * copyright notice and this permission notice appear in all copies.
9 | *
10 | * THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
13 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
15 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
16 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 | */
18 |
19 | /*
20 | * This file contains functionality related to converting WebCodecs VideoFrames
21 | * and AudioDatas to libav.js Frames.
22 | */
23 |
24 | import type * as LibAVJS from "@libav.js/types";
25 | import type * as LibAVJSWebCodecs from "libavjs-webcodecs-polyfill";
26 | declare let LibAV: any;
27 |
28 | /**
29 | * Convert a VideoFrame to a libav.js Frame. The libav.js frame will use the
30 | * same timebase as WebCodecs, 1/1000000.
31 | * @param frame VideoFrame to convert.
32 | */
33 | export async function videoFrameToLAFrame(frame: LibAVJSWebCodecs.VideoFrame) {
34 | // First just naively extract all the data
35 | const data = new Uint8Array(frame.allocationSize());
36 | await frame.copyTo(data);
37 |
38 | /* libav.js ≥ 5 changed the format of frames, harmonizing it with the format
39 | * of WebCodecs. This bridge is still compatible with libav.js < 5, but
40 | * assumes ≥ 5 unless it can prove otherwise. */
41 | let libavjs5 = true;
42 | if (
43 | typeof LibAV !== "undefined" && LibAV && LibAV.VER &&
44 | parseInt(LibAV.VER) < 5
45 | ) {
46 | libavjs5 = false;
47 | }
48 |
49 | // Then figure out how that corresponds to planes
50 | let libavFormat = 5, bpp = 1, planes = 3, cwlog2 = 0, chlog2 = 0;
51 | switch (frame.format) {
52 | case "I420":
53 | libavFormat = 0;
54 | cwlog2 = chlog2 = 1;
55 | break;
56 |
57 | case "I420A":
58 | libavFormat = 33;
59 | planes = 4;
60 | cwlog2 = chlog2 = 1;
61 | break;
62 |
63 | case "I422":
64 | libavFormat = 4;
65 | cwlog2 = 1;
66 | break;
67 |
68 | case "NV12":
69 | libavFormat = 23;
70 | planes = 2;
71 | chlog2 = 1;
72 | break;
73 |
74 | case "RGBA":
75 | case "RGBX":
76 | libavFormat = 26;
77 | planes = 1;
78 | bpp = 4;
79 | break;
80 |
81 | case "BGRA":
82 | case "BGRX":
83 | libavFormat = 28;
84 | planes = 1;
85 | bpp = 4;
86 | break;
87 | }
88 |
89 | // And copy out the data
90 | const laFrame: LibAVJS.Frame = {
91 | format: libavFormat,
92 | data: null,
93 | pts: ~~frame.timestamp,
94 | ptshi: Math.floor(frame.timestamp / 0x100000000),
95 | time_base_num: 1,
96 | time_base_den: 1000000,
97 | width: frame.visibleRect.width,
98 | height: frame.visibleRect.height
99 | };
100 |
101 | if (libavjs5) {
102 | // Make our layout
103 | const layout: LibAVJSWebCodecs.PlaneLayout[] = [];
104 | let offset = 0;
105 | for (let p = 0; p < planes; p++) {
106 | let w = frame.visibleRect.width;
107 | let h = frame.visibleRect.height;
108 | if (p === 1 || p === 2) {
109 | w >>= cwlog2;
110 | h >>= chlog2;
111 | }
112 | layout.push({offset, stride: w * bpp});
113 | offset += w * h * bpp;
114 | }
115 | laFrame.data = data;
116 | laFrame.layout = layout;
117 |
118 | } else {
119 | // libav.js < 5 format: one array per row
120 | laFrame.data = [];
121 | let offset = 0;
122 | for (let p = 0; p < planes; p++) {
123 | const plane: Uint8Array[] = [];
124 | laFrame.data.push(plane);
125 | let wlog2 = 0, hlog2 = 0;
126 | if (p === 1 || p === 2) {
127 | wlog2 = cwlog2;
128 | hlog2 = chlog2;
129 | }
130 | for (let y = 0; y < frame.visibleRect.height >>> hlog2; y++) {
131 | const w = (frame.visibleRect.width * bpp) >>> wlog2;
132 | plane.push(data.subarray(offset, offset + w));
133 | offset += w;
134 | }
135 | }
136 |
137 | }
138 |
139 | return laFrame;
140 | }
141 |
142 | /**
143 | * Convert an AudioData to a libav.js Frame. The libav.js frame will use the
144 | * same timebase as WebCodecs, 1/1000000.
145 | * @param frame AudioFrame to convert.
146 | */
147 | export async function audioDataToLAFrame(frame: LibAVJSWebCodecs.AudioData) {
148 | // Figure out how the data corresponds to frames
149 | let libavFormat = 6;
150 | let TypedArray: any = Int16Array;
151 | const planar = /-planar$/.test(frame.format);
152 | switch (frame.format) {
153 | case "u8":
154 | case "u8-planar":
155 | libavFormat = planar ? 5 : 0;
156 | TypedArray = Uint8Array;
157 | break;
158 |
159 | case "s16":
160 | case "s16-planar":
161 | libavFormat = planar ? 6 : 1;
162 | break;
163 |
164 | case "s32":
165 | case "s32-planar":
166 | libavFormat = planar ? 7 : 2;
167 | TypedArray = Int32Array;
168 | break;
169 |
170 | case "f32":
171 | case "f32-planar":
172 | libavFormat = planar ? 8 : 3;
173 | TypedArray = Float32Array;
174 | break;
175 | }
176 |
177 | // And copy out the data
178 | const laFrame: LibAVJS.Frame = {
179 | format: libavFormat,
180 | data: null,
181 | pts: ~~frame.timestamp,
182 | ptshi: Math.floor(frame.timestamp / 0x100000000),
183 | time_base_num: 1,
184 | time_base_den: 1000000,
185 | sample_rate: frame.sampleRate,
186 | nb_samples: frame.numberOfFrames,
187 | channels: frame.numberOfChannels
188 | };
189 | if (planar) {
190 | laFrame.data = [];
191 | for (let p = 0; p < frame.numberOfChannels; p++) {
192 | const plane = new TypedArray(frame.numberOfFrames);
193 | laFrame.data.push(plane);
194 | await frame.copyTo(plane.buffer, {planeIndex: p, format: frame.format});
195 | }
196 | } else {
197 | const data = laFrame.data = new TypedArray(frame.numberOfFrames * frame.numberOfChannels);
198 | await frame.copyTo(data.buffer, {planeIndex: 0, format: frame.format});
199 | }
200 |
201 | return laFrame;
202 | }
203 |
--------------------------------------------------------------------------------
/samples/transcoder/transcoder.js:
--------------------------------------------------------------------------------
1 | /*
2 | * This (un)license applies only to this sample code, and not to
3 | * libavjs-webcodecs-bridge as a whole:
4 | *
5 | * This is free and unencumbered software released into the public domain.
6 | *
7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute
8 | * this software, either in source code form or as a compiled binary, for any
9 | * purpose, commercial or non-commercial, and by any means.
10 | *
11 | * In jurisdictions that recognize copyright laws, the author or authors of
12 | * this software dedicate any and all copyright interest in the software to the
13 | * public domain. We make this dedication for the benefit of the public at
14 | * large and to the detriment of our heirs and successors. We intend this
15 | * dedication to be an overt act of relinquishment in perpetuity of all present
16 | * and future rights to this software under copyright law.
17 | *
18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 | */
25 |
26 | importScripts("../worker-util.js");
27 |
28 | onmessage = async ev => {
29 | const file = ev.data;
30 | let streams, configs, allPackets;
31 |
32 | // Demux the file
33 | try {
34 | [streams, configs, allPackets] =
35 | await sampleDemux(file, {unify: true});
36 | } catch (ex) {
37 | console.error(ex);
38 | return;
39 | }
40 |
41 | // Prepare for transcoding
42 | const libav = await LibAV.LibAV({noworker: true});
43 | let outStreams = [];
44 | let decoders = new Array(streams.length);
45 | let oc, pb, pkt;
46 | let transcodePromise = Promise.all([]);
47 | for (let si = 0; si < streams.length; si++) {
48 | const inStream = streams[si];
49 | if (inStream.codec_type === 0 /* video */) {
50 | // ENCODING
51 | const oi = outStreams.length;
52 | const enc = new VideoEncoder({
53 | output: (chunk, metadata) => {
54 | transcodePromise = transcodePromise.then(async () => {
55 | const packet =
56 | await LibAVWebCodecsBridge.encodedVideoChunkToPacket(
57 | libav, chunk, metadata, outStreams[oi].stream, oi);
58 | await libav.ff_write_multi(oc, pkt, [packet]);
59 | });
60 | },
61 | error: err => {
62 | console.error(`video encoder ${oi}: ${err.toString()}`);
63 | }
64 | });
65 | const config = {
66 | codec: "vp8",
67 | width: configs[si].codedWidth,
68 | height: configs[si].codedHeight
69 | };
70 | enc.configure(config);
71 | const stream = await LibAVWebCodecsBridge.configToVideoStream(
72 | libav, config);
73 |
74 | // DECODING
75 | const dec = new VideoDecoder({
76 | output: frame => {
77 | enc.encode(frame);
78 | frame.close();
79 | },
80 | error: err => {
81 | console.error(`video decoder ${oi}: ${err.toString()}`);
82 | }
83 | });
84 | decoders[si] = dec;
85 | dec.configure(configs[si]);
86 |
87 | outStreams.push({
88 | inIdx: si,
89 | type: "video",
90 | encoder: enc,
91 | config,
92 | stream,
93 | decoder: dec
94 | });
95 |
96 | } else if (inStream.codec_type === 1 /* audio */) {
97 | // ENCODING
98 | const oi = outStreams.length;
99 | const enc = new AudioEncoder({
100 | output: (chunk, metadata) => {
101 | transcodePromise = transcodePromise.then(async () => {
102 | const packet =
103 | await LibAVWebCodecsBridge.encodedAudioChunkToPacket(
104 | libav, chunk, metadata, outStreams[oi].stream, oi);
105 | await libav.ff_write_multi(oc, pkt, [packet]);
106 | });
107 | },
108 | error: err => {
109 | console.error(`audio encoder ${oi}: ${err.toString()}`);
110 | }
111 | });
112 | const config = {
113 | codec: "opus",
114 | sampleRate: configs[si].sampleRate,
115 | numberOfChannels: configs[si].numberOfChannels
116 | };
117 | enc.configure(config);
118 | const stream = await LibAVWebCodecsBridge.configToAudioStream(
119 | libav, config);
120 |
121 | // DECODING
122 | const dec = new AudioDecoder({
123 | output: frame => {
124 | enc.encode(frame);
125 | frame.close();
126 | },
127 | error: err => {
128 | console.error(`audio decoder ${oi}: ${err.toString()}`);
129 | }
130 | });
131 | decoders[si] = dec;
132 | dec.configure(configs[si]);
133 |
134 | outStreams.push({
135 | inIdx: si,
136 | type: "audio",
137 | encoder: enc,
138 | config,
139 | stream,
140 | decoder: dec
141 | });
142 |
143 | }
144 | }
145 |
146 | // Prepare for muxing
147 | [oc, , pb] = await libav.ff_init_muxer({
148 | format_name: "webm",
149 | filename: "out.webm",
150 | open: true,
151 | codecpars: true
152 | }, outStreams.map(x => x.stream));
153 | await libav.avformat_write_header(oc, 0);
154 | pkt = await libav.av_packet_alloc();
155 |
156 | // Transcode
157 | for (const packet of allPackets[0]) {
158 | const dec = decoders[packet.stream_index];
159 | if (!dec)
160 | continue;
161 | const inStream = streams[packet.stream_index];
162 | if (inStream.codec_type === 0 /* video */) {
163 | dec.decode(LibAVWebCodecsBridge.packetToEncodedVideoChunk(
164 | packet, inStream));
165 | } else if (inStream.codec_type === 1 /* audio */) {
166 | dec.decode(LibAVWebCodecsBridge.packetToEncodedAudioChunk(
167 | packet, inStream));
168 | }
169 | }
170 |
171 | // Flush
172 | for (const stream of outStreams) {
173 | await stream.decoder.flush();
174 | stream.decoder.close();
175 | await stream.encoder.flush();
176 | stream.encoder.close();
177 | }
178 | await transcodePromise;
179 | await libav.av_write_trailer(oc);
180 | await libav.av_packet_free(pkt);
181 | await libav.ff_free_muxer(oc, pb);
182 |
183 | // Read out the file
184 | const data = await libav.readFile("out.webm");
185 | postMessage({c: "chunk", chunk: data});
186 | libav.terminate();
187 |
188 | postMessage({c: "done"});
189 | };
190 |
--------------------------------------------------------------------------------
/docs/API.md:
--------------------------------------------------------------------------------
1 | libavjs-webcodecs-bridge has two directions of use: demuxing and muxing. The
2 | demuxing functions are designed to aid when using libav.js for demuxing and
3 | WebCodecs for decoding. The muxing functions are designed to aid when using
4 | WebCodecs for encoding and libav.js for muxing. In either case, it's a BYOL
5 | (bring your own library) system: you must provide your own instance of libav.js,
6 | and if WebCodecs isn't built into your browser, you must bring a polyfill
7 | (presumably libavjs-webcodecs-polyfill).
8 |
9 |
10 | ## Demuxing
11 |
12 | If you are demuxing in libav.js, you will have a libav.js `Stream` object for
13 | each stream, and libav.js `Packet` objects for each packet in the file. Convert
14 | the `Stream` object to a configuration to configure a WebCodecs decoder, and
15 | convert each `Packet` to an encoded chunk to be decoded in WebCodecs.
16 |
17 | ### `audioStreamToConfig`, `videoStreamToConfig`
18 |
19 | ```js
20 | async function audioStreamToConfig(
21 | libav: LibAVJS.LibAV, stream: LibAVJS.Stream
22 | ): Promise;
23 |
24 | async function videoStreamToConfig(
25 | libav: LibAVJS.LibAV, stream: LibAVJS.Stream
26 | ): Promise;
27 | ```
28 |
29 | libav.js `Stream`s can be converted to WebCodecs configurations, to be passed to
30 | `AudioDecoder.configure` or `VideoDecoder.configure`. You must determine whether
31 | you have an audio or video stream yourself, by checking `stream.codec_type`.
32 |
33 | To convert an audio stream to a suitable configuration, use
34 | `await audioStreamToConfig(libav, stream)`. Use `videoStreamToConfig` for video
35 | streams. These functions will *always* return something, regardless of whether
36 | WebCodecs actually supports the codec in question, so make sure to check whether
37 | the configuration is actually supported.
38 |
39 | ### `packetToEncodedAudioChunk`, `packetToEncodedVideoChunk`
40 |
41 | ```js
42 | function packetToEncodedAudioChunk(
43 | packet: LibAVJS.Packet, stream: LibAVJS.Stream, opts: {
44 | EncodedAudioChunk?: any
45 | } = {}
46 | ): LibAVJSWebCodecs.EncodedAudioChunk;
47 |
48 | function packetToEncodedVideoChunk(
49 | packet: LibAVJS.Packet, stream: LibAVJS.Stream, opts: {
50 | EncodedVideoChunk?: any
51 | } = {}
52 | ): LibAVJSWebCodecs.EncodedVideoChunk;
53 | ```
54 |
55 | libav.js `Packet`s can be converted to WebCodecs `EncodedAudioChunk`s or
56 | `EncodedVideoChunk`s.
57 |
58 | To convert an audio packet to an `EncodedAudioChunk`, use
59 | `packetToEncodedAudioChunk(packet, stream)`. Use `packetToEncodedVideoChunk` for
60 | video packets. Note that these functions are synchronous.
61 |
62 | If you're using a polyfill, you can pass the `EncodedAudioChunk` or
63 | `EncodedVideoChunk` constructor as the appropriate field of the third (`opts`)
64 | argument.
65 |
66 | Note that FFmpeg (and thus libav.js) and WebCodecs disagree on the definition of
67 | keyframe with both H.264 and H.265. WebCodecs requires a non-recovery frame,
68 | i.e., a keyframe with no B-frames, whereas FFmpeg takes the keyframe status from
69 | the container, and all container formats mark recovery frames as keyframes
70 | (because they are keyframes). No implementation of WebCodecs actually cares
71 | whether you mark a frame as a keyframe or a delta frame *except* for the first
72 | frame sent to the decoder. The consequence of this is that if you seek to the
73 | middle of an H.264 or H.265 file and read a frame that libav.js indicates is a
74 | keyframe, you may not actually be able to start decoding with that frame. There
75 | is no practical way to fix this on the libavjs-webcodecs-bridge side, because
76 | FFmpeg offers no API to distinguish these frame types; it would be necessary to
77 | manually parse frame data instead. See [issue
78 | 3](https://github.com/Yahweasel/libavjs-webcodecs-bridge/issues/3) for some
79 | suggested workarounds.
80 |
81 |
82 | ## Muxing
83 |
84 | If you are encoding with WebCodecs, you will have a WebCodecs configuration, and
85 | a stream of `EncodedAudioChunk`s or `EncodedVideoChunk`s. Convert the
86 | configuration to a stream configuration used in libav.js's `ff_init_muxer`, and
87 | the encoded chunks to libav.js `Packet`s.
88 |
89 | ### `configToAudioStream`, `configToVideoStream`
90 |
91 | ```js
92 | async function configToAudioStream(
93 | libav: LibAVJS.LibAV, config: LibAVJSWebCodecs.AudioEncoderConfig
94 | ): Promise<[number, number, number]>;
95 |
96 | async function configToVideoStream(
97 | libav: LibAVJS.LibAV, config: LibAVJSWebCodecs.VideoEncoderConfig
98 | ): Promise<[number, number, number]>;
99 | ```
100 |
101 | Configurations for audio or video encoders in WebCodecs can be converted to
102 | stream information sufficient for `ff_init_muxer`. Note that `ff_init_muxer`
103 | expects three pieces of information for each stream: a pointer to stream
104 | information (in this case, `AVCodecParameters`), and the numerator and
105 | denominator of the timebase used. Thus, these functions return those three
106 | numbers, in the array demanded by `ff_init_muxer`.
107 |
108 | To convert an audio configuration to a suitable stream, use
109 | `await configToAudioStream(libav, config)`. Use `configToVideoStream` for video
110 | streams. These functions will *always* return something, regardless of whether
111 | the codec is recognized or libav.js supports it, so make sure to check whether
112 | `ff_init_muxer` actually succeeds.
113 |
114 | Two things of note about this function:
115 | - These return `AVCodecParameters`, so you must set the `codecpars`
116 | option to `true` when calling `ff_init_muxer`.
117 | - Because of differences between libav.js and WebCodecs, you must convert at
118 | least one chunk from each stream to a packet *before* starting the muxer.
119 | This is because of codec parameters that are only passed with the first
120 | encoded chunk. `demo/demo.js` demonstrates this.
121 |
122 | ### `encodedAudioChunkToPacket`, `encodedVideoChunkToPacket`
123 |
124 | ```js
125 | async function encodedAudioChunkToPacket(
126 | libav: LibAVJS.LibAV, chunk: LibAVJSWebCodecs.EncodedAudioChunk, metadata: any,
127 | stream: [number, number, number], streamIndex: number
128 | ): Promise;
129 |
130 | async function encodedVideoChunkToPacket(
131 | libav: LibAVJS.LibAV, chunk: LibAVJSWebCodecs.EncodedVideoChunk, metadata: any,
132 | stream: [number, number, number], streamIndex: number
133 | ): Promise;
134 | ```
135 |
136 | WebCodecs encoded chunks (`EncodedAudioChunk`s and `EncodedVideoChunk`s) can be
137 | converted to libav.js `Packet`s, for use in `ff_write_multi`.
138 |
139 | To convert an audio chunk to a libav.js packet, use
140 | `encodedAudioChunkToPacket(libav, chunk, metadata, stream, streamIndex)`.
141 | `libav` is the libav.js instance in use, `chunk` is the encoded chunk,
142 | `metadata` is the metadata sent with the chunk, `stream` is the stream
143 | information returned by `configToAudioStream`, and `streamIndex` is the index of
144 | the stream in your call to `ff_init_muxer`. Use `encodedVideoChunkToPacket` for
145 | video chunks. Note that these functions are asynchronous, unlike their demuxing
146 | counterparts.
147 |
148 | Due to differences in how libav and WebCodecs handle extra data for codecs, *you
149 | must convert at least one packet from each stream before initializing the
150 | muxer*. These functions convert the packet, but also initialize the extra codec
151 | data (because in WebCodecs it's sent with the first packet), and that extra
152 | codec data must be initialized before the muxer. This can make the ordering of
153 | tasks a bit awkward, but is unavoidable. You may want to look at the demo in
154 | `demo/` for an example of the correct ordering of steps.
155 |
--------------------------------------------------------------------------------
/src/latowc.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * This file is part of the libav.js WebCodecs Bridge implementation.
3 | *
4 | * Copyright (c) 2024 Yahweasel and contributors
5 | *
6 | * Permission to use, copy, modify, and/or distribute this software for any
7 | * purpose with or without fee is hereby granted, provided that the above
8 | * copyright notice and this permission notice appear in all copies.
9 | *
10 | * THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
13 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
15 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
16 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 | */
18 |
19 | /*
20 | * This file contains functionality related to converting libav.js Frames to
21 | * WebCodecs VideoFrames and AudioDatas.
22 | */
23 |
24 | import type * as LibAVJS from "@libav.js/types";
25 | import * as LibAVJSWebCodecs from "libavjs-webcodecs-polyfill";
26 |
27 | declare let VideoFrame: any, AudioData: any;
28 |
29 | // (Duplicated from libav.js)
30 | function i64tof64(lo: number, hi: number) {
31 | // Common positive case
32 | if (!hi && lo >= 0) return lo;
33 |
34 | // Common negative case
35 | if (hi === -1 && lo < 0) return lo;
36 |
37 | /* Lo bit negative numbers are really just the 32nd bit being
38 | * set, so we make up for that with an additional 2^32 */
39 | return (
40 | hi * 0x100000000 +
41 | lo +
42 | ((lo < 0) ? 0x100000000 : 0)
43 | );
44 | }
45 |
46 | /**
47 | * Convert a libav.js timestamp to a WebCodecs timestamp.
48 | * @param lo Low bits of the timestamp.
49 | * @param hi High bits of the timestamp.
50 | * @param timeBase Optional timebase to use for conversion.
51 | */
52 | function laTimeToWCTime(lo: number, hi: number, timeBase?: [number, number]) {
53 | let ret = i64tof64(lo, hi);
54 | if (timeBase)
55 | ret = Math.round(ret * 1000000 * timeBase[0] / timeBase[1]);
56 | return ret;
57 | }
58 |
59 | /**
60 | * Convert a libav.js Frame to a VideoFrame. If not provided in opts, the
61 | * libav.js frame is assumed to use the same timebase as WebCodecs, 1/1000000.
62 | * @param frame libav.js Frame.
63 | * @param opts Optional options, namely a VideoFrame constructor and timebase
64 | * to use.
65 | */
66 | export function laFrameToVideoFrame(
67 | frame: LibAVJS.Frame, opts: {
68 | VideoFrame?: any,
69 | timeBase?: [number, number],
70 | transfer?: boolean
71 | } = {}
72 | ) {
73 | let VF: any;
74 | if (opts.VideoFrame)
75 | VF = opts.VideoFrame;
76 | else
77 | VF = VideoFrame;
78 |
79 | let layout: LibAVJSWebCodecs.PlaneLayout[];
80 | let data: Uint8Array;
81 | let transfer: ArrayBuffer[] = [];
82 |
83 | let timeBase = opts.timeBase;
84 | if (!timeBase && frame.time_base_num)
85 | timeBase = [frame.time_base_num||1, frame.time_base_den||1000000];
86 |
87 | if (frame.layout) {
88 | // Modern (libav.js ≥ 5) frame in WebCodecs-like format
89 | data = frame.data;
90 | layout = frame.layout;
91 | if (opts.transfer)
92 | transfer.push(data.buffer);
93 |
94 | } else {
95 | // Pre-libavjs-5 frame with one array per row
96 | // Combine all the frame data into a single object
97 | layout = [];
98 | let size = 0;
99 | for (let p = 0; p < frame.data.length; p++) {
100 | const plane = frame.data[p];
101 | layout.push({
102 | offset: size,
103 | stride: plane[0].length
104 | });
105 | size += plane.length * plane[0].length;
106 | }
107 | data = new Uint8Array(size);
108 | let offset = 0;
109 | for (let p = 0; p < frame.data.length; p++) {
110 | const plane = frame.data[p];
111 | const linesize = plane[0].length;
112 | for (let y = 0; y < plane.length; y++) {
113 | data.set(plane[y], offset);
114 | offset += linesize;
115 | }
116 | }
117 | transfer.push(data.buffer);
118 |
119 | }
120 |
121 | // Choose the format
122 | let format: LibAVJSWebCodecs.VideoPixelFormat = "I420";
123 | switch (frame.format) {
124 | case 0:
125 | format = "I420";
126 | break;
127 |
128 | case 33:
129 | format = "I420A";
130 | break;
131 |
132 | case 4:
133 | format = "I422";
134 | break;
135 |
136 | case 23:
137 | format = "NV12";
138 | break;
139 |
140 | case 26:
141 | format = "RGBA";
142 | break;
143 |
144 | case 28:
145 | format = "BGRA";
146 | break;
147 |
148 | default:
149 | throw new Error("Unsupported pixel format");
150 | }
151 |
152 | // And make the VideoFrame
153 | return new VF(data, {
154 | format,
155 | codedWidth: frame.width,
156 | codedHeight: frame.height,
157 | timestamp: laTimeToWCTime(frame.pts||0, frame.ptshi||0, timeBase),
158 | layout,
159 | transfer
160 | });
161 | }
162 |
163 | /**
164 | * Convert a libav.js Frame to an AudioData. If not provide din opts, the
165 | * libav.js frame is assumed to use the same timebase as WebCodecs, 1/1000000.
166 | * @param frame libav.js Frame.
167 | * @param opts Optional options, namely an AudioData constructor and timebase
168 | * to use.
169 | */
170 | export function laFrameToAudioData(
171 | frame: LibAVJS.Frame, opts: {
172 | AudioData?: any,
173 | timeBase?: [number, number]
174 | } = {}
175 | ) {
176 | let AD: any;
177 | if (opts.AudioData)
178 | AD = opts.AudioData;
179 | else
180 | AD = AudioData;
181 |
182 | let timeBase = opts.timeBase;
183 | if (!timeBase && frame.time_base_num)
184 | timeBase = [frame.time_base_num||1, frame.time_base_den||1000000];
185 |
186 | // Combine all the frame data into a single object
187 | let size = 0;
188 | if (( frame.data).buffer) {
189 | // Non-planar
190 | size = ( frame.data).byteLength;
191 | } else {
192 | // Planar
193 | for (let p = 0; p < frame.data.length; p++)
194 | size += frame.data[p].byteLength;
195 | }
196 | const data = new Uint8Array(size);
197 | let offset = 0;
198 | if (( frame.data).buffer) {
199 | const rd = frame.data;
200 | data.set(new Uint8Array(rd.buffer, rd.byteOffset, rd.byteLength));
201 |
202 | } else {
203 | let offset = 0;
204 | for (let p = 0; p < frame.data.length; p++) {
205 | const rp = frame.data[p];
206 | const plane = new Uint8Array(rp.buffer, rp.byteOffset, rp.byteLength);
207 | data.set(plane, offset);
208 | offset += plane.length;
209 | }
210 | }
211 |
212 | // Choose the format
213 | let format: LibAVJSWebCodecs.AudioSampleFormat = "s16";
214 | switch (frame.format) {
215 | case 0: format = "u8"; break;
216 | case 1: format = "s16"; break;
217 | case 2: format = "s32"; break;
218 | case 3: format = "f32"; break;
219 | case 5: format = "u8-planar"; break;
220 | case 6: format = "s16-planar"; break;
221 | case 7: format = "s32-planar"; break;
222 | case 8: format = "f32-planar"; break;
223 |
224 | default:
225 | throw new Error("Unsupported sample format");
226 | }
227 |
228 | // And make the AudioData
229 | return new AD({
230 | format,
231 | data,
232 | sampleRate: frame.sample_rate,
233 | numberOfFrames: frame.nb_samples,
234 | numberOfChannels: frame.channels,
235 | timestamp: laTimeToWCTime(frame.pts||0, frame.ptshi||0, timeBase)
236 | });
237 | }
238 |
--------------------------------------------------------------------------------
/src/mux.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * This file is part of the libav.js WebCodecs Bridge implementation.
3 | *
4 | * Copyright (c) 2023, 2024 Yahweasel
5 | *
6 | * Permission to use, copy, modify, and/or distribute this software for any
7 | * purpose with or without fee is hereby granted, provided that the above
8 | * copyright notice and this permission notice appear in all copies.
9 | *
10 | * THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
13 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
15 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
16 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 | */
18 |
19 | /*
20 | * This file contains functionality related to using libav.js for converting
21 | * WebCodecs data to libav.js's formats and muxing.
22 | */
23 |
24 | import type * as LibAVJS from "@libav.js/types";
25 | import type * as LibAVJSWebCodecs from "libavjs-webcodecs-polyfill";
26 | declare let LibAV : LibAVJS.LibAVWrapper;
27 | declare let LibAVWebCodecs : any;
28 | declare let EncodedAudioChunk : any;
29 | declare let EncodedVideoChunk : any;
30 |
31 | /**
32 | * Convert a WebCodecs audio configuration to stream context sufficient for
33 | * libav.js, namely codecpar and timebase.
34 | *
35 | * @param libav The libav.js instance that created this stream.
36 | * @param config The configuration to convert.
37 | * @returns [address of codecpar, timebase numerator, timebase denominator]
38 | */
39 | export async function configToAudioStream(
40 | libav: LibAVJS.LibAV, config: LibAVJSWebCodecs.AudioEncoderConfig
41 | ): Promise<[number, number, number]> {
42 | const codecLong = config.codec;
43 | let codec: string;
44 | if (typeof codecLong === "object")
45 | codec = codecLong.libavjs.codec;
46 | else
47 | codec = codecLong.replace(/\..*/, "");
48 |
49 | // Convert the codec to a libav name
50 | switch (codec) {
51 | case "mp4a": codec = "aac"; break;
52 | case "pcm-u8": codec = "pcm_u8"; break;
53 | case "pcm-s16": codec = "pcm_s16le"; break;
54 | case "pcm-s24": codec = "pcm_s24le"; break;
55 | case "pcm-s32": codec = "pcm_s32le"; break;
56 | case "pcm-f32": codec = "pcm_f32le"; break;
57 | }
58 |
59 | // Find the associated codec
60 | const desc = await libav.avcodec_descriptor_get_by_name(codec);
61 |
62 | // Make the codecpar
63 | const codecpar = await libav.avcodec_parameters_alloc();
64 | if (desc) {
65 | await libav.AVCodecParameters_codec_type_s(codecpar,
66 | await libav.AVCodecDescriptor_type(desc));
67 | await libav.AVCodecParameters_codec_id_s(codecpar,
68 | await libav.AVCodecDescriptor_id(desc));
69 | if (config.sampleRate) {
70 | await libav.AVCodecParameters_sample_rate_s(codecpar,
71 | config.sampleRate);
72 | }
73 | if (config.numberOfChannels) {
74 | await libav.AVCodecParameters_channels_s(codecpar,
75 | config.numberOfChannels);
76 | }
77 | }
78 |
79 | // And the timebase
80 | let timebaseNum = 1, timebaseDen = 1000000;
81 | if (config.sampleRate)
82 | timebaseDen = config.sampleRate;
83 |
84 | return [codecpar, timebaseNum, timebaseDen];
85 | }
86 |
87 | /**
88 | * Convert a WebCodecs video configuration to stream context sufficient for
89 | * libav.js, namely codecpar and timebase.
90 | *
91 | * @param libav The libav.js instance that created this stream.
92 | * @param config The configuration to convert.
93 | * @returns [address of codecpar, timebase numerator, timebase denominator]
94 | */
95 | export async function configToVideoStream(
96 | libav: LibAVJS.LibAV, config: LibAVJSWebCodecs.VideoEncoderConfig | VideoEncoderConfig
97 | ): Promise<[number, number, number]> {
98 | const codecLong = config.codec;
99 | let codec: string;
100 | if (typeof codecLong === "object")
101 | codec = codecLong.libavjs.codec;
102 | else
103 | codec = codecLong.replace(/\..*/, "");
104 |
105 | // Convert the codec to a libav name
106 | switch (codec) {
107 | case "av01": codec = "av1"; break;
108 | case "avc1":
109 | case "avc3":
110 | codec = "h264";
111 | break;
112 | case "hev1":
113 | case "hvc1":
114 | codec = "hevc";
115 | break;
116 | case "vp09": codec = "vp9"; break;
117 | }
118 |
119 | // Find the associated codec
120 | const desc = await libav.avcodec_descriptor_get_by_name(codec);
121 |
122 | // Make the codecpar
123 | const codecpar = await libav.avcodec_parameters_alloc();
124 | if (desc) {
125 | await libav.AVCodecParameters_codec_type_s(codecpar,
126 | await libav.AVCodecDescriptor_type(desc));
127 | await libav.AVCodecParameters_codec_id_s(codecpar,
128 | await libav.AVCodecDescriptor_id(desc));
129 | await libav.AVCodecParameters_width_s(codecpar, config.width);
130 | await libav.AVCodecParameters_height_s(codecpar, config.height);
131 | // FIXME: Use displayWidth and displayHeight to make SAR
132 | }
133 |
134 | // And the timebase
135 | let timebaseNum = 1, timebaseDen = 1000000;
136 | if (config.framerate) {
137 | // Simple if it's an integer
138 | if (config.framerate === ~~config.framerate) {
139 | timebaseDen = config.framerate;
140 | } else {
141 | /* Need to find an integer ratio. First try 1001, as many common
142 | * framerates are x/1001 */
143 | const fr1001 = config.framerate * 1001;
144 | if (Math.abs(fr1001 - ~~fr1001) <= 0.000001) {
145 | timebaseNum = 1001;
146 | timebaseDen = fr1001;
147 | } else {
148 | /* Just look for a power of two. This will always work because
149 | * of how floating point works. */
150 | timebaseDen = config.framerate;
151 | while (
152 | Math.abs(timebaseDen - Math.floor(timebaseDen)) >= 0.000001 &&
153 | timebaseDen < 1000000
154 | ) {
155 | timebaseNum *= 2;
156 | timebaseDen *= 2;
157 | }
158 | timebaseDen = Math.floor(timebaseDen);
159 | }
160 | }
161 | }
162 |
163 | return [codecpar, timebaseNum, timebaseDen];
164 | }
165 |
166 | /*
167 | * Convert the timestamp and duration from microseconds to an arbitrary timebase
168 | * given by libav.js (internal)
169 | */
170 | function times(
171 | chunk: LibAVJSWebCodecs.EncodedAudioChunk | LibAVJSWebCodecs.EncodedVideoChunk | EncodedVideoChunk,
172 | stream: [number, number, number]
173 | ) {
174 | const num = stream[1];
175 | const den = stream[2];
176 | return {
177 | timestamp: Math.round(chunk.timestamp * den / num / 1000000),
178 | duration: Math.round((chunk.duration||0) * den / num / 1000000)
179 | };
180 | }
181 |
182 | /*
183 | * Convert a WebCodecs Encoded{Audio,Video}Chunk to a libav.js packet for muxing. Internal.
184 | */
185 | function encodedChunkToPacket(
186 | chunk: LibAVJSWebCodecs.EncodedAudioChunk | LibAVJSWebCodecs.EncodedVideoChunk | EncodedVideoChunk,
187 | stream: [number, number, number], streamIndex: number
188 | ): LibAVJS.Packet {
189 | const {timestamp, duration} = times(chunk, stream);
190 |
191 | // Convert into high and low bits
192 | let pts: number, ptshi: number, dur: number, durhi: number;
193 | if (typeof LibAV !== "undefined" && LibAV.f64toi64) {
194 | [pts, ptshi] = LibAV.f64toi64(timestamp);
195 | [dur, durhi] = LibAV.f64toi64(duration);
196 | } else {
197 | pts = ~~timestamp;
198 | ptshi = Math.floor(timestamp / 0x100000000);
199 | dur = ~~duration;
200 | durhi = Math.floor(duration / 0x100000000);
201 | }
202 |
203 | // Create a buffer for it
204 | const data = new Uint8Array(chunk.byteLength);
205 | chunk.copyTo(data.buffer);
206 |
207 | // And make a packet
208 | return {
209 | data,
210 | pts, ptshi,
211 | dts: pts, dtshi: ptshi,
212 | time_base_num: stream[1], time_base_den: stream[2],
213 | stream_index: streamIndex,
214 | flags: 0,
215 | duration: dur, durationhi: durhi
216 | };
217 | }
218 |
219 | /**
220 | * Convert a WebCodecs EncodedAudioChunk to a libav.js packet for muxing.
221 | * @param libav The libav.js instance that created this stream.
222 | * @param chunk The chunk itself.
223 | * @param metadata The metadata sent with this chunk.
224 | * @param stream The stream this packet belongs to (necessary for timestamp conversion).
225 | * @param streamIndex The stream index to inject into the packet
226 | */
227 | export async function encodedAudioChunkToPacket(
228 | libav: LibAVJS.LibAV, chunk: LibAVJSWebCodecs.EncodedAudioChunk, metadata: any,
229 | stream: [number, number, number], streamIndex: number
230 | ): Promise {
231 | // NOTE: libav and metadata are not currently used for audio
232 | return encodedChunkToPacket(chunk, stream, streamIndex);
233 | }
234 |
235 | /**
236 | * Convert a WebCodecs EncodedVideoChunk to a libav.js packet for muxing. Note
237 | * that this also may modify codecpar, if the packet comes with extradata.
238 | * @param libav The libav.js instance that created this stream.
239 | * @param chunk The chunk itself.
240 | * @param metadata The metadata sent with this chunk.
241 | * @param stream The stream this packet belongs to (necessary for timestamp conversion).
242 | * @param streamIndex The stream index to inject into the packet
243 | */
244 | export async function encodedVideoChunkToPacket(
245 | libav: LibAVJS.LibAV, chunk: LibAVJSWebCodecs.EncodedVideoChunk | EncodedVideoChunk, metadata: any,
246 | stream: [number, number, number], streamIndex: number
247 | ): Promise {
248 | const ret = encodedChunkToPacket(chunk, stream, streamIndex);
249 | if (chunk.type === "key")
250 | ret.flags = 1;
251 |
252 | // Copy in the extradata if applicable
253 | if (stream[0] && metadata && metadata.decoderConfig && metadata.decoderConfig.description) {
254 | const codecpar = stream[0];
255 | const oldExtradata = await libav.AVCodecParameters_extradata(codecpar);
256 | if (!oldExtradata) {
257 | let description: any = metadata.decoderConfig.description;
258 | if (description.buffer)
259 | description = description.slice(0);
260 | else
261 | description = (new Uint8Array(description)).slice(0);
262 | const extradata =
263 | await libav.malloc(description.length);
264 | await libav.copyin_u8(extradata, description);
265 | await libav.AVCodecParameters_extradata_s(
266 | codecpar, extradata);
267 | await libav.AVCodecParameters_extradata_size_s(
268 | codecpar, description.length);
269 | }
270 | }
271 |
272 | return ret;
273 | }
274 |
--------------------------------------------------------------------------------
/demo/demo.js:
--------------------------------------------------------------------------------
1 | /*
2 | * This (un)license applies only to this sample code, and not to
3 | * libavjs-webcodecs-bridge as a whole:
4 | *
5 | * This is free and unencumbered software released into the public domain.
6 | *
7 | * Anyone is free to copy, modify, publish, use, compile, sell, or distribute
8 | * this software, either in source code form or as a compiled binary, for any
9 | * purpose, commercial or non-commercial, and by any means.
10 | *
11 | * In jurisdictions that recognize copyright laws, the author or authors of
12 | * this software dedicate any and all copyright interest in the software to the
13 | * public domain. We make this dedication for the benefit of the public at
14 | * large and to the detriment of our heirs and successors. We intend this
15 | * dedication to be an overt act of relinquishment in perpetuity of all present
16 | * and future rights to this software under copyright law.
17 | *
18 | * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21 | * AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 | */
25 |
26 | class BufferStream extends ReadableStream {
27 | buf = [];
28 | res = null;
29 |
30 | constructor() {
31 | super({
32 | pull: async (controller) => {
33 | while (!this.buf.length) {
34 | await new Promise(res => this.res = res);
35 | }
36 | const next = this.buf.shift();
37 | if (next !== null)
38 | controller.enqueue(next);
39 | else
40 | controller.close();
41 | }
42 | });
43 | }
44 |
45 | push(next) {
46 | this.buf.push(next);
47 | if (this.res) {
48 | const res = this.res;
49 | this.res = null;
50 | res();
51 | }
52 | }
53 | }
54 |
55 | async function main() {
56 | // Get an input file
57 | const fileBox = document.getElementById("file");
58 | await new Promise(res => {
59 | fileBox.onchange = function() {
60 | if (fileBox.files.length)
61 | res();
62 | };
63 | });
64 | const file = fileBox.files[0];
65 | document.getElementById("input-box").style.display = "none";
66 |
67 | // Codec info
68 | const vc = document.getElementById("vc").value;
69 | const ac = document.getElementById("ac").value;
70 |
71 | /* Prepare libav. We're using noworker here because libav is
72 | * loaded from a different origin, but you should simply
73 | * load libav from the same origin! */
74 | const libav = await LibAV.LibAV({noworker: true});
75 | await libav.mkreadaheadfile("input", file);
76 |
77 | // Start demuxer
78 | const [ifc, istreams] =
79 | await libav.ff_init_demuxer_file("input");
80 | const rpkt = await libav.av_packet_alloc();
81 | const wpkt = await libav.av_packet_alloc();
82 |
83 | // Translate all the streams
84 | const iToO = [];
85 | const decoders = [];
86 | const decoderStreams = [];
87 | const decConfigs = [];
88 | const packetToChunks = [];
89 | const encoders = [];
90 | const encoderStreams = [];
91 | const encoderReaders = [];
92 | const encConfigs = [];
93 | const chunkToPackets = [];
94 | const ostreams = [];
95 | for (let streamI = 0; streamI < istreams.length; streamI++) {
96 | const istream = istreams[streamI];
97 | iToO.push(-1);
98 | let streamToConfig, Decoder, packetToChunk,
99 | configToStream, Encoder, chunkToPacket;
100 | if (istream.codec_type === libav.AVMEDIA_TYPE_VIDEO) {
101 | streamToConfig = LibAVWebCodecsBridge.videoStreamToConfig;
102 | Decoder = VideoDecoder;
103 | packetToChunk = LibAVWebCodecsBridge.packetToEncodedVideoChunk;
104 | configToStream = LibAVWebCodecsBridge.configToVideoStream;
105 | Encoder = VideoEncoder;
106 | chunkToPacket = LibAVWebCodecsBridge.encodedVideoChunkToPacket;
107 | } else if (istream.codec_type === libav.AVMEDIA_TYPE_AUDIO) {
108 | streamToConfig = LibAVWebCodecsBridge.audioStreamToConfig;
109 | Decoder = AudioDecoder;
110 | packetToChunk = LibAVWebCodecsBridge.packetToEncodedAudioChunk;
111 | configToStream = LibAVWebCodecsBridge.configToAudioStream;
112 | Encoder = AudioEncoder;
113 | chunkToPacket = LibAVWebCodecsBridge.encodedAudioChunkToPacket;
114 | } else {
115 | continue;
116 | }
117 |
118 | // Convert the config
119 | const config = await streamToConfig(libav, istream);
120 | let supported;
121 | try {
122 | supported = await Decoder.isConfigSupported(config);
123 | } catch (ex) {}
124 | if (!supported || !supported.supported)
125 | continue;
126 | iToO[streamI] = decConfigs.length;
127 | decConfigs.push(config);
128 |
129 | // Make the decoder
130 | const stream = new BufferStream();
131 | decoderStreams.push(stream);
132 | const decoder = new Decoder({
133 | output: frame => stream.push(frame),
134 | error: error =>
135 | alert("Decoder " + JSON.stringify(config) + ":\n" + error)
136 | });
137 | decoder.configure(config);
138 | decoders.push(decoder);
139 | packetToChunks.push(packetToChunk);
140 |
141 | // Make the encoder config
142 | const encConfig = {
143 | codec: (istream.codec_type === libav.AVMEDIA_TYPE_VIDEO)
144 | ? vc : ac,
145 | width: config.codedWidth,
146 | height: config.codedHeight,
147 | numberOfChannels: config.numberOfChannels,
148 | sampleRate: config.sampleRate
149 | };
150 | encConfigs.push(encConfig);
151 |
152 | // Make the encoder
153 | const encStream = new BufferStream();
154 | encoderStreams.push(encStream);
155 | encoderReaders.push(encStream.getReader());
156 | const encoder = new Encoder({
157 | output: (chunk, metadata) => encStream.push({chunk, metadata}),
158 | error: error =>
159 | alert("Encoder " + JSON.stringify(encConfig) + ":\n" + error)
160 | });
161 | encoder.configure(encConfig);
162 | encoders.push(encoder);
163 | chunkToPackets.push(chunkToPacket);
164 |
165 | // Make the output stream
166 | ostreams.push(await configToStream(libav, encConfig));
167 | }
168 |
169 | if (!decoders.length)
170 | throw new Error("No decodable streams found!");
171 |
172 | // Demuxer -> decoder
173 | (async () => {
174 | while (true) {
175 | const [res, packets] =
176 | await libav.ff_read_frame_multi(ifc, rpkt, {limit: 1});
177 | if (res !== -libav.EAGAIN &&
178 | res !== 0 &&
179 | res !== libav.AVERROR_EOF)
180 | break;
181 |
182 | for (const idx in packets) {
183 | if (iToO[idx] < 0)
184 | continue;
185 | const o = iToO[idx];
186 | const dec = decoders[o];
187 | const p2c = packetToChunks[o];
188 | for (const packet of packets[idx]) {
189 | const chunk = p2c(packet, istreams[idx]);
190 | while (dec.decodeQueueSize) {
191 | await new Promise(res => {
192 | dec.addEventListener("dequeue", res, {once: true});
193 | });
194 | }
195 | dec.decode(chunk);
196 | }
197 | }
198 |
199 | if (res === libav.AVERROR_EOF)
200 | break;
201 | }
202 |
203 | for (let i = 0; i < decoders.length; i++) {
204 | await decoders[i].flush();
205 | decoders[i].close();
206 | decoderStreams[i].push(null);
207 | }
208 | })();
209 |
210 | // Decoder -> encoder
211 | for (let i = 0; i < decoders.length; i++) {
212 | (async () => {
213 | const decStream = decoderStreams[i];
214 | const decRdr = decStream.getReader();
215 | const enc = encoders[i];
216 | const encStream = encoderStreams[i];
217 |
218 | while (true) {
219 | const {done, value} = await decRdr.read();
220 | if (done)
221 | break;
222 |
223 | /* Pointlessly convert back and forth, just to demonstrate those
224 | * functions */
225 | let frame;
226 | if (value.codedWidth) {
227 | frame = await LibAVWebCodecsBridge.videoFrameToLAFrame(value);
228 | value.close();
229 | frame = LibAVWebCodecsBridge.laFrameToVideoFrame(frame);
230 | } else {
231 | frame = await LibAVWebCodecsBridge.audioDataToLAFrame(value);
232 | value.close();
233 | frame = LibAVWebCodecsBridge.laFrameToAudioData(frame);
234 | }
235 |
236 | enc.encode(frame);
237 | frame.close();
238 | }
239 |
240 | await enc.flush();
241 | enc.close();
242 | encStream.push(null);
243 | })();
244 | }
245 |
246 | // We need to get at least one packet from each stream before we can mux
247 | let ofc, pb;
248 | {
249 | const starterPackets = [];
250 | for (let i = 0; i < ostreams.length; i++) {
251 | const encRdr = encoderReaders[i];
252 | const {done, value} = await encRdr.read();
253 | if (done)
254 | continue;
255 | const chunkToPacket = chunkToPackets[i];
256 | const ostream = ostreams[i];
257 |
258 | // Convert it
259 | const packet = await chunkToPacket(
260 | libav, value.chunk, value.metadata, ostream, i);
261 | starterPackets.push(packet);
262 | }
263 |
264 | // Make the muxer
265 | [ofc, , pb] = await libav.ff_init_muxer({
266 | filename: "output.mkv",
267 | open: true,
268 | codecpars: true
269 | }, ostreams);
270 | await libav.avformat_write_header(ofc, 0);
271 |
272 | // And pass in the starter packets
273 | await libav.ff_write_multi(ofc, wpkt, starterPackets);
274 | }
275 |
276 | // Now pass through everything else
277 | const encPromises = [];
278 | let writePromise = Promise.all([]);
279 | for (let i = 0; i < encoderReaders.length; i++) {
280 | encPromises.push((async () => {
281 | const encRdr = encoderReaders[i];
282 | const chunkToPacket = chunkToPackets[i];
283 | const ostream = ostreams[i];
284 | while (true) {
285 | const {done, value} = await encRdr.read();
286 | if (done)
287 | break;
288 | writePromise = writePromise.then(async () => {
289 | const packet = await chunkToPacket(
290 | libav, value.chunk, value.metadata, ostream, i);
291 | await libav.ff_write_multi(ofc, wpkt, [packet]);
292 | });
293 | }
294 | })());
295 | }
296 |
297 | await Promise.all(encPromises);
298 | await writePromise;
299 |
300 | // And end the stream
301 | await libav.av_write_trailer(ofc);
302 |
303 | // Clean up
304 | await libav.avformat_close_input_js(ifc);
305 | await libav.ff_free_muxer(ofc, pb);
306 | await libav.av_packet_free(rpkt);
307 | await libav.av_packet_free(wpkt);
308 |
309 | // And fetch the file
310 | const output = await libav.readFile("output.mkv");
311 | await libav.terminate();
312 | const ofile = new File([output.buffer], "output.mkv",
313 | {type: "video/x-matroska"});
314 | document.location.href = URL.createObjectURL(ofile);
315 | }
316 |
317 | main();
318 |
319 |
--------------------------------------------------------------------------------
/src/demux.ts:
--------------------------------------------------------------------------------
1 | /*
2 | * This file is part of the libav.js WebCodecs Bridge implementation.
3 | *
4 | * Copyright (c) 2023, 2024 Yahweasel
5 | *
6 | * Permission to use, copy, modify, and/or distribute this software for any
7 | * purpose with or without fee is hereby granted, provided that the above
8 | * copyright notice and this permission notice appear in all copies.
9 | *
10 | * THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
13 | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
15 | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
16 | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 | */
18 |
19 | /*
20 | * This file contains functionality related to using libav.js for demuxing, and
21 | * then converting everything to WebCodecs for decoding.
22 | */
23 |
24 | import type * as LibAVJS from "@libav.js/types";
25 | import type * as LibAVJSWebCodecs from "libavjs-webcodecs-polyfill";
26 | declare let LibAV : LibAVJS.LibAVWrapper;
27 | declare let LibAVWebCodecs : any;
28 | declare let EncodedAudioChunk : any;
29 | declare let EncodedVideoChunk : any;
30 |
31 | /**
32 | * Convert a libav.js audio stream to a WebCodecs configuration.
33 | *
34 | * @param libav The libav.js instance that created this stream.
35 | * @param stream The stream to convert.
36 | */
37 | export async function audioStreamToConfig(
38 | libav: LibAVJS.LibAV, stream: LibAVJS.Stream | LibAVJS.CodecParameters
39 | ): Promise {
40 | let codecpar: LibAVJS.CodecParameters;
41 | if (( stream).codecpar) {
42 | codecpar = await libav.ff_copyout_codecpar(
43 | ( stream).codecpar
44 | );
45 | } else {
46 | codecpar = stream;
47 | }
48 |
49 | const codecString = await libav.avcodec_get_name(codecpar.codec_id);
50 |
51 | // Start with the basics
52 | const ret: LibAVJSWebCodecs.AudioDecoderConfig = {
53 | codec: "unknown",
54 | sampleRate: codecpar.sample_rate!,
55 | numberOfChannels: codecpar.channels!
56 | };
57 | const extradata = codecpar.extradata;
58 |
59 | // Then convert the actual codec
60 | switch (codecString) {
61 | case "flac":
62 | ret.codec = "flac";
63 | ret.description = extradata;
64 | break;
65 |
66 | case "mp3":
67 | ret.codec = "mp3";
68 | break;
69 |
70 | case "aac":
71 | {
72 | const profile = codecpar.profile!;
73 | switch (profile) {
74 | case 1: // AAC_LOW
75 | default:
76 | ret.codec = "mp4a.40.2";
77 | break;
78 |
79 | case 4: // AAC_HE
80 | ret.codec = "mp4a.40.5";
81 | break;
82 |
83 | case 28: // AAC_HE_V2
84 | ret.codec = "mp4a.40.29";
85 | break;
86 | }
87 | if (extradata)
88 | ret.description = extradata;
89 | break;
90 | }
91 |
92 | case "opus":
93 | ret.codec = "opus";
94 | break;
95 |
96 | case "vorbis":
97 | ret.codec = "vorbis";
98 | ret.description = extradata;
99 | break;
100 |
101 | default:
102 | // Best we can do is a libavjs-webcodecs-polyfill-specific config
103 | if (typeof LibAVWebCodecs !== "undefined") {
104 | ret.codec = {libavjs:{
105 | codec: codecString,
106 | ctx: {
107 | channels: codecpar.channels!,
108 | sample_rate: codecpar.sample_rate!
109 | }
110 | }};
111 | if (extradata)
112 | ret.description = extradata;
113 | }
114 | break;
115 | }
116 |
117 | if (ret.codec)
118 | return ret;
119 | return null;
120 | }
121 |
122 | /**
123 | * Convert a libav.js video stream to a WebCodecs configuration.
124 | *
125 | * @param libav The libav.js instance that created this stream.
126 | * @param stream The stream to convert.
127 | */
128 | export async function videoStreamToConfig(
129 | libav: LibAVJS.LibAV, stream: LibAVJS.Stream | LibAVJS.CodecParameters
130 | ): Promise {
131 | let codecpar: LibAVJS.CodecParameters;
132 | if (( stream).codecpar) {
133 | codecpar = await libav.ff_copyout_codecpar(
134 | ( stream).codecpar
135 | );
136 | } else {
137 | codecpar = stream;
138 | }
139 |
140 | const codecString = await libav.avcodec_get_name(codecpar.codec_id);
141 |
142 | // Start with the basics
143 | const ret: LibAVJSWebCodecs.VideoDecoderConfig = {
144 | codec: "unknown",
145 | codedWidth: codecpar.width!,
146 | codedHeight: codecpar.height!
147 | };
148 | const extradata = codecpar.extradata;
149 |
150 | // Some commonly needed data
151 | let profile = codecpar.profile!;
152 | let level = codecpar.level!;
153 |
154 | // Then convert the actual codec
155 | switch (codecString) {
156 | case "av1":
157 | {
158 | let codec = "av01";
159 |
160 | //
161 | if (profile < 0)
162 | profile = 0;
163 | codec += `.${profile}`;
164 |
165 | //
166 | if (level < 0)
167 | level = 0;
168 | let levelS = level.toString();
169 | if (levelS.length < 2)
170 | levelS = `0${level}`;
171 | const tier = "M"; // FIXME: Is this exposed by ffmpeg?
172 | codec += `.${levelS}${tier}`;
173 |
174 | //
175 | const format = codecpar.format;
176 | const desc = await libav.av_pix_fmt_desc_get(format);
177 | let bitDepth = (await libav.AVPixFmtDescriptor_comp_depth(desc, 0)).toString();
178 | if (bitDepth.length < 2)
179 | bitDepth = `0${bitDepth}`;
180 | codec += `.${bitDepth}`;
181 |
182 | // FIXME: the rest are technically optional, so left out
183 | ret.codec = codec;
184 | break;
185 | }
186 |
187 | case "h264": // avc1
188 | {
189 | let codec = "avc1";
190 |
191 | // Technique extracted from hlsenc.c
192 | if (extradata &&
193 | (extradata[0] | extradata[1] | extradata[2]) === 0 &&
194 | extradata[3] === 1 &&
195 | (extradata[4] & 0x1F) === 7) {
196 | codec += ".";
197 | for (let i = 5; i <= 7; i++) {
198 | let s = extradata[i].toString(16);
199 | if (s.length < 2)
200 | s = "0" + s;
201 | codec += s;
202 | }
203 |
204 | } else {
205 | // Do it from the stream data alone
206 |
207 | //
208 | if (profile < 0)
209 | profile = 77;
210 | const profileB = profile & 0xFF;
211 | let profileS = profileB.toString(16);
212 | if (profileS.length < 2)
213 | profileS = `0${profileS}`;
214 | codec += `.${profileS}`;
215 |
216 | //
217 | let constraints = 0;
218 | if (profile & 0x100 /* FF_PROFILE_H264_CONSTRAINED */) {
219 | // One or more of the constraint bits should be set
220 | if (profileB === 66 /* FF_PROFILE_H264_BASELINE */) {
221 | // All three
222 | constraints |= 0xE0;
223 | } else if (profileB === 77 /* FF_PROFILE_H264_MAIN */) {
224 | // Only constrained to main
225 | constraints |= 0x60;
226 | } else if (profile === 88 /* FF_PROFILE_H264_EXTENDED */) {
227 | // Only constrained to extended
228 | constraints |= 0x20;
229 | } else {
230 | // Constrained, but we don't understand how
231 | break;
232 | }
233 | }
234 | let constraintsS = constraints.toString(16);
235 | if (constraintsS.length < 2)
236 | constraintsS = `0${constraintsS}`;
237 | codec += constraintsS;
238 |
239 | //
240 | if (level < 0)
241 | level = 10;
242 | let levelS = level.toString(16);
243 | if (levelS.length < 2)
244 | levelS = `0${levelS}`;
245 | codec += levelS;
246 | }
247 |
248 | ret.codec = codec;
249 | if (extradata && extradata[0])
250 | ret.description = extradata;
251 | break;
252 | }
253 |
254 | case "hevc": // hev1/hvc1
255 | {
256 | let codec;
257 |
258 | if (extradata && extradata.length > 12) {
259 | codec = "hvc1";
260 | const dv = new DataView(extradata.buffer);
261 | ret.description = extradata;
262 |
263 | // Extrapolated from MP4Box.js
264 | codec += ".";
265 | const profileSpace = extradata[1] >> 6;
266 | switch (profileSpace) {
267 | case 1: codec += "A"; break;
268 | case 2: codec += "B"; break;
269 | case 3: codec += "C"; break;
270 | }
271 |
272 | const profileIDC = extradata[1] & 0x1F;
273 | codec += profileIDC + ".";
274 |
275 | const profileCompatibility = dv.getUint32(2);
276 | let val = profileCompatibility;
277 | let reversed = 0;
278 | for (let i = 0; i < 32; i++) {
279 | reversed |= val & 1;
280 | if (i === 31) break;
281 | reversed <<= 1;
282 | val >>= 1;
283 | }
284 | codec += reversed.toString(16) + ".";
285 |
286 | const tierFlag = (extradata[1] & 0x20) >> 5;
287 | if (tierFlag === 0)
288 | codec += 'L';
289 | else
290 | codec += 'H';
291 |
292 | const levelIDC = extradata[12];
293 | codec += levelIDC;
294 |
295 | let constraintString = "";
296 | for (let i = 11; i >= 6; i--) {
297 | const b = extradata[i];
298 | if (b || constraintString)
299 | constraintString = "." + b.toString(16) + constraintString;
300 | }
301 | codec += constraintString;
302 |
303 | } else {
304 | /* NOTE: This string was extrapolated from hlsenc.c, but is clearly
305 | * not valid for every possible H.265 stream. */
306 | if (profile < 0)
307 | profile = 0;
308 | if (level < 0)
309 | level = 0;
310 | codec = `hev1.${profile}.4.L${level}.B01`;
311 |
312 | }
313 |
314 | ret.codec = codec;
315 | break;
316 | }
317 |
318 | case "vp8":
319 | ret.codec = "vp8";
320 | break;
321 |
322 | case "vp9":
323 | {
324 | let codec = "vp09";
325 |
326 | //
327 | let profileS = profile.toString();
328 | if (profile < 0)
329 | profileS = "00";
330 | if (profileS.length < 2)
331 | profileS = `0${profileS}`;
332 | codec += `.${profileS}`;
333 |
334 | //
335 | let levelS = level.toString();
336 | if (level < 0)
337 | levelS = "10";
338 | if (levelS.length < 2)
339 | levelS = `0${levelS}`;
340 | codec += `.${levelS}`;
341 |
342 | //
343 | const format = codecpar.format;
344 | const desc = await libav.av_pix_fmt_desc_get(format);
345 | let bitDepth = (await libav.AVPixFmtDescriptor_comp_depth(desc, 0)).toString();
346 | if (bitDepth === "0")
347 | bitDepth = "08";
348 | if (bitDepth.length < 2)
349 | bitDepth = `0${bitDepth}`;
350 | codec += `.${bitDepth}`;
351 |
352 | //
353 | const subX = await libav.AVPixFmtDescriptor_log2_chroma_w(desc);
354 | const subY = await libav.AVPixFmtDescriptor_log2_chroma_h(desc);
355 | let chromaSubsampling = 0;
356 | if (subX > 0 && subY > 0) {
357 | chromaSubsampling = 1; // YUV420
358 | } else if (subX > 0 || subY > 0) {
359 | chromaSubsampling = 2; // YUV422
360 | } else {
361 | chromaSubsampling = 3; // YUV444
362 | }
363 | codec += `.0${chromaSubsampling}`;
364 |
365 | codec += ".1.1.1.0";
366 |
367 | ret.codec = codec;
368 | break;
369 | }
370 |
371 | default:
372 | // Best we can do is a libavjs-webcodecs-polyfill-specific config
373 | if (typeof LibAVWebCodecs !== "undefined") {
374 | ret.codec = {libavjs:{
375 | codec: codecString,
376 | ctx: {
377 | pix_fmt: codecpar.format,
378 | width: codecpar.width!,
379 | height: codecpar.height!
380 | }
381 | }};
382 | if (extradata)
383 | ret.description = extradata;
384 | }
385 | break;
386 | }
387 |
388 | if (ret.codec)
389 | return ret;
390 | return null;
391 | }
392 |
393 | /*
394 | * Convert the timestamp and duration from a libav.js packet to microseconds for
395 | * WebCodecs.
396 | */
397 | function times(
398 | packet: LibAVJS.Packet,
399 | timeBaseSrc: {time_base_num: number, time_base_den: number} | [number, number]
400 | ) {
401 | // Convert from lo, hi to f64
402 | let pDuration = (packet.durationhi||0) * 0x100000000 + (packet.duration||0);
403 | let pts = (packet.ptshi||0) * 0x100000000 + (packet.pts||0);
404 | if (typeof LibAV !== "undefined" && LibAV.i64tof64) {
405 | pDuration = LibAV.i64tof64(packet.duration||0, packet.durationhi||0);
406 | pts = LibAV.i64tof64(packet.pts||0, packet.ptshi||0);
407 | }
408 |
409 | // Get the appropriate time base
410 | let tbNum = packet.time_base_num || 0;
411 | let tbDen = packet.time_base_den || 1000000;
412 | if (!tbNum) {
413 | if ((<[number, number]> timeBaseSrc).length) {
414 | const timeBase = <[number, number]> timeBaseSrc;
415 | tbNum = timeBase[0];
416 | tbDen = timeBase[1];
417 | } else {
418 | const timeBase = <{time_base_num: number, time_base_den: number}> timeBaseSrc;
419 | tbNum = timeBase.time_base_num;
420 | tbDen = timeBase.time_base_den;
421 | }
422 | }
423 |
424 | // Convert the duration
425 | const duration = Math.round(
426 | pDuration * tbNum / tbDen * 1000000
427 | );
428 |
429 | // Convert the timestamp
430 | let timestamp = Math.round(
431 | pts * tbNum / tbDen * 1000000
432 | );
433 |
434 | return {timestamp, duration};
435 | }
436 |
437 | /**
438 | * Convert a libav.js audio packet to a WebCodecs EncodedAudioChunk.
439 | * @param packet The packet itself.
440 | * @param timeBaseSrc Source for time base, which can be a Stream or just a
441 | * timebase.
442 | * @param opts Extra options. In particular, if using a polyfill, you can set
443 | * the EncodedAudioChunk constructor here.
444 | */
445 | export function packetToEncodedAudioChunk(
446 | packet: LibAVJS.Packet,
447 | timeBaseSrc: {time_base_num: number, time_base_den: number} | [number, number],
448 | opts: {
449 | EncodedAudioChunk?: any
450 | } = {}
451 | ): LibAVJSWebCodecs.EncodedAudioChunk {
452 | let EAC: any;
453 | if (opts.EncodedAudioChunk)
454 | EAC = opts.EncodedAudioChunk;
455 | else
456 | EAC = EncodedAudioChunk;
457 |
458 | const {timestamp, duration} = times(packet, timeBaseSrc);
459 |
460 | return new EAC({
461 | type: "key", // all audio chunks are keyframes in all audio codecs
462 | timestamp,
463 | duration,
464 | data: packet.data.buffer
465 | });
466 | }
467 |
468 | /**
469 | * Convert a libav.js video packet to a WebCodecs EncodedVideoChunk.
470 | * @param packet The packet itself.
471 | * @param timeBaseSrc Source for time base, which can be a Stream or just a
472 | * timebase.
473 | * @param opts Extra options. In particular, if using a polyfill, you can set
474 | * the EncodedVideoChunk constructor here.
475 | */
476 | export function packetToEncodedVideoChunk(
477 | packet: LibAVJS.Packet,
478 | timeBaseSrc: {time_base_num: number, time_base_den: number} | [number, number],
479 | opts: {
480 | EncodedVideoChunk?: any
481 | } = {}
482 | ): LibAVJSWebCodecs.EncodedVideoChunk {
483 | let EVC: any;
484 | if (opts.EncodedVideoChunk)
485 | EVC = opts.EncodedVideoChunk;
486 | else
487 | EVC = EncodedVideoChunk;
488 |
489 | const {timestamp, duration} = times(packet, timeBaseSrc);
490 |
491 | return new EVC({
492 | type: ((packet.flags||0) & 1) ? "key" : "delta",
493 | timestamp,
494 | duration,
495 | data: packet.data.buffer
496 | });
497 | }
498 |
--------------------------------------------------------------------------------