├── .gitignore
├── README.md
├── exported
└── placeholder
├── package-lock.json
├── package.json
└── src
├── approaches
├── basic
│ └── index.js
├── imagedata
│ └── index.js
├── imagedataworker
│ ├── index.js
│ └── worker.js
├── webgl
│ ├── index.js
│ └── worker.js
├── webm
│ └── index.js
├── webmloop
│ └── index.js
├── webmstreams
│ └── index.js
├── webmworker
│ ├── index.js
│ └── worker.js
└── worker
│ ├── index.js
│ └── worker.js
├── index.css
├── index.html
├── index.js
├── lib
└── p5.js
├── sketch.js
└── util
└── executeFfmpeg.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # Custom
2 | exported/*
3 | !exported/placeholder
4 | src/lib/ffmpeg
5 | src/lib/ffmpeg.exe
6 |
7 |
8 | # Logs
9 | logs
10 | *.log
11 | npm-debug.log*
12 | yarn-debug.log*
13 | yarn-error.log*
14 | lerna-debug.log*
15 |
16 | # Diagnostic reports (https://nodejs.org/api/report.html)
17 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
18 |
19 | # Runtime data
20 | pids
21 | *.pid
22 | *.seed
23 | *.pid.lock
24 | .DS_Store
25 |
26 | # Directory for instrumented libs generated by jscoverage/JSCover
27 | lib-cov
28 |
29 | # Coverage directory used by tools like istanbul
30 | coverage
31 | *.lcov
32 |
33 | # nyc test coverage
34 | .nyc_output
35 |
36 | # node-waf configuration
37 | .lock-wscript
38 |
39 | # Compiled binary addons (https://nodejs.org/api/addons.html)
40 | build/Release
41 |
42 | # Dependency directories
43 | node_modules/
44 | jspm_packages/
45 |
46 | # TypeScript v1 declaration files
47 | typings/
48 |
49 | # TypeScript cache
50 | *.tsbuildinfo
51 |
52 | # Optional npm cache directory
53 | .npm
54 |
55 | # Optional eslint cache
56 | .eslintcache
57 |
58 | # Optional REPL history
59 | .node_repl_history
60 |
61 | # Output of 'npm pack'
62 | *.tgz
63 |
64 | # Yarn Integrity file
65 | .yarn-integrity
66 |
67 | # dotenv environment variables file
68 | .env
69 | .env.test
70 |
71 | # parcel-bundler cache (https://parceljs.org/)
72 | .cache
73 |
74 | # next.js build output
75 | .next
76 |
77 | # nuxt.js build output
78 | .nuxt
79 |
80 | # vuepress build output
81 | .vuepress/dist
82 |
83 | # Serverless directories
84 | .serverless/
85 |
86 | # FuseBox cache
87 | .fusebox/
88 |
89 | # DynamoDB Local files
90 | .dynamodb/
91 |
92 | # Webpack
93 | .webpack/
94 |
95 | # Electron-Forge
96 | out/
97 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Canvas to stream speed test
2 |
3 | Testing different approaches for sending canvas frames to ffmpeg efficiently within Electron JS
4 |
5 | ## Instructions
6 |
7 | - Add ffmpeg executable of your platform to src/lib/
8 | - run npm install and npm start (see progress percent for each approach on top left corner)
9 | - Watch results in console
10 | - Compare the video results in the exports folder
11 | - Try with different amounts of frames, complexity, clearing the background, resolutions...
12 |
13 | ## For adding approaches:
14 |
15 | - The approach consists of either:
16 | a) An async conversion function (canvasToFrame) that takes the canvas and returns a single frame and a function (canvasToFrame) that gets withd and height and returns an array of arguments that allow ffmepg to understand the data (see included approaches for reference)
17 | b) A handleAll function that takes the canvas and some metadata (first or last frame, name) and handles the entire video recording process
18 | - Save the approach in src/approaches within its own named folder, with an index.js containing an object with the chosen functions
19 | - Add new approaches to the approaches array, by folder name, in sketch.js
20 |
21 | ## Conclusions
22 |
23 | - The new webstream approaches work well but memory becomes hard to control, usually resulting in a crash for long projects
24 | - Using imageData as opposed to converting the canvas to PNG and using ffmpeg in a worker provide the best results for complex images (hard to compress). PNG approaches perform a bit better if the background is reset in every frame (fewer painted pixels are converted to PNG). There, the "worker" approach seems to be faster in most cases
25 | - Drawing to a hidden canvas (p5.Renderer object) seems to provide a clear advantage in some approaches, not in webgl
26 | - Using webgl and reading the data with readpixels also seems to be efficient, but this might not be viable in some use cases
27 |
--------------------------------------------------------------------------------
/exported/placeholder:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JuanIrache/canvas-stream-test/f2b6d44ab9ee2d641517a3974f28ceb08b362945/exported/placeholder
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "canvas-stream-test",
3 | "productName": "canvas-stream-test",
4 | "version": "1.0.0",
5 | "description": "My Electron application description",
6 | "main": "src/index.js",
7 | "scripts": {
8 | "start": "electron-forge start",
9 | "package": "electron-forge package",
10 | "make": "electron-forge make",
11 | "publish": "electron-forge publish",
12 | "lint": "echo \"No linting configured\""
13 | },
14 | "keywords": [],
15 | "author": {
16 | "name": "JuanIrache",
17 | "email": "yrache@gmail.com"
18 | },
19 | "license": "MIT",
20 | "config": {
21 | "forge": {
22 | "packagerConfig": {},
23 | "makers": [
24 | {
25 | "name": "@electron-forge/maker-squirrel",
26 | "config": {
27 | "name": "canvas_stream_test"
28 | }
29 | },
30 | {
31 | "name": "@electron-forge/maker-zip",
32 | "platforms": [
33 | "darwin"
34 | ]
35 | },
36 | {
37 | "name": "@electron-forge/maker-deb",
38 | "config": {}
39 | },
40 | {
41 | "name": "@electron-forge/maker-rpm",
42 | "config": {}
43 | }
44 | ]
45 | }
46 | },
47 | "dependencies": {
48 | "electron-squirrel-startup": "^1.0.0"
49 | },
50 | "devDependencies": {
51 | "@electron-forge/cli": "^6.0.0-beta.54",
52 | "@electron-forge/maker-deb": "^6.0.0-beta.54",
53 | "@electron-forge/maker-rpm": "^6.0.0-beta.54",
54 | "@electron-forge/maker-squirrel": "^6.0.0-beta.54",
55 | "@electron-forge/maker-zip": "^6.0.0-beta.54",
56 | "electron": "12.0.7"
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/src/approaches/basic/index.js:
--------------------------------------------------------------------------------
1 | // Standard browser approach to convert canvas frames to png
2 |
3 | module.exports = {
4 | ffmpegArgs: () => ['-f', 'image2pipe'],
5 | canvasToFrame: canvas =>
6 | new Promise(resolve =>
7 | canvas.toBlob(
8 | async blob => resolve(Buffer.from(await blob.arrayBuffer(), 'base64')),
9 | 'image/png'
10 | )
11 | )
12 | };
13 |
--------------------------------------------------------------------------------
/src/approaches/imagedata/index.js:
--------------------------------------------------------------------------------
1 | // Use ctx.getImageData to send the raw image data quickly to ffmpeg
2 |
3 | module.exports = {
4 | ffmpegArgs: (w, h) => [
5 | '-f',
6 | 'rawvideo',
7 | '-pix_fmt',
8 | 'rgba',
9 | '-s',
10 | `${w}x${h}`
11 | ],
12 | canvasToFrame: canvas =>
13 | Buffer.from(
14 | canvas.getContext('2d').getImageData(0, 0, canvas.width, canvas.height)
15 | .data
16 | )
17 | };
18 |
--------------------------------------------------------------------------------
/src/approaches/imagedataworker/index.js:
--------------------------------------------------------------------------------
1 | // Use imageData approach but handle everything on worker, including ffmpeg
2 |
3 | const path = require('path');
4 |
5 | const worker = new Worker(path.resolve(__dirname, 'worker.js'));
6 |
7 | module.exports = {
8 | handleAll: ({ canvas, first, last, name }) =>
9 | new Promise(async resolve => {
10 | const imageData = canvas
11 | .getContext('2d')
12 | .getImageData(0, 0, canvas.width, canvas.height).data.buffer;
13 |
14 | const { width: w, height: h } = canvas;
15 |
16 | worker.onmessage = ({ data }) =>
17 | resolve(/* actually, handle success or error here */);
18 |
19 | worker.postMessage(
20 | {
21 | action: 'saveFrame',
22 | payload: { first, last, name, imageData, w, h }
23 | },
24 | [imageData]
25 | );
26 | })
27 | };
28 |
--------------------------------------------------------------------------------
/src/approaches/imagedataworker/worker.js:
--------------------------------------------------------------------------------
1 | const { PassThrough } = require('stream');
2 | const { execFile } = require('child_process');
3 | const path = require('path');
4 |
5 | let imagesStream;
6 |
7 | const addFrameToStream = ({ imageData, imagesStream }) =>
8 | new Promise(resolve => {
9 | const ok = imagesStream.write(Buffer.from(imageData), 'utf8', () => {});
10 | if (ok) resolve();
11 | else imagesStream.once('drain', resolve);
12 | });
13 |
14 | const executeFfmpeg = ({ imagesStream, name, w, h }) => {
15 | const child = execFile(
16 | path.resolve(__dirname, '../../lib/ffmpeg'),
17 | [
18 | '-f',
19 | 'rawvideo',
20 | '-pix_fmt',
21 | 'rgba',
22 | '-s',
23 | `${w}x${h}`,
24 | '-i',
25 | '-',
26 | '-c:v',
27 | 'libx264',
28 | '-preset',
29 | 'ultrafast',
30 | '-y',
31 | path.resolve(__dirname, `../../../exported/${name}.mp4`)
32 | ],
33 | err => {
34 | if (err) console.error(err);
35 | else self.postMessage({ action: 'success' });
36 | }
37 | );
38 |
39 | imagesStream.pipe(child.stdin);
40 | };
41 |
42 | onmessage = async ({ data }) => {
43 | const { action, payload } = data;
44 | const { first, last, name, imageData, w, h } = payload;
45 | if (first) {
46 | imagesStream = new PassThrough();
47 | executeFfmpeg({ imagesStream, name, w, h });
48 | }
49 |
50 | await addFrameToStream({ imagesStream, imageData });
51 |
52 | if (last) imagesStream.end();
53 | else self.postMessage({ action: 'success' });
54 | };
55 |
--------------------------------------------------------------------------------
/src/approaches/webgl/index.js:
--------------------------------------------------------------------------------
1 | // Render in webgl, extract data with readPixels and do ffmpeg in worker. In theory should be fast
2 |
3 | const path = require('path');
4 |
5 | const worker = new Worker(path.resolve(__dirname, 'worker.js'));
6 |
7 | module.exports = {
8 | webgl: true,
9 | handleAll: ({ canvas, first, last, name }) =>
10 | new Promise(async resolve => {
11 | const pixels = new Uint8Array(canvas.width * canvas.height * 4);
12 | const gl = canvas.getContext('webgl', {
13 | preserveDrawingBuffer: true
14 | });
15 | gl.readPixels(
16 | 0,
17 | 0,
18 | canvas.width,
19 | canvas.height,
20 | gl.RGBA,
21 | gl.UNSIGNED_BYTE,
22 | pixels
23 | );
24 | const imageData = pixels.buffer;
25 |
26 | const { width: w, height: h } = canvas;
27 |
28 | worker.onmessage = ({ data }) =>
29 | resolve(/* actually, handle success or error here */);
30 |
31 | worker.postMessage(
32 | {
33 | action: 'saveFrame',
34 | payload: { first, last, name, imageData, w, h }
35 | },
36 | [imageData]
37 | );
38 | })
39 | };
40 |
--------------------------------------------------------------------------------
/src/approaches/webgl/worker.js:
--------------------------------------------------------------------------------
1 | const { PassThrough } = require('stream');
2 | const { execFile } = require('child_process');
3 | const path = require('path');
4 |
5 | let imagesStream;
6 |
7 | const addFrameToStream = ({ imageData, imagesStream }) =>
8 | new Promise(resolve => {
9 | const ok = imagesStream.write(Buffer.from(imageData), 'utf8', () => {});
10 | if (ok) resolve();
11 | else imagesStream.once('drain', resolve);
12 | });
13 |
14 | const executeFfmpeg = ({ imagesStream, name, w, h }) => {
15 | const child = execFile(
16 | path.resolve(__dirname, '../../lib/ffmpeg'),
17 | [
18 | '-f',
19 | 'rawvideo',
20 | '-pix_fmt',
21 | 'rgba',
22 | '-s',
23 | `${w}x${h}`,
24 | '-i',
25 | '-',
26 | '-c:v',
27 | 'libx264',
28 | '-preset',
29 | 'ultrafast',
30 | '-y',
31 | path.resolve(__dirname, `../../../exported/${name}.mp4`)
32 | ],
33 | err => {
34 | if (err) console.error(err);
35 | else self.postMessage({ action: 'success' });
36 | }
37 | );
38 |
39 | imagesStream.pipe(child.stdin);
40 | };
41 |
42 | onmessage = async ({ data }) => {
43 | const { action, payload } = data;
44 | const { first, last, name, imageData, w, h } = payload;
45 | if (first) {
46 | imagesStream = new PassThrough();
47 | executeFfmpeg({ imagesStream, name, w, h });
48 | }
49 |
50 | await addFrameToStream({ imagesStream, imageData });
51 |
52 | if (last) imagesStream.end();
53 | else self.postMessage({ action: 'success' });
54 | };
55 |
--------------------------------------------------------------------------------
/src/approaches/webm/index.js:
--------------------------------------------------------------------------------
1 | // Create a webm file from the browser and only use ffmpeg to adjust the frame rate
2 | // Inspired by https://stackoverflow.com/questions/58907270/record-at-constant-fps-with-canvascapturemediastream-even-on-slow-computers
3 |
4 | const path = require('path');
5 | const { writeFile } = require('fs/promises');
6 | const { unlink } = require('fs');
7 | const executeFfmpeg = require('../../util/executeFfmpeg');
8 |
9 | const wait = ms => new Promise(res => setTimeout(res, ms));
10 |
11 | const waitForEvent = (target, type) =>
12 | new Promise(res =>
13 | target.addEventListener(type, res, {
14 | once: true
15 | })
16 | );
17 |
18 | class FrameByFrameCanvasRecorder {
19 | constructor(source_canvas, FPS) {
20 | this.FPS = FPS;
21 | this.source = source_canvas;
22 | const canvas = (this.canvas = source_canvas.cloneNode());
23 | const ctx = (this.drawingContext = canvas.getContext('2d'));
24 |
25 | ctx.drawImage(source_canvas, 0, 0);
26 | const stream = (this.stream = canvas.captureStream(0));
27 | this.track = stream.getVideoTracks()[0];
28 |
29 | const rec = (this.recorder = new MediaRecorder(stream, {
30 | mimeType: 'video/webm;codecs=vp9',
31 | videoBitsPerSecond: 25000000
32 | }));
33 | const chunks = (this.chunks = []);
34 | rec.ondataavailable = evt => chunks.push(evt.data);
35 | rec.start();
36 | waitForEvent(rec, 'start').then(evt => rec.pause());
37 | this._init = waitForEvent(rec, 'pause');
38 | }
39 | async recordFrame() {
40 | await this._init;
41 | const rec = this.recorder;
42 | const canvas = this.canvas;
43 | const source = this.source;
44 | const ctx = this.drawingContext;
45 | if (canvas.width !== source.width || canvas.height !== source.height) {
46 | canvas.width = source.width;
47 | canvas.height = source.height;
48 | }
49 |
50 | const timer = wait(1000 / this.FPS);
51 |
52 | rec.resume();
53 | await waitForEvent(rec, 'resume');
54 |
55 | ctx.clearRect(0, 0, canvas.width, canvas.height);
56 | ctx.drawImage(source, 0, 0);
57 | this.track.requestFrame();
58 |
59 | await timer;
60 |
61 | rec.pause();
62 | await waitForEvent(rec, 'pause');
63 | }
64 | async export() {
65 | this.recorder.stop();
66 | this.stream.getTracks().forEach(track => track.stop());
67 | await waitForEvent(this.recorder, 'stop');
68 | return new Blob(this.chunks);
69 | }
70 | }
71 |
72 | let recorder;
73 |
74 | module.exports = {
75 | handleAll: ({ canvas, first, last, name }) =>
76 | new Promise(async resolve => {
77 | if (first) {
78 | recorder = new FrameByFrameCanvasRecorder(canvas, 25);
79 | }
80 |
81 | await recorder.recordFrame();
82 |
83 | if (last) {
84 | await wait(500);
85 | const blob = await recorder.export();
86 |
87 | const buffer = Buffer.from(await blob.arrayBuffer());
88 |
89 | const tempPath = path.resolve(__dirname, `../../../exported/temp.webm`);
90 | await writeFile(tempPath, buffer);
91 | executeFfmpeg({
92 | args: ['-fflags', '+genpts', '-r', '25', '-i', tempPath],
93 | name,
94 | done: () => {
95 | unlink(tempPath, () => {});
96 | resolve();
97 | }
98 | });
99 | } else resolve();
100 | })
101 | };
102 |
--------------------------------------------------------------------------------
/src/approaches/webmloop/index.js:
--------------------------------------------------------------------------------
1 | // Create a browser WEBM and use ffmpeg to adjust the frame rate. Use p5js standard loop to decide frame rate
2 | // Inspired by https://stackoverflow.com/questions/58907270/record-at-constant-fps-with-canvascapturemediastream-even-on-slow-computers (but not frame by frame)
3 |
4 | const path = require('path');
5 | const { writeFile } = require('fs/promises');
6 | const { unlink } = require('fs');
7 | const executeFfmpeg = require('../../util/executeFfmpeg');
8 |
9 | const wait = ms => new Promise(res => setTimeout(res, ms));
10 |
11 | const waitForEvent = (target, type) =>
12 | new Promise(res =>
13 | target.addEventListener(type, res, {
14 | once: true
15 | })
16 | );
17 |
18 | class WEBMRecorder {
19 | constructor(source_canvas, resolve, pending) {
20 | const stream = (this.stream = source_canvas.captureStream());
21 | const rec = (this.recorder = new MediaRecorder(stream, {
22 | mimeType: 'video/webm;codecs=vp9',
23 | videoBitsPerSecond: 25000000
24 | }));
25 | const chunks = (this.chunks = []);
26 | rec.ondataavailable = evt => chunks.push(evt.data);
27 | rec.start();
28 | waitForEvent(rec, 'start').then(() => resolve({ pending }));
29 | }
30 | async export() {
31 | this.recorder.stop();
32 | this.stream.getTracks().forEach(track => track.stop());
33 | await waitForEvent(this.recorder, 'stop');
34 | return new Blob(this.chunks);
35 | }
36 | }
37 |
38 | let recorder, onceDone;
39 |
40 | module.exports = {
41 | loopStart: ({ canvas }) =>
42 | new Promise(resolve => {
43 | const pending = new Promise(function (resolve) {
44 | onceDone = resolve;
45 | });
46 | recorder = new WEBMRecorder(canvas, resolve, pending);
47 | }),
48 | loopEnd: async ({ name }) => {
49 | await wait(500);
50 | const blob = await recorder.export();
51 |
52 | const buffer = Buffer.from(await blob.arrayBuffer());
53 |
54 | const tempPath = path.resolve(__dirname, `../../../exported/temp.webm`);
55 | await writeFile(tempPath, buffer);
56 | executeFfmpeg({
57 | args: ['-fflags', '+genpts', '-r', '25', '-i', tempPath],
58 | name,
59 | done: () => {
60 | unlink(tempPath, () => {});
61 | onceDone();
62 | }
63 | });
64 | }
65 | };
66 |
--------------------------------------------------------------------------------
/src/approaches/webmstreams/index.js:
--------------------------------------------------------------------------------
1 | // Create a browser WEBM and use ffmpeg to adjust the frame rate. Use p5js standard loop to decide frame rate. Sent to ffmpeg via a stream
2 | // Inspired by https://stackoverflow.com/questions/58907270/record-at-constant-fps-with-canvascapturemediastream-even-on-slow-computers (but not frame by frame)
3 |
4 | const executeFfmpeg = require('../../util/executeFfmpeg');
5 |
6 | const wait = ms => new Promise(res => setTimeout(res, ms));
7 |
8 | const waitForEvent = (target, type) =>
9 | new Promise(res =>
10 | target.addEventListener(type, res, {
11 | once: true
12 | })
13 | );
14 |
15 | const chunksToBuf = async chunks => {
16 | const blob = await new Blob(chunks);
17 | const arrbuf = await blob.arrayBuffer();
18 | const buf = Buffer.from(arrbuf);
19 | chunks.length = 0;
20 | return buf;
21 | };
22 |
23 | class WEBMRecorder {
24 | constructor({ canvas, resolve, pending }) {
25 | const imagesStream = (this.imagesStream = new PassThrough());
26 | const stream = (this.stream = canvas.captureStream());
27 | this.writable = true;
28 | const rec = (this.recorder = new MediaRecorder(stream, {
29 | mimeType: 'video/webm;codecs=vp9',
30 | videoBitsPerSecond: 25000000
31 | }));
32 | const chunks = (this.chunks = []);
33 | rec.ondataavailable = async evt => {
34 | chunks.push(evt.data);
35 | const doWrite = async () => {
36 | if (chunks.length) {
37 | const buf = await chunksToBuf(chunks);
38 | const ok = imagesStream.write(buf, 'utf8', () => {});
39 | if (!ok) {
40 | this.writable = false;
41 | imagesStream.once('drain', () => {
42 | this.writable = true;
43 | doWrite();
44 | });
45 | }
46 | }
47 | };
48 | if (this.writable) doWrite();
49 | };
50 | rec.start(60000);
51 | waitForEvent(rec, 'start').then(() => resolve({ pending }));
52 | }
53 | async export() {
54 | this.recorder.stop();
55 | this.stream.getTracks().forEach(track => track.stop());
56 | await waitForEvent(this.recorder, 'stop');
57 | while (this.chunks.length) await new Promise(setImmediate);
58 | this.imagesStream.end();
59 | }
60 | }
61 |
62 | let recorder;
63 |
64 | module.exports = {
65 | loopStart: ({ canvas, name }) =>
66 | new Promise(resolve => {
67 | let onceDone;
68 | const pending = new Promise(function (resolve) {
69 | onceDone = resolve;
70 | });
71 | recorder = new WEBMRecorder({ canvas, resolve, pending });
72 | executeFfmpeg({
73 | args: ['-fflags', '+genpts', '-r', '25'],
74 | name,
75 | done: onceDone,
76 | imagesStream: recorder.imagesStream
77 | });
78 | }),
79 | loopEnd: async () => {
80 | await wait(500);
81 | await recorder.export();
82 | }
83 | };
84 |
--------------------------------------------------------------------------------
/src/approaches/webmworker/index.js:
--------------------------------------------------------------------------------
1 | // Create a browser WEBM and use ffmpeg to adjust the frame rate. Use p5js standard loop to decide frame rate. Send to ffmpeg (in a worker) with a stream
2 | // Inspired by https://stackoverflow.com/questions/58907270/record-at-constant-fps-with-canvascapturemediastream-even-on-slow-computers (but not frame by frame)
3 |
4 | const path = require('path');
5 |
6 | const worker = new Worker(path.resolve(__dirname, 'worker.js'));
7 |
8 | const wait = ms => new Promise(res => setTimeout(res, ms));
9 |
10 | const waitForEvent = (target, type) =>
11 | new Promise(res =>
12 | target.addEventListener(type, res, {
13 | once: true
14 | })
15 | );
16 |
17 | class WEBMRecorder {
18 | constructor({ canvas, resolve, pending }) {
19 | const stream = (this.stream = canvas.captureStream());
20 | this.writable = true;
21 | const rec = (this.recorder = new MediaRecorder(stream, {
22 | mimeType: 'video/webm;codecs=vp9',
23 | videoBitsPerSecond: 25000000
24 | }));
25 | rec.ondataavailable = async evt => {
26 | const blob = await new Blob([evt.data]);
27 | const arrbuf = await blob.arrayBuffer();
28 | worker.postMessage({ action: 'write', payload: { arrbuf } }, [arrbuf]);
29 | };
30 | rec.start(1000);
31 | waitForEvent(rec, 'start').then(() => resolve({ pending }));
32 | }
33 | async export() {
34 | this.recorder.stop();
35 | this.stream.getTracks().forEach(track => track.stop());
36 | await waitForEvent(this.recorder, 'stop');
37 | worker.postMessage({ action: 'end' });
38 | }
39 | }
40 |
41 | let recorder;
42 |
43 | module.exports = {
44 | loopStart: ({ canvas, name }) =>
45 | new Promise(resolve => {
46 | let onceDone;
47 | const pending = new Promise(function (resolve) {
48 | onceDone = resolve;
49 | });
50 | worker.onmessage = ({ data }) => {
51 | const { action, payload } = data;
52 | switch (action) {
53 | case 'done':
54 | onceDone();
55 | break;
56 |
57 | default:
58 | break;
59 | }
60 | };
61 | worker.postMessage({ action: 'start', payload: { name } });
62 | recorder = new WEBMRecorder({ canvas, resolve, pending });
63 | }),
64 | loopEnd: async () => {
65 | await wait(500);
66 | await recorder.export();
67 | }
68 | };
69 |
--------------------------------------------------------------------------------
/src/approaches/webmworker/worker.js:
--------------------------------------------------------------------------------
1 | const { PassThrough } = require('stream');
2 | const { execFile } = require('child_process');
3 | const path = require('path');
4 |
5 | let imagesStream;
6 |
7 | const pendingBufs = [];
8 | let writable = true;
9 |
10 | const addToStream = ({ imagesStream, arrbuf }) => {
11 | pendingBufs.push(Buffer.from(arrbuf));
12 | const doWrite = async () => {
13 | if (pendingBufs.length) {
14 | const buf = Buffer.concat(pendingBufs);
15 | pendingBufs.length = 0;
16 | const ok = imagesStream.write(buf, 'utf8', () => {});
17 | if (!ok) {
18 | writable = false;
19 | imagesStream.once('drain', () => {
20 | writable = true;
21 | doWrite();
22 | });
23 | }
24 | }
25 | };
26 | if (writable) doWrite();
27 | };
28 |
29 | const executeFfmpeg = ({ imagesStream, name }) => {
30 | const child = execFile(
31 | path.resolve(__dirname, '../../lib/ffmpeg'),
32 | [
33 | '-fflags',
34 | '+genpts',
35 | '-r',
36 | '25',
37 | '-i',
38 | '-',
39 | '-c:v',
40 | 'libx264',
41 | '-preset',
42 | 'ultrafast',
43 | '-y',
44 | path.resolve(__dirname, `../../../exported/${name}.mp4`)
45 | ],
46 | err => {
47 | if (err) console.error(err);
48 | else self.postMessage({ action: 'done' });
49 | }
50 | );
51 |
52 | imagesStream.pipe(child.stdin);
53 | };
54 |
55 | onmessage = async ({ data }) => {
56 | const { action, payload } = data;
57 | switch (action) {
58 | case 'start': {
59 | const { name } = payload;
60 | imagesStream = new PassThrough();
61 | executeFfmpeg({ imagesStream, name });
62 | break;
63 | }
64 | case 'write':
65 | const { arrbuf } = payload;
66 | addToStream({ imagesStream, arrbuf });
67 | break;
68 |
69 | case 'end':
70 | while (pendingBufs.length) await new Promise(setImmediate);
71 | imagesStream.end();
72 | break;
73 |
74 | default:
75 | break;
76 | }
77 | };
78 |
--------------------------------------------------------------------------------
/src/approaches/worker/index.js:
--------------------------------------------------------------------------------
1 | // Do the conversion from canvas to PNG in parallel in a Worker and send that to ffmpeg
2 |
3 | const path = require('path');
4 |
5 | const worker = new Worker(path.resolve(__dirname, 'worker.js'));
6 |
7 | let initialised;
8 |
9 | module.exports = {
10 | ffmpegArgs: () => ['-f', 'image2pipe'],
11 | canvasToFrame: canvas =>
12 | new Promise(async resolve => {
13 | if (!initialised) {
14 | initialised = true;
15 | worker.postMessage({
16 | action: 'setSize',
17 | payload: { size: [canvas.width, canvas.height] }
18 | });
19 | }
20 |
21 | const bitmap = await createImageBitmap(canvas);
22 |
23 | worker.onmessage = ({ data }) => {
24 | resolve(Buffer.from(data.payload, 'base64'));
25 | };
26 |
27 | worker.postMessage({ action: 'saveFrame', payload: { bitmap } }, [
28 | bitmap
29 | ]);
30 | })
31 | };
32 |
--------------------------------------------------------------------------------
/src/approaches/worker/worker.js:
--------------------------------------------------------------------------------
1 | const canvas = new OffscreenCanvas(300, 300);
2 | const ctx = canvas.getContext('bitmaprenderer');
3 |
4 | const saveFrame = async ({ bitmap }) => {
5 | ctx.transferFromImageBitmap(bitmap);
6 | const blob = await canvas.convertToBlob();
7 | const arrBuf = await blob.arrayBuffer();
8 | self.postMessage({ action: 'success', payload: arrBuf }, [arrBuf]);
9 | };
10 |
11 | onmessage = ({ data }) => {
12 | const { action, payload } = data;
13 | if (action === 'setSize') {
14 | const { size } = payload;
15 | canvas.width = size[0];
16 | canvas.height = size[1];
17 | } else if (action === 'saveFrame') saveFrame(payload);
18 | };
19 |
--------------------------------------------------------------------------------
/src/index.css:
--------------------------------------------------------------------------------
1 | body {
2 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
3 | margin: auto;
4 | max-width: 38rem;
5 | padding: 2rem;
6 | }
7 |
--------------------------------------------------------------------------------
/src/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/src/index.js:
--------------------------------------------------------------------------------
1 | const { app, BrowserWindow } = require('electron');
2 | const path = require('path');
3 |
4 | // Handle creating/removing shortcuts on Windows when installing/uninstalling.
5 | if (require('electron-squirrel-startup')) {
6 | // eslint-disable-line global-require
7 | app.quit();
8 | }
9 |
10 | const createWindow = () => {
11 | // Create the browser window.
12 | const mainWindow = new BrowserWindow({
13 | width: 800,
14 | height: 600,
15 | webPreferences: {
16 | nodeIntegration: true,
17 | nodeIntegrationInWorker: true,
18 | contextIsolation: false,
19 | backgroundThrottling: false,
20 | enableRemoteModule: true
21 | }
22 | });
23 |
24 | mainWindow.once('ready-to-show', mainWindow.maximize);
25 |
26 | // and load the index.html of the app.
27 | mainWindow.loadFile(path.join(__dirname, 'index.html'));
28 |
29 | // Open the DevTools.
30 | mainWindow.webContents.openDevTools();
31 | };
32 |
33 | // This method will be called when Electron has finished
34 | // initialization and is ready to create browser windows.
35 | // Some APIs can only be used after this event occurs.
36 | app.on('ready', createWindow);
37 |
38 | // Quit when all windows are closed, except on macOS. There, it's common
39 | // for applications and their menu bar to stay active until the user quits
40 | // explicitly with Cmd + Q.
41 | app.on('window-all-closed', () => {
42 | if (process.platform !== 'darwin') {
43 | app.quit();
44 | }
45 | });
46 |
47 | app.on('activate', () => {
48 | // On OS X it's common to re-create a window in the app when the
49 | // dock icon is clicked and there are no other windows open.
50 | if (BrowserWindow.getAllWindows().length === 0) {
51 | createWindow();
52 | }
53 | });
54 |
55 | // In this file you can include the rest of your app's specific main process
56 | // code. You can also put them in separate files and import them here.
57 |
--------------------------------------------------------------------------------
/src/sketch.js:
--------------------------------------------------------------------------------
1 | // Import approaches
2 | const benchmarkApproach = 'worker';
3 | const approaches = [
4 | 'webm',
5 | 'webmloop',
6 | 'webmstreams',
7 | 'webmworker',
8 | 'basic',
9 | 'imagedata',
10 | 'imagedataworker',
11 | 'webgl'
12 | ];
13 |
14 | // Try with more frames or complex images for a solid solution
15 |
16 | const frames = 1000;
17 | const complexity = 10;
18 | const clearBackground = false;
19 | const frameWidth = 1920;
20 | const frameHeight = 1080;
21 | const visualize = true; // View the frames as they render
22 |
23 | ////////////////////////// Do not edit below this line
24 | const { PassThrough } = require('stream');
25 | const executeFfmpeg = require('./util/executeFfmpeg');
26 |
27 | let drawable;
28 |
29 | function setup() {
30 | if (visualize) createCanvas(frameWidth, frameHeight);
31 | else createCanvas(30, 12);
32 | noLoop();
33 |
34 | const paint = ({ i, percent, graph }) => {
35 | if (clearBackground) g.clear();
36 |
37 | const paintOnce = ii => {
38 | const a = noise(ii / 100);
39 | const b = noise(ii / 100 + 10);
40 | const c = noise(ii / 100 + 10000);
41 | const d = noise(ii / 100 + 1000000);
42 | const e = noise(ii / 100 + 100000000);
43 | const f = noise(ii / 100 + 1000000000);
44 | const g = noise(ii / 100 + 1000000000);
45 | graph.fill(a * 255, b * 255, c * 255);
46 | graph.stroke(0, f * 100);
47 | graph.ellipse(d * frameWidth, e * frameHeight, 80 * g, 80 * g);
48 | };
49 |
50 | for (let j = 0; j < complexity; j++) {
51 | noiseSeed(j);
52 | paintOnce(i);
53 | }
54 |
55 | if (visualize) {
56 | image(graph, 0, 0);
57 | noStroke();
58 | fill(255);
59 | rect(0, 0, 30, 12);
60 | } else background(255);
61 | fill(0);
62 | text(`${percent}%`, 0, 10);
63 | };
64 |
65 | const processVideo = async (
66 | { canvasToFrame, ffmpegArgs, handleAll, webgl, loopStart, loopEnd },
67 | name
68 | ) => {
69 | const graph = createGraphics(frameWidth, frameHeight, webgl ? WEBGL : P2D);
70 | if (webgl) graph.translate(-frameWidth / 2, -frameHeight / 2);
71 |
72 | clear();
73 | console.log('Start rendering', name, 'approach');
74 | const startTime = Date.now();
75 |
76 | if (loopStart) {
77 | const i = 0;
78 | paint({ i, percent: Math.round((100 * i) / frames), graph });
79 | const { pending } = await loopStart({ canvas: graph.elt, name });
80 | drawable = { i: i + 1, paint, loopEnd, frames, graph, name };
81 | loop();
82 | await pending;
83 | } else if (handleAll) {
84 | for (let i = 0; i <= frames; i++) {
85 | paint({ i, percent: Math.round((100 * i) / frames), graph });
86 | await handleAll({
87 | canvas: graph.elt,
88 | first: i === 0,
89 | last: i === frames,
90 | name
91 | });
92 | }
93 | } else {
94 | const imagesStream = new PassThrough();
95 | let done = false;
96 | executeFfmpeg({
97 | args: ffmpegArgs(frameWidth, frameHeight),
98 | imagesStream,
99 | name,
100 | done: () => (done = true)
101 | });
102 |
103 | const addFrameToStream = frameData =>
104 | new Promise(resolve => {
105 | const ok = imagesStream.write(frameData, 'utf8', () => {});
106 | if (ok) resolve();
107 | else imagesStream.once('drain', resolve);
108 | });
109 |
110 | for (let i = 0; i <= frames; i++) {
111 | paint({ i, percent: Math.round((100 * i) / frames), graph });
112 | const frameData = await canvasToFrame(graph.elt);
113 | await addFrameToStream(frameData);
114 | }
115 |
116 | imagesStream.end();
117 |
118 | while (!done) await new Promise(setImmediate);
119 | }
120 | const duration = Date.now() - startTime;
121 | return duration;
122 | };
123 |
124 | const runTest = async () => {
125 | const benchmark = await processVideo(
126 | require(`./approaches/${benchmarkApproach}/index`),
127 | benchmarkApproach
128 | );
129 |
130 | const toSec = ms => Math.round(ms / 1000);
131 |
132 | const results = [[benchmarkApproach, toSec(benchmark) + ' s', '100%']];
133 |
134 | console.log(
135 | `Benchmark (${benchmarkApproach}) duration is ${toSec(benchmark)}s`
136 | );
137 |
138 | for (const dir of approaches) {
139 | const approach = require(`./approaches/${dir}/index`);
140 | const duration = await processVideo(approach, dir);
141 | if (duration < benchmark) {
142 | console.log(
143 | `${dir} approach is faster than the benchmark! (${toSec(
144 | duration
145 | )}s) Check if videos look the same`
146 | );
147 | } else {
148 | console.log(`${dir} approach is not fast enough (${toSec(duration)}s)`);
149 | }
150 | results.push([
151 | dir,
152 | toSec(duration) + ' s',
153 | Math.round((100 * duration) / benchmark) + '%'
154 | ]);
155 | }
156 | console.table(results);
157 | };
158 | setImmediate(runTest);
159 | }
160 |
161 | function draw() {
162 | if (drawable) {
163 | const { i, paint, loopEnd, frames, graph, name } = drawable;
164 | if (i <= frames) {
165 | paint({ i, percent: Math.round((100 * i) / frames), graph });
166 | const last = i === frames;
167 | if (last) {
168 | loopEnd({ name });
169 | noLoop();
170 | drawable = null;
171 | } else drawable.i++;
172 | }
173 | }
174 | }
175 |
--------------------------------------------------------------------------------
/src/util/executeFfmpeg.js:
--------------------------------------------------------------------------------
1 | const { execFile } = require('child_process');
2 | const path = require('path');
3 |
4 | module.exports = ({ args, imagesStream, name, done }) => {
5 | const child = execFile(
6 | path.resolve(__dirname, '../lib/ffmpeg'),
7 | [
8 | ...args,
9 | ...(imagesStream ? ['-i', '-'] : []),
10 | '-c:v',
11 | 'libx264',
12 | '-preset',
13 | 'ultrafast',
14 | '-y',
15 | path.resolve(__dirname, `../../exported/${name}.mp4`)
16 | ],
17 | err => {
18 | if (err) console.error(err);
19 | else if (done) done();
20 | }
21 | );
22 |
23 | // child.stderr.on('data', console.log);
24 |
25 | // child.stdout.on('data', console.log);
26 |
27 | if (imagesStream) imagesStream.pipe(child.stdin);
28 | };
29 |
--------------------------------------------------------------------------------