├── www ├── assets │ ├── back.png │ ├── logo.png │ ├── favicon.ico │ ├── common.css │ └── logo.svg └── index.html ├── tsconfig.json ├── .gitignore ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── README.md ├── LICENSE ├── webpack.config.js ├── package.json └── src ├── shader.ts └── app.ts /www/assets/back.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sebavan/BabylonjsVideoProcessingSample/HEAD/www/assets/back.png -------------------------------------------------------------------------------- /www/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sebavan/BabylonjsVideoProcessingSample/HEAD/www/assets/logo.png -------------------------------------------------------------------------------- /www/assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sebavan/BabylonjsVideoProcessingSample/HEAD/www/assets/favicon.ico -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "esNext", 4 | "target": "es5", 5 | "moduleResolution": "node", 6 | "sourceMap": true, 7 | "types" : [], 8 | "lib": ["dom", "es2015"] 9 | } 10 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build Folders (you can keep bin if you'd like, to store dlls and pdbs) 2 | # And OS or Editor folders 3 | [Bb]in/ 4 | *.tmp 5 | *.log 6 | *.DS_Store 7 | ._* 8 | Thumbs.db 9 | .cache 10 | .tmproj 11 | nbproject 12 | *.sublime-project 13 | *.sublime-workspace 14 | .idea 15 | .directory 16 | build 17 | .history 18 | .tempChromeProfileForDebug 19 | 20 | # Node Modules 21 | node_modules/* 22 | 23 | # Local dist 24 | dist/* 25 | www/scripts/* -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "configurations": [ 4 | { 5 | "name": "Launch Index (Chrome)", 6 | "type": "chrome", 7 | "request": "launch", 8 | "url": "http://localhost:8080/test/dev/index.html", 9 | "webRoot": "${workspaceRoot}", 10 | "sourceMaps": true, 11 | "preLaunchTask": "devServer", 12 | "userDataDir": "${workspaceRoot}/.tempChromeProfileForDebug", 13 | "runtimeArgs": [ 14 | "--enable-unsafe-es3-apis" 15 | ] 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /www/assets/common.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --main-bg-color: #2a2342; 3 | --main-bg-color-dark: #201936; 4 | --main-txt-color: #ffffff; 5 | --main-padding: 15px; 6 | } 7 | 8 | html, 9 | body { 10 | width: 100%; 11 | padding: 0; 12 | margin: 0; 13 | background-color: var(--main-bg-color-dark); 14 | overflow: hidden; 15 | color: var(--main-txt-color); 16 | font-family: "acumin-pro"; 17 | } 18 | 19 | a { 20 | color: #BB464B; 21 | text-decoration: unset; 22 | } 23 | a:hover { 24 | color: #BB464B; 25 | } 26 | a:visited { 27 | color: #BB464B; 28 | } 29 | a:focus { 30 | color: #BB464B; 31 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | // Configure glob patterns for excluding files and folders. 4 | "files.exclude": { 5 | "**/.git": true, 6 | "**/.svn": true, 7 | "**/.hg": true, 8 | "**/.DS_Store": true, 9 | "**/.vs": true, 10 | "**/.tempChromeProfileForDebug": true, 11 | "**/node_modules": true, 12 | "**/.temp": true 13 | }, 14 | "search.exclude": { 15 | "**/.tempChromeProfileForDebug": true, 16 | "**/node_modules": true, 17 | "**/.temp": true, 18 | "**/.dist": true, 19 | "**/*.map": true 20 | }, 21 | "editor.tabSize": 4, 22 | "typescript.tsdk": "node_modules\\typescript\\lib" 23 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Babylon.js Video Processing DEMO 2 | 3 | [![Twitter](https://img.shields.io/twitter/follow/babylonjs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=babylonjs) 4 | 5 | **Any questions?** Here is our official [forum](https://forum.babylonjs.com/). 6 | 7 | You can find documentation for the controls [here](https://doc.babylonjs.com/features/controls) 8 | 9 | ## Running locally 10 | 11 | After cloning the repo, running locally during development is all the simplest: 12 | ``` 13 | npm install 14 | npm start 15 | ``` 16 | 17 | For VSCode users, if you have installed the Chrome Debugging extension, you can start debugging within VSCode by using the appropriate launch menu. 18 | 19 | ## It does not work ! 20 | 21 | Please ensure you are running on a Chromium based browser and that your WebCam is plugged in :-) 22 | 23 | ## Live Demo 24 | 25 | All available in the famous [Babylon.js website](https://www.babylonjs.com/demos/videoprocessing/). 26 | 27 | And please have a read at the associated article on [Medium](https://medium.com/@babylonjs/web-video-processing-made-easy-714487f7443b) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 sebavan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | 3 | var SRC_DIR = path.resolve(__dirname, "./src"); 4 | var DIST_DIR = path.resolve(__dirname, "./www"); 5 | var DEV_DIR = path.resolve(__dirname, "./.temp"); 6 | 7 | var buildConfig = function(env) { 8 | var isProd = env.prod; 9 | return { 10 | context: __dirname, 11 | entry: { 12 | index: SRC_DIR + "/app.ts" 13 | }, 14 | output: { 15 | path: (isProd ? DIST_DIR : DEV_DIR) + "/scripts/", 16 | filename: "[name].js", 17 | publicPath: "/scripts/", 18 | }, 19 | devtool: isProd ? false : "source-map", 20 | devServer: { 21 | static: ['www'] 22 | }, 23 | resolve: { 24 | extensions: [".ts", ".js"] 25 | }, 26 | module: { 27 | rules: [{ 28 | test: /\.tsx?$/, 29 | loader: "ts-loader", 30 | }, 31 | { 32 | test: /\.css$/, 33 | use: ["style-loader", "css-loader"], 34 | }] 35 | }, 36 | mode: isProd ? "production" : "development" 37 | }; 38 | } 39 | 40 | module.exports = buildConfig; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "babylonjs-video-processing-sample", 3 | "version": "1.0.0", 4 | "description": "Quick Demo of using the Babylon.js controls to process videos in real time.", 5 | "author": { 6 | "name": "Sebastien VANDENBERGHE" 7 | }, 8 | "contributors": [ 9 | "Sebastien VANDENBERGHE" 10 | ], 11 | "keywords": [ 12 | "controls", 13 | "video", 14 | "acceleration", 15 | "3D", 16 | "2D", 17 | "javascript", 18 | "html5", 19 | "webgl", 20 | "webgl2", 21 | "webgpu", 22 | "babylon" 23 | ], 24 | "license": "MIT", 25 | "readme": "README.md", 26 | "typings": "dist/src/index.d.ts", 27 | "main": "dist/src/index.js", 28 | "files": [ 29 | "dist/src/**", 30 | "README.md" 31 | ], 32 | "scripts": { 33 | "build": "webpack --env=prod", 34 | "start": "npx webpack-dev-server --open", 35 | "devServer": "npx webpack-dev-server" 36 | }, 37 | "devDependencies": { 38 | "@babylonjs/controls": "^2.0.0-alpha.1", 39 | "@babylonjs/core": "^5.0.0-alpha.1", 40 | "ts-loader": "^9.2.6", 41 | "typescript": "^4.4.4", 42 | "webpack": "^5.94.0", 43 | "webpack-cli": "^4.9.0", 44 | "webpack-dev-server": "^4.13.3" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "command": "npm", 6 | "type": "shell", 7 | "presentation": { 8 | "echo": true, 9 | "reveal": "always", 10 | "focus": false, 11 | "panel": "shared" 12 | }, 13 | "tasks": [ 14 | { 15 | "label": "start", 16 | "args": [ "start" ], 17 | "group": { 18 | "kind": "build", 19 | "isDefault": true 20 | }, 21 | "isBackground": true, 22 | "problemMatcher": { 23 | "owner": "typescript", 24 | "fileLocation": "relative", 25 | "pattern": { 26 | "regexp": "^([^\\s].*)\\((\\d+|\\,\\d+|\\d+,\\d+,\\d+,\\d+)\\):\\s+(error|warning|info)\\s+(TS\\d+)\\s*:\\s*(.*)$", 27 | "file": 1, 28 | "location": 2, 29 | "severity": 3, 30 | "code": 4, 31 | "message": 5 32 | }, 33 | "background": { 34 | "activeOnStart": true, 35 | "beginsPattern": "start", 36 | "endsPattern": "Compiled successfully" 37 | } 38 | } 39 | }, 40 | { 41 | "label": "devServer", 42 | "args": [ "run", "devServer" ], 43 | "group": { 44 | "kind": "build", 45 | "isDefault": true 46 | }, 47 | "isBackground": true, 48 | "problemMatcher": { 49 | "owner": "typescript", 50 | "fileLocation": "relative", 51 | "pattern": { 52 | "regexp": "^([^\\s].*)\\((\\d+|\\,\\d+|\\d+,\\d+,\\d+,\\d+)\\):\\s+(error|warning|info)\\s+(TS\\d+)\\s*:\\s*(.*)$", 53 | "file": 1, 54 | "location": 2, 55 | "severity": 3, 56 | "code": 4, 57 | "message": 5 58 | }, 59 | "background": { 60 | "activeOnStart": true, 61 | "beginsPattern": "devServer", 62 | "endsPattern": "Compiled successfully" 63 | } 64 | } 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /src/shader.ts: -------------------------------------------------------------------------------- 1 | 2 | // We are trying to render a video looking like an old V1 GameBoy. 3 | // 4 | // We therefore need to draw quads in only 4 colors: 5 | // 1. We first find the closest tile we are in: 6 | // const float screenSize = 512.; 7 | // vec2 tileUV = floor(gl_FragCoord.xy / gridSize) * gridSize / screenSize; 8 | // 2. Then extract the luminance of the current video texel: 9 | // vec4 tileColor = texture2D(textureSampler, tileUV); 10 | // float tileLuminance = getLuminance(tileColor.rgb); 11 | // 3. And finally match the luminance with our color palette of 4 colors: 12 | // vec4 finalColor = palette[int(tileLuminance * 3. + lumaOffset)]; 13 | // 14 | // And draw a grid on top of it to simulate the old pixel spaces: 15 | // 1. We check horizontally or vertically if we are on the grid 16 | // onGridline(gl_FragCoord.x, gridSize) 17 | // 2. By using a simple modulo operation 18 | // return mod(floor(distFrom), spacing) == 0.0; 19 | // 3. And if we are we use the gridColor: 20 | // gl_FragColor = gridLineColor; 21 | // return; 22 | 23 | const fragmentShader = ` 24 | varying vec2 vUV; 25 | 26 | // Default Sampler 27 | uniform sampler2D textureSampler; 28 | 29 | // Transform color to luminance. 30 | float getLuminance(vec3 color) 31 | { 32 | return clamp(dot(color, vec3(0.2126, 0.7152, 0.0722)), 0., 1.); 33 | } 34 | 35 | // Returns whether the given fragment position lies on a grid line. 36 | bool onGridline(float distFrom, float spacing) 37 | { 38 | return mod(floor(distFrom), spacing) == 0.0; 39 | } 40 | 41 | void main(void) 42 | { 43 | // Color Palette 44 | vec4 palette[4]; 45 | palette[0] = vec4( 22, 30, 87, 255.) / 255.; 46 | palette[1] = vec4( 78, 109, 90, 255.) / 255.; 47 | palette[2] = vec4(106, 149, 50, 255.) / 255.; 48 | palette[3] = vec4(115, 161, 43, 255.) / 255.; 49 | 50 | // Luminance adaptation 51 | const float lumaOffset = 0.2; 52 | 53 | // Grid lines 54 | const vec4 gridLineColor = vec4(vec3(120, 168, 51) / 255. , 1.0); 55 | const float gridSize = 8.; 56 | 57 | // Screen Definition 58 | const float screenSize = 512.; 59 | vec2 tileUV = floor(gl_FragCoord.xy / gridSize) * gridSize / screenSize; 60 | 61 | // Mirror for selfie; 62 | tileUV.x = 1. - tileUV.x; 63 | 64 | // Square fetch of luminance 65 | vec4 tileColor = texture2D(textureSampler, tileUV); 66 | float tileLuminance = getLuminance(tileColor.rgb); 67 | 68 | // Adapt luma to effect and pick from the palette 69 | int colorChoice = int(tileLuminance * 3. + lumaOffset); 70 | #ifdef WEBGL2 71 | vec4 finalColor = palette[colorChoice]; 72 | #else 73 | vec4 finalColor = vec4(0.); 74 | if (colorChoice == 0) { 75 | finalColor = palette[0]; 76 | } else if (colorChoice == 1) { 77 | finalColor = palette[1]; 78 | } else if (colorChoice == 2) { 79 | finalColor = palette[2]; 80 | } else { 81 | finalColor = palette[3]; 82 | } 83 | #endif 84 | 85 | // Are we on a line. 86 | if (onGridline(gl_FragCoord.x, gridSize) || onGridline(gl_FragCoord.y, gridSize)) 87 | { 88 | gl_FragColor = gridLineColor; 89 | return; 90 | } 91 | 92 | gl_FragColor = finalColor; 93 | }`; 94 | 95 | export const ShaderConfiguration = { 96 | name: "GameBoy", 97 | fragmentShader, 98 | samplerNames: ["textureSampler"], 99 | } -------------------------------------------------------------------------------- /www/assets/logo.svg: -------------------------------------------------------------------------------- 1 | babylonjs_identity_color_dark -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import { ThinEngine } from "@babylonjs/core/Engines/thinEngine"; 2 | import { Constants } from "@babylonjs/core/Engines/constants"; 3 | import { HtmlElementTexture } from "@babylonjs/core/Materials/Textures/htmlElementTexture"; 4 | import { EffectWrapper } from "@babylonjs/core/Materials/effectRenderer"; 5 | 6 | import { ImageFilter } from "@babylonjs/controls/dist/src/imageFilter"; 7 | import { Resizer } from "@babylonjs/controls/dist/src/resizer"; 8 | 9 | import { ShaderConfiguration } from "./shader"; 10 | 11 | // Find back the rendering Canvas 12 | const mainCanvas = document.getElementById("renderCanvas") as HTMLCanvasElement; 13 | 14 | // Filter video in realtime by reusing the babylon controls and our thinEngine... cause size matters. 15 | async function main() { 16 | const customFilter = new ImageFilter(mainCanvas); 17 | const resizer = new Resizer(customFilter); 18 | const engine = customFilter.engine; 19 | 20 | const customEffectWrapper = new EffectWrapper({ 21 | ...ShaderConfiguration, 22 | engine: engine, 23 | }); 24 | 25 | // Creates the required input for the effect. 26 | const webcamTexture = await getWebcamTextureAsync(engine).catch(() => { 27 | alert('Please ensure you are running on a Chromium based browser and that your Webcam is plugged in.'); 28 | }); 29 | 30 | // Early exit in case of setup issues. 31 | if (!webcamTexture) { 32 | return; 33 | } 34 | 35 | // Creates an off screen texture to resize the webcam texture for a better pixellation. 36 | const resizedTexture = resizer.createOffscreenTexture({ 37 | width: Math.floor((webcamTexture.element as HTMLVideoElement).videoWidth / 8), 38 | height: Math.floor((webcamTexture.element as HTMLVideoElement).videoHeight / 8), 39 | }, Constants.TEXTURE_NEAREST_NEAREST); 40 | 41 | // Rely on the underlying engine render loop to update the filter result every frame. 42 | engine.runRenderLoop(() => { 43 | // Render. Please note we are using render instead of filter to improve 44 | // performances of real time filter. filter is creating a promise and will therefore 45 | // generate some lags and garbage. 46 | webcamTexture.update(); 47 | 48 | // Resize the texture to pixellate. 49 | resizer.resizeToTexture(webcamTexture, resizedTexture); 50 | 51 | // Renders the pixelate texture with our custom effect. 52 | customFilter.render(resizedTexture, customEffectWrapper); 53 | }); 54 | } 55 | 56 | /** 57 | * Creates a video texture from a stream. 58 | * This is fully available in the Babylon Video Texture and is only here for education purpose. 59 | */ 60 | function createVideoTextureFromStreamAsync(engine: ThinEngine, stream: MediaStream): Promise { 61 | var video = document.createElement("video"); 62 | video.setAttribute('autoplay', ''); 63 | video.setAttribute('muted', 'true'); 64 | video.setAttribute('playsinline', ''); 65 | video.muted = true; 66 | 67 | if (video.mozSrcObject !== undefined) { 68 | // hack for Firefox < 19 69 | video.mozSrcObject = stream; 70 | } else { 71 | if (typeof video.srcObject == "object") { 72 | video.srcObject = stream; 73 | } else { 74 | window.URL = window.URL || window.webkitURL || window.mozURL || window.msURL; 75 | video.src = (window.URL && window.URL.createObjectURL(stream)); 76 | } 77 | } 78 | 79 | return new Promise((resolve) => { 80 | let onPlaying = () => { 81 | const webcamTexture = new HtmlElementTexture("video", video, { 82 | engine, 83 | scene: null, 84 | }); 85 | 86 | // No repeat is needed here for WebGL1. 87 | webcamTexture.wrapU = Constants.TEXTURE_CLAMP_ADDRESSMODE; 88 | webcamTexture.wrapV = Constants.TEXTURE_CLAMP_ADDRESSMODE; 89 | 90 | video.removeEventListener("playing", onPlaying); 91 | 92 | resolve(webcamTexture); 93 | }; 94 | 95 | video.addEventListener("playing", onPlaying); 96 | video.play(); 97 | }); 98 | } 99 | 100 | /** 101 | * Gets a webcam feed and converts it into a video texture. 102 | * This is fully available in the Babylon Video Texture and is only here for education purpose. 103 | */ 104 | function getWebcamTextureAsync(engine: ThinEngine): Promise { 105 | if (navigator.mediaDevices) { 106 | return navigator.mediaDevices.getUserMedia({ 107 | video: { facingMode: "user" }, 108 | audio: false 109 | }) 110 | .then((stream) => { 111 | return createVideoTextureFromStreamAsync(engine, stream); 112 | }); 113 | } 114 | 115 | return Promise.reject("navigator.mediaDevices.getUserMedia is not supported by your browser."); 116 | } 117 | 118 | main(); -------------------------------------------------------------------------------- /www/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Video Processing Sample 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 130 | 131 | 132 | 133 |
134 | 139 |
140 | This demo highlights how to apply video filter in real time to any video in a Web Page. 141 |
142 |
143 | Using the power of the GPU with Babylon.js can dramatically improve the user experience and make possible real time filtering of videos. 144 |
145 |
146 | 147 | 148 |
149 |
150 | You can have a look at the associated article on our blog. 151 |
152 |
153 | 154 | 155 | 156 | 157 | --------------------------------------------------------------------------------