├── .gitignore ├── README.md └── web ├── Cargo.lock ├── Cargo.toml ├── js ├── App.tsx ├── components │ └── Video.tsx ├── index.tsx ├── stores │ ├── MediaEngine.ts │ ├── WasmEngine.ts │ └── use_engines.tsx └── utils │ └── config.ts ├── package.json ├── src ├── lib.rs ├── macros.rs ├── rendering_engine.rs └── utils.rs ├── static ├── index.css └── index.html ├── tests └── app.rs ├── tsconfig.json ├── webpack.config.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | pkg 3 | target -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pixel Processing With Rust 2 | 3 | Positioning remote artifacts in AR involves a lot of computation between animation frames. You need some pixel data from every frame, and a bunch of other data and secrets. This repo shows you how to obtain raw pixel data from each frame. 4 | 5 | [A blog article about this work can be found here](https://dev.to/fallenstedt/using-rust-and-webassembly-to-process-pixels-from-a-video-feed-4hhg) 6 | 7 | ## Setup 8 | 9 | 1. Clone this repo and run `yarn` inside the `web` dir. 10 | 2. `yarn start` will build rust and the web example 11 | -------------------------------------------------------------------------------- /web/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "bumpalo" 5 | version = "3.4.0" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" 8 | 9 | [[package]] 10 | name = "cfg-if" 11 | version = "0.1.10" 12 | source = "registry+https://github.com/rust-lang/crates.io-index" 13 | checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 14 | 15 | [[package]] 16 | name = "console_error_panic_hook" 17 | version = "0.1.6" 18 | source = "registry+https://github.com/rust-lang/crates.io-index" 19 | checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" 20 | dependencies = [ 21 | "cfg-if", 22 | "wasm-bindgen", 23 | ] 24 | 25 | [[package]] 26 | name = "futures" 27 | version = "0.1.29" 28 | source = "registry+https://github.com/rust-lang/crates.io-index" 29 | checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" 30 | 31 | [[package]] 32 | name = "js-sys" 33 | version = "0.3.40" 34 | source = "registry+https://github.com/rust-lang/crates.io-index" 35 | checksum = "ce10c23ad2ea25ceca0093bd3192229da4c5b3c0f2de499c1ecac0d98d452177" 36 | dependencies = [ 37 | "wasm-bindgen", 38 | ] 39 | 40 | [[package]] 41 | name = "lazy_static" 42 | version = "1.4.0" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 45 | 46 | [[package]] 47 | name = "libc" 48 | version = "0.2.71" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" 51 | 52 | [[package]] 53 | name = "log" 54 | version = "0.4.8" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" 57 | dependencies = [ 58 | "cfg-if", 59 | ] 60 | 61 | [[package]] 62 | name = "memory_units" 63 | version = "0.4.0" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" 66 | 67 | [[package]] 68 | name = "optical-character-recognition-wasm" 69 | version = "0.1.0" 70 | dependencies = [ 71 | "console_error_panic_hook", 72 | "futures", 73 | "js-sys", 74 | "wasm-bindgen", 75 | "wasm-bindgen-futures", 76 | "wasm-bindgen-test", 77 | "web-sys", 78 | "wee_alloc", 79 | ] 80 | 81 | [[package]] 82 | name = "proc-macro2" 83 | version = "0.4.30" 84 | source = "registry+https://github.com/rust-lang/crates.io-index" 85 | checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" 86 | dependencies = [ 87 | "unicode-xid 0.1.0", 88 | ] 89 | 90 | [[package]] 91 | name = "proc-macro2" 92 | version = "1.0.18" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" 95 | dependencies = [ 96 | "unicode-xid 0.2.0", 97 | ] 98 | 99 | [[package]] 100 | name = "quote" 101 | version = "0.6.13" 102 | source = "registry+https://github.com/rust-lang/crates.io-index" 103 | checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" 104 | dependencies = [ 105 | "proc-macro2 0.4.30", 106 | ] 107 | 108 | [[package]] 109 | name = "quote" 110 | version = "1.0.7" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" 113 | dependencies = [ 114 | "proc-macro2 1.0.18", 115 | ] 116 | 117 | [[package]] 118 | name = "scoped-tls" 119 | version = "1.0.0" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" 122 | 123 | [[package]] 124 | name = "syn" 125 | version = "1.0.33" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" 128 | dependencies = [ 129 | "proc-macro2 1.0.18", 130 | "quote 1.0.7", 131 | "unicode-xid 0.2.0", 132 | ] 133 | 134 | [[package]] 135 | name = "unicode-xid" 136 | version = "0.1.0" 137 | source = "registry+https://github.com/rust-lang/crates.io-index" 138 | checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" 139 | 140 | [[package]] 141 | name = "unicode-xid" 142 | version = "0.2.0" 143 | source = "registry+https://github.com/rust-lang/crates.io-index" 144 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 145 | 146 | [[package]] 147 | name = "wasm-bindgen" 148 | version = "0.2.63" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0" 151 | dependencies = [ 152 | "cfg-if", 153 | "wasm-bindgen-macro", 154 | ] 155 | 156 | [[package]] 157 | name = "wasm-bindgen-backend" 158 | version = "0.2.63" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "ded84f06e0ed21499f6184df0e0cb3494727b0c5da89534e0fcc55c51d812101" 161 | dependencies = [ 162 | "bumpalo", 163 | "lazy_static", 164 | "log", 165 | "proc-macro2 1.0.18", 166 | "quote 1.0.7", 167 | "syn", 168 | "wasm-bindgen-shared", 169 | ] 170 | 171 | [[package]] 172 | name = "wasm-bindgen-futures" 173 | version = "0.3.27" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "83420b37346c311b9ed822af41ec2e82839bfe99867ec6c54e2da43b7538771c" 176 | dependencies = [ 177 | "cfg-if", 178 | "futures", 179 | "js-sys", 180 | "wasm-bindgen", 181 | "web-sys", 182 | ] 183 | 184 | [[package]] 185 | name = "wasm-bindgen-macro" 186 | version = "0.2.63" 187 | source = "registry+https://github.com/rust-lang/crates.io-index" 188 | checksum = "838e423688dac18d73e31edce74ddfac468e37b1506ad163ffaf0a46f703ffe3" 189 | dependencies = [ 190 | "quote 1.0.7", 191 | "wasm-bindgen-macro-support", 192 | ] 193 | 194 | [[package]] 195 | name = "wasm-bindgen-macro-support" 196 | version = "0.2.63" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "3156052d8ec77142051a533cdd686cba889537b213f948cd1d20869926e68e92" 199 | dependencies = [ 200 | "proc-macro2 1.0.18", 201 | "quote 1.0.7", 202 | "syn", 203 | "wasm-bindgen-backend", 204 | "wasm-bindgen-shared", 205 | ] 206 | 207 | [[package]] 208 | name = "wasm-bindgen-shared" 209 | version = "0.2.63" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "c9ba19973a58daf4db6f352eda73dc0e289493cd29fb2632eb172085b6521acd" 212 | 213 | [[package]] 214 | name = "wasm-bindgen-test" 215 | version = "0.2.50" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "a2d9693b63a742d481c7f80587e057920e568317b2806988c59cd71618bc26c1" 218 | dependencies = [ 219 | "console_error_panic_hook", 220 | "futures", 221 | "js-sys", 222 | "scoped-tls", 223 | "wasm-bindgen", 224 | "wasm-bindgen-futures", 225 | "wasm-bindgen-test-macro", 226 | ] 227 | 228 | [[package]] 229 | name = "wasm-bindgen-test-macro" 230 | version = "0.2.50" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "0789dac148a8840bbcf9efe13905463b733fa96543bfbf263790535c11af7ba5" 233 | dependencies = [ 234 | "proc-macro2 0.4.30", 235 | "quote 0.6.13", 236 | ] 237 | 238 | [[package]] 239 | name = "web-sys" 240 | version = "0.3.40" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "7b72fe77fd39e4bd3eaa4412fd299a0be6b3dfe9d2597e2f1c20beb968f41d17" 243 | dependencies = [ 244 | "js-sys", 245 | "wasm-bindgen", 246 | ] 247 | 248 | [[package]] 249 | name = "wee_alloc" 250 | version = "0.4.5" 251 | source = "registry+https://github.com/rust-lang/crates.io-index" 252 | checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" 253 | dependencies = [ 254 | "cfg-if", 255 | "libc", 256 | "memory_units", 257 | "winapi", 258 | ] 259 | 260 | [[package]] 261 | name = "winapi" 262 | version = "0.3.8" 263 | source = "registry+https://github.com/rust-lang/crates.io-index" 264 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 265 | dependencies = [ 266 | "winapi-i686-pc-windows-gnu", 267 | "winapi-x86_64-pc-windows-gnu", 268 | ] 269 | 270 | [[package]] 271 | name = "winapi-i686-pc-windows-gnu" 272 | version = "0.4.0" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 275 | 276 | [[package]] 277 | name = "winapi-x86_64-pc-windows-gnu" 278 | version = "0.4.0" 279 | source = "registry+https://github.com/rust-lang/crates.io-index" 280 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 281 | -------------------------------------------------------------------------------- /web/Cargo.toml: -------------------------------------------------------------------------------- 1 | # You must change these to your own details. 2 | [package] 3 | name = "optical-character-recognition-wasm" 4 | description = "todo add desc" 5 | version = "0.1.0" 6 | authors = ["Alex Fallenstedt ", "Adam Michel "] 7 | categories = ["wasm"] 8 | readme = "README.md" 9 | edition = "2018" 10 | 11 | [lib] 12 | crate-type = ["cdylib"] 13 | 14 | [features] 15 | # If you uncomment this line, it will enable `wee_alloc`: 16 | #default = ["wee_alloc"] 17 | 18 | [dependencies] 19 | # The `wasm-bindgen` crate provides the bare minimum functionality needed 20 | # to interact with JavaScript. 21 | wasm-bindgen = "0.2.45" 22 | 23 | # `wee_alloc` is a tiny allocator for wasm that is only ~1K in code size 24 | # compared to the default allocator's ~10K. However, it is slower than the default 25 | # allocator, so it's not enabled by default. 26 | wee_alloc = { version = "0.4.2", optional = true } 27 | 28 | [dependencies.web-sys] 29 | version = "0.3.40" 30 | features = [ 31 | 'console', 32 | 'CanvasRenderingContext2d', 33 | 'Document', 34 | 'EventTarget', 35 | 'Element', 36 | 'HtmlCanvasElement', 37 | 'HtmlVideoElement', 38 | 'HtmlElement', 39 | 'ImageData', 40 | 'MediaStream', 41 | 'MessageEvent', 42 | 'Performance', 43 | 'RtcDataChannel', 44 | 'RtcDataChannelEvent', 45 | 'Window', 46 | ] 47 | 48 | # The `console_error_panic_hook` crate provides better debugging of panics by 49 | # logging them with `console.error`. This is great for development, but requires 50 | # all the `std::fmt` and `std::panicking` infrastructure, so it's only enabled 51 | # in debug mode. 52 | [target."cfg(debug_assertions)".dependencies] 53 | console_error_panic_hook = "0.1.5" 54 | 55 | # These crates are used for running unit tests. 56 | [dev-dependencies] 57 | wasm-bindgen-test = "0.2.45" 58 | futures = "0.1.27" 59 | js-sys = "0.3.40" 60 | wasm-bindgen-futures = "0.3.22" 61 | -------------------------------------------------------------------------------- /web/js/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import { Observer } from 'mobx-react' 3 | import { Video } from './components/Video'; 4 | import { useEngines } from './stores/use_engines'; 5 | 6 | function App() { 7 | const { wasmEngine } = useEngines() 8 | 9 | useEffect(() => { 10 | console.log("SUP") 11 | async function loadWasm() { 12 | await wasmEngine.initialize() 13 | } 14 | loadWasm() 15 | }, []) 16 | 17 | return ( 18 | 19 | {() => wasmEngine.loading ?

Loading...

: 21 | ) 22 | 23 | } 24 | 25 | export default App; 26 | -------------------------------------------------------------------------------- /web/js/components/Video.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useRef, useCallback } from 'react'; 2 | import Stats from 'stats.js'; 3 | import { useEngines } from '../stores/use_engines'; 4 | import { config } from '../utils/config'; 5 | 6 | export function Video() { 7 | const canvasRef = useRef(null) 8 | const { mediaEngine, wasmEngine } = useEngines() 9 | 10 | 11 | useEffect(() => { 12 | // Render each frame to a canvas element for Rust to see 13 | mediaEngine.getMedia().then((data) => { 14 | if (mediaEngine.initalized && mediaEngine.instance.media && canvasRef.current !== null) { 15 | const renderingEngine = new wasmEngine.instance!.RenderingEngine(); 16 | renderingEngine.add_target_canvas(canvasRef.current); 17 | renderingEngine.start(mediaEngine.instance.media); 18 | } 19 | }) 20 | 21 | }, [canvasRef, mediaEngine]) 22 | 23 | return ( 24 |
25 | 26 |
27 | ); 28 | } -------------------------------------------------------------------------------- /web/js/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import App from './App'; 4 | 5 | ReactDOM.render( 6 | 7 | 8 | , 9 | document.getElementById('root') 10 | ); 11 | -------------------------------------------------------------------------------- /web/js/stores/MediaEngine.ts: -------------------------------------------------------------------------------- 1 | import { observable, action, runInAction } from 'mobx'; 2 | 3 | export interface VideoData { 4 | media: MediaStream | null; 5 | } 6 | 7 | export class MediaEngine { 8 | @observable 9 | public initalized = false; 10 | 11 | @observable 12 | public instance!: VideoData; 13 | 14 | constructor(private readonly constraints: MediaStreamConstraints) {} 15 | 16 | @action 17 | public async getMedia() { 18 | const payload: VideoData = { 19 | media: null, 20 | } 21 | 22 | try { 23 | const stream = await navigator.mediaDevices.getUserMedia(this.constraints); 24 | if (stream) { 25 | payload.media = stream 26 | } 27 | 28 | } catch (err) { 29 | throw err; 30 | } 31 | 32 | runInAction(() => { 33 | this.instance = payload 34 | this.initalized = true; 35 | }) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /web/js/stores/WasmEngine.ts: -------------------------------------------------------------------------------- 1 | import { observable, runInAction, action } from "mobx"; 2 | 3 | type ConverterType = typeof import("../../pkg/index.js"); 4 | 5 | export class WasmEngine { 6 | @observable 7 | public instance: ConverterType | undefined = undefined; 8 | 9 | @observable 10 | public loading: boolean = true; 11 | 12 | @observable 13 | public error: Error | undefined = undefined; 14 | 15 | @action 16 | public async initialize() { 17 | try { 18 | //@ts-ignore 19 | const wasm = await import("../../pkg/index.js"); 20 | runInAction(() => { 21 | this.loading = false; 22 | this.instance = wasm 23 | }) 24 | } catch (error) { 25 | runInAction(() => { 26 | this.loading = false; 27 | this.error = error 28 | }) 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /web/js/stores/use_engines.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { MediaEngine } from "./MediaEngine"; 3 | import { config } from '../utils/config'; 4 | import { WasmEngine } from './WasmEngine'; 5 | 6 | 7 | const globalContext = React.createContext({ 8 | mediaEngine: new MediaEngine({ 9 | audio: false, 10 | video: { width: config.video.width, height: config.video.height } 11 | }), 12 | wasmEngine: new WasmEngine() 13 | }); 14 | 15 | export const useEngines = () => React.useContext(globalContext); 16 | 17 | -------------------------------------------------------------------------------- /web/js/utils/config.ts: -------------------------------------------------------------------------------- 1 | export const config = { 2 | video: { 3 | width: 1280, 4 | height: 720 5 | } 6 | } as const; -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "You ", 3 | "name": "rust-webpack-template", 4 | "version": "0.1.0", 5 | "scripts": { 6 | "build": "rimraf dist pkg && webpack", 7 | "start": "rimraf dist pkg && webpack-dev-server --open -d", 8 | "test": "cargo test && wasm-pack test --headless" 9 | }, 10 | "devDependencies": { 11 | "@types/react": "^16.9.38", 12 | "@types/react-dom": "^16.9.8", 13 | "@types/stats": "^0.16.30", 14 | "@wasm-tool/wasm-pack-plugin": "^1.1.0", 15 | "copy-webpack-plugin": "^5.0.3", 16 | "rimraf": "^3.0.0", 17 | "ts-loader": "^7.0.5", 18 | "typescript": "^3.9.5", 19 | "webpack": "^4.42.0", 20 | "webpack-cli": "^3.3.3", 21 | "webpack-dev-server": "^3.7.1" 22 | }, 23 | "dependencies": { 24 | "mobx": "^5.15.4", 25 | "mobx-react": "^6.2.2", 26 | "react": "^16.13.1", 27 | "react-dom": "^16.13.1", 28 | "stats.js": "^0.17.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /web/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod macros; 2 | mod utils; 3 | 4 | mod rendering_engine; 5 | use wasm_bindgen::prelude::*; 6 | 7 | // When the `wee_alloc` feature is enabled, this uses `wee_alloc` as the global 8 | // allocator. 9 | // 10 | // If you don't want to use `wee_alloc`, you can safely delete this. 11 | #[cfg(feature = "wee_alloc")] 12 | #[global_allocator] 13 | static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; 14 | 15 | #[wasm_bindgen(start)] 16 | pub fn main_js() -> Result<(), JsValue> { 17 | utils::set_panic_hook(); 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /web/src/macros.rs: -------------------------------------------------------------------------------- 1 | // A macro to provide `println!(..)`-style syntax for `console.log` logging. 2 | #[macro_export] 3 | macro_rules! log { 4 | ( $( $t:tt )* ) => { 5 | web_sys::console::log_1(&format!( $( $t )* ).into()); 6 | } 7 | } 8 | 9 | // A macro to provide `println!(..)`-style syntax for `console.error` logging. 10 | #[macro_export] 11 | macro_rules! err { 12 | ( $( $t:tt )* ) => { 13 | web_sys::console::error_1(&format!( $( $t )* ).into()); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /web/src/rendering_engine.rs: -------------------------------------------------------------------------------- 1 | use std::cell::RefCell; 2 | use std::rc::Rc; 3 | use wasm_bindgen::prelude::*; 4 | use wasm_bindgen::JsCast; 5 | use web_sys::{ 6 | CanvasRenderingContext2d, Document, Event, HtmlCanvasElement, HtmlVideoElement, MediaStream, 7 | Window, 8 | }; 9 | 10 | #[derive(Debug)] 11 | struct Dimensions { 12 | width: f64, 13 | height: f64, 14 | } 15 | 16 | #[derive(Debug)] 17 | struct RenderingEngineCanvas { 18 | element: HtmlCanvasElement, 19 | context_2d: CanvasRenderingContext2d, 20 | } 21 | 22 | #[wasm_bindgen] 23 | #[derive(Debug)] 24 | pub struct RenderingEngine { 25 | canvas: Rc, 26 | render_targets: Rc>>, 27 | cancel: Rc>, 28 | } 29 | 30 | #[wasm_bindgen] 31 | impl RenderingEngine { 32 | #[wasm_bindgen(constructor)] 33 | pub fn new() -> RenderingEngine { 34 | let canvas = Rc::new(RenderingEngine::create_buffer_canvas()); 35 | let render_targets = Rc::new(RefCell::new(Vec::new())); 36 | let cancel = Rc::new(RefCell::new(false)); 37 | 38 | RenderingEngine { 39 | canvas, 40 | render_targets, 41 | cancel, 42 | } 43 | } 44 | 45 | #[wasm_bindgen(method)] 46 | pub fn start(&self, media_stream: &MediaStream) { 47 | let video = RenderingEngine::create_video_element(media_stream); 48 | &self.start_animation_loop(&video); 49 | } 50 | 51 | fn start_animation_loop(&self, video: &Rc) { 52 | let video = video.clone(); 53 | let canvas = self.canvas.clone(); 54 | let cancel = self.cancel.clone(); 55 | let render_targets = self.render_targets.clone(); 56 | 57 | let f = Rc::new(RefCell::new(None)); 58 | let g = f.clone(); 59 | 60 | *g.borrow_mut() = Some(Closure::wrap(Box::new(move || { 61 | if *cancel.borrow() == true { 62 | let _ = f.borrow_mut().take(); 63 | return; 64 | } 65 | 66 | // Ready state should be >= HAVE_CURRENT_DATA, see: https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/readyState 67 | if video.ready_state() >= 2 { 68 | let video_dimensions = Dimensions { 69 | width: video.video_width() as f64, 70 | height: video.video_height() as f64, 71 | }; 72 | // draw frame onto buffer canvas 73 | // perform any pixel manipulation you need on this canvas 74 | canvas.element.set_width(video_dimensions.width as u32); 75 | canvas.element.set_height(video_dimensions.height as u32); 76 | canvas 77 | .context_2d 78 | .draw_image_with_html_video_element(&video, 0.0, 0.0) 79 | .expect("failed to draw video frame to element"); 80 | 81 | // render resulting image onto target canvas 82 | for target in render_targets.borrow().iter() { 83 | // Use scrollWidth/scrollHeight so we fill the canvas element. 84 | let target_dimensions = Dimensions { 85 | width: target.element.scroll_width() as f64, 86 | height: target.element.scroll_height() as f64, 87 | }; 88 | let scaled_dimensions = RenderingEngine::get_scaled_video_size( 89 | &video_dimensions, 90 | &target_dimensions, 91 | ); 92 | let offset = Dimensions { 93 | width: (target_dimensions.width - scaled_dimensions.width) / 2.0, 94 | height: (target_dimensions.height - scaled_dimensions.height) / 2.0, 95 | }; 96 | 97 | // Ensure the target canvas has a set width/height, otherwise rendering breaks. 98 | target.element.set_width(target_dimensions.width as u32); 99 | target.element.set_height(target_dimensions.height as u32); 100 | 101 | target 102 | .context_2d 103 | .draw_image_with_html_canvas_element_and_dw_and_dh( 104 | &canvas.element, 105 | offset.width, 106 | offset.height, 107 | scaled_dimensions.width, 108 | scaled_dimensions.height, 109 | ) 110 | .expect("failed to draw buffer to target "); 111 | } 112 | } 113 | 114 | RenderingEngine::request_animation_frame(f.borrow().as_ref().unwrap()); 115 | }) as Box)); 116 | RenderingEngine::request_animation_frame(g.borrow().as_ref().unwrap()); 117 | } 118 | 119 | #[wasm_bindgen(method)] 120 | pub fn add_target_canvas(&mut self, canvas: HtmlCanvasElement) { 121 | let context = canvas 122 | .get_context("2d") 123 | .unwrap() 124 | .unwrap() 125 | .dyn_into::() 126 | .expect("failed to obtain 2d rendering context for target "); 127 | 128 | let container = RenderingEngineCanvas { 129 | element: canvas, 130 | context_2d: context, 131 | }; 132 | 133 | let mut render_targets = self.render_targets.borrow_mut(); 134 | render_targets.push(container); 135 | } 136 | 137 | #[wasm_bindgen(method)] 138 | pub fn stop(&mut self) { 139 | *self.cancel.borrow_mut() = true; 140 | } 141 | 142 | fn get_scaled_video_size(video: &Dimensions, container: &Dimensions) -> Dimensions { 143 | let video_landscape = video.width > video.height; 144 | let container_landscape = container.width > container.height; 145 | let same_orientation = video_landscape == container_landscape; 146 | 147 | let aspect_ratio = Dimensions { 148 | width: video.width, 149 | height: video.height, 150 | }; 151 | let mut size = Dimensions { 152 | width: container.width, 153 | height: container.height, 154 | }; 155 | 156 | if same_orientation { 157 | let m_w = size.width / aspect_ratio.width; 158 | let m_h = size.height / aspect_ratio.height; 159 | 160 | if m_h > m_w { 161 | size.width = (size.height / aspect_ratio.height) * aspect_ratio.width; 162 | } else if m_w > m_h { 163 | size.height = (size.width / aspect_ratio.width) * aspect_ratio.height; 164 | } 165 | } else { 166 | let m_w = size.width / aspect_ratio.width; 167 | let m_h = size.height / aspect_ratio.height; 168 | 169 | if m_h < m_w { 170 | size.width = (size.height / aspect_ratio.height) * aspect_ratio.width; 171 | } else if m_w < m_h { 172 | size.height = (size.width / aspect_ratio.width) * aspect_ratio.height; 173 | } 174 | } 175 | 176 | size 177 | } 178 | 179 | fn create_video_element(media_stream: &MediaStream) -> Rc { 180 | let video = RenderingEngine::get_document() 181 | .create_element("video") 182 | .unwrap() 183 | .dyn_into::() 184 | .expect("failed to create