├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── build_ffmpeg.sh ├── build_wasm.sh ├── doc ├── .gitignore ├── README.md ├── babel.config.js ├── blog │ ├── 2019-05-28-first-blog-post_md │ ├── 2021-08-01-mdx-blog-post_mdx │ ├── 2023-02-3-intro │ │ ├── docusaurus-plushie-banner.jpeg │ │ └── index_md │ ├── 2023-2-27-libav-oop.md │ ├── 2023-2-6-why-frameflow.md │ └── authors.yml ├── docs │ ├── api │ │ ├── _category_.yml │ │ ├── index.md │ │ └── modules.md │ ├── guides │ │ ├── assets │ │ │ └── download analysis.png │ │ ├── codec.md │ │ ├── dataType.md │ │ ├── download.md │ │ └── licenses.md │ └── intro │ │ ├── filters.md │ │ └── getStarted.md ├── docusaurus.config.js ├── package-lock.json ├── package.json ├── sidebars.js ├── src │ ├── components │ │ ├── HomepageFeatures │ │ │ ├── index.js │ │ │ └── styles.module.css │ │ └── codeEditor │ │ │ └── index.tsx │ ├── css │ │ └── custom.css │ └── pages │ │ ├── index.module.css │ │ ├── index.tsx │ │ └── markdown-page.md ├── static │ ├── .nojekyll │ ├── img │ │ ├── diagram.png │ │ ├── docusaurus-social-card.jpg │ │ ├── docusaurus.png │ │ ├── favicon.ico │ │ ├── home_icon.png │ │ ├── logo.svg │ │ ├── undraw_docusaurus_mountain.svg │ │ ├── undraw_docusaurus_react.svg │ │ └── undraw_docusaurus_tree.svg │ ├── static.d.ts │ └── video │ │ ├── audio.mp3 │ │ └── flame.avi └── tsconfig.json ├── examples ├── assets │ ├── Bunny.mkv │ ├── Bunny.mp4 │ ├── CantinaBand3.wav │ ├── audio.mp3 │ └── flame.avi ├── browser │ ├── codec.html │ ├── demo.html │ ├── index.html │ ├── todo.html │ └── transmux.html └── node │ └── transcode.js ├── jest.config.js ├── package-lock.json ├── package.json ├── src ├── cpp │ ├── bind.cpp │ ├── decode.cpp │ ├── decode.h │ ├── demuxer.cpp │ ├── demuxer.h │ ├── encode.cpp │ ├── encode.h │ ├── filter.cpp │ ├── filter.h │ ├── frame.cpp │ ├── frame.h │ ├── metadata.cpp │ ├── metadata.h │ ├── muxer.cpp │ ├── muxer.h │ ├── packet.h │ ├── resample.h │ ├── stream.h │ ├── utils.cpp │ └── utils.h └── ts │ ├── __test__ │ ├── filters.test.js │ └── streamIO.test.js │ ├── codecs.ts │ ├── filters.ts │ ├── globals.ts │ ├── graph.ts │ ├── hls.ts │ ├── loader.ts │ ├── main.ts │ ├── message.ts │ ├── metadata.ts │ ├── streamIO.ts │ ├── transcoder.worker.ts │ ├── types │ ├── WebCodecs.d.ts │ ├── ffmpeg.d.ts │ ├── flags.ts │ ├── graph.ts │ ├── wasm.d.ts │ └── worker-loader.d.ts │ └── utils.ts ├── tsconfig.json └── webpack.config.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # vscode 10 | .vscode 11 | 12 | # Diagnostic reports (https://nodejs.org/api/report.html) 13 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 14 | 15 | # Runtime data 16 | pids 17 | *.pid 18 | *.seed 19 | *.pid.lock 20 | 21 | # Directory for instrumented libs generated by jscoverage/JSCover 22 | lib-cov 23 | 24 | # Coverage directory used by tools like istanbul 25 | coverage 26 | *.lcov 27 | 28 | # nyc test coverage 29 | .nyc_output 30 | 31 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 32 | .grunt 33 | 34 | # Bower dependency directory (https://bower.io/) 35 | bower_components 36 | 37 | # node-waf configuration 38 | .lock-wscript 39 | 40 | # Compiled binary addons (https://nodejs.org/api/addons.html) 41 | build/Release 42 | 43 | # wasm build 44 | src/wasm 45 | 46 | # Dependency directories 47 | node_modules/ 48 | jspm_packages/ 49 | FFmpeg/ 50 | emsdk/ 51 | ffmpeg_libraries/ 52 | 53 | # TypeScript v1 declaration files 54 | typings/ 55 | 56 | # TypeScript cache 57 | *.tsbuildinfo 58 | 59 | # Optional npm cache directory 60 | .npm 61 | 62 | # Optional eslint cache 63 | .eslintcache 64 | 65 | # Microbundle cache 66 | .rpt2_cache/ 67 | .rts2_cache_cjs/ 68 | .rts2_cache_es/ 69 | .rts2_cache_umd/ 70 | 71 | # Optional REPL history 72 | .node_repl_history 73 | 74 | # Output of 'npm pack' 75 | *.tgz 76 | 77 | # Yarn Integrity file 78 | .yarn-integrity 79 | 80 | # dotenv environment variables file 81 | .env 82 | .env.test 83 | 84 | # parcel-bundler cache (https://parceljs.org/) 85 | .cache 86 | 87 | # Next.js build output 88 | .next 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # Serverless directories 104 | .serverless/ 105 | 106 | # FuseBox cache 107 | .fusebox/ 108 | 109 | # DynamoDB Local files 110 | .dynamodb/ 111 | 112 | # TernJS port file 113 | .tern-port 114 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGET_EXEC := ffmpeg-wrapper.o 2 | BUILD_DIR := ./build 3 | SRC_DIRS := ./src/cpp 4 | FFMPEG_DIRS := ./FFmpeg 5 | 6 | # Find all the C and C++ files we want to compile 7 | # Note the single quotes around the * expressions. Make will incorrectly expand these otherwise. 8 | SRCS := $(shell find $(SRC_DIRS) -name '*.cpp') 9 | # String substitution for every C/C++ file. 10 | # As an example, hello.cpp turns into ./build/hello.cpp.o 11 | OBJS := $(SRCS:$(SRC_DIRS)/%=$(BUILD_DIR)/%.o) 12 | 13 | # Every folder in ./src will need to be passed to GCC so that it can find header files 14 | INC_DIRS := $(FFMPEG_DIRS) $(SRC_DIRS) 15 | # Add a prefix to INC_DIRS. So moduleA would become -ImoduleA. GCC understands this -I flag 16 | INC_FLAGS := $(addprefix -I,$(INC_DIRS)) 17 | 18 | # The -MMD and -MP flags together generate Makefiles for us! 19 | # These files will have .d instead of .o as the output. 20 | CPPFLAGS := $(INC_FLAGS) -MMD -MP -Wall -lembind 21 | LDFLAGS := -lembind 22 | 23 | # # The final build (link) step. 24 | # $(BUILD_DIR)/$(TARGET_EXEC): $(OBJS) 25 | # $(CXX) $(OBJS) -o $@ $(LDFLAGS) 26 | 27 | # Build step for C++ source (seperately) 28 | $(BUILD_DIR)/%.cpp.o: $(SRC_DIRS)/%.cpp 29 | mkdir -p $(dir $@) 30 | $(CXX) $(CPPFLAGS) -c $< -o $@ 31 | 32 | 33 | # .PHONY: clean 34 | # clean: 35 | # rm -r $(BUILD_DIR) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [FrameFlow](https://frameflow.netlify.app/) 2 | 3 | A both speedy and compatible video processing library for Web Browser, based on WebCodecs and FFmpeg (WebAssembly). 4 | 5 | For encoding/decoding, it is hardware accelerated by WebCodecs as default, which works in Chromium-based clients (Chrome, Edge, Electron...). And also provides fallback solutions by FFmpeg (WebAssembly). 6 | 7 | For muxing/demuxing (without loss), it uses FFmpeg to do it on the fly efficiently. For example, convert HLS stream to mp4. 8 | 9 | It also provides some usual filters (trim, loop, concat...), based on FFmpeg Filters. These are useful for lightweight tasks, like audio processing. 10 | 11 | ## Features 12 | - Process videos in stream way, without video size limitation. 13 | - Accept stream input `MediaStream` (from canvas, Camera, ...), and output stream of frames (to canvas...) as well. 14 | - Use `WebCodecs` to have hardware acceleration for Chromium-based client (Chrome (>=106), Edge, Opera, Electron...). 15 | - Get detailed metadata of video file by reading only several chunks, either from local disk or remote url. 16 | - Processing speed can be controlled either automatically or manually. 17 | 18 | ## Install 19 | 20 | ### NPM 21 | ```bash 22 | npm i frameflow 23 | ``` 24 | 25 | ### HTML script 26 | ```html 27 | 28 | ``` 29 | 30 | ## Basic example 31 | 32 | ```JavaScript 33 | import fflow from 'frameflow' 34 | 35 | let video = await fflow.source(videoBlob) // use web File api to get File handler. 36 | let audio = await fflow.source(audioURL) // remote media file (no need to download entirely beforehand) 37 | let audioTrim = audio.trim({start: 10, duration: video.duration}) // use metadata of video 38 | let blob = await fflow.group([video, audioTrim]).exportTo(Blob, {format: 'mp4'}) // group and trancode to 39 | videoDom.src = URL.createObjectURL(blob) 40 | // now can play in the browser 41 | ``` 42 | Although this example writes to blob entirely, then play. 43 | But underhood, it streams out chunks and then put togather. 44 | So you can customize to write to any other targets in stream way. 45 | 46 | ```JavaScript 47 | // `exportTo` actually use `export` underhood 48 | const target = await this.export({format: 'mp4'}) 49 | for await (const chunk of target) { 50 | if (!chunk?.data) continue 51 | // write to any target with chunk.data and chunk.offset (since not always in sequence) 52 | chunk.close() // for memory efficiency 53 | } 54 | ``` 55 | 56 | ## Transmux without re-encoding 57 | When you want to change only file format without loss, it will automatically copy packets without re-encoding. 58 | 59 | ### HLS to mp4 60 | Currently mainly support static `m3u8` file url as input. 61 | 62 | ```JavaScript 63 | const hlsSource = await fflow.source(`http://hls-example.m3u8`) 64 | const blob = await hlsSource.exportTo(Blob, { format: 'mp4' }) 65 | ``` 66 | 67 | ## Transcoding 68 | When you set export video/audio configuration (like codec, bitrate, etc), it will decode and encode. 69 | 70 | ```JavaScript 71 | const source = await fflow.source(`http://video.webm`) 72 | const out_1 = await source.exportTo(Blob, { format: 'mp4' }) 73 | const out_2 = await source.exportTo(Blob, { format: 'webm', bitrate: 10000000 }) 74 | ``` 75 | For `out_1`, if `webm` and `mp4` have different codecs (usually), so it will transcode. 76 | For `out2`, setting different output bitrate from input's, will also transcode. 77 | 78 | 79 | ### More examples 80 | More detailed browser examples are in the `./examples/browser/`. 81 | 82 | If you want to run them, please use latest release version. And then, at the root directory of the project. 83 | ``` 84 | npm install 85 | npm start 86 | ``` 87 | In dev mode, it will serve `./examples` as root directory. 88 | 89 | ## Worker setting 90 | Underhood, each export task uses an unique worker for each time. 91 | But this will cause too long to load (~1 second), and also cost too much memory 92 | if processing many videos at the same time. 93 | - use `fflow.load()` to load the default worker with loaded ffmpeg module, shared by all use cases. 94 | - use `fflow.load({newWorker: true})` to create an new worker with loaded ffmpeg module, and assign to `export` as a parameter. This multi-thread case will accelerate if having many `export` tasks. 95 | And sometimes, as a more complex case, if two `export` have dependencies, there may be a dead lock 96 | if they share the same worker (thread). 97 | 98 | ```JavaScript 99 | let worker = fflow.load({newWorker: true}) 100 | let blob = await video.exportTo(Blob, {format: 'mp4', worker }) // group and trancode to 101 | ``` 102 | 103 | ## Difference with FFmpeg library 104 | This library consists of two parts: 105 | - JavaScript(TypeScript) as a wrapper orchestrates entire workflow, and also handles all input/output logic. 106 | - WASM migrated from core library (`libav`) of FFmpeg. Not include FFmpeg tools like Commandline. So cannot use commandline in FFmpeg to process. 107 | 108 | 109 | ## Document 110 | All tutorials and documents are in [FrameFlow Doc](https://frameflow.netlify.app/docs/intro/getStarted). 111 | 112 | ## [Problems](https://frameflow.netlify.app/blog/why-frameflow/#problems-of-frameflow) 113 | 114 | 115 | ## How to build 116 | *Warning: [webpack dev mode cannot hot reload in WSL2 (windows).](https://mbuotidem.github.io/blog/2021/01/09/how-to-hot-reload-auto-refresh-react-app-on-WSL.html)* 117 | 118 | ### Dependencies (Ubuntu) 119 | Tools dependencies install 120 | ``` 121 | sudo apt-get update -y 122 | sudo apt-get install -y pkg-config 123 | ``` 124 | 125 | ### Emscripten 126 | ``` 127 | git clone https://github.com/emscripten-core/emsdk.git --branch 3.1.52 128 | rm -r emsdk/.git 129 | ``` 130 | [Install Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) 131 | 132 | ### FFmpeg version (n5.0 release) 133 | ``` 134 | git clone https://github.com/FFmpeg/FFmpeg --depth 1 --branch n5.0 135 | rm -r FFmpeg/.git 136 | ``` 137 | 138 | ### External FFmpeg Libraries 139 | All external libraries sources are under `./ffmpeg_libraries` 140 | ``` 141 | cd ffmpeg_libraries 142 | ``` 143 | 144 | x264 145 | ``` 146 | git clone https://github.com/mirror/x264.git --depth 1 --branch stable 147 | ``` 148 | Libvpx 149 | ``` 150 | git clone https://github.com/webmproject/libvpx.git --depth 1 --branch v1.12.0 151 | ``` 152 | 153 | ### Compilation 154 | ``` 155 | ./build_ffmpeg.sh 156 | ./build_wasm.sh 157 | ``` 158 | 159 | -------------------------------------------------------------------------------- /build_ffmpeg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT=$PWD 4 | EMSDK_ROOT=$ROOT/emsdk 5 | FFMPEG=$ROOT/FFmpeg 6 | LLVM_RANLIB=$EMSDK_ROOT/upstream/bin/llvm-ranlib 7 | LLVM_NM=$EMSDK_ROOT/upstream/bin/llvm-nm 8 | EXT_LIB=$ROOT/ffmpeg_libraries 9 | EXT_LIB_BUILD=$EXT_LIB/build 10 | # pkgconfig path: https://emscripten.org/docs/compiling/Building-Projects.html#pkg-config 11 | EXT_LIB_BUILD_PKG_CONFIG=$EXT_LIB_BUILD/lib/pkgconfig 12 | # activate emcc 13 | source $EMSDK_ROOT/emsdk_env.sh 14 | 15 | CFLAGS="-s USE_PTHREADS=1 -O3" 16 | 17 | 18 | ################### 19 | # External libraries build 20 | ################### 21 | 22 | # external libraries 23 | # x264 24 | cd "$EXT_LIB"/x264 && emconfigure ./configure \ 25 | --prefix="${EXT_LIB_BUILD}" \ 26 | --host=i686-gnu \ 27 | --enable-static \ 28 | --disable-cli \ 29 | --disable-asm \ 30 | --extra-cflags="$CFLAGS" 31 | cd "$EXT_LIB"/x264 && emmake make clean 32 | cd "$EXT_LIB"/x264 && emmake make install-lib-static -j4 33 | 34 | # libvpx 35 | cd "$EXT_LIB"/libvpx && emconfigure ./configure \ 36 | --prefix="${EXT_LIB_BUILD}" \ 37 | --target=generic-gnu \ 38 | --disable-install-bins \ 39 | --disable-examples \ 40 | --disable-tools \ 41 | --disable-docs \ 42 | --disable-unit-tests \ 43 | --disable-dependency-tracking \ 44 | --extra-cflags="$CFLAGS" \ 45 | --extra-cxxflags="$CFLAGS" 46 | cd "$EXT_LIB"/libvpx && emmake make install -j4 47 | 48 | # export global env variable for FFmpeg to detect 49 | export EM_PKG_CONFIG_PATH=$EXT_LIB_BUILD_PKG_CONFIG 50 | # export STRIP="llvm-strip" 51 | 52 | ################### 53 | # FFmpeg build 54 | ################### 55 | 56 | # configure FFmpeg with Emscripten 57 | CFLAGS="$CFLAGS -I$EXT_LIB_BUILD/include" 58 | LDFLAGS="$CFLAGS -s INITIAL_MEMORY=33554432 -L$EXT_LIB_BUILD/lib" # 33554432 bytes = 32 MB 59 | CONFIG_ARGS=( 60 | --disable-autodetect 61 | --disable-runtime-cpudetect 62 | --target-os=none # use none to prevent any os specific configurations 63 | --arch=x86_32 # use x86_32 to achieve minimal architectural optimization 64 | --enable-cross-compile # enable cross compile 65 | --disable-asm # disable asm optimization 66 | --disable-stripping # disable stripping 67 | --enable-gpl # for x264 68 | # protocal 69 | --disable-protocols 70 | --enable-protocol=file 71 | 72 | --disable-programs 73 | --disable-avdevice 74 | --disable-bsfs 75 | --disable-network 76 | --disable-debug 77 | 78 | # selected protocols 79 | --disable-protocols 80 | --enable-protocol=file 81 | 82 | # external libraries 83 | --enable-libx264 84 | --enable-libvpx 85 | 86 | --disable-sdl2 87 | --disable-hwaccels 88 | --disable-doc 89 | --extra-cflags="$CFLAGS" 90 | --extra-cxxflags="$CFLAGS" 91 | --extra-ldflags="$LDFLAGS" 92 | --pkg-config-flags="--static" 93 | --nm="$LLVM_NM -g" 94 | --ar=emar 95 | --as=llvm-as 96 | --ranlib="$LLVM_RANLIB" 97 | --cc=emcc 98 | --cxx=em++ 99 | --objcc=emcc 100 | --dep-cc=emcc 101 | ) 102 | # build FFmpeg library 103 | cd "$FFMPEG" && emconfigure ./configure "${CONFIG_ARGS[@]}" 104 | read -r -p "Check the FFmpeg configure, and press key to continue..." 105 | cd "$FFMPEG" && emmake make -j4 -------------------------------------------------------------------------------- /build_wasm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT=$PWD 4 | EMSDK_ROOT=$ROOT/emsdk 5 | FFMPEG=$ROOT/FFmpeg 6 | EXT_LIB_BUILD=$ROOT/ffmpeg_libraries/build 7 | 8 | # activate emcc 9 | source $EMSDK_ROOT/emsdk_env.sh 10 | 11 | # verify Emscripten version 12 | emcc -v 13 | 14 | 15 | NAME="ffmpeg_built" 16 | WASM_DIR="./src/wasm" 17 | 18 | # build ffmpeg.wasm (FFmpeg library + src/cpp/*) 19 | mkdir -p $WASM_DIR 20 | ARGS=( 21 | -Isrc/cpp -I$FFMPEG src/cpp/*.cpp 22 | -L$FFMPEG/libavcodec -L$FFMPEG/libavfilter -L$FFMPEG/libavformat -L$FFMPEG/libavutil -L$FFMPEG/libswresample -L$FFMPEG/libswscale -L$FFMPEG/libpostproc -L$EXT_LIB_BUILD/lib 23 | -lavfilter -lavformat -lavcodec -lavutil -lswresample -lswscale -lpostproc -lx264 -lvpx 24 | # -Wno-deprecated-declarations -Wno-pointer-sign -Wno-implicit-int-float-conversion -Wno-switch -Wno-parentheses -Qunused-arguments 25 | # -fno-rtti -fno-exceptions 26 | -lembind 27 | -o $WASM_DIR/$NAME.js 28 | 29 | # all settings can be see at: https://github.com/emscripten-core/emscripten/blob/main/src/settings.js 30 | -s INITIAL_MEMORY=33554432 # 33554432 bytes = 32 MB 31 | -s MODULARIZE=1 32 | -s EXPORT_ES6=1 33 | -s EXPORT_NAME=$NAME 34 | -s FILESYSTEM=0 35 | -s WASM_BIGINT=1 # need platform support JS BigInt 36 | -s ENVIRONMENT='web,worker' # node? 37 | -s ALLOW_MEMORY_GROWTH=1 38 | 39 | -s ASYNCIFY # need -O3 when enable asyncify 40 | -O3 41 | ) 42 | 43 | echo "${ARGS[@]}" 44 | em++ "${ARGS[@]}" 45 | 46 | # copy *.d.ts to enable typescript 47 | TYPE_WASM=src/ts/types/ffmpeg.d.ts 48 | echo "copy $TYPE_WASM to $WASM_DIR/$NAME.d.ts" 49 | cp $TYPE_WASM $WASM_DIR/$NAME.d.ts -------------------------------------------------------------------------------- /doc/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | Using SSH: 30 | 31 | ``` 32 | $ USE_SSH=true yarn deploy 33 | ``` 34 | 35 | Not using SSH: 36 | 37 | ``` 38 | $ GIT_USER= yarn deploy 39 | ``` 40 | 41 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 42 | -------------------------------------------------------------------------------- /doc/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /doc/blog/2019-05-28-first-blog-post_md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: first-blog-post 3 | title: First Blog Post 4 | authors: 5 | name: Gao Wei 6 | title: Docusaurus Core Team 7 | url: https://github.com/wgao19 8 | image_url: https://github.com/wgao19.png 9 | tags: [hola, docusaurus] 10 | --- 11 | 12 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque elementum dignissim ultricies. Fusce rhoncus ipsum tempor eros aliquam consequat. Lorem ipsum dolor sit amet 13 | -------------------------------------------------------------------------------- /doc/blog/2021-08-01-mdx-blog-post_mdx: -------------------------------------------------------------------------------- 1 | --- 2 | slug: mdx-blog-post 3 | title: MDX Blog Post 4 | authors: [slorber] 5 | tags: [docusaurus] 6 | --- 7 | 8 | Blog posts support [Docusaurus Markdown features](https://docusaurus.io/docs/markdown-features), such as [MDX](https://mdxjs.com/). 9 | 10 | :::tip 11 | 12 | Use the power of React to create interactive blog posts. 13 | 14 | ```js 15 | 16 | ``` 17 | 18 | 19 | 20 | ::: 21 | -------------------------------------------------------------------------------- /doc/blog/2023-02-3-intro/docusaurus-plushie-banner.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/blog/2023-02-3-intro/docusaurus-plushie-banner.jpeg -------------------------------------------------------------------------------- /doc/blog/2023-02-3-intro/index_md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: intro 3 | title: Introduction 4 | authors: [carson] 5 | tags: [frameflow] 6 | --- 7 | 8 | [Docusaurus blogging features](https://docusaurus.io/docs/blog) are powered by the [blog plugin](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-content-blog). 9 | 10 | Simply add Markdown files (or folders) to the `blog` directory. 11 | 12 | Regular blog authors can be added to `authors.yml`. 13 | 14 | The blog post date can be extracted from filenames, such as: 15 | 16 | - `2019-05-30-welcome.md` 17 | - `2019-05-30-welcome/index.md` 18 | 19 | A blog post folder can be convenient to co-locate blog post images: 20 | 21 | ![Docusaurus Plushie](./docusaurus-plushie-banner.jpeg) 22 | 23 | The blog supports tags as well! 24 | 25 | **And if you don't want a blog**: just delete this directory, and use `blog: false` in your Docusaurus config. 26 | -------------------------------------------------------------------------------- /doc/blog/2023-2-27-libav-oop.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: libav-oop 3 | title: Libav (FFmpeg) C API understanding from Object-Oriented view 4 | authors: [carson] 5 | --- 6 | 7 | Recently, I was developing WebAssembly based FFmpeg library, [FrameFlow](https://github.com/carsonDB/frameflow). It directly uses low-level C API of libav* folders from FFmpeg, to give more power to web browser. I want to share some development experience of using those C APIs. 8 | 9 | FFmpeg mainly has two ways to use it. Command-line way or C API. Actually Command-line program is also based on C API. Now when your first time to learn those APIs, it would be confused why there are multiple steps to create one thing. Because C language only has functions to do something. Why not use just one function to init something? 10 | Here is an example (C++), from [encode.cpp](https://github.com/carsonDB/frameflow/blob/6681b44073a65e5ab612e0bf6f24f71742095d5d/src/cpp/encode.cpp#L14). 11 | ```cpp 12 | auto codec = avcodec_find_encoder_by_name(info.codec_name.c_str()); 13 | auto codec_ctx = avcodec_alloc_context3(codec); 14 | set_avcodec_context_from_streamInfo(info, codec_ctx); 15 | auto ret = avcodec_open2(codec_ctx, codec, NULL); 16 | ``` 17 | This is minimum requirements to create an encoder. Let me explain one by one. 18 | First `avcodec_find_encoder_by_name` find the `Codec` by its name. This `Codec` is just like a class. You cannot change any value in it. It gives you some meta information about the codec (like `libx264` codec), and also has pointers to functions to encode for example. Its type is `AVCodec`. 19 | Second line `avcodec_alloc_context3`, is just `malloc` a memory block, with every value in the struct set to default value. It is called `codec_ctx` (codec context). The name is a convention in FFmpeg. Because its type is `AVCodecContext`. This is just like using `new` to create a new object (instance). 20 | The third line is to set all values from `info` which I defined before. And this function is my defined function. Don't care about it. This step is just like giving parameters to `constructor` of the class. 21 | The last line `avcodec_open2` is to initiate the object (instance). Just like calling constructor of the class. 22 | 23 | So although, FFmpeg is written in pure C language. But it actually uses some Object-oriented style to organize the codebase. You can also see other similar examples about `demuxer`, `muxer`, `decoder` in my project. 24 | 25 | ## Changes after init 26 | 27 | ### Decoder: Time_base 28 | In my experience of developing, there are some annoying bugs that seem weird, at first glance. Then after understanding the init process as I explained above, there is a key step that we should care about, last step `avcodec_open2`. Because it starts a contructor function, and init. It may change some fields that you set at the previous step. 29 | For example, here when you call `avcodec_open2`. It will use specifed codec algorithm to init. And often, `time_base` will be changed to another value. That may let us surprised. So any output frames' `time_base` is according to the new one, not the one you set. So after calling `avcodec_open2`, you may need to retrieve current `time_base` value from `codec_ctx`, to do further stuff. 30 | By the way, you may wonder what is `time_base` ? It might be worth to write another blog to explain. And now, simply explained, it is just a time unit, like second, microsecond, etc. 31 | 32 | ### Encoder: format (pixel format / sample format)... 33 | There is another example. For encoder, pixel format (video) or sample format (audio) may be changed, by specified codec algorithm, which the decoder uses. So after init, the encoder may only accept another pixel format frame. So before encoding, you need to `rescale` video frames to the specified pixel format, or `resample` audio frames to the specified sample format. 34 | 35 | ## Conclusion 36 | Overall, having an Object-oriented view would better understand those C APIs. And You can see all cpp codes in [FrameFlow-0.1.1 release](https://github.com/carsonDB/frameflow/blob/6681b44073a65e5ab612e0bf6f24f71742095d5d/src/cpp/). 37 | -------------------------------------------------------------------------------- /doc/blog/2023-2-6-why-frameflow.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: why-frameflow 3 | title: Why FrameFlow 4 | authors: [carson] 5 | --- 6 | 7 | ### Background 8 | 9 | Several years ago, I was developing a video editor. As a fan of front-end development, 10 | my primary choice is to make it on web page. However, several components that I need drop the idea. 11 | One of them is video processing tool. I needed FFmpeg, which cannot run in browser directly. 12 | So I had to use Electron. 13 | 14 | Then it seems feasible. But it made me exhausted, since I heavily relied on FFmpeg. 15 | First, to use FFmpeg in Electron, actually Nodejs. We need to use it through Node provided api to 16 | start a child process and send commands to it. It looks like we remote control something. 17 | Although with the help of [node-fluent-ffmpeg](https://github.com/fluent-ffmpeg/node-fluent-ffmpeg), 18 | everything became easier. But underhood, it still uses command-line interface. 19 | 20 | Command-line interface of FFmpeg is very easy to use at first glance. 21 | However, in my use case, I needed several recorded audio files, trimmed, concat togather and merge with video. Then as for CMD, I needed to learn how to use `filter_complex` to build a complex graph, which costs a lot of time. 22 | 23 | After these finished, exporting video worked. Since my editor generated video frames from canvas-like place. 24 | I used ReadableStream to feed images into FFmpeg process. Because the process looked like a black box. 25 | I cannot optimized the speed further. 26 | 27 | Through the development, I also found that there is another way to use FFmpeg. To call low-level C API from `FFmpeg/libav*` folders. Actually FFmpeg is a collection of several tools, FFmpeg command-line program, FFprobe, FFplay. They are all based on `libav*` libraries. These APIs are flexible enough when we are not satisfied by CMD way. But learning curve is too high, we need to learn fundamentally [how video processing works](https://github.com/leandromoreira/ffmpeg-libav-tutorial). 28 | 29 | ### Inspiration from [FFmpeg.wasm](https://github.com/ffmpegwasm/ffmpeg.wasm) 30 | Someday, I accidently got that FFmpeg had been ported to WebAssembly. And it worked. 31 | I was excited about the open source project. Hoped it can evetually allow my project move to web page. 32 | However, I found that it only allows processing after an entire video file loaded into memory. 33 | So my stream of input images is not applicable. 34 | 35 | ### Solution: Custom I/O 36 | After a while, a [discussion](https://github.com/ffmpegwasm/ffmpeg.wasm/issues/58#issuecomment-879278640) of FFmpeg.wasm issue gave me a better solution. We can use WebAssembly to directly use libav APIs. In other words, reimplement input and output. Thus wasm-based FFmpeg can interact with any JavaScript data. This will give us enough flexiblity, and can real fit into browser environment. 37 | 38 | There is another project [BeamCoder](https://github.com/Streampunk/beamcoder) gave me guides about how to wrap those low-level api and expose to JS. So in my case, I use C++ to wrap C api and use [Emscripten embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html) to expose self defined FFmpeg classes to JS world. So I can build video processing workflow in a JS worker. 39 | 40 | ### Inspired from TensorFlow / Pytorch 41 | Initially, I just wanted to build something similar to BeamCoder. But maybe we can do it further. Since I know the experience of learning FFmpeg basic concepts and API is painful. Like, `container` / `codec`, `demux` / `decode`, `encode` / `mux`, and `pts`, `dts`, `time_base`, etc. So if this project abstracts those concepts while also keeps the same flexibility, others can avoid same headache experience. 42 | 43 | Then, I figured out that we can build a library like machine learning frameworks (`Tensorflow`, `Pytorch`). 44 | Each frame of a video, no matter whether it is compressed (`packet`), or not (`frame`). They can all be viewed as `Tensor`. And entire process is through a tensors' graph. First build a graph. And when processing video, for each iteration, feed data (images/file/stream), execute the graph, and get the output (images/chunks). 45 | 46 | ### Additional gain 47 | So it is just a `for...loop`. We can keep it loop until ends. Also we can `break` in the middle, or skip by `continue`. Here is an example using FrameFlow api. 48 | ```js 49 | for await (let chunk of target) { 50 | if (/* cond */) 51 | continue 52 | else if (/* cond */) 53 | break 54 | 55 | // do something regularly ... 56 | } 57 | ``` 58 | 59 | ### API design logic 60 | The goal of FrameFlow is to keep flexible and simple. It is designed to support all JavaScript data, all will be processed in stream way. So video, audio or sequence of images can be viewed as an array. Each element in the array is an image (frame). Sometimes it is in compressed format, sometimes in uncompressed, and sometimes multiple arrays (video track/audio track) zip togather, like using python `zip` function. 61 | 62 | Also, building a filter graph should also go in JS way. 63 | Here is an example of comparing FFmpeg command-line style and FrameFlow JS style. 64 | 65 | [CMD style](https://stackoverflow.com/a/56109487/6690269) 66 | ```bash 67 | ffmpeg -i test.mp4 -filter_complex \ 68 | '[0:v] trim=start=5:end=10, setpts=PTS-STARTPTS [v0]; \ 69 | [0:a]atrim=start=5:end=10,asetpts=PTS-STARTPTS [a0]'\ 70 | -map [v0] -map [a0] output.mp4 71 | ``` 72 | 73 | FrameFlow style 74 | ```js 75 | let video = await fflow.source('./test.mp4') 76 | await video.trim({start: 5, duration: 5}).exportTo('./output.mp4') 77 | ``` 78 | 79 | You don't need to understand why we need `trim` and `atrim`, what is `setpts`, what is `-map`... 80 | Internally, frameflow actually converts JS style to FFmpeg style to build a filter graph. 81 | 82 | ### Problems of FrameFlow 83 | After talking about advantages of frameflow, let's talk about some critic problems that still exist in the project. 84 | 85 | ### Speed 86 | It is the top issue that decides how impactful it will be. Is it just a toy, a demo or a killer app ? Although WebAssembly is designed to have near-native speed performance. But in reality, things are not that simple. In my current version, since it hasn't done any optimization. The speed is roughly 10x slower than native FFmpeg one. Why, let me explain. 87 | 88 | After doing some initial speed tests, I found out the bottlenecks are encode and decode phases. 89 | Especially the encode phase. The gap between frameflow and FFmpeg is from three aspects. 90 | 91 | - WebAssembly speed is a little bit slower than native one. Especially when there are many interactions between JS and WASM. The speed will slow down to half speed of the native one. 92 | - FFmpeg have multi-threads enabled, frameflow currently haven't enabled. 93 | - FFmpeg has SIMD optimization for various CPU architectures. FrameFlow hasn't. 94 | 95 | #### Solutions 96 | So here are some solutions for above each problem. 97 | 98 | - Since frameflow directly manipulates FFmpeg low-level api. There is no need to mount any `Emscripten FS`. 99 | Every interaction between JS and Wasm is under control, even `log` to `stderr`. We can optimize if needed. 100 | 101 | - Enable multi-threads, if enable `SharedArrayBuffer` and [cross-origin isolation](https://web.dev/i18n/en/cross-origin-isolation-guide/). Most cases are ok with that, except [some few use cases](https://blog.logrocket.com/understanding-sharedarraybuffer-and-cross-origin-isolation/#:~:text=The%20COEP%20header%20breaks%20every%20integration%20that%20requires%20communication%20with%20cross%2Dorigin%20windows%20in%20the%20browser%2C%20such%20as%20authentications%20from%20third%2Dparty%20servers%20and%20payments%20(checkouts).). 102 | 103 | - Write WebAssembly SIMD codes. Since FFmpeg uses assembly SIMD code, which cannot port to wasm, because Emscripten only allow [`C intrinsics` codes](https://emscripten.org/docs/porting/simd.html#:~:text=Emscripten%20does%20not%20support%20x86%20or%20any%20other%20native%20inline%20SIMD%20assembly%20or%20building%20.s%20assembly%20files%2C%20so%20all%20code%20should%20be%20written%20to%20use%20SIMD%20intrinsic%20functions%20or%20compiler%20vector%20extensions.). So rewrite all the optimization codes would need a lot of time. 104 | And Safari currently hasn't fully supported it. Check the [browser compatibility](https://webassembly.org/roadmap/#:~:text=0.1-,Fixed%2Dwidth%20SIMD,-91). 105 | 106 | - Additional, use WebCodecs API when browsers support specific codec. This will directly have native encode and decode power. That is estimated to be have near-native speed, without any limitation. But not all browser support it. check the [compatibility](https://caniuse.com/webcodecs). 107 | 108 | Altogather, not each one solution can solve the issue perfectly, but they togather will accelerate a lot. 109 | That will be estimated to satisfied most of our daily use cases. 110 | 111 | ### Packet size 112 | FrameFlow heavily relies on FFmpeg as basic component. 113 | However, FFmpeg library itself is huge size, from the perspective of web developers. So frameflow current wasm version is about 22 MB size. 114 | Why? Because, during web developing, codes are downloaded and run when need. But FFmpeg downloads all before running, which is dozens of MB size. If you build the library without any components, 115 | like encoder/decoder, demuxer/muxer and filters. Size of the library reduces to 1~2 MB, even <1 MB. 116 | 117 | #### Solutions 118 | 119 | - So do we really need all those components? No, most of time, we only need very small fraction of it. 120 | So why not download on demand? Like streaming media. In the future, we can attempt to use [Emscripten split feature](https://emscripten.org/docs/optimizing/Module-Splitting.html) to lazily load each fragment on demand. 121 | 122 | - Current version has `loadWASM` api, which can preload the wasm binary. 123 | 124 | 125 | ## Conclusion 126 | 127 | FrameFlow is designed to support any JavaScript data in stream way. And will do most of things FFmpeg can do. 128 | And also gives more friendly API than either FFmpeg command-line way or low-level C API. 129 | Your words will shape the future of FrameFlow. 130 | -------------------------------------------------------------------------------- /doc/blog/authors.yml: -------------------------------------------------------------------------------- 1 | endi: 2 | name: Endilie Yacop Sucipto 3 | title: Maintainer of Docusaurus 4 | url: https://github.com/endiliey 5 | image_url: https://github.com/endiliey.png 6 | 7 | yangshun: 8 | name: Yangshun Tay 9 | title: Front End Engineer @ Facebook 10 | url: https://github.com/yangshun 11 | image_url: https://github.com/yangshun.png 12 | 13 | slorber: 14 | name: Sébastien Lorber 15 | title: Docusaurus maintainer 16 | url: https://sebastienlorber.com 17 | image_url: https://github.com/slorber.png 18 | 19 | carson: 20 | name: Siyuan Wu 21 | title: Creator of FrameFlow 22 | url: https://github.com/carsondb 23 | image_url: https://avatars.githubusercontent.com/u/11520061?v=4 24 | -------------------------------------------------------------------------------- /doc/docs/api/_category_.yml: -------------------------------------------------------------------------------- 1 | label: "API" -------------------------------------------------------------------------------- /doc/docs/api/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "index" 3 | title: "frameflow" 4 | sidebar_label: "Readme" 5 | sidebar_position: 0 6 | custom_edit_url: null 7 | --- 8 | 9 | # [FrameFlow](https://frameflow.netlify.app/) 10 | 11 | A both speedy and compatible video processing library for Web Browser, based on WebCodecs and FFmpeg (WebAssembly). It is hardware accelerated by WebCodecs as default, which works in Chromium-based clients (Chrome, Edge, Electron...). And also provides fallback solutions by FFmpeg (WebAssembly). It also provides some usual filters (trim, concat...). 12 | 13 | ## Features 14 | - Process videos in stream way, without video size limitation. 15 | - Accept stream input `MediaStream` (from canvas, Camera, ...), and output stream of frames (to canvas...) as well. 16 | - Use `WebCodecs` to have hardware acceleration for Chromium-based client (Chrome (>=106), Edge, Opera, Electron...). 17 | - Get detailed metadata of video file by reading only several chunks, either from local disk or remote url. 18 | - Processing speed can be controlled either automatically or manually. 19 | 20 | ⚠️ Note: **web browser** examples are tested. Nodejs hasn't been tested yet. 21 | 22 | ## Demo 23 | 24 | ```JavaScript 25 | import fflow from 'frameflow' 26 | 27 | let video = await fflow.source(videoBlob) // use web File api to get File handler. 28 | let audio = await fflow.source(audioURL) // remote media file (no need to download entirely beforehand) 29 | let audioTrim = audio.trim({start: 10, duration: video.duration}) // use metadata of video 30 | let blob = await fflow.group([video, audioTrim]).exportTo(Blob, {format: 'mp4'}) // group and trancode to 31 | videoDom.src = URL.createObjectURL(blob) 32 | // now can play in the browser 33 | ``` 34 | Although this example writes to blob entirely, then play. 35 | But underhood, it streams out chunks and then put togather. 36 | 37 | ### More examples 38 | More detailed browser examples are in the `./examples/browser/`. 39 | If you want to run them, please use latest release version. And then, at the root directory of the project, 40 | ``` 41 | npm install 42 | npm start 43 | ``` 44 | In dev mode, it will serve `./examples` as root directory. 45 | 46 | ## Install 47 | 48 | ### NPM 49 | ```bash 50 | npm i frameflow 51 | ``` 52 | 53 | ### HTML script 54 | ```html 55 | 56 | ``` 57 | 58 | ## Document 59 | All tutorials and documents are in [FrameFlow Doc](https://frameflow.netlify.app/docs/intro/getStarted). 60 | 61 | ## [Problems](https://frameflow.netlify.app/blog/why-frameflow/#problems-of-frameflow) 62 | 63 | ## How to build 64 | *Warning: [webpack dev mode cannot hot reload in WSL2 (windows).](https://mbuotidem.github.io/blog/2021/01/09/how-to-hot-reload-auto-refresh-react-app-on-WSL.html)* 65 | 66 | ### Dependencies (Ubuntu) 67 | Tools dependencies install 68 | ``` 69 | sudo apt-get update -y 70 | sudo apt-get install -y pkg-config 71 | ``` 72 | 73 | ### Emscripten 74 | ``` 75 | git clone https://github.com/emscripten-core/emsdk.git --branch 3.1.30 76 | rm -r emsdk/.git 77 | ``` 78 | [Install Emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) 79 | 80 | ### FFmpeg version (n5.0 release) 81 | ``` 82 | git clone https://github.com/FFmpeg/FFmpeg --depth 1 --branch n5.0 83 | rm -r FFmpeg/.git 84 | ``` 85 | 86 | ### External FFmpeg Libraries 87 | All external libraries sources are under `./ffmpeg_libraries` 88 | ``` 89 | cd ffmpeg_libraries 90 | ``` 91 | 92 | x264 93 | ``` 94 | git clone https://github.com/mirror/x264.git --depth 1 --branch stable 95 | ``` 96 | Libvpx 97 | ``` 98 | git clone https://github.com/webmproject/libvpx.git --depth 1 --branch v1.12.0 99 | ``` 100 | 101 | ### Compilation 102 | ``` 103 | ./build_ffmpeg.sh 104 | ./build_wasm.sh 105 | ``` 106 | -------------------------------------------------------------------------------- /doc/docs/api/modules.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "modules" 3 | title: "frameflow" 4 | sidebar_label: "Exports" 5 | sidebar_position: 0.5 6 | custom_edit_url: null 7 | --- 8 | 9 | ## Variables 10 | 11 | ### default 12 | 13 | • **default**: `Object` 14 | 15 | #### Type declaration 16 | 17 | | Name | Type | 18 | | :------ | :------ | 19 | | `concat` | (`trackArr`: (`TrackGroup` \| `Track`)[]) => `FilterTrackGroup` | 20 | | `group` | (`trackArr`: (`TrackGroup` \| `Track`)[]) => `TrackGroup` | 21 | | `loadWASM` | () => `Promise`<`ArrayBuffer`\> | 22 | | `merge` | (`trackArr`: (`TrackGroup` \| `Track`)[]) => `FilterTrackGroup` | 23 | | `setFlags` | (`flags`: `Flags`) => `void` | 24 | | `source` | (`src`: `SourceType`, `options?`: {}) => `Promise`<`SourceTrackGroup`\> | 25 | 26 | #### Defined in 27 | 28 | [main.ts:346](https://github.com/carsonDB/frameflow/blob/b731bd0/src/ts/main.ts#L346) 29 | -------------------------------------------------------------------------------- /doc/docs/guides/assets/download analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/docs/guides/assets/download analysis.png -------------------------------------------------------------------------------- /doc/docs/guides/codec.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "codec" 3 | title: "Encode / Decode" 4 | sidebar_position: 0 5 | --- 6 | 7 | ⚠️*This section's examples only works based on `WebCodecs`, which Chromium-based clients support. * 8 | 9 | In this section, we will talk about encoding and decoding, which involve uncompressed frames I/O. 10 | For example, we can input frames from camera or screen recording, or output frames to canvas. 11 | Here below introduce two examples, which you can check in the [examples/browser/codec.html](https://github.com/carsonDB/frameflow/blob/main/examples/browser/codec.html). To run this example, please follow the [guides](https://github.com/carsonDB/frameflow#more-examples). 12 | 13 | TODO... 14 | -------------------------------------------------------------------------------- /doc/docs/guides/dataType.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "dataType" 3 | title: "File/Stream" 4 | sidebar_position: 0 5 | --- 6 | 7 | # File / Stream 8 | 9 | ## Overview 10 | FrameFlow supports a lot of data as input or output, if only they are supported by browsers natively. 11 | For example, `string (url/path)`, `ReadableStream`, `ArrayBuffer/TypedArray`, etc. 12 | But essentially, they can be divided into two groups, `file` or `stream`. 13 | 14 | ## File 15 | As for FFmpeg users, file type is very common to understand. They are stored in local disks. We can get enough metadata information from them, before the running. 16 | 17 | However, in web environment, things change a little differnt. 18 | File type becomes more general, if only meet the following criteria. 19 | - Can get total length (byteLength) of the source, before running. 20 | - Source can be seeked to any position, within the total length. 21 | 22 | Given these two properties, we actually don't need to give the library additional information, except source itself. Provided the source, the library will seek to different positions and probe a little data. 23 | Then get enough metadata information of the source, which will be necessary for running. 24 | 25 | For example, if we give it a remote url. Then it will fetch with only `HEAD` required, to get totoal length of the source. If given `undefined` or `0`, this source will be seen as a `stream`. 26 | So you need to give it additional information to run. 27 | 28 | ## Stream (TODO) 29 | ⚠️Stream haven't been implemented yet. 30 | 31 | Contrary to `File`, stream has no above two properties, which can be used for real-time processing. 32 | But you need to give the source with additional information, like format of the source, etc. 33 | 34 | -------------------------------------------------------------------------------- /doc/docs/guides/download.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "download" 3 | title: "WASM Download" 4 | sidebar_position: 0 5 | --- 6 | 7 | # WASM Download 8 | 9 | ## WASM file size 10 | The library includes binary wasm file, you can see from [`unpkg`](https://unpkg.com/browse/frameflow/dist/) or [`jsDelivr`](https://cdn.jsdelivr.net/npm/frameflow/dist/). 11 | And during downloading, it actually transfers ~8MB, instead of ~22MB. 12 | Because servers automatically compress to gzip format, and browsers uncompress it as well. 13 | We can check in our `Chrome dev tool -> network`: 14 | 15 | ![download analysis](./assets/download%20analysis.png) 16 | 17 | ## Download strategy 18 | By default, the library will download when it really needs. So don't need to download manually. 19 | But if you care about the download latency (a few seconds in my network environment), you can preload it using `loadWASM` function. Like this: 20 | 21 | ```js 22 | import fflow from 'frameflow' 23 | 24 | fflow.loadWASM() // no need to use await 25 | /** 26 | * do others 27 | **/ 28 | 29 | // start to use it 30 | const src = await fflow.source('...') 31 | ``` 32 | 33 | `loadWASM` will return a `promise` from `fetch`, you can get wasm file in `ArrayBuffer`, which can be for your customized cache. But usually, there is no need to. 34 | You can call `loadWASM` multiple times, and only the first `fetch` caches the `promise`. 35 | All later calls will just `await` the first `promise`. So don't worry about multiple calls of it. 36 | 37 | ## Lazily download (TODO) 38 | Actually, with the library including more codecs, container formats, the size of wasm file will increase as well. So ultimate solution to this, is to split the wasm module and lazily download on demand. 39 | Each part can be reduced to <1 MB. So we can ignore the downloading process. 40 | However, it is the future work, which haven't done yet. -------------------------------------------------------------------------------- /doc/docs/guides/licenses.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "licenses" 3 | title: "Licenses" 4 | sidebar_position: 0 5 | --- 6 | 7 | # Licenses 8 | 9 | ## FrameFlow license 10 | 11 | This open source project is `LGPL` license. But it actually doesn't matter. 12 | Because it is based on FFmpeg, which has two license, `GPL` or `LGPL`. 13 | Roughly speaking, `GPL` is stricter than `LGPL`. So if only you comply with these licenses. 14 | You also comply with FrameFlow license. 15 | 16 | ## FFmpeg licenses 17 | Now lets' talk about FFmpeg licenses. It may be confusing to you if you are not familiar with these licenses. 18 | I'll explain them in plain English. 19 | 20 | ### GPL 21 | If you use FFmpeg GPL license, you need to either open source your project or purchase a GPL commercial license for your proprietary software. And the most important point is, it is **contagious**. 22 | That means, any project contains GPL FFmpeg, also need to comply with this rule. 23 | So be careful about your usage. 24 | 25 | ### LGPL 26 | `LGPL = Lesser GPL`. So it only requires that, any modifications in FFmpeg library must be open source. 27 | But has no requirement of any codes **outside** its library, even for commecial use cases, for free. 28 | 29 | ## How to avoid GPL 30 | By default, this project releases GPL FFmpeg compiled wasm file. Because it contains most components and chooses best. 31 | 32 | However, if you want to only use `LGPL`, then at current time, you need to build wasm file by yourself. 33 | Just remove `--enable-gpl` option in `./build_ffmpeg.sh`. Create an issue if you have any questions. 34 | 35 | Preparing multiple versions of wasm file would be the future work of this project. 36 | 37 | By the way, these are my understanding of licenses. Please correct me if there are some mistakes. -------------------------------------------------------------------------------- /doc/docs/intro/filters.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: 'filters' 3 | title: 'Filters' 4 | sidebar_position: 2 5 | --- 6 | 7 | # Filters 8 | 9 | In FFmpeg, filters is useful when we need to manipulate each frame of a video. 10 | But using FFmpeg command-line to build a filter link or graph can be difficult sometimes. 11 | Now here, we can use JavaScript way to build a graph, both flexible and easy to read. 12 | 13 | ## Example 14 | ```js 15 | let video = await fflow.source('./test.avi') 16 | await video.trim({start: 1, duration: video.duration}) 17 | .setVolume(0.5) 18 | .exportTo('./out_test.mp4') 19 | ``` 20 | 21 | This example apply `trim` and `setVolume` filter operations which support chainable operation. 22 | Each filter operation returns a new TrackGroup. 23 | 24 | ⚠️ Note that there are some difference between these filter operations and FFmpeg filters. 25 | They are not one-to-one correspondence. 26 | For example, in FFmpeg, we apply `trim` to video track (stream) and `atrim` to audio track (stream). 27 | But here, `TrackGroup.trim()` apply either `trim` or `atrim` to each internal track (stream). 28 | They are smart enough to build and process. 29 | 30 | ## Filter list 31 | 32 | Here is list of current filters (functions) avaible: 33 | - trim 34 | - loop 35 | - setVolume 36 | - setDataFormat 37 | - concat 38 | - group 39 | - merge 40 | 41 | Each one can be check in [Filters API](/docs/API/modules) 42 | -------------------------------------------------------------------------------- /doc/docs/intro/getStarted.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: 'getStarted' 3 | title: 'Get Started' 4 | sidebar_position: 1 5 | --- 6 | 7 | # Get Started 8 | 9 | ## Install 10 | 11 | ### NPM 12 | ```bash 13 | npm i frameflow 14 | ``` 15 | ### HTML script 16 | ```html 17 | 18 | ``` 19 | 20 | ## Create source 21 | Accept multiple type of sources, e.g. url / path / Blob / ArrayBuffer / ReadableStream. 22 | TypeScript will give hints. 23 | ```js 24 | let video = await fflow.source('./test.avi') 25 | ``` 26 | 27 | ## Get metadata 28 | Metadata can provide you some information, which can be used for filters' arguments. 29 | And internally, they are also used for checking your created filter graph, which will explain in [filters](#filters). 30 | ```js 31 | let video = await fflow.source('./test.avi') 32 | console.log(video.duration) // get one item in metadata 33 | console.log(video.metadata) // get all metadata information (container + tracks) 34 | ``` 35 | 36 | ## Tracks selection 37 | Usually, we can directly operate on multiple tracks as a group. 38 | And one track is seen as a group in which only one element. 39 | Thus, created source is also a group. 40 | And it is also convenient to apply filters to a group of tracks at a time. 41 | ```js 42 | let video = await fflow.source('./test.avi') // return a TrackGroup 43 | let audioTrackGroup = video.filter('audio') // return a new TrackGroup (contain only audio tracks) 44 | let audioTracks = audioTrackGroup.tracks() // return an array of Tracks. (audio) 45 | let newGroup = fflow.group([audioTracks[0], audioTracks[1]]) // group multiple tracks into one group 46 | ``` 47 | 48 | ## Transcode 49 | One of core use cases is to convert audio/video files into another with different parameters, e.g. format, codec. Here you just need to focus on source and target, using `Export` function. 50 | It will build a graph internally to execute. 51 | There are three ways to `export`, meeting your various use cases. 52 | 53 | ### `exportTo`: fastest way 54 | This api can export to multiple types of outputs, e.g. url / path / Blob / ArrayBuffer. 55 | ```js 56 | let video = await fflow.source('./test.avi') 57 | await video.exportTo('./out_test.mp4') // no return 58 | let blob = await video.exportTo(Blob) // return a new blob 59 | ``` 60 | ### `export + for...of`: flexible way with automatic pace 61 | ```js 62 | let target = await video.export() 63 | for await (let chunk of target) { 64 | /* post-processing chunk */ 65 | chunk.data // Uint8Array (browser) / Buffer (node) 66 | chunk.offset // chunk offset position (bytes) in the output file 67 | } 68 | ``` 69 | ### `export + next`: most flexible way with manual control 70 | Actually this way is the basic method for above two ways. 71 | ```js 72 | let target = await video.export() 73 | let chunk = await target.next() 74 | while (chunk.data) { 75 | /* post-processing chunk */ 76 | chunk = await target.next() 77 | } 78 | // execute this line if quitting halfway. 79 | await target.close() 80 | ``` 81 | -------------------------------------------------------------------------------- /doc/docusaurus.config.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | // Note: type annotations allow type checking and IDEs autocompletion 3 | 4 | const lightCodeTheme = require('prism-react-renderer/themes/github'); 5 | const darkCodeTheme = require('prism-react-renderer/themes/dracula'); 6 | 7 | /** @type {import('@docusaurus/types').Config} */ 8 | const config = { 9 | title: 'FrameFlow', 10 | tagline: 'Both speedy and compatible video process library for Web Browser', 11 | // favicon: 'img/favicon.ico', 12 | 13 | // Set the production url of your site here 14 | url: 'https://frameflow.netlify.app', 15 | // Set the // pathname under which your site is served 16 | // For GitHub pages deployment, it is often '//' 17 | baseUrl: '/', 18 | 19 | // GitHub pages deployment config. 20 | // If you aren't using GitHub pages, you don't need these. 21 | organizationName: 'carsonDB', // Usually your GitHub org/user name. 22 | projectName: 'frameflow', // Usually your repo name. 23 | 24 | onBrokenLinks: 'throw', 25 | onBrokenMarkdownLinks: 'warn', 26 | 27 | // Even if you don't use internalization, you can use this field to set useful 28 | // metadata like html lang. For example, if your site is Chinese, you may want 29 | // to replace "en" with "zh-Hans". 30 | i18n: { 31 | defaultLocale: 'en', 32 | locales: ['en'], 33 | }, 34 | 35 | presets: [ 36 | [ 37 | 'classic', 38 | /** @type {import('@docusaurus/preset-classic').Options} */ 39 | ({ 40 | docs: { 41 | sidebarPath: require.resolve('./sidebars.js'), 42 | // Please change this to your repo. 43 | // Remove this to remove the "edit this page" links. 44 | editUrl: 45 | 'https://github.com/carsondb/frameflow/tree/main/doc/', 46 | }, 47 | blog: { 48 | showReadingTime: true, 49 | // Please change this to your repo. 50 | // Remove this to remove the "edit this page" links. 51 | editUrl: 52 | 'https://github.com/carsondb/frameflow/tree/main/doc/', 53 | }, 54 | theme: { 55 | customCss: require.resolve('./src/css/custom.css'), 56 | }, 57 | }), 58 | ], 59 | ], 60 | 61 | themeConfig: 62 | /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ 63 | ({ 64 | // Replace with your project's social card 65 | // image: 'img/docusaurus-social-card.jpg', 66 | navbar: { 67 | // title: 'My Site', 68 | logo: { 69 | alt: 'Home Logo', 70 | src: 'img/home_icon.png', 71 | }, 72 | items: [ 73 | { 74 | type: 'doc', 75 | docId: 'intro/getStarted', 76 | position: 'left', 77 | label: 'Docs', 78 | }, 79 | { 80 | type: 'doc', 81 | docId: 'api/index', 82 | position: 'left', 83 | label: 'API', 84 | }, 85 | {to: '/blog', label: 'Blog', position: 'left'}, 86 | { 87 | href: 'https://github.com/carsonDB/frameflow', 88 | label: 'GitHub', 89 | position: 'right', 90 | }, 91 | ], 92 | }, 93 | footer: { 94 | style: 'dark', 95 | links: [ 96 | { 97 | title: 'Docs', 98 | items: [ 99 | // { 100 | // label: 'Tutorial', 101 | // to: '/docs/intro', 102 | // }, 103 | ], 104 | }, 105 | // { 106 | // title: 'Community', 107 | // items: [ 108 | // { 109 | // label: 'Stack Overflow', 110 | // href: 'https://stackoverflow.com/questions/tagged/docusaurus', 111 | // }, 112 | // { 113 | // label: 'Discord', 114 | // href: 'https://discordapp.com/invite/docusaurus', 115 | // }, 116 | // { 117 | // label: 'Twitter', 118 | // href: 'https://twitter.com/docusaurus', 119 | // }, 120 | // ], 121 | // }, 122 | { 123 | title: 'More', 124 | items: [ 125 | { 126 | label: 'Blog', 127 | to: '/blog', 128 | }, 129 | { 130 | label: 'GitHub', 131 | href: 'https://github.com/carsonDB/frameflow', 132 | }, 133 | ], 134 | }, 135 | ], 136 | // copyright: `Copyright © ${new Date().getFullYear()} FrameFlow, Inc.`, 137 | }, 138 | prism: { 139 | theme: lightCodeTheme, 140 | darkTheme: darkCodeTheme, 141 | }, 142 | }), 143 | 144 | themes: ['@docusaurus/theme-live-codeblock'], 145 | 146 | plugins: [ 147 | [ 148 | 'docusaurus-plugin-typedoc', 149 | // Plugin / TypeDoc options 150 | { 151 | entryPoints: ['../src/ts/main.ts'], 152 | tsconfig: '../tsconfig.json', 153 | out: 'api', 154 | }, 155 | ], 156 | ] 157 | }; 158 | 159 | module.exports = config; 160 | -------------------------------------------------------------------------------- /doc/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frameflow", 3 | "version": "0.1.6", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "clear": "docusaurus clear", 12 | "serve": "docusaurus serve", 13 | "write-translations": "docusaurus write-translations", 14 | "write-heading-ids": "docusaurus write-heading-ids" 15 | }, 16 | "dependencies": { 17 | "@docusaurus/core": "^2.3.1", 18 | "@docusaurus/preset-classic": "^2.3.1", 19 | "@docusaurus/theme-live-codeblock": "^2.3.1", 20 | "@mdx-js/react": "^1.6.22", 21 | "clsx": "^1.2.1", 22 | "frameflow": "^0.1.6", 23 | "react": "^17.0.2", 24 | "react-dom": "^17.0.2", 25 | "use-editable": "^2.3.3" 26 | }, 27 | "devDependencies": { 28 | "@docusaurus/module-type-aliases": "^2.3.1", 29 | "@tsconfig/docusaurus": "^1.0.6", 30 | "@types/prismjs": "^1.26.0", 31 | "docusaurus-plugin-typedoc": "^0.18.0", 32 | "typedoc": "^0.23.24", 33 | "typedoc-plugin-markdown": "^3.14.0", 34 | "typescript": "^4.9.5" 35 | }, 36 | "browserslist": { 37 | "production": [ 38 | ">0.5%", 39 | "not dead", 40 | "not op_mini all" 41 | ], 42 | "development": [ 43 | "last 1 chrome version", 44 | "last 1 firefox version", 45 | "last 1 safari version" 46 | ] 47 | }, 48 | "engines": { 49 | "node": ">=16.14" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /doc/sidebars.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creating a sidebar enables you to: 3 | - create an ordered group of docs 4 | - render a sidebar for each doc of that group 5 | - provide next/previous navigation 6 | 7 | The sidebars can be generated from the filesystem, or explicitly defined here. 8 | 9 | Create as many sidebars as you want. 10 | */ 11 | 12 | // @ts-check 13 | 14 | /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ 15 | const sidebars = { 16 | tutorialSidebar: [ 17 | { 18 | type: 'category', 19 | label: 'Introduction', 20 | items: [{type: 'autogenerated', dirName: 'intro'}], 21 | 22 | }, 23 | { 24 | type: 'category', 25 | label: 'Guides', 26 | items: [{type: 'autogenerated', dirName: 'guides'}] 27 | }, 28 | { 29 | type: 'category', 30 | label: 'API', 31 | items: [{type: 'autogenerated', dirName: 'api'}] 32 | } 33 | ] 34 | 35 | }; 36 | 37 | module.exports = sidebars; 38 | -------------------------------------------------------------------------------- /doc/src/components/HomepageFeatures/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import CodeBlock from '@theme/CodeBlock'; 3 | import clsx from 'clsx'; 4 | import styles from './styles.module.css'; 5 | 6 | const FeatureList = [ 7 | { 8 | title: 'Stream I/O', 9 | code: `let src = await fflow.source('...')\nconsole.log(src.metadata)`, 10 | description: ( 11 | <> 12 | FrameFlow was designed to support all JavaScript I/O as stream way. 13 | Just one simple line, with metadata as side effects. 14 | 15 | ), 16 | }, 17 | { 18 | title: 'Build filter graph in JS way', 19 | code: `src.trim({start: 1, duration: 10})\n` + 20 | ` .setVolume(0.5)\n` 21 | , 22 | description: ( 23 | <> 24 | Instead of building filter graph using FFmpeg command-line, 25 | frameflow use a simple way to construct. Here is to trim a video input. 26 | 27 | ), 28 | }, 29 | { 30 | title: 'Control progress by yourself', 31 | code: `// method 1 \n` + 32 | `await src.exportTo('...')\n` + 33 | `// method 2 \n` + 34 | `let target = await src.export()\n` + 35 | `for await (let chunk of target) {\n` + 36 | ` // do something... \n` + 37 | `}\n` + 38 | `// method 3 \n` + 39 | `let target = await src.export()\n` + 40 | `// one at a time\n` + 41 | `let chunk = await target.next()\n` 42 | , 43 | description: ( 44 | <> 45 | You can choose either exportTo to export video automatically, 46 | or export to stream output. 47 | 48 | ), 49 | }, 50 | ]; 51 | 52 | function Feature({code, title, description}) { 53 | return ( 54 |
55 |
56 | {/* */} 57 |
58 | {code} 59 |
60 |
61 |
62 |

{title}

63 |

{description}

64 |
65 |
66 | ); 67 | } 68 | 69 | export default function HomepageFeatures() { 70 | return ( 71 |
72 |
73 |
74 | {FeatureList.map((props, idx) => ( 75 | 76 | ))} 77 |
78 |
79 |
80 | ); 81 | } 82 | -------------------------------------------------------------------------------- /doc/src/components/HomepageFeatures/styles.module.css: -------------------------------------------------------------------------------- 1 | .features { 2 | display: flex; 3 | align-items: center; 4 | padding: 2rem 0; 5 | width: 100%; 6 | } 7 | 8 | .featureSvg { 9 | height: 200px; 10 | width: 200px; 11 | } 12 | -------------------------------------------------------------------------------- /doc/src/components/codeEditor/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { CSSProperties, useRef } from 'react'; 2 | import Highlight, { defaultProps } from "prism-react-renderer"; 3 | import { useEditable } from 'use-editable' 4 | import theme from 'prism-react-renderer/themes/github'; 5 | 6 | export default function CodeEditor(props: {children: string, onChange: (code: string) => void, style?: CSSProperties}) { 7 | const ref = useRef(null) 8 | useEditable(ref, props.onChange) 9 | 10 | return ( 11 | 12 | {({ className, style, tokens, getLineProps, getTokenProps }) => ( 13 | 
14 |           {tokens.map((line, i) => (
15 |             

16 |               {line.map((token, key) => (
17 |                 
18 |               ))}
19 |             

20 |           ))}
21 |
22 | )} 23 | 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /doc/src/css/custom.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Any CSS included here will be global. The classic template 3 | * bundles Infima by default. Infima is a CSS framework designed to 4 | * work well for content-centric websites. 5 | */ 6 | 7 | /* You can override the default Infima variables here. */ 8 | :root { 9 | --ifm-color-primary: #2e8555; 10 | --ifm-color-primary-dark: #29784c; 11 | --ifm-color-primary-darker: #277148; 12 | --ifm-color-primary-darkest: #205d3b; 13 | --ifm-color-primary-light: #33925d; 14 | --ifm-color-primary-lighter: #359962; 15 | --ifm-color-primary-lightest: #3cad6e; 16 | --ifm-code-font-size: 95%; 17 | --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); 18 | } 19 | 20 | /* For readability concerns, you should choose a lighter palette in dark mode. */ 21 | [data-theme='dark'] { 22 | --ifm-color-primary: #25c2a0; 23 | --ifm-color-primary-dark: #21af90; 24 | --ifm-color-primary-darker: #1fa588; 25 | --ifm-color-primary-darkest: #1a8870; 26 | --ifm-color-primary-light: #29d5b0; 27 | --ifm-color-primary-lighter: #32d8b4; 28 | --ifm-color-primary-lightest: #4fddbf; 29 | --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); 30 | } 31 | -------------------------------------------------------------------------------- /doc/src/pages/index.module.css: -------------------------------------------------------------------------------- 1 | /** 2 | * CSS files with the .module.css suffix will be treated as CSS modules 3 | * and scoped locally. 4 | */ 5 | 6 | .heroBanner { 7 | padding: 4rem 0; 8 | text-align: center; 9 | position: relative; 10 | overflow: hidden; 11 | background-color: #269392; 12 | } 13 | 14 | @media screen and (max-width: 996px) { 15 | .heroBanner { 16 | padding: 2rem; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /doc/src/pages/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useRef, useState } from 'react'; 2 | import Link from '@docusaurus/Link'; 3 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; 4 | import CodeEditor from '@site/src/components/codeEditor'; 5 | import HomepageFeatures from '@site/src/components/HomepageFeatures'; 6 | import Frameflow_diagram from '@site/static/img/diagram.png'; 7 | import Layout from '@theme/Layout'; 8 | import clsx from 'clsx'; 9 | import fflow from 'frameflow'; 10 | 11 | import styles from './index.module.css'; 12 | 13 | function HomepageHeader() { 14 | const {siteConfig} = useDocusaurusContext(); 15 | return ( 16 |
17 |
18 |

{siteConfig.title}

19 |

{siteConfig.tagline}

20 | 21 |
22 | 25 | Get Started 26 | 27 | 30 | Try a demo 31 | 32 |
33 |
34 |
35 | ); 36 | } 37 | 38 | 39 | const demoVideoURL = require('@site/static/video/flame.avi').default 40 | const demoAudioURL = require('@site/static/video/audio.mp3').default 41 | const demoCode = ` 42 | // import fflow from 'frameflow' 43 | // Given Variables: (fflow, console, onProgress, videoDom) 44 | let videoURL = '${demoVideoURL}' 45 | let audioURL = '${demoAudioURL}' 46 | let video = await fflow.source(videoURL) 47 | let audio = await fflow.source(audioURL) 48 | let trimAudio = audio.trim({start: 10, duration: video.duration}) 49 | let newVideo = fflow.group([video, trimAudio]) 50 | let outBlob = await newVideo.exportTo(Blob, {format: 'mp4', progress: onProgress}) 51 | videoDom.src = URL.createObjectURL(outBlob) 52 | ` 53 | 54 | 55 | 56 | function HomepageDemo() { 57 | const [code, setCode] = useState(demoCode) 58 | const [msg, setMsg] = useState('') 59 | const videoRef = useRef(null) 60 | // preload 61 | useEffect(() => { fflow.loadWASM() }) 62 | 63 | const onClick = () => { 64 | setMsg(' ...') 65 | Function(`"use strict"; 66 | const {fflow, console, Blob, onProgress, videoDom} = this; 67 | (async () => { ${code} })() 68 | `).bind({fflow, console, Blob, 69 | onProgress: (p: number) => setMsg(` (${(p*100).toFixed(1)}%)`), videoDom: videoRef.current})() 70 | } 71 | 72 | return ( 73 |
74 |
75 |

Try a demo

76 |

Trim a audio and group with avi video, to mp4 file, which can play in HTMLVideoElement.

77 | 81 |
82 |
83 |
86 |
87 | setCode(c)} style={{width: '65%'}} > 88 | {code} 89 | 90 |
91 |
92 | ) 93 | } 94 | 95 | export default function Home() { 96 | const {siteConfig} = useDocusaurusContext(); 97 | return ( 98 | 101 | 102 |
103 | 104 | 105 |
106 | 107 | ); 108 | } 109 | -------------------------------------------------------------------------------- /doc/src/pages/markdown-page.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Markdown page example 3 | --- 4 | 5 | # Markdown page example 6 | 7 | You don't need React to write simple standalone pages. 8 | -------------------------------------------------------------------------------- /doc/static/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/.nojekyll -------------------------------------------------------------------------------- /doc/static/img/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/img/diagram.png -------------------------------------------------------------------------------- /doc/static/img/docusaurus-social-card.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/img/docusaurus-social-card.jpg -------------------------------------------------------------------------------- /doc/static/img/docusaurus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/img/docusaurus.png -------------------------------------------------------------------------------- /doc/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/img/favicon.ico -------------------------------------------------------------------------------- /doc/static/img/home_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/img/home_icon.png -------------------------------------------------------------------------------- /doc/static/img/logo.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/static/static.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.png' { 2 | export = string; 3 | } -------------------------------------------------------------------------------- /doc/static/video/audio.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/video/audio.mp3 -------------------------------------------------------------------------------- /doc/static/video/flame.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/doc/static/video/flame.avi -------------------------------------------------------------------------------- /doc/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/docusaurus/tsconfig.json", 3 | "compilerOptions": { 4 | "baseUrl": "." 5 | } 6 | } -------------------------------------------------------------------------------- /examples/assets/Bunny.mkv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/examples/assets/Bunny.mkv -------------------------------------------------------------------------------- /examples/assets/Bunny.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/examples/assets/Bunny.mp4 -------------------------------------------------------------------------------- /examples/assets/CantinaBand3.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/examples/assets/CantinaBand3.wav -------------------------------------------------------------------------------- /examples/assets/audio.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/examples/assets/audio.mp3 -------------------------------------------------------------------------------- /examples/assets/flame.avi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsonDB/frameflow/031b7c5d15ac376d4406202dbd287ab303268bbc/examples/assets/flame.avi -------------------------------------------------------------------------------- /examples/browser/codec.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | encode 7 | 8 | 9 | 10 | 11 | 14 | 15 |

use WebCodecs: Only available in Chromium-base client

16 | 17 |

👉Decode: mp4 Video to canvas (real-time)

18 | 19 | 20 | 49 | 50 | 51 |

👉Encode: Screen record to mp4 video (real-time)

52 |
53 | 54 | 65 | 66 | 67 |

👉Decode (mp4) + Canvas (add watermark) + Encode (mp4)

68 |
69 | 70 | 71 | 76 | 77 | 78 | 122 | 123 | -------------------------------------------------------------------------------- /examples/browser/demo.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | frameflow dev 7 | 8 | 12 | 13 | 14 | 15 | 22 | 23 | 24 |

👉Fetch and get metadata

25 |

Please open console to see metadata.

26 | 27 | 34 | 35 | 36 |

👉Trancode

37 |
38 | 39 | 40 | 48 | 49 | 50 |

👉Filter: trim

51 |
52 | 53 | 54 | 63 | 64 | 65 |

👉Hybrid: trim + transcode + mux

66 |
67 | 68 | 69 | 70 | 82 | 83 |

👉mp4 to webm

84 |
85 | 86 | 87 | 95 | 96 |

👉MediaRecoder to wav file

97 |
98 | 99 | 100 | 118 | 119 |

👉Audio: loop

120 |
121 | 122 | 129 | 130 |

👉Audio: trim + loop + concat

131 |
132 | 133 | 143 | 144 |

👉wav to mp4

145 |
146 | 147 | 154 | 155 | -------------------------------------------------------------------------------- /examples/browser/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | frameflow dev 7 | 11 | 12 | 13 | 14 |

Please use GitHub latest release codes (tested)

15 | 16 |

Demo

17 |

Encode / Decode

18 |

Transmux

19 |

TODO (developing)

20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /examples/browser/todo.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | encode 7 | 8 | 9 | 10 | 11 | 14 | 15 |

👉Upload video (developing...)

16 |
17 | 18 | 19 | 20 | 41 | 42 | -------------------------------------------------------------------------------- /examples/browser/transmux.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Transmux Examples 7 | 8 | 9 | 10 | 11 | 14 | 15 |

The following conversion cases do not require encoding or decoding.

16 | 17 |

👉Convert MKV to MP4

18 |
19 | 20 | 21 | 38 | 39 |

👉Convert HLS Stream to MP4

40 |
41 | 42 | 43 | 55 | 56 |

👉mp4 extract only video

57 |
58 | 59 | 60 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /examples/node/transcode.js: -------------------------------------------------------------------------------- 1 | const fflow = require('../../src/cpp/ts/main') 2 | 3 | 4 | (async () => { 5 | let src = await fflow.source('../assets/flame.avi') 6 | await src.exportTo('flame.mp4') 7 | }) -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} */ 2 | module.exports = { 3 | rootDir: 'src/ts', 4 | preset: 'ts-jest', 5 | testEnvironment: 'node', 6 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frameflow", 3 | "version": "0.3.5", 4 | "description": "Audio/Video stream processing library for JavaScript world", 5 | "main": "./dist/frameflow.min.js", 6 | "homepage": "https://github.com/carsondb/frameflow", 7 | "url": "https://github.com/carsondb/frameflow/issues", 8 | "email": "carsonrundb@gmail.com", 9 | "types": "./dist/main.d.ts", 10 | "files": [ 11 | "dist" 12 | ], 13 | "directories": { 14 | "example": "examples" 15 | }, 16 | "scripts": { 17 | "start": "webpack serve --open --mode development", 18 | "build": "webpack --mode production", 19 | "prepublishOnly": "npm run build", 20 | "test": "jest --coverage" 21 | }, 22 | "author": "carsonDB", 23 | "license": "LGPL", 24 | "dependencies": { 25 | "mime": "^3.0.0", 26 | "uuid": "^9.0.0" 27 | }, 28 | "devDependencies": { 29 | "@types/dom-webcodecs": "^0.1.5", 30 | "@types/emscripten": "^1.39.6", 31 | "@types/node": "^22.15.21", 32 | "@types/uuid": "^9.0.0", 33 | "html-webpack-plugin": "^5.5.0", 34 | "jest": "^29.4.2", 35 | "terser-webpack-plugin": "^5.3.6", 36 | "ts-jest": "^29.0.5", 37 | "ts-loader": "^9.4.2", 38 | "typescript": "^4.7.4", 39 | "webpack": "^5.81.0", 40 | "webpack-cli": "^5.0.1", 41 | "webpack-dev-server": "^4.11.1", 42 | "worker-loader": "^3.0.8" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/cpp/bind.cpp: -------------------------------------------------------------------------------- 1 | #ifndef BIND_H 2 | #define BIND_H 3 | 4 | #include 5 | #include "utils.h" 6 | #include "metadata.h" 7 | #include "stream.h" 8 | #include "encode.h" 9 | #include "demuxer.h" 10 | #include "decode.h" 11 | #include "filter.h" 12 | #include "muxer.h" 13 | using namespace emscripten; 14 | 15 | 16 | EMSCRIPTEN_BINDINGS(metadata) { 17 | value_object("StreamInfo") 18 | .field("index", &StreamInfo::index) 19 | .field("timeBase", &StreamInfo::time_base) 20 | .field("bitRate", &StreamInfo::bit_rate) 21 | .field("startTime", &StreamInfo::start_time) 22 | .field("duration", &StreamInfo::duration) 23 | .field("mediaType", &StreamInfo::codec_type) 24 | .field("codecName", &StreamInfo::codec_name) 25 | .field("format", &StreamInfo::format) 26 | .field("extraData", &StreamInfo::extraData) 27 | .field("width", &StreamInfo::width) 28 | .field("height", &StreamInfo::height) 29 | .field("frameRate", &StreamInfo::frame_rate) 30 | .field("sampleAspectRatio", &StreamInfo::sample_aspect_ratio) 31 | .field("sampleRate", &StreamInfo::sample_rate) 32 | .field("channelLayout", &StreamInfo::channel_layout) 33 | .field("channels", &StreamInfo::channels) 34 | ; 35 | 36 | value_object("DataFormat") 37 | .field("format", &DataFormat::format) 38 | .field("channelLayout", &DataFormat::channelLayout) 39 | .field("channels", &DataFormat::channels) 40 | .field("sampleRate", &DataFormat::sampleRate) 41 | ; 42 | 43 | value_object("FormatInfo") 44 | .field("formatName", &FormatInfo::format_name) 45 | .field("bitRate", &FormatInfo::bit_rate) 46 | .field("duration", &FormatInfo::duration) 47 | .field("streamInfos", &FormatInfo::streamInfos) 48 | ; 49 | } 50 | 51 | EMSCRIPTEN_BINDINGS(demuxer) { 52 | 53 | class_("Demuxer") 54 | // .constructor() 55 | .constructor<>() 56 | .function("build", &Demuxer::build) 57 | .function("seek", &Demuxer::seek) 58 | .function("read", &Demuxer::read, allow_raw_pointers()) 59 | .function("dump", &Demuxer::dump) 60 | .function("getTimeBase", &Demuxer::getTimeBase) 61 | .function("getMetadata", &Demuxer::getMetadata) 62 | .function("currentTime", &Demuxer::currentTime) 63 | ; 64 | } 65 | 66 | EMSCRIPTEN_BINDINGS(decode) { 67 | class_("Decoder") 68 | .constructor(allow_raw_pointers()) 69 | .constructor() 70 | .property("name", &Decoder::name) 71 | .property("timeBase", &Decoder::timeBase) 72 | .property("dataFormat", &Decoder::dataFormat) 73 | .function("decode", &Decoder::decode, allow_raw_pointers()) 74 | .function("flush", &Decoder::flush, allow_raw_pointers()) 75 | ; 76 | 77 | } 78 | 79 | EMSCRIPTEN_BINDINGS(packet) { 80 | value_object("TimeInfo") 81 | .field("pts", &TimeInfo::pts) 82 | .field("dts", &TimeInfo::dts) 83 | .field("duration", &TimeInfo::duration) 84 | ; 85 | 86 | class_("Packet") 87 | .constructor() 88 | .property("key", &Packet::key) 89 | .property("size", &Packet::size) 90 | .property("streamIndex", &Packet::stream_index) 91 | .function("getData", &Packet::getData) 92 | .function("getTimeInfo", &Packet::getTimeInfo) 93 | .function("setTimeInfo", &Packet::setTimeInfo) 94 | .function("dump", &Packet::dump) 95 | ; 96 | } 97 | 98 | EMSCRIPTEN_BINDINGS(frame) { 99 | value_object("FrameInfo") 100 | .field("format", &FrameInfo::format) 101 | .field("height", &FrameInfo::height) 102 | .field("width", &FrameInfo::width) 103 | .field("channels", &FrameInfo::channels) 104 | .field("sampleRate", &FrameInfo::sample_rate) 105 | .field("nbSamples", &FrameInfo::nb_samples) 106 | .field("channelLayout", &FrameInfo::channel_layout) 107 | ; 108 | 109 | class_("Frame") 110 | .constructor() 111 | .function("getFrameInfo", &Frame::getFrameInfo) 112 | .class_function("inferChannelLayout", &Frame::inferChannelLayout) 113 | .property("key", &Frame::key) 114 | .property("pts", &Frame::doublePTS) 115 | .property("name", &Frame::name) 116 | .function("getPlanes", &Frame::getPlanes) 117 | .function("dump", &Frame::dump) 118 | ; 119 | } 120 | 121 | EMSCRIPTEN_BINDINGS(filter) { 122 | class_("Filterer") 123 | .constructor, std::map, std::map, std::string>() 124 | .function("filter", &Filterer::filter, allow_raw_pointers()) 125 | .function("flush", &Filterer::flush, allow_raw_pointers()) 126 | ; 127 | 128 | class_("BitstreamFilterer") 129 | .constructor() 130 | .function("filter", &BitstreamFilterer::filter, allow_raw_pointers()) 131 | ; 132 | } 133 | 134 | EMSCRIPTEN_BINDINGS(encode) { 135 | value_object("AVRational") 136 | .field("num", &AVRational::num) 137 | .field("den", &AVRational::den) 138 | ; 139 | 140 | class_("Encoder") 141 | .constructor() 142 | .property("timeBase", &Encoder::timeBase) 143 | .property("dataFormat", &Encoder::dataFormat) 144 | .function("encode", &Encoder::encode, allow_raw_pointers()) 145 | .function("flush", &Encoder::flush, allow_raw_pointers()) 146 | ; 147 | } 148 | 149 | EMSCRIPTEN_BINDINGS(muxer) { 150 | class_("Muxer") 151 | .constructor() 152 | .class_function("inferFormatInfo", &Muxer::inferFormatInfo) 153 | .function("dump", &Muxer::dump) 154 | .function("newStreamWithDemuxer", select_overload(&Muxer::newStream), allow_raw_pointers()) 155 | .function("newStreamWithEncoder", select_overload(&Muxer::newStream), allow_raw_pointers()) 156 | .function("newStreamWithInfo", select_overload(&Muxer::newStream), allow_raw_pointers()) 157 | .function("writeHeader", &Muxer::writeHeader) 158 | .function("writeTrailer", &Muxer::writeTrailer) 159 | .function("writeFrame", &Muxer::writeFrame, allow_raw_pointers()) 160 | ; 161 | 162 | value_object("InferredFormatInfo") 163 | .field("format", &InferredFormatInfo::format) 164 | .field("video", &InferredFormatInfo::video) 165 | .field("audio", &InferredFormatInfo::audio) 166 | ; 167 | 168 | value_object("InferredStreamInfo") 169 | .field("codecName", &InferredStreamInfo::codec_name) 170 | .field("format", &InferredStreamInfo::format) 171 | ; 172 | } 173 | 174 | EMSCRIPTEN_BINDINGS(utils) { 175 | emscripten::function("createFrameVector", &createVector); 176 | emscripten::function("createStringStringMap", &createMap); 177 | 178 | register_vector("vector"); 179 | register_vector("vector"); 180 | register_vector("vector"); 181 | register_vector("vector"); 182 | register_vector("vector"); // map.keys() 183 | register_map("MapStringString"); 184 | 185 | emscripten::function("setConsoleLogger", &setConsoleLogger); 186 | } 187 | 188 | #endif -------------------------------------------------------------------------------- /src/cpp/decode.cpp: -------------------------------------------------------------------------------- 1 | #include "decode.h" 2 | 3 | 4 | Decoder::Decoder(Demuxer* demuxer, int stream_index, string name) { 5 | this->_name = name; 6 | auto stream = demuxer->av_stream(stream_index); 7 | auto codecpar = stream->codecpar; 8 | auto codec = avcodec_find_decoder(codecpar->codec_id); 9 | CHECK(codec != NULL, "Could not find input codec"); 10 | codec_ctx = avcodec_alloc_context3(codec); 11 | avcodec_parameters_to_context(codec_ctx, codecpar); 12 | codec_ctx->framerate = av_guess_frame_rate(demuxer->av_format_context(), stream, NULL); 13 | avcodec_open2(codec_ctx, codec, NULL); 14 | } 15 | 16 | Decoder::Decoder(StreamInfo info, string name) { 17 | this->_name = name; 18 | // create codec 19 | auto codec = avcodec_find_decoder(avcodec_descriptor_get_by_name(info.codec_name.c_str())->id); 20 | // auto codec = avcodec_find_decoder_by_name(info.codec_name.c_str()); 21 | CHECK(codec != NULL, "Could not find input codec"); 22 | codec_ctx = avcodec_alloc_context3(codec); 23 | // set parameters 24 | set_avcodec_context_from_streamInfo(info, codec_ctx); 25 | avcodec_open2(codec_ctx, codec, NULL); 26 | } 27 | 28 | std::vector Decoder::decodePacket(Packet* pkt) { 29 | int ret = avcodec_send_packet(codec_ctx, pkt->av_packet()); 30 | // get all the available frames from the decoder 31 | std::vector frames; 32 | 33 | while (1) { 34 | auto frame = new Frame(this->name()); 35 | ret = avcodec_receive_frame(codec_ctx, frame->av_ptr()); 36 | if (ret < 0) { 37 | // those two return values are special and mean there is no output 38 | // frame available, but there were no errors during decoding 39 | delete frame; 40 | if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)) 41 | break; 42 | CHECK(false, "decode frame failed"); 43 | } 44 | frame->av_ptr()->pts = frame->av_ptr()->best_effort_timestamp; 45 | frames.push_back(frame); 46 | } 47 | return frames; 48 | } 49 | 50 | 51 | std::vector Decoder::decode(Packet* pkt) { 52 | // rescale packet from demuxer stream to encoder 53 | av_packet_rescale_ts(pkt->av_packet(), AV_TIME_BASE_Q, codec_ctx->time_base); 54 | auto frames = this->decodePacket(pkt); 55 | // rescale frame to request time_base 56 | for (const auto& f : frames) 57 | f->set_pts(av_rescale_q(f->pts(), codec_ctx->time_base, AV_TIME_BASE_Q)); 58 | 59 | return frames; 60 | } 61 | 62 | -------------------------------------------------------------------------------- /src/cpp/decode.h: -------------------------------------------------------------------------------- 1 | #ifndef DECODE_H 2 | #define DECODE_H 3 | 4 | #include 5 | extern "C" { 6 | #include 7 | #include 8 | } 9 | 10 | #include "utils.h" 11 | #include "frame.h" 12 | #include "packet.h" 13 | #include "demuxer.h" 14 | using namespace std; 15 | 16 | 17 | class Decoder { 18 | AVCodecContext* codec_ctx; 19 | std::string _name; 20 | 21 | public: 22 | Decoder(Demuxer* demuxer, int stream_index, std::string name); 23 | Decoder(StreamInfo info, std::string name); 24 | ~Decoder() { avcodec_free_context(&codec_ctx); }; 25 | std::string name() const { return _name; } 26 | AVRational timeBase() const { return codec_ctx->time_base; } 27 | DataFormat dataFormat() const { return createDataFormat(codec_ctx); } 28 | std::vector decodePacket(Packet* pkt); 29 | std::vector decode(Packet* pkt); 30 | std::vector flush() { 31 | auto pkt = new Packet(); 32 | pkt->av_packet()->data = NULL; 33 | pkt->av_packet()->size = 0; 34 | auto frames = decode(pkt); 35 | delete pkt; 36 | 37 | return frames; 38 | } 39 | }; 40 | 41 | 42 | #endif -------------------------------------------------------------------------------- /src/cpp/demuxer.cpp: -------------------------------------------------------------------------------- 1 | #include "demuxer.h" 2 | 3 | 4 | // Custom reading avio https://www.codeproject.com/Tips/489450/Creating-Custom-FFmpeg-IO-Context 5 | static int read_packet(void *opaque, uint8_t *buf, int buf_size) 6 | { 7 | auto& reader = *reinterpret_cast(opaque); 8 | auto data = val(typed_memory_view(buf_size, buf)); 9 | auto read_size = reader.call("read", data).await().as(); 10 | 11 | if (!read_size) 12 | return AVERROR_EOF; 13 | 14 | return read_size; 15 | } 16 | 17 | /** 18 | * Warning: any function involve this call, will give promise (async). 19 | * Warning: enable asyncify will disable bigInt, so be careful that binding int64_t not allowed 20 | */ 21 | static int64_t seek_for_read(void* opaque, int64_t pos, int whence) { 22 | auto& reader = *reinterpret_cast(opaque); 23 | auto size = (int64_t)reader["size"].as(); 24 | auto offset = (int64_t)reader["offset"].as(); 25 | 26 | switch (whence) { 27 | case AVSEEK_SIZE: 28 | return size; 29 | case SEEK_SET: 30 | if (pos >= size) return AVERROR_EOF; 31 | reader.call("seek", (double)pos).await(); break; 32 | case SEEK_CUR: 33 | pos += offset; 34 | if (pos >= size) return AVERROR_EOF; 35 | reader.call("seek", (double)pos).await(); break; 36 | case SEEK_END: 37 | if (pos >= size) return AVERROR_EOF; 38 | pos = size - pos; 39 | reader.call("seek", (double)pos).await(); break; 40 | default: 41 | CHECK(false, "cannot process seek_for_read"); 42 | } 43 | 44 | return pos; 45 | } 46 | 47 | 48 | void Demuxer::build(val _reader) { 49 | reader = std::move(_reader); // reader will be destroyed at end of this function 50 | auto buffer = (uint8_t*)av_malloc(buf_size); 51 | auto readerPtr = reinterpret_cast(&reader); 52 | if ((int64_t)reader["size"].as() <= 0) 53 | io_ctx = avio_alloc_context(buffer, buf_size, 0, readerPtr, &read_packet, NULL, NULL); 54 | else 55 | io_ctx = avio_alloc_context(buffer, buf_size, 0, readerPtr, &read_packet, NULL, &seek_for_read); 56 | format_ctx->pb = io_ctx; 57 | // open and get metadata 58 | auto ret = avformat_open_input(&format_ctx, NULL, NULL, NULL); 59 | 60 | CHECK(ret == 0, "Could not open input file."); 61 | ret = avformat_find_stream_info(format_ctx, NULL); 62 | CHECK(ret >= 0, "Could not open find stream info."); 63 | // init currentStreamsPTS 64 | for (int i = 0; i < format_ctx->nb_streams; i++) 65 | currentStreamsPTS[format_ctx->streams[i]->index] = 0; 66 | } 67 | 68 | 69 | Packet* Demuxer::read() { 70 | auto pkt = new Packet(); 71 | auto ret = av_read_frame(format_ctx, pkt->av_packet()); 72 | 73 | if (pkt->size() <= 0) return pkt; 74 | // update current stream pts (avoid end of file where pkt is empty with uninit values) 75 | auto av_pkt = pkt->av_packet(); 76 | // convert to microseconds 77 | auto& time_base = format_ctx->streams[pkt->stream_index()]->time_base; 78 | av_packet_rescale_ts(av_pkt, time_base, AV_TIME_BASE_Q); 79 | 80 | auto next_pts = av_pkt->pts + av_pkt->duration; 81 | currentStreamsPTS[pkt->stream_index()] = next_pts / (double)AV_TIME_BASE; 82 | 83 | return pkt; 84 | } -------------------------------------------------------------------------------- /src/cpp/demuxer.h: -------------------------------------------------------------------------------- 1 | #ifndef DEMUXER_H 2 | #define DEMUXER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | extern "C" { 9 | #include 10 | } 11 | 12 | #include "metadata.h" 13 | #include "utils.h" 14 | #include "stream.h" 15 | #include "packet.h" 16 | using namespace emscripten; 17 | 18 | 19 | class Demuxer { 20 | AVFormatContext* format_ctx; 21 | AVIOContext* io_ctx; 22 | std::map currentStreamsPTS; 23 | int buf_size = 32*1024; 24 | val reader; 25 | public: 26 | Demuxer() { 27 | format_ctx = avformat_alloc_context(); 28 | } 29 | /* async */ 30 | void build(val _reader); 31 | 32 | ~Demuxer() { 33 | avformat_close_input(&format_ctx); 34 | if (io_ctx) 35 | av_freep(&io_ctx->buffer); 36 | avio_context_free(&io_ctx); 37 | } 38 | 39 | /* async */ 40 | void seek(int64_t timestamp, int stream_index) { 41 | av_seek_frame(format_ctx, stream_index, timestamp, AVSEEK_FLAG_BACKWARD); 42 | } 43 | 44 | /* async */ 45 | Packet* read(); 46 | 47 | void dump() { 48 | av_dump_format(format_ctx, 0, NULL, 0); 49 | } 50 | 51 | AVRational getTimeBase(int stream_index) { 52 | return format_ctx->streams[stream_index]->time_base; 53 | } 54 | 55 | FormatInfo getMetadata() { 56 | return createFormatInfo(format_ctx); 57 | } 58 | 59 | /* timestamp of current first packet of the stream, which will be parsed next */ 60 | double currentTime(int stream_index) { 61 | CHECK(currentStreamsPTS.count(stream_index) > 0, "stream_index not in valid currentStreamsPTS"); 62 | return currentStreamsPTS[stream_index]; 63 | } 64 | 65 | // only for c++ 66 | AVFormatContext* av_format_context() { return format_ctx; } 67 | AVStream* av_stream(int i) { 68 | CHECK(i >= 0 && i < format_ctx->nb_streams, "get av stream i error"); 69 | return format_ctx->streams[i]; 70 | } 71 | }; 72 | 73 | 74 | #endif -------------------------------------------------------------------------------- /src/cpp/encode.cpp: -------------------------------------------------------------------------------- 1 | #include "encode.h" 2 | 3 | 4 | Encoder::Encoder(StreamInfo info) { 5 | /* codec_ctx.time_base should smaller than 1/sample_rate (maybe change when open...??) 6 | * Because we need high resolution if using audio fifo to encode smaller sample size frame. 7 | */ 8 | if (info.codec_type == "audio") { 9 | AVRational max_time_base = {1, info.sample_rate}; 10 | info.time_base = av_cmp_q(max_time_base, info.time_base) < 0 ? max_time_base : info.time_base; 11 | } 12 | 13 | // use codec id to specified codec name (h264 -> libx264) 14 | auto codec = avcodec_find_encoder(avcodec_descriptor_get_by_name(info.codec_name.c_str())->id); 15 | // auto codec = avcodec_find_encoder_by_name(info.codec_name.c_str()); 16 | 17 | CHECK(codec, "Could not allocate video codec context"); 18 | codec_ctx = avcodec_alloc_context3(codec); 19 | CHECK(codec_ctx, "Could not allocate video codec context"); 20 | set_avcodec_context_from_streamInfo(info, codec_ctx); 21 | /* Allow the use of the experimental encoder. */ 22 | codec_ctx->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL; 23 | auto ret = avcodec_open2(codec_ctx, codec, NULL); 24 | CHECK(ret == 0, "could not open codec"); 25 | // create fifo for audio (after codec_ctx init) 26 | if (codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) 27 | this->fifo = new AudioFrameFIFO(codec_ctx); 28 | } 29 | 30 | 31 | /** 32 | * refer: FFmpeg/doc/examples/encode_video.c 33 | */ 34 | vector Encoder::encodeFrame(Frame* frame) { 35 | auto avframe = frame == NULL ? NULL : frame->av_ptr(); 36 | auto ret = avcodec_send_frame(codec_ctx, avframe); 37 | CHECK(ret >= 0, "Error sending a frame for encoding"); 38 | vector packets; 39 | while (1) { 40 | auto pkt = new Packet(); 41 | ret = avcodec_receive_packet(codec_ctx, pkt->av_packet()); 42 | if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 43 | delete pkt; 44 | break; 45 | } 46 | CHECK(ret >= 0, "Error during encoding"); 47 | packets.push_back(pkt); 48 | } 49 | return packets; 50 | } 51 | 52 | /** 53 | * Audio frame buffer push to fifo first. 54 | * refer: FFmpeg/doc/examples/transcode_aac.c 55 | */ 56 | vector Encoder::encode(Frame* frame) { 57 | // rescale pts 58 | frame->set_pts(av_rescale_q(frame->pts(), AV_TIME_BASE_Q, codec_ctx->time_base)); 59 | 60 | vector outVec; 61 | /* Make sure that there is one frame worth of samples in the FIFO 62 | * buffer so that the encoder can do its work. 63 | * Since the decoder's and the encoder's frame size may differ, we 64 | * need to FIFO buffer to store as many frames worth of input samples 65 | * that they make up at least one frame worth of output samples. 66 | * */ 67 | if (codec_ctx->codec_type == AVMEDIA_TYPE_AUDIO && codec_ctx->frame_size > 0) { 68 | if (frame != NULL) 69 | fifo->push(frame); 70 | /* Read as many samples from the FIFO buffer as required to fill the frame.*/ 71 | while (fifo->size() >= codec_ctx->frame_size || (frame == NULL && fifo->size() > 0)) { 72 | auto out_frame = fifo->pop(codec_ctx, codec_ctx->frame_size); 73 | const auto& pkt_vec = this->encodeFrame(frame != NULL ? out_frame : NULL); 74 | outVec.insert(std::end(outVec), std::begin(pkt_vec), std::end(pkt_vec)); 75 | } 76 | } 77 | else { 78 | const auto& pkt_vec = this->encodeFrame(frame); 79 | outVec.insert(std::end(outVec), std::begin(pkt_vec), std::end(pkt_vec)); 80 | } 81 | // rescale back to base time_base 82 | for (auto p : outVec) 83 | av_packet_rescale_ts(p->av_packet(), codec_ctx->time_base, AV_TIME_BASE_Q); 84 | 85 | return outVec; 86 | } 87 | 88 | -------------------------------------------------------------------------------- /src/cpp/encode.h: -------------------------------------------------------------------------------- 1 | #ifndef ENCODE_H 2 | #define ENCODE_H 3 | 4 | #include 5 | #include 6 | 7 | extern "C" { 8 | #include 9 | } 10 | 11 | 12 | #include "metadata.h" 13 | #include "packet.h" 14 | #include "frame.h" 15 | #include "utils.h" 16 | 17 | 18 | class Encoder { 19 | /** 20 | * @brief encoder for a video/audio stream. 21 | * 22 | */ 23 | AVCodecContext* codec_ctx; 24 | AudioFrameFIFO* fifo = NULL; 25 | 26 | public: 27 | Encoder(StreamInfo info); 28 | ~Encoder() { 29 | if (fifo != NULL) 30 | delete fifo; 31 | avcodec_free_context(&codec_ctx); 32 | }; 33 | AVRational timeBase() const { return codec_ctx->time_base; } 34 | DataFormat dataFormat() const { return createDataFormat(codec_ctx); } 35 | vector encodeFrame(Frame* frame); 36 | vector encode(Frame* frame); 37 | vector flush() { return encode(NULL); } 38 | // c++ only 39 | void setFlags(int flag) { codec_ctx->flags |= flag; } 40 | const AVCodecContext* av_codecContext_ptr() { return codec_ctx; } 41 | }; 42 | 43 | 44 | #endif -------------------------------------------------------------------------------- /src/cpp/filter.cpp: -------------------------------------------------------------------------------- 1 | #include "filter.h" 2 | 3 | 4 | void InOut::addEntry(std::string name, AVFilterContext* filter_ctx, int pad_idx) { 5 | // create and init new entry 6 | auto entry = avfilter_inout_alloc(); 7 | entry->name = strdup(name.c_str()); 8 | entry->filter_ctx = filter_ctx; 9 | entry->pad_idx = 0; 10 | entry->next = NULL; 11 | // add to the tail of entries 12 | if (entries == NULL) entries = entry; 13 | else { 14 | auto e = entries; 15 | while (e->next != NULL) e = e->next; 16 | e->next = entry; 17 | } 18 | } 19 | 20 | 21 | /** 22 | * inParams: map 23 | * outParams: map 24 | * mediaTypes: map 25 | */ 26 | Filterer::Filterer( 27 | map inParams, 28 | map outParams, 29 | map mediaTypes, 30 | string filterSpec 31 | ) { 32 | AVFilterGraph* graph = filterGraph.av_FilterGraph(); 33 | // create input nodes 34 | for (auto const& [id, params] : inParams) { 35 | CHECK(id.length() > 0, "Filterer: buffersrc id should not be empty"); 36 | AVFilterContext *buffersrc_ctx; 37 | const AVFilter *buffersrc = avfilter_get_by_name(mediaTypes[id] == "video" ? "buffer" : "abuffer"); 38 | avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, id.c_str(), params.c_str(), NULL, graph); 39 | outputs.addEntry(id.c_str(), buffersrc_ctx, 0); 40 | buffersrc_ctx_map[id] = buffersrc_ctx; 41 | } 42 | // create end nodes 43 | for (auto const& [id, params] : outParams) { 44 | CHECK(id.length() > 0, "Filterer: buffersink id should not be empty"); 45 | AVFilterContext *buffersink_ctx; 46 | const AVFilter *buffersink = avfilter_get_by_name(mediaTypes[id] == "video" ? "buffersink" : "abuffersink"); 47 | avfilter_graph_create_filter(&buffersink_ctx, buffersink, id.c_str(), NULL, NULL, graph); 48 | // todo... may be set out args 49 | // ret = av_opt_set_int_list(buffersink_ctx, "sample_rates", out_sample_rates, -1, AV_OPT_SEARCH_CHILDREN); 50 | inputs.addEntry(id.c_str(), buffersink_ctx, 0); 51 | buffersink_ctx_map[id] = buffersink_ctx; 52 | } 53 | // create graph and valid 54 | auto ins = inputs.av_filterInOut(); 55 | auto outs = outputs.av_filterInOut(); 56 | auto ret = avfilter_graph_parse_ptr(graph, filterSpec.c_str(), &ins, &outs, NULL); 57 | CHECK(ret >= 0, "cannot parse filter graph"); 58 | ret = avfilter_graph_config(graph, NULL); 59 | CHECK(ret >= 0, "cannot configure graph"); 60 | } 61 | 62 | 63 | /** 64 | * process once 65 | * In/Out frames should all have non-empty Frame::name. 66 | */ 67 | vector Filterer::filter(vector frames) { 68 | std::vector out_frames; 69 | 70 | // At each time, send a frame, and pull frames as much as possible. 71 | for (auto const& frame : frames) { 72 | // feed to graph 73 | const auto& id = frame->name(); 74 | if (buffersrc_ctx_map.count(id) == 0) continue; 75 | auto ctx = buffersrc_ctx_map[id]; 76 | auto ret = av_buffersrc_add_frame_flags(ctx, frame->av_ptr(), AV_BUFFERSRC_FLAG_KEEP_REF); 77 | CHECK(ret >= 0, "Error while feeding the filtergraph"); 78 | // pull filtered frames from each entry of filtergraph outputs 79 | for (auto const& [id, ctx] : buffersink_ctx_map) { 80 | while (1) { 81 | auto out_frame = new Frame(id); 82 | auto ret = av_buffersink_get_frame(ctx, out_frame->av_ptr()); 83 | if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 84 | delete out_frame; 85 | break; 86 | } 87 | CHECK(ret >= 0, "error get filtered frames from buffersink"); 88 | out_frame->av_ptr()->pict_type = AV_PICTURE_TYPE_NONE; 89 | out_frames.push_back(out_frame); 90 | } 91 | } 92 | } 93 | 94 | return out_frames; 95 | } 96 | 97 | 98 | vector Filterer::flush() { 99 | std::vector out_frames; 100 | for (const auto& [id, ctx] : buffersrc_ctx_map) { 101 | auto ret = av_buffersrc_add_frame_flags(ctx, NULL, AV_BUFFERSRC_FLAG_KEEP_REF); 102 | CHECK(ret >= 0, "Error while flushing the filtergraph"); 103 | // pull filtered frames from each entry of filtergraph outputs 104 | for (auto const& [id, ctx] : buffersink_ctx_map) { 105 | while (1) { 106 | auto out_frame = new Frame(id); 107 | auto ret = av_buffersink_get_frame(ctx, out_frame->av_ptr()); 108 | if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 109 | delete out_frame; 110 | break; 111 | } 112 | CHECK(ret >= 0, "error get filtered frames from buffersink"); 113 | out_frame->av_ptr()->pict_type = AV_PICTURE_TYPE_NONE; 114 | out_frames.push_back(out_frame); 115 | } 116 | } 117 | } 118 | 119 | return out_frames; 120 | } 121 | 122 | // BitstreamFilterer implementation 123 | void BitstreamFilterer::filter(Packet* packet) { 124 | int ret = av_bsf_send_packet(bsf_ctx, packet->av_packet()); 125 | CHECK(ret >= 0, "Error sending packet to bitstream filter"); 126 | ret = av_bsf_receive_packet(bsf_ctx, packet->av_packet()); 127 | CHECK(ret >= 0, "Error receiving packet from bitstream filter"); 128 | } 129 | -------------------------------------------------------------------------------- /src/cpp/filter.h: -------------------------------------------------------------------------------- 1 | #ifndef FILTER_H 2 | #define FILTER_H 3 | 4 | #include 5 | #include 6 | extern "C" { 7 | #include 8 | #include 9 | #include 10 | #include 11 | } 12 | 13 | #include "frame.h" 14 | #include "stream.h" 15 | #include "demuxer.h" 16 | #include "muxer.h" 17 | using namespace std; 18 | 19 | 20 | class FilterGraph { 21 | AVFilterGraph* graph; 22 | 23 | public: 24 | FilterGraph() { graph = avfilter_graph_alloc(); } 25 | ~FilterGraph() { avfilter_graph_free(&graph); } 26 | 27 | AVFilterGraph* av_FilterGraph() { return graph; } 28 | }; 29 | 30 | 31 | class InOut { 32 | AVFilterInOut *entries = NULL; 33 | public: 34 | AVFilterInOut *av_filterInOut() { return entries; } 35 | void addEntry(string name, AVFilterContext* filter_ctx, int pad_idx); 36 | }; 37 | 38 | 39 | class Filterer { 40 | FilterGraph filterGraph; 41 | InOut inputs; 42 | InOut outputs; 43 | map buffersrc_ctx_map; 44 | map buffersink_ctx_map; 45 | 46 | public: 47 | /** 48 | * @brief Build a filter graph, either video or audio. 49 | * 50 | * @param type `video` or `audio` 51 | * @param inParams map 52 | * @param outParams map 53 | * @param filterSpec 54 | */ 55 | Filterer(map inParams, map outParams, map mediaTypes, string filterSpec); 56 | vector filter(vector); 57 | vector flush(); 58 | }; 59 | 60 | 61 | class BitstreamFilterer { 62 | AVBSFContext* bsf_ctx; 63 | 64 | public: 65 | BitstreamFilterer(string filter_name, Demuxer* demuxer, int in_stream_index, Muxer* muxer, int out_stream_index) { 66 | const AVBitStreamFilter* bsf = av_bsf_get_by_name(filter_name.c_str()); 67 | CHECK(bsf != NULL, "Could not find bitstream filter"); 68 | av_bsf_alloc(bsf, &bsf_ctx); 69 | 70 | // copy codec parameters 71 | auto istream = demuxer->av_stream(in_stream_index); 72 | auto ostream = muxer->av_stream(out_stream_index); 73 | auto ret = avcodec_parameters_copy(bsf_ctx->par_in, istream->codecpar); 74 | CHECK(ret >= 0, "Failed to copy codec parameters to bitstream filter"); 75 | ret = avcodec_parameters_copy(bsf_ctx->par_out, ostream->codecpar); 76 | CHECK(ret >= 0, "Failed to copy codec parameters to bitstream filter"); 77 | 78 | ret = av_bsf_init(bsf_ctx); 79 | CHECK(ret >= 0, "Failed to initialize bitstream filter"); 80 | } 81 | ~BitstreamFilterer() { av_bsf_free(&bsf_ctx); } 82 | 83 | AVBSFContext* av_bsfContext() { return bsf_ctx; } 84 | void filter(Packet* packet); 85 | }; 86 | 87 | 88 | #endif -------------------------------------------------------------------------------- /src/cpp/frame.cpp: -------------------------------------------------------------------------------- 1 | #include "frame.h" 2 | 3 | 4 | 5 | Frame::Frame(FrameInfo info, double pts, std::string name) { 6 | this->_name = name; 7 | av_frame = av_frame_alloc(); 8 | auto isVideo = info.height > 0 && info.width > 0; 9 | if (isVideo) { 10 | av_frame->format = av_get_pix_fmt(info.format.c_str()); 11 | av_frame->height = info.height; 12 | av_frame->width = info.width; 13 | auto ret = av_frame_get_buffer(av_frame, 0); 14 | CHECK(ret >= 0, "Could not allocate output frame samples (error '%s')"); 15 | } 16 | else { 17 | // if channel_layout given, infer default channel_layout given channels. 18 | auto channel_layout = info.channel_layout != "" ? 19 | av_get_channel_layout(info.channel_layout.c_str()) : 20 | av_get_default_channel_layout(info.channels); 21 | 22 | this->audio_reinit( 23 | av_get_sample_fmt(info.format.c_str()), 24 | info.sample_rate, 25 | channel_layout, 26 | info.nb_samples 27 | ); 28 | } 29 | av_frame->pts = (int64_t)pts; 30 | } 31 | 32 | 33 | FrameInfo Frame::getFrameInfo() { 34 | auto isVideo = av_frame->height > 0 && av_frame->width > 0; 35 | auto format = isVideo ? 36 | av_get_pix_fmt_name((AVPixelFormat)av_frame->format) : 37 | av_get_sample_fmt_name((AVSampleFormat)av_frame->format); 38 | 39 | return { 40 | .format = format, 41 | .height = av_frame->height, 42 | .width = av_frame->width, 43 | .sample_rate = av_frame->sample_rate, 44 | .channels = av_frame->channels, 45 | .channel_layout = get_channel_layout_name(av_frame->channels, av_frame->channel_layout), 46 | .nb_samples = av_frame->nb_samples 47 | }; 48 | } 49 | 50 | void Frame::audio_reinit(AVSampleFormat sample_fmt, int sample_rate, uint64_t channel_layout, int nb_samples) { 51 | av_frame_unref(av_frame); 52 | av_frame->channel_layout = channel_layout; 53 | 54 | av_frame->format = sample_fmt; 55 | av_frame->sample_rate = sample_rate; 56 | av_frame->nb_samples = nb_samples; 57 | auto ret = av_frame_get_buffer(av_frame, 0); 58 | CHECK(ret >= 0, "Could not allocate output frame samples"); 59 | } 60 | 61 | 62 | void AudioFrameFIFO::push(Frame* in_frame) { 63 | auto num_sample = in_frame->av_ptr()->nb_samples; 64 | auto fifo_size = this->size() + num_sample; 65 | if (fifo_size <= 0) return; 66 | /* Make the FIFO as large as it needs to be to hold both, the old and the new samples. */ 67 | auto ret = av_audio_fifo_realloc(fifo, fifo_size); 68 | CHECK(ret >= 0, "Could not reallocate FIFO"); 69 | /* Store the new samples in the FIFO buffer. */ 70 | auto num_writes = av_audio_fifo_write(fifo, (void **)in_frame->av_ptr()->data, num_sample); 71 | CHECK(num_writes == num_sample, "Could not write data to FIFO\n"); 72 | } 73 | 74 | Frame* AudioFrameFIFO::pop(AVCodecContext* codec_ctx, int request_size) { 75 | const int frame_size = FFMIN(this->size(), request_size); 76 | // release previous buffer and create new buffer for current frame_size 77 | out_frame.audio_reinit(codec_ctx->sample_fmt, codec_ctx->sample_rate, codec_ctx->channel_layout, frame_size); 78 | 79 | auto write_size = av_audio_fifo_read(fifo, (void **)out_frame.av_ptr()->data, frame_size); 80 | CHECK(frame_size == write_size, "Could not read data from FIFO"); 81 | 82 | auto pts = this->acc_samples * (double)this->sample_duration.num / this->sample_duration.den; 83 | out_frame.set_pts((int64_t)std::round(pts)); 84 | this->acc_samples += out_frame.av_ptr()->nb_samples; 85 | 86 | return &out_frame; 87 | } -------------------------------------------------------------------------------- /src/cpp/frame.h: -------------------------------------------------------------------------------- 1 | #ifndef FRAME_H 2 | #define FRAME_H 3 | 4 | #include 5 | #include 6 | extern "C" { 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | } 14 | 15 | #include "utils.h" 16 | using namespace emscripten; 17 | 18 | 19 | struct FrameInfo { 20 | std::string format; 21 | int height; 22 | int width; 23 | int sample_rate; 24 | int channels; 25 | std::string channel_layout; 26 | int nb_samples; 27 | }; 28 | 29 | class Frame { 30 | AVFrame* av_frame = NULL; 31 | int align = 32; 32 | std::string _name; // streamId 33 | public: 34 | Frame() { 35 | av_frame = av_frame_alloc(); // don't remove alloc (called by Class default constructor) 36 | } 37 | Frame(std::string name) { 38 | this->_name = name; 39 | av_frame = av_frame_alloc(); 40 | } 41 | Frame(FrameInfo info, double pts, std::string name); 42 | ~Frame() { av_frame_free(&av_frame); } 43 | 44 | FrameInfo getFrameInfo(); 45 | bool key() const { return av_frame->key_frame; } 46 | double doublePTS() const { return av_frame->pts; } 47 | std::string name() const { return this->_name; } 48 | int64_t pts() const { return av_frame->pts; } 49 | void set_pts(int64_t pts) { av_frame->pts = pts; } 50 | static std::string inferChannelLayout(int channels) { 51 | return get_channel_layout_name(channels, 0); 52 | } 53 | 54 | void audio_reinit(AVSampleFormat sample_fmt, int sample_rate, uint64_t channel_layout, int nb_samples); 55 | 56 | std::vector getPlanes() { 57 | std::vector data; 58 | auto isVideo = av_frame->height > 0 && av_frame->width > 0; 59 | 60 | if (isVideo) { 61 | size_t sizes[4] = {0}; 62 | // video frame only has max 4 planes 63 | av_image_fill_plane_sizes( 64 | sizes, (AVPixelFormat)av_frame->format, av_frame->height, (ptrdiff_t*)av_frame->linesize); 65 | for (int i = 0; i < 4; i++) { 66 | if (sizes[i] <= 0) break; 67 | auto plane = val(typed_memory_view(sizes[i], av_frame->data[i])); 68 | data.push_back(plane); 69 | } 70 | } 71 | else { 72 | // audio frame may has >8 planes (extended_data) 73 | auto planar = av_sample_fmt_is_planar((AVSampleFormat)av_frame->format); 74 | auto planes = planar ? av_frame->channels : 1; 75 | for (int i = 0; i < planes; i++) { 76 | auto size = av_samples_get_buffer_size( 77 | &av_frame->linesize[0], av_frame->channels, 78 | av_frame->nb_samples, (AVSampleFormat)av_frame->format, 0); 79 | auto plane = val(typed_memory_view((size_t)av_frame->linesize[0], av_frame->extended_data[i])); 80 | CHECK(size >= 0, "failed on av_samples_get_buffer_size"); 81 | data.push_back(plane); 82 | } 83 | } 84 | 85 | return data; 86 | } 87 | 88 | void dump() { 89 | auto& time_base = av_frame->time_base; 90 | printf("Frame (pts:%s pts_time:%s)\n", 91 | av_ts2str(av_frame->pts), av_ts2timestr(av_frame->pts, &time_base) 92 | ); 93 | } 94 | 95 | AVFrame* av_ptr() { return av_frame; }; 96 | }; 97 | 98 | 99 | class AudioFrameFIFO { 100 | AVAudioFifo* fifo; 101 | Frame out_frame; 102 | int64_t acc_samples = 0; 103 | AVRational sample_duration; // number of unit per audio sample 104 | 105 | public: 106 | AudioFrameFIFO(AVCodecContext* codec_ctx) { 107 | fifo = av_audio_fifo_alloc(codec_ctx->sample_fmt, codec_ctx->channels, 1); 108 | CHECK(fifo != NULL, "Could not allocate FIFO"); 109 | this->sample_duration = {codec_ctx->time_base.den, codec_ctx->time_base.num * codec_ctx->sample_rate}; 110 | } 111 | ~AudioFrameFIFO() { 112 | av_audio_fifo_free(fifo); 113 | } 114 | 115 | int size() const { return av_audio_fifo_size(fifo); } 116 | 117 | void push(Frame* in_frame); 118 | Frame* pop(AVCodecContext* codec_ctx, int request_size); 119 | }; 120 | 121 | 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/cpp/metadata.cpp: -------------------------------------------------------------------------------- 1 | #include "metadata.h" 2 | 3 | /* from timestamp in time_base to seconds */ 4 | inline double toSeconds(int64_t time_ts, AVRational& time_base) { 5 | return time_ts != AV_NOPTS_VALUE ? 6 | time_ts * (double)time_base.num / time_base.den : 0; 7 | } 8 | 9 | StreamInfo createStreamInfo(AVFormatContext* format_ctx, AVStream* s) { 10 | StreamInfo info; 11 | info.index = s->index; 12 | auto par = s->codecpar; 13 | info.time_base = s->time_base; 14 | info.bit_rate = par->bit_rate; 15 | info.start_time = toSeconds(s->start_time, s->time_base); 16 | info.duration = toSeconds(s->duration, s->time_base); 17 | // info.codec_name = avcodec_find_decoder(par->codec_id)->name; 18 | info.codec_name = avcodec_descriptor_get(par->codec_id)->name; 19 | info.extraData = emscripten::val(emscripten::typed_memory_view(par->extradata_size, par->extradata)); 20 | 21 | if (par->codec_type == AVMEDIA_TYPE_VIDEO) { 22 | info.codec_type = "video"; 23 | info.width = par->width; 24 | info.height = par->height; 25 | info.frame_rate = av_q2d(av_guess_frame_rate(format_ctx, s, NULL)); 26 | info.sample_aspect_ratio = s->sample_aspect_ratio; 27 | info.format = av_get_pix_fmt_name((AVPixelFormat)par->format); 28 | } 29 | else if (par->codec_type == AVMEDIA_TYPE_AUDIO) { 30 | info.codec_type = "audio"; 31 | info.sample_rate = par->sample_rate; 32 | info.channels = par->channels; 33 | info.format = av_get_sample_fmt_name((AVSampleFormat)par->format); 34 | info.channel_layout = get_channel_layout_name(par->channels, par->channel_layout); 35 | 36 | } 37 | 38 | return info; 39 | } 40 | 41 | void set_avstream_from_streamInfo(AVStream* s, StreamInfo& info) { 42 | auto par = s->codecpar; 43 | s->time_base = info.time_base; 44 | par->bit_rate = info.bit_rate; 45 | // par->codec_id = avcodec_find_encoder_by_name(info.codec_name.c_str())->id; 46 | par->codec_id = avcodec_descriptor_get_by_name(info.codec_name.c_str())->id; 47 | 48 | if (info.codec_type == "video") { 49 | par->codec_type = AVMEDIA_TYPE_VIDEO; 50 | par->height = info.height; 51 | par->width = info.width; 52 | s->sample_aspect_ratio = info.sample_aspect_ratio; 53 | par->format = av_get_pix_fmt(info.format.c_str()); 54 | } 55 | else if (info.codec_type == "audio") { 56 | par->codec_type = AVMEDIA_TYPE_AUDIO; 57 | par->sample_rate = info.sample_rate; 58 | par->channels = info.channels; 59 | par->format = av_get_sample_fmt(info.format.c_str()); 60 | par->channel_layout = av_get_channel_layout(info.channel_layout.c_str()); 61 | } 62 | } 63 | 64 | // /** 65 | // * @param num_list ended with 0 66 | // * @return index of num_list 67 | // */ 68 | // int find_nearest_number(int num, const int* num_list) { 69 | // int id = 0, diff = abs(num - num_list[0]); 70 | // for (int i = 1; num_list[i] != 0; i++) { 71 | // id = abs(num - num_list[i]) < diff ? i : id; 72 | // } 73 | // return id; 74 | // } 75 | 76 | void set_avcodec_context_from_streamInfo(StreamInfo& info, AVCodecContext* ctx) { 77 | ctx->bit_rate = info.bit_rate; 78 | ctx->time_base = info.time_base; 79 | auto codec = ctx->codec; 80 | if (info.codec_type == "video") { 81 | ctx->codec_type = AVMEDIA_TYPE_VIDEO; 82 | ctx->width = info.width; 83 | ctx->height = info.height; 84 | ctx->sample_aspect_ratio = info.sample_aspect_ratio; 85 | ctx->pix_fmt = av_get_pix_fmt(info.format.c_str()); 86 | // find proper frame_rate 87 | auto frame_rate = av_d2q(info.frame_rate, INT_MAX); 88 | // if (codec->supported_framerates != NULL) { 89 | // auto id = av_find_nearest_q_idx(frame_rate, codec->supported_framerates); 90 | // ctx->framerate = codec->supported_framerates[id]; 91 | // } 92 | // else 93 | ctx->framerate = frame_rate; 94 | } 95 | else if (info.codec_type == "audio") { 96 | ctx->codec_type = AVMEDIA_TYPE_AUDIO; 97 | ctx->sample_fmt = av_get_sample_fmt(info.format.c_str()); 98 | ctx->channel_layout = av_get_channel_layout(info.channel_layout.c_str()); 99 | ctx->channels = av_get_channel_layout_nb_channels(ctx->channel_layout); 100 | // // find proper sample_rate 101 | // if (codec->supported_samplerates != NULL) { 102 | // auto id = find_nearest_number(info.sample_rate, codec->supported_samplerates); 103 | // ctx->sample_rate = codec->supported_samplerates[id]; 104 | // } 105 | // else 106 | ctx->sample_rate = info.sample_rate; 107 | } 108 | } 109 | 110 | DataFormat createDataFormat(AVCodecContext* ctx) { 111 | DataFormat df; 112 | if (ctx->codec_type == AVMEDIA_TYPE_VIDEO) { 113 | df.format = av_get_pix_fmt_name(ctx->pix_fmt); 114 | } 115 | else if (ctx->codec_type == AVMEDIA_TYPE_AUDIO) { 116 | df.sampleRate = ctx->sample_rate; 117 | df.channels = ctx->channels; 118 | df.format = av_get_sample_fmt_name(ctx->sample_fmt); 119 | df.channelLayout = get_channel_layout_name(ctx->channels, ctx->channel_layout); 120 | } 121 | 122 | return df; 123 | } 124 | 125 | 126 | FormatInfo createFormatInfo(AVFormatContext* p) { 127 | FormatInfo info; 128 | info.format_name = p->iformat->name; 129 | info.bit_rate = p->bit_rate; 130 | auto time_base_q = AV_TIME_BASE_Q; 131 | info.duration = toSeconds(p->duration, time_base_q); 132 | for (int i = 0; i < p->nb_streams; i++) { 133 | auto codec_type = p->streams[i]->codecpar->codec_type; 134 | if (codec_type == AVMEDIA_TYPE_VIDEO || codec_type == AVMEDIA_TYPE_AUDIO) 135 | info.streamInfos.push_back(createStreamInfo(p, p->streams[i])); 136 | } 137 | 138 | return info; 139 | } 140 | -------------------------------------------------------------------------------- /src/cpp/metadata.h: -------------------------------------------------------------------------------- 1 | #ifndef METADATA_H 2 | #define METADATA_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "utils.h" 10 | 11 | extern "C" { 12 | #include 13 | #include 14 | #include 15 | #include 16 | } 17 | using namespace std; 18 | 19 | 20 | struct StreamInfo { 21 | int index; 22 | AVRational time_base; 23 | 24 | // int64_t bit_rate; 25 | // int64_t start_time; 26 | int bit_rate; 27 | double start_time; 28 | double duration; 29 | 30 | string codec_type; 31 | string codec_name; 32 | string format; 33 | emscripten::val extraData; 34 | // video 35 | int width; 36 | int height; 37 | double frame_rate; 38 | AVRational sample_aspect_ratio; 39 | // audio 40 | int sample_rate; 41 | string channel_layout; 42 | int channels; 43 | 44 | }; 45 | 46 | 47 | StreamInfo createStreamInfo(AVFormatContext* p, AVStream* s); 48 | void set_avstream_from_streamInfo(AVStream* stream, StreamInfo& info); 49 | void set_avcodec_context_from_streamInfo(StreamInfo& info, AVCodecContext* ctx); 50 | 51 | 52 | struct DataFormat { 53 | string format; // AVSampleFormat / AVPixelFormat 54 | string channelLayout; 55 | int channels; 56 | int sampleRate; 57 | }; 58 | 59 | DataFormat createDataFormat(AVCodecContext* ctx); 60 | 61 | 62 | struct FormatInfo { 63 | std::string format_name; 64 | // int64_t bit_rate; 65 | int bit_rate; 66 | double duration; 67 | vector streamInfos; 68 | }; 69 | 70 | FormatInfo createFormatInfo(AVFormatContext* p); 71 | 72 | #endif -------------------------------------------------------------------------------- /src/cpp/muxer.cpp: -------------------------------------------------------------------------------- 1 | #include "muxer.h" 2 | 3 | 4 | 5 | // Custom writing avio https://ffmpeg.org/pipermail/ffmpeg-devel/2014-November/165014.html 6 | static int write_packet(void* opaque, uint8_t* buf, int buf_size) { 7 | auto writer = *reinterpret_cast(opaque); 8 | auto data = val(typed_memory_view(buf_size, buf)); 9 | writer.call("write", data); 10 | return buf_size; 11 | 12 | } 13 | 14 | static int64_t seek_for_write(void* opaque, int64_t pos, int whence) { 15 | auto writer = *reinterpret_cast(opaque); 16 | 17 | switch (whence) { 18 | case SEEK_SET: 19 | writer.call("seek", (double)pos); break; 20 | case SEEK_CUR: 21 | pos += (int64_t)writer["offset"].as(); 22 | writer.call("seek", (double)pos); break; 23 | default: 24 | CHECK(false, "cannot process seek_for_read"); 25 | } 26 | 27 | return pos; 28 | } 29 | 30 | 31 | Muxer::Muxer(string format, val _writer) { 32 | writer = std::move(_writer); // writer will be destroyed at end of this function 33 | auto writerPtr = reinterpret_cast(&writer); 34 | // create buffer for writing 35 | auto buffer = (uint8_t*)av_malloc(buf_size); 36 | io_ctx = avio_alloc_context(buffer, buf_size, 1, writerPtr, NULL, write_packet, seek_for_write); 37 | avformat_alloc_output_context2(&format_ctx, NULL, format.c_str(), NULL); 38 | CHECK(format_ctx != NULL, "Could not create output format context"); 39 | format_ctx->pb = io_ctx; 40 | format_ctx->flags |= AVFMT_FLAG_CUSTOM_IO; 41 | } 42 | 43 | 44 | InferredFormatInfo Muxer::inferFormatInfo(string format_name, string filename) { 45 | auto format = av_guess_format(format_name.c_str(), filename.c_str(), NULL); 46 | if (format == NULL) 47 | // maybe format_name is extension of the filename 48 | format = av_guess_format("", (filename + "." + format_name).c_str(), NULL); 49 | auto video_codec = avcodec_find_encoder(format->video_codec); 50 | auto audio_codec = avcodec_find_encoder(format->audio_codec); 51 | 52 | InferredStreamInfo videoInfo = { 53 | .codec_name = avcodec_descriptor_get(video_codec->id)->name, 54 | .format = av_get_pix_fmt_name(*video_codec->pix_fmts), 55 | }; 56 | InferredStreamInfo audioInfo = { 57 | .codec_name = avcodec_descriptor_get(audio_codec->id)->name, 58 | .format = av_get_sample_fmt_name(*audio_codec->sample_fmts) 59 | }; 60 | 61 | return { 62 | .format = format->name, 63 | .video = videoInfo, 64 | .audio = audioInfo }; 65 | } 66 | 67 | 68 | void Muxer::writeFrame(Packet* packet, int stream_i) { 69 | auto av_pkt = packet->av_packet(); 70 | CHECK(stream_i >= 0 && stream_i < streams.size(), "stream_index of packet not in valid streams"); 71 | auto av_stream = streams[stream_i]->av_stream_ptr(); 72 | // rescale packet to muxer stream 73 | av_packet_rescale_ts(av_pkt, AV_TIME_BASE_Q, av_stream->time_base); 74 | av_pkt->stream_index = stream_i; 75 | 76 | int ret = av_interleaved_write_frame(format_ctx, av_pkt); 77 | CHECK(ret >= 0, "interleave write frame error"); 78 | } -------------------------------------------------------------------------------- /src/cpp/muxer.h: -------------------------------------------------------------------------------- 1 | #ifndef MUXER_H 2 | #define MUXER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include "stream.h" 8 | #include "packet.h" 9 | #include "demuxer.h" 10 | 11 | extern "C" { 12 | #include 13 | #include 14 | } 15 | 16 | #include "encode.h" 17 | #include "utils.h" 18 | using namespace emscripten; 19 | 20 | 21 | struct InferredStreamInfo { 22 | string codec_name; 23 | // AVRational time_base; 24 | string format; 25 | }; 26 | 27 | struct InferredFormatInfo { 28 | string format; 29 | InferredStreamInfo video; 30 | InferredStreamInfo audio; 31 | }; 32 | 33 | 34 | class Muxer { 35 | AVFormatContext* format_ctx; 36 | AVIOContext* io_ctx; 37 | std::vector streams; 38 | int buf_size = 32*1024; 39 | val writer; 40 | 41 | public: 42 | Muxer(string format, val _writer); 43 | ~Muxer() { 44 | for (const auto& s : streams) 45 | delete s; 46 | avformat_free_context(format_ctx); 47 | if (io_ctx) 48 | av_freep(&io_ctx->buffer); 49 | avio_context_free(&io_ctx); 50 | } 51 | 52 | static InferredFormatInfo inferFormatInfo(string format_name, string filename); 53 | 54 | void dump() { 55 | av_dump_format(format_ctx, 0, "", 1); 56 | } 57 | 58 | // transmux 59 | void newStream(Demuxer* demuxer, int streamIndex) { 60 | auto stream = demuxer->av_stream(streamIndex); 61 | streams.push_back(new Stream(format_ctx, stream)); 62 | } 63 | 64 | // transcode 65 | void newStream(Encoder* encoder) { 66 | /* Some formats want stream headers to be separate. */ 67 | if (format_ctx->oformat->flags & AVFMT_GLOBALHEADER) 68 | encoder->setFlags(AV_CODEC_FLAG_GLOBAL_HEADER); 69 | 70 | streams.push_back(new Stream(format_ctx, encoder)); 71 | } 72 | 73 | // for not FF.encoder (e.g. WebCodecs encoder) 74 | void newStream(StreamInfo streamInfo) { 75 | streams.push_back(new Stream(format_ctx, streamInfo)); 76 | } 77 | 78 | void writeHeader() { 79 | auto ret = avformat_write_header(format_ctx, NULL); 80 | CHECK(ret >= 0, "Error occurred when opening output file"); 81 | } 82 | void writeTrailer() { 83 | auto ret = av_write_trailer(format_ctx); 84 | CHECK(ret == 0, "Error when writing trailer"); 85 | } 86 | void writeFrame(Packet* packet, int stream_i); 87 | 88 | // only for C++ 89 | AVStream* av_stream(int index) { 90 | CHECK(index >= 0 && index < format_ctx->nb_streams, "index out of range"); 91 | return format_ctx->streams[index]; 92 | } 93 | }; 94 | 95 | 96 | #endif -------------------------------------------------------------------------------- /src/cpp/packet.h: -------------------------------------------------------------------------------- 1 | #ifndef PACKET_H 2 | #define PACKET_H 3 | 4 | #include 5 | #include 6 | extern "C" { 7 | #include 8 | #include 9 | } 10 | 11 | 12 | /** 13 | * all int64_t should be converted double (otherwise will become int32) 14 | */ 15 | struct TimeInfo { 16 | double pts; 17 | double dts; 18 | double duration; 19 | }; 20 | 21 | 22 | class Packet { 23 | AVPacket* packet; 24 | public: 25 | Packet() { packet = av_packet_alloc(); } 26 | Packet(int bufSize, TimeInfo info) { 27 | packet = av_packet_alloc(); 28 | av_new_packet(packet, bufSize); 29 | packet->pts = info.pts; 30 | packet->dts = info.dts; 31 | packet->duration = info.duration; 32 | } 33 | 34 | ~Packet() { av_packet_free(&packet); }; 35 | 36 | bool key() const { return packet->flags | AV_PKT_FLAG_KEY; } 37 | 38 | int size() const { return packet->size; } 39 | 40 | int stream_index() const { return packet->stream_index; } 41 | 42 | emscripten::val getData() { 43 | return emscripten::val(emscripten::typed_memory_view(packet->size, packet->data)); // check length of data 44 | } 45 | 46 | TimeInfo getTimeInfo() { 47 | return {.pts = (double)packet->pts, .dts = (double)packet->dts, .duration = (double)packet->duration}; 48 | } 49 | 50 | void setTimeInfo(TimeInfo info) { 51 | packet->pts = info.pts; 52 | packet->dts = info.dts; 53 | packet->duration = info.duration; 54 | } 55 | 56 | void dump() { 57 | auto& time_base = packet->time_base; 58 | printf("Packet (pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d)\n", 59 | av_ts2str(packet->pts), av_ts2timestr(packet->pts, &time_base), 60 | av_ts2str(packet->dts), av_ts2timestr(packet->dts, &time_base), 61 | av_ts2str(packet->duration), av_ts2timestr(packet->duration, &time_base), 62 | packet->stream_index 63 | ); 64 | } 65 | 66 | AVPacket* av_packet() { return packet; } 67 | }; 68 | 69 | #endif -------------------------------------------------------------------------------- /src/cpp/resample.h: -------------------------------------------------------------------------------- 1 | 2 | extern "C" { 3 | #include 4 | #include 5 | } 6 | 7 | #include "frame.h" 8 | #include "utils.h" 9 | 10 | 11 | class Resampler { 12 | SwrContext* resample_ctx; 13 | Frame out_frame; 14 | 15 | public: 16 | Resampler(AVCodecContext* from_codec_ctx, AVCodecContext* to_codec_ctx) { 17 | 18 | resample_ctx = swr_alloc_set_opts(NULL, 19 | av_get_default_channel_layout(to_codec_ctx->channels), 20 | to_codec_ctx->sample_fmt, 21 | to_codec_ctx->sample_rate, 22 | av_get_default_channel_layout(from_codec_ctx->channels), 23 | from_codec_ctx->sample_fmt, 24 | from_codec_ctx->sample_rate, 25 | 0, NULL); 26 | 27 | CHECK(resample_ctx != NULL, "Could not allocate resample context"); 28 | 29 | /* 30 | * Perform a sanity check so that the number of converted samples is 31 | * not greater than the number of samples to be converted. 32 | * If the sample rates differ, this case has to be handled differently 33 | */ 34 | CHECK(to_codec_ctx->sample_rate == from_codec_ctx->sample_rate, "sample rate differ"); 35 | 36 | /* Open the resampler with the specified parameters. */ 37 | auto ret = swr_init(resample_ctx); 38 | CHECK(ret >= 0, "Could not open resample context"); 39 | } 40 | 41 | ~Resampler() { swr_free(&resample_ctx); } 42 | 43 | /* Convert the samples using the resampler. */ 44 | Frame* convert(Frame* frame) { 45 | auto av_frame = frame->av_ptr(); 46 | // out_frame.audio_reinit(); 47 | // auto ret = swr_convert( 48 | // resample_ctx, out_frame->data, out_frame->nb_samples, av_frame->extended_data, av_frame->nb_samples); 49 | // CHECK(ret >= 0, "Could not convert input samples"); 50 | 51 | return &out_frame; 52 | } 53 | }; -------------------------------------------------------------------------------- /src/cpp/stream.h: -------------------------------------------------------------------------------- 1 | #ifndef STREAM_H 2 | #define STREAM_H 3 | 4 | // #include 5 | #include "encode.h" 6 | #include "utils.h" 7 | #include "metadata.h" 8 | using namespace std; 9 | 10 | 11 | 12 | class Stream { 13 | AVStream* av_stream; 14 | 15 | public: 16 | 17 | /** 18 | * refer: FFmpeg/doc/examples/remuxing.c 19 | */ 20 | Stream(AVFormatContext* format_ctx, AVStream* avstream) { 21 | av_stream = avformat_new_stream(format_ctx, NULL); 22 | auto ret = avcodec_parameters_copy(av_stream->codecpar, avstream->codecpar); 23 | CHECK(ret >= 0, "Failed to copy stream parameters to output stream."); 24 | av_stream->codecpar->codec_tag = 0; 25 | } 26 | 27 | Stream(AVFormatContext* format_ctx, Encoder* encoder) { 28 | auto av_encoder = encoder->av_codecContext_ptr(); 29 | av_stream = avformat_new_stream(format_ctx, NULL); 30 | auto ret = avcodec_parameters_from_context(av_stream->codecpar, av_encoder); 31 | CHECK(ret >= 0, "Failed to copy encoder parameters to output stream."); 32 | av_stream->time_base = av_encoder->time_base; 33 | // av_stream->id = format_ctx->nb_streams - 1; 34 | } 35 | 36 | Stream(AVFormatContext* format_ctx, StreamInfo& info) { 37 | av_stream = avformat_new_stream(format_ctx, NULL); 38 | set_avstream_from_streamInfo(av_stream, info); 39 | // av_stream->codecpar->codec_tag = avcodec_pix_fmt_to_codec_tag((AVPixelFormat)av_stream->codecpar->format); 40 | } 41 | 42 | ~Stream() { av_free(av_stream); } 43 | 44 | AVStream* av_stream_ptr() { return av_stream; } 45 | 46 | }; 47 | 48 | 49 | #endif -------------------------------------------------------------------------------- /src/cpp/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | 3 | 4 | static void log_callback(void *ptr, int level, const char *fmt, va_list vl) { 5 | // skip when less important messages 6 | if (level > av_log_get_level()) return; 7 | 8 | va_list vl2; 9 | char line[1024]; 10 | static int print_prefix = 1; 11 | va_copy(vl2, vl); 12 | av_log_format_line(ptr, level, fmt, vl2, line, sizeof(line), &print_prefix); 13 | va_end(vl2); 14 | string msg = line; 15 | auto console = emscripten::val::global("console"); 16 | 17 | if (level <= AV_LOG_ERROR) 18 | console.call("error", msg); 19 | else if (level <= AV_LOG_WARNING) 20 | console.call("warn", msg); 21 | else 22 | console.call("log", msg); 23 | } 24 | 25 | void setConsoleLogger(bool verbose) { 26 | av_log_set_level(verbose ? AV_LOG_VERBOSE : AV_LOG_INFO); 27 | av_log_set_callback(log_callback); 28 | } 29 | 30 | 31 | 32 | string get_channel_layout_name(int channels, uint64_t channel_layout) { 33 | if (!channel_layout) 34 | channel_layout = (uint64_t)av_get_default_channel_layout(channels); 35 | int buf_size = 256; 36 | char buf[buf_size]; 37 | av_get_channel_layout_string(buf, buf_size, channels, channel_layout); 38 | return buf; 39 | } 40 | -------------------------------------------------------------------------------- /src/cpp/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_H 2 | #define UTILS_H 3 | 4 | #define CHECK(cond, msg) assert(cond && msg) 5 | 6 | #include 7 | #include 8 | #include 9 | extern "C" { 10 | #include 11 | #include 12 | } 13 | using namespace std; 14 | 15 | 16 | template 17 | vector createVector() { 18 | return vector(); 19 | } 20 | 21 | 22 | template 23 | map createMap() { 24 | return map(); 25 | } 26 | 27 | /* set custom (console) Logger */ 28 | void setConsoleLogger(bool verbose); 29 | 30 | 31 | /* get description of channel_layout */ 32 | string get_channel_layout_name(int channels, uint64_t channel_layout); 33 | 34 | #endif -------------------------------------------------------------------------------- /src/ts/__test__/filters.test.js: -------------------------------------------------------------------------------- 1 | const { applySingleFilter, applyMulitpleFilter } = require('../filters') 2 | 3 | 4 | describe('Single input filters', () => { 5 | 6 | it('Should trim video and audio streams', () => { 7 | const sourceNode = {outStreams: [ 8 | {mediaType: 'video', startTime: 1, duration: 10 }, 9 | {mediaType: 'audio', startTime: 2, duration: 20}, 10 | ]} 11 | const streamRefs = [ 12 | {from: sourceNode, index: 0}, 13 | {from: sourceNode, index: 1}, 14 | ] 15 | 16 | const trim = { 17 | type: "trim", 18 | args: { start: 2, duration: 3, } 19 | } 20 | const outRefs = applySingleFilter(streamRefs, trim) 21 | expect(outRefs).toHaveLength(2) 22 | expect(outRefs[0].from.outStreams[0].mediaType).toBe('video') 23 | expect(outRefs[1].from.outStreams[0].mediaType).toBe('audio') 24 | }) 25 | }) 26 | 27 | 28 | describe('Multiple input filters', () => { 29 | 30 | }) 31 | -------------------------------------------------------------------------------- /src/ts/__test__/streamIO.test.js: -------------------------------------------------------------------------------- 1 | jest.mock('../graph', () => ({ 2 | buildGraphConfig: () => {} 3 | })) 4 | jest.mock('../loader', () => ({ 5 | loadWASM: async () => new Uint8Array() 6 | })) 7 | jest.mock('../message') 8 | 9 | 10 | 11 | 12 | 13 | describe('class Reader (browser)', () => { 14 | 15 | jest.mock('../utils', () => ({ isBrowser: true, isNode: false })) 16 | const { Reader, Exporter } = require('../streamIO') 17 | 18 | const totalLength = 100 19 | jest.mock(fetch, async () => ({ 20 | headers: { 21 | get: () => totalLength 22 | } 23 | })) 24 | 25 | beforeEach(async () => { 26 | const reply = jest.fn((msgType, callback) => {}) 27 | const reader = new Reader('reader_id', './video.mp4', { reply }) 28 | await reader.build() 29 | }) 30 | 31 | it('Should ready to reply for `read` and `seek` request', async () => { 32 | expect(reply.mock.calls).toHaveLength(2) 33 | expect(reply.mock.calls[0][0]).toBe('read') 34 | expect(reply.mock.calls[1][0]).toBe('seek') 35 | }) 36 | 37 | it('Should reply data for `read` request', async () => { 38 | const {inputs} = await reply.mock.calls[0][1]() 39 | expect(inputs.length).toBeGreaterThan(0) 40 | // todo... 41 | }) 42 | 43 | it('Should reply for `seek` request', async () => { 44 | // todo... 45 | }) 46 | 47 | }) -------------------------------------------------------------------------------- /src/ts/filters.ts: -------------------------------------------------------------------------------- 1 | import { AudioStreamMetadata, FilterNode, StreamMetadata, StreamRef } from "./types/graph"; 2 | 3 | export type FilterArgs = Extract['args'] 4 | export type Filter = 5 | { type: 'tracks', args: 'video' | 'audio' } | 6 | { type: 'trim', args: { start: number, duration: number } } | 7 | { type: 'setpts' } | 8 | { type: 'fifo' } | 9 | { type: 'volume', args: number } | 10 | { type: 'merge' } | // implicit args: {number of inputs} 11 | { type: 'concat' } | 12 | { type: 'format', args: { pixelFormat?: string, sampleFormat?: string, sampleRate?: number, channelLayout?: string } } 13 | 14 | 15 | /** 16 | * valid `args` based on previous streams, then create FilterNodes (update streams metadata) 17 | * @returns array of streamRef, ref to created FilterNodes' outStreams, or unchanged streamRefs. 18 | */ 19 | export function applySingleFilter(streamRefs: StreamRef[], filter: Filter): StreamRef[] { 20 | const outStreams = streamRefs.map(streamRef => { 21 | const s = streamRef.from.outStreams[streamRef.index] 22 | switch (filter.type) { 23 | case 'trim': { 24 | const name = s.mediaType == 'audio' ? 'atrim' : 'trim' 25 | const start = Math.max(filter.args.start, s.startTime) 26 | const end = Math.min(start + filter.args.duration, s.startTime + s.duration) 27 | const duration = Math.max(end - start, 0) 28 | const from: FilterNode = { 29 | type: 'filter', filter: {name, ffmpegArgs: {start, duration}}, 30 | inStreams: [streamRef], outStreams: [{...s, startTime: start, duration}] } 31 | return {from, index: 0} 32 | } 33 | // first frame pts reset to 0 34 | case 'setpts': { 35 | const name = s.mediaType == 'audio' ? 'asetpts' : 'setpts' 36 | const from: FilterNode = { 37 | type: 'filter', filter: {name: name, ffmpegArgs: 'PTS-STARTPTS'}, 38 | inStreams: [streamRef], outStreams: [{...s}] } 39 | return {from, index: 0} 40 | } 41 | case 'fifo': { 42 | const name = s.mediaType == 'audio' ? 'afifo' : 'fifo' 43 | const from: FilterNode = { 44 | type: 'filter', filter: {name: name, ffmpegArgs: ''}, 45 | inStreams: [streamRef], outStreams: [{...s}] } 46 | return {from, index: 0} 47 | } 48 | case 'volume': { 49 | const volume = filter.args 50 | if (s.mediaType == 'video') return streamRef 51 | const from: FilterNode = { 52 | type: 'filter', filter: {name: 'volume', ffmpegArgs: {volume}}, 53 | inStreams: [streamRef], outStreams: [{...s, volume}]} 54 | return {from, index: 0} 55 | } 56 | case 'format': { 57 | const {pixelFormat, channelLayout, sampleFormat, sampleRate} = filter.args 58 | const name = s.mediaType == 'audio' ? 'aformat' : 'format' 59 | const ffmpegArgs = s.mediaType == 'audio' ? 60 | {sample_fmts: sampleFormat, channel_layouts: channelLayout, sample_rates: sampleRate} : 61 | {pix_fmts: pixelFormat} 62 | const metadata: StreamMetadata = s.mediaType == 'audio' ? 63 | {...s, sampleFormat: sampleFormat ?? s.sampleFormat, 64 | channelLayout: channelLayout ?? s.channelLayout, 65 | sampleRate: sampleRate ?? s.sampleRate} : 66 | {...s, pixelFormat: pixelFormat ?? s.pixelFormat} 67 | const from: FilterNode = { 68 | type: 'filter', filter: {name, ffmpegArgs}, 69 | inStreams: [streamRef], outStreams: [metadata] } 70 | return {from, index: 0} 71 | } 72 | default: throw `${filter.type}: not support single input filter` 73 | } 74 | }) 75 | 76 | if (streamRefs.every((r, i) => r == outStreams[i])) 77 | throw `${filter.type}: no nothing filtering` 78 | 79 | return outStreams 80 | } 81 | 82 | 83 | /** 84 | * valid `args` based on previous streams, then create FilterNodes (update streams metadata) 85 | * @returns array of streamRef, ref to created FilterNodes' outStreams, or unchanged streamRefs. 86 | */ 87 | export function applyMulitpleFilter(streamRefsArr: StreamRef[][], filter: Filter): StreamRef[] { 88 | 89 | switch (filter.type) { 90 | case 'concat': { 91 | const n = streamRefsArr.length 92 | const segment = streamRefsArr[0] 93 | const v = segment.filter(r => r.from.outStreams[r.index].mediaType == 'video').length 94 | const a = segment.filter(r => r.from.outStreams[r.index].mediaType == 'audio').length 95 | // todo... check more 96 | if (streamRefsArr.some(refs => refs.length != streamRefsArr[0].length)) 97 | throw `${filter.type}: all segments should have same audio/video tracks` 98 | // concat 99 | const duration = streamRefsArr.reduce((acc, refs) => acc + refs[0].from.outStreams[refs[0].index].duration, 0) 100 | const from: FilterNode = {type: 'filter', inStreams: streamRefsArr.flat(), 101 | outStreams: streamRefsArr[0].map(r => ({...r.from.outStreams[r.index], duration})), 102 | filter: {name: 'concat', ffmpegArgs: {n, v, a}} } 103 | return from.outStreams.map((_, i) => ({from, index: i})) 104 | }; 105 | case 'merge': { 106 | const streamRefs = streamRefsArr.flat() 107 | const audioStreamRefs = streamRefs.filter(ref => ref.from.outStreams[ref.index].mediaType == 'audio') 108 | const inAudioStreams = audioStreamRefs.map(r => r.from.outStreams[r.index]) as AudioStreamMetadata[] 109 | // choose smallest duration 110 | const duration = inAudioStreams.reduce((acc, s) => Math.min(s.duration, acc), inAudioStreams[0].duration) 111 | // All inputs must have the same sample rate, and format 112 | if (inAudioStreams.some(m => m.sampleRate != inAudioStreams[0].sampleRate)) 113 | throw `${filter.type}: all inputs must have same sampleRate` 114 | if (inAudioStreams.some(m => m.sampleFormat != inAudioStreams[0].sampleFormat)) 115 | throw `${filter.type}: all inputs must have same sampleFormat` 116 | // out stream metadata mainly use first one 117 | const from: FilterNode = {type: 'filter', inStreams: audioStreamRefs, 118 | outStreams: [{...inAudioStreams[0], duration}], 119 | filter: {name: 'amerge', ffmpegArgs: {inputs: audioStreamRefs.length}}} 120 | return [...streamRefs.filter(ref => ref.from.outStreams[ref.index].mediaType != 'audio'), 121 | {from, index: 0}] 122 | } 123 | default: throw `${filter.type}: not found multiple input filter` 124 | } 125 | 126 | } 127 | 128 | -------------------------------------------------------------------------------- /src/ts/globals.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Only for browser environment. Deprecated for nodejs environment. 3 | * TODO... 4 | */ 5 | 6 | import { Flags } from './types/flags' 7 | 8 | 9 | class GlobalFlags { 10 | #flags: Flags = {} 11 | set(obj: Flags) { 12 | Object.assign(this.#flags, obj) 13 | } 14 | get() { 15 | return structuredClone(this.#flags) 16 | } 17 | } 18 | export const globalFlags = new GlobalFlags() 19 | 20 | export const Worker = window.Worker 21 | 22 | export class SourceStream { 23 | stream: ReadableStreamDefaultReader 24 | #end = false 25 | 26 | constructor(stream: globalThis.ReadableStream) { 27 | this.stream = stream.getReader() 28 | } 29 | 30 | async read() { 31 | const { done, value } = await this.stream.read() 32 | if (done) this.#end = true 33 | return value 34 | } 35 | 36 | get end() { return this.#end } 37 | 38 | async cancel() { 39 | await this.stream.cancel() 40 | } 41 | 42 | 43 | } -------------------------------------------------------------------------------- /src/ts/graph.ts: -------------------------------------------------------------------------------- 1 | import { v4 as uuid } from 'uuid' 2 | import { applySingleFilter } from './filters' 3 | import { FilterInstance, GraphInstance, StreamInstanceRef, StreamRef, TargetNode, UserNode } from './types/graph' 4 | 5 | /** 6 | * given endpoints, backtrace until sources. 7 | * - make up some filterNodes (e.g. resample) 8 | * - tree shake and optimize the graph (todo...) 9 | * - convert to graphInstance (easy for further work) 10 | */ 11 | export function buildGraphInstance(target: TargetNode): [GraphInstance, Map] { 12 | // make up graph 13 | // target = completeGraph(target) 14 | 15 | // add uuid for each UserNode, and build map 16 | const node2id = new Map() 17 | node2id.set(target, uuid()) 18 | const traversalGraph = (streamRefs: StreamRef[]) => streamRefs.forEach(({from}) => { 19 | if (!node2id.has(from)) node2id.set(from, uuid()) 20 | if (from.type != 'source') 21 | traversalGraph(from.inStreams) 22 | }) 23 | traversalGraph(target.inStreams) 24 | 25 | // convert to graphInstance 26 | const sources: string[] = [] 27 | const targets: string[] = [] 28 | let nodeInstances: GraphInstance['nodes'] = {} 29 | let filterInstance: NonNullable = { 30 | inputs: [], 31 | outputs: [], 32 | filters: [], 33 | } 34 | node2id.forEach((id, node) => { 35 | if (node.type == 'source') { 36 | sources.push(id) 37 | const {data: {source: _, ...dataRest}, ...rest} = node 38 | nodeInstances[id] = {...rest, id, data: {...dataRest}} 39 | } 40 | else if (node.type == 'filter') { 41 | filterInstance.filters.push(id) 42 | const inStreams = node.inStreams.map(({from, index}) => { 43 | const instanceRef = {from: node2id.get(from) ?? '', index} 44 | if (from.type == 'source') 45 | filterInstance.inputs.push(instanceRef) 46 | return instanceRef 47 | }) 48 | nodeInstances[id] = {...node, inStreams, id} 49 | } 50 | else { 51 | targets.push(id) 52 | const inStreams = node.inStreams.map(({from, index}) => { 53 | const instanceRef = {from: node2id.get(from) ?? '', index} 54 | if (from.type == 'filter') 55 | filterInstance.outputs.push(instanceRef) 56 | return instanceRef 57 | }) 58 | nodeInstances[id] = {...node, inStreams, id} 59 | } 60 | 61 | }); 62 | // complete filters (+ split) 63 | [filterInstance.filters, nodeInstances] = filtersComplete(filterInstance.filters, nodeInstances) 64 | 65 | // reverse filters 66 | filterInstance.filters.reverse() 67 | 68 | const grapInstance = { 69 | nodes: nodeInstances, 70 | sources, 71 | filterInstance: (filterInstance.filters.length > 0 ? filterInstance: undefined), 72 | targets 73 | } 74 | 75 | return [grapInstance, node2id] 76 | } 77 | 78 | 79 | /** 80 | * if one stream is used multiple times, then prepend `split` filter to clone 81 | * */ 82 | function filtersComplete(filters: string[], nodes: GraphInstance['nodes']) { 83 | const streamId = (from: string, index: number) => `${from}:${index}` 84 | const stats: {[streamId in string]?: 85 | {streamEntries: {filter: string, index: number}[], stream: StreamInstanceRef, isVideo: boolean}} = {} 86 | // stats all inStreams 87 | for (const filterId of filters) { 88 | const node = nodes[filterId] 89 | if (node?.type !== 'filter') continue 90 | node.inStreams.forEach((r, i) => { 91 | const id = streamId(r.from, r.index) 92 | const fromInstance = nodes[r.from]?.outStreams[r.index] 93 | if (!stats[id]) { 94 | stats[id] = {streamEntries: [], stream: r, isVideo: fromInstance?.mediaType == 'video' } 95 | } 96 | stats[id]?.streamEntries.push({filter: filterId, index: i}) 97 | }) 98 | } 99 | Object.values(stats) 100 | .filter((v) => (v?.streamEntries.length??0) > 1) 101 | .forEach((v) => { 102 | const fromStream = nodes[v?.stream.from??'']?.outStreams[v?.stream.index??0] 103 | const numSplit = v?.streamEntries.length 104 | if (!v || !fromStream || !numSplit) return 105 | // add `split/asplit` filters 106 | const splitInstance: FilterInstance = { 107 | type: 'filter', inStreams: [v.stream], id: uuid(), 108 | outStreams: Array(numSplit).fill(fromStream), 109 | filter: {name: v?.isVideo ? 'split' : 'asplit', ffmpegArgs: `${numSplit}`}} 110 | 111 | filters = [...filters, splitInstance.id] 112 | nodes = {...nodes, [splitInstance.id]: splitInstance} 113 | // update inStream entries 114 | v.streamEntries.forEach((e, i) => { 115 | const instance = nodes[e.filter] 116 | if (instance?.type != 'filter') return 117 | const inStreams = [...instance.inStreams] 118 | inStreams[e.index] = {from: splitInstance.id, index: i} 119 | nodes[e.filter] = {...instance, inStreams} 120 | }) 121 | }) 122 | 123 | return [filters, nodes] as const 124 | } 125 | 126 | 127 | function completeGraph(target: TargetNode): TargetNode { 128 | // check target inStream / outStream, see if need to add filterNode (format) 129 | const inStreams = target.inStreams.map((inRef, i) => { 130 | const outS = target.outStreams[i] 131 | const inS = inRef.from.outStreams[inRef.index] 132 | 133 | if (inS.mediaType == 'video' && outS.mediaType == 'video') { 134 | // console.warn('disable pixel format convert') 135 | if (inS.pixelFormat != outS.pixelFormat) 136 | return applySingleFilter([inRef], {type: 'format', args: {pixelFormat: outS.pixelFormat}})[0] 137 | } 138 | else if (inS.mediaType == 'audio' && outS.mediaType == 'audio') { 139 | const keys = ['sampleFormat', 'sampleRate', 'channelLayout'] as const 140 | if (keys.some(k => inS[k] != outS[k])) { 141 | const { channelLayout, sampleFormat, sampleRate } = outS 142 | return applySingleFilter([inRef], {type: 'format', args: {channelLayout, sampleFormat, sampleRate }})[0] 143 | } 144 | } 145 | return inRef 146 | }) 147 | 148 | return {...target, inStreams} 149 | } -------------------------------------------------------------------------------- /src/ts/hls.ts: -------------------------------------------------------------------------------- 1 | import { BufferData } from "./types/graph" 2 | 3 | export async function isHLSStream(url: string): Promise { 4 | try { 5 | const response = await fetch(url) 6 | const contentType = response.headers.get('content-type') 7 | return contentType?.includes('application/vnd.apple.mpegurl') || 8 | contentType?.includes('application/x-mpegurl') || 9 | url.endsWith('.m3u8') 10 | } catch { 11 | return false 12 | } 13 | } 14 | 15 | export async function getStaticHLSMetadata(url: string): Promise<{ segmentCount: number; totalDuration: number }> { 16 | try { 17 | const baseUrl = url.substring(0, url.lastIndexOf('/') + 1) 18 | const playlistInfo = await parsePlaylist(url, baseUrl) 19 | 20 | 21 | const getMetadata = async (listInfo: PlaylistInfo) => { 22 | if (listInfo.segmentUrls.length == 0) return { segmentCount: 0, totalDuration: 0 } 23 | const segmentCount = listInfo.segmentUrls.length 24 | // Calculate total duration from the playlist 25 | let totalDuration = 0 26 | const response = await fetch(url) 27 | const text = await response.text() 28 | const lines = text.split('\n') 29 | 30 | for (const line of lines) { 31 | if (line.startsWith('#EXTINF:')) { 32 | const durationMatch = line.match(/#EXTINF:([\d.]+)/) 33 | if (durationMatch) { 34 | totalDuration += parseFloat(durationMatch[1]) 35 | } 36 | } 37 | } 38 | 39 | return { segmentCount, totalDuration } 40 | } 41 | 42 | // If it's a master playlist, we need to get the segments from the best variant 43 | if (playlistInfo.isMaster && playlistInfo.variantUrls.length > 0) { 44 | const bestVariantUrl = await selectBestVariant(playlistInfo.variantUrls, baseUrl) 45 | const variantBaseUrl = bestVariantUrl.substring(0, bestVariantUrl.lastIndexOf('/') + 1) 46 | const variantInfo = await parsePlaylist(bestVariantUrl, variantBaseUrl) 47 | return getMetadata(variantInfo) 48 | } else { 49 | // Direct segment playlist 50 | return getMetadata(playlistInfo) 51 | } 52 | } catch (error) { 53 | console.error('Error getting HLS metadata:', error) 54 | return { segmentCount: 0, totalDuration: 0 } 55 | } 56 | } 57 | 58 | interface PlaylistInfo { 59 | isMaster: boolean 60 | variantUrls: { url: string; bandwidth: number }[] 61 | segmentUrls: string[] 62 | } 63 | 64 | async function parsePlaylist(url: string, baseUrl: string): Promise { 65 | const response = await fetch(url) 66 | const text = await response.text() 67 | const lines = text.split('\n') 68 | 69 | const isMaster = lines.some(line => line.includes('#EXT-X-STREAM-INF')) 70 | const variantUrls: { url: string; bandwidth: number }[] = [] 71 | const segmentUrls: string[] = [] 72 | 73 | let currentLine = 0 74 | while (currentLine < lines.length) { 75 | const line = lines[currentLine].trim() 76 | 77 | if (line.startsWith('#EXT-X-STREAM-INF')) { 78 | // Master playlist variant 79 | const bandwidthMatch = line.match(/BANDWIDTH=(\d+)/) 80 | const bandwidth = bandwidthMatch ? parseInt(bandwidthMatch[1]) : 0 81 | 82 | currentLine++ 83 | const variantUrl = lines[currentLine].trim() 84 | if (variantUrl && !variantUrl.startsWith('#')) { 85 | variantUrls.push({ 86 | url: new URL(variantUrl, baseUrl).href, 87 | bandwidth: bandwidth 88 | }) 89 | } 90 | } else if (line.startsWith('#EXTINF')) { 91 | // Segment duration info 92 | currentLine++ 93 | const segmentUrl = lines[currentLine].trim() 94 | if (segmentUrl && !segmentUrl.startsWith('#')) { 95 | segmentUrls.push(new URL(segmentUrl, baseUrl).href) 96 | } 97 | } 98 | currentLine++ 99 | } 100 | 101 | return { isMaster, variantUrls, segmentUrls } 102 | } 103 | 104 | async function measureNetworkSpeed(url: string): Promise { 105 | const startTime = performance.now() 106 | const response = await fetch(url) 107 | const data = await response.arrayBuffer() 108 | const endTime = performance.now() 109 | 110 | // Calculate speed in bits per second 111 | const durationInSeconds = (endTime - startTime) / 1000 112 | const sizeInBits = data.byteLength * 8 113 | return sizeInBits / durationInSeconds 114 | } 115 | 116 | async function selectBestVariant(variants: { url: string; bandwidth: number }[], baseUrl: string): Promise { 117 | // First try the highest bandwidth variant 118 | const sortedVariants = [...variants].sort((a, b) => b.bandwidth - a.bandwidth) 119 | 120 | for (const variant of sortedVariants) { 121 | try { 122 | // Get the first segment URL from the variant playlist 123 | const variantInfo = await parsePlaylist(variant.url, baseUrl) 124 | if (variantInfo.segmentUrls.length > 0) { 125 | const segmentUrl = variantInfo.segmentUrls[0] 126 | const actualSpeed = await measureNetworkSpeed(segmentUrl) 127 | 128 | // If actual speed is at least 80% of the variant's bandwidth, use this variant 129 | if (actualSpeed >= variant.bandwidth * 0.8) { 130 | return variant.url 131 | } 132 | } 133 | } catch (error) { 134 | console.warn(`Failed to measure speed for variant ${variant.url}:`, error) 135 | continue 136 | } 137 | } 138 | 139 | // If no variant meets the speed requirement, fall back to the lowest bandwidth 140 | return sortedVariants[0].url 141 | } 142 | 143 | export async function createHLSStream(url: string, options: { maxBufferSize?: number } = {}): Promise> { 144 | let currentSegment = 0 145 | let playlist: string[] = [] 146 | let baseUrl = url.substring(0, url.lastIndexOf('/') + 1) 147 | let isLive = false 148 | let lastUpdateTime = Date.now() 149 | const maxBufferSize = options.maxBufferSize || 3 // Default segments in buffer stream 150 | 151 | const fetchSegment = async (segmentUrl: string): Promise => { 152 | const response = await fetch(segmentUrl) 153 | return await response.arrayBuffer() 154 | } 155 | 156 | const updatePlaylist = async () => { 157 | try { 158 | const masterInfo = await parsePlaylist(url, baseUrl) 159 | 160 | if (masterInfo.isMaster && masterInfo.variantUrls.length > 0) { 161 | const bestVariantUrl = await selectBestVariant(masterInfo.variantUrls, baseUrl) 162 | const variantBaseUrl = bestVariantUrl.substring(0, bestVariantUrl.lastIndexOf('/') + 1) 163 | const variantInfo = await parsePlaylist(bestVariantUrl, variantBaseUrl) 164 | playlist = variantInfo.segmentUrls 165 | } else { 166 | playlist = masterInfo.segmentUrls 167 | } 168 | 169 | // Check if this is a live stream 170 | isLive = playlist.some(url => url.includes('live') || url.includes('event')) 171 | lastUpdateTime = Date.now() 172 | } catch (error) { 173 | console.error('Error updating playlist:', error) 174 | } 175 | } 176 | 177 | return new ReadableStream({ 178 | async start(controller) { 179 | await updatePlaylist() 180 | }, 181 | 182 | async pull(controller) { 183 | try { 184 | // For live streams, check for playlist updates every 5 seconds 185 | if (isLive && Date.now() - lastUpdateTime > 5000) { 186 | await updatePlaylist() 187 | } 188 | 189 | // If we've reached the end of the playlist 190 | if (currentSegment >= playlist.length) { 191 | if (isLive) { 192 | // For live streams, wait for new segments 193 | await new Promise(resolve => setTimeout(resolve, 1000)) 194 | await updatePlaylist() 195 | } else { 196 | // For VOD, close the stream 197 | controller.close() 198 | return 199 | } 200 | } 201 | 202 | // Calculate how many segments to buffer 203 | const segmentsToBuffer = Math.min( 204 | maxBufferSize, 205 | playlist.length - currentSegment 206 | ) 207 | 208 | // Fetch and enqueue multiple segments up to the buffer limit 209 | for (let i = 0; i < segmentsToBuffer; i++) { 210 | const segmentUrl = playlist[currentSegment + i] 211 | const data = await fetchSegment(segmentUrl) 212 | controller.enqueue(new Uint8Array(data)) 213 | } 214 | currentSegment += segmentsToBuffer 215 | 216 | // If we have more segments to buffer, schedule another pull 217 | if (currentSegment < playlist.length) { 218 | // The stream will automatically call pull again when the consumer is ready 219 | return 220 | } 221 | } catch (error) { 222 | controller.error(error) 223 | } 224 | }, 225 | 226 | cancel() { 227 | // Clean up any resources if needed 228 | currentSegment = 0 229 | playlist = [] 230 | } 231 | }) 232 | } 233 | -------------------------------------------------------------------------------- /src/ts/loader.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * WASM Module manager 3 | * Note: this file import from built wasm files, which should be built beforehand. 4 | */ 5 | import Worker from 'worker-loader?inline=no-fallback!./transcoder.worker.ts' 6 | // @ts-ignore 7 | import pkgJSON from '../../package.json' 8 | import { FFWorker } from './message' 9 | 10 | // Warning: webpack 5 only support pattern: new Worker(new URL('', import.meta.url)) 11 | // const createWorker = () => new Worker(new URL('./transcoder.worker.ts', import.meta.url)) 12 | /* use worker inline way to avoid bundle issue as dependency for further bundle. */ 13 | export const createWorker = () => new Worker() 14 | 15 | 16 | const wasmFileName = `ffmpeg_built.wasm` 17 | // production wasm from remote CDN 18 | let DefaultURL = `https://unpkg.com/frameflow@${pkgJSON.version}/dist/${wasmFileName}` 19 | 20 | // this if branch will be removed after built 21 | if (process.env.NODE_ENV !== 'production') { 22 | DefaultURL = new URL(`../wasm/ffmpeg_built.wasm`, import.meta.url).href 23 | console.assert(DefaultURL.includes(wasmFileName)) // keep same wasm name with prod one 24 | } 25 | 26 | // store default global things here 27 | const defaults = { 28 | wasm: undefined as Promise | undefined, 29 | worker: undefined as Promise | undefined 30 | } 31 | 32 | function loadWASM(url: RequestInfo = DefaultURL) { 33 | if (defaults.wasm) return defaults.wasm 34 | console.log('Fetch WASM start...') 35 | // assign to global variable 36 | defaults.wasm = fetch(url).then(async res => { 37 | if (!res.ok) throw `WASM binary fetch failed.` 38 | const wasm = await res.arrayBuffer() 39 | console.log(`Fetch WASM (${wasm.byteLength}) done.`) 40 | return wasm 41 | }) 42 | 43 | return defaults.wasm 44 | } 45 | 46 | export interface LoadArgs { newWorker?: boolean, url?: string } 47 | 48 | export function loadWorker(args?: LoadArgs) { 49 | const {newWorker, url} = args ?? {} 50 | if (!newWorker && defaults.worker) return defaults.worker 51 | // assign to global variable 52 | const ffWorker = loadWASM(url).then(async wasm => { 53 | const ffWorker = new FFWorker(createWorker()) 54 | // pass wasm to used and must return for future uses 55 | const loadResult = ffWorker.send('load', {wasm}, [wasm]) 56 | defaults.wasm = loadResult.then(({wasm}) => wasm) 57 | await loadResult 58 | return ffWorker 59 | }) 60 | if (!newWorker && !defaults.worker) { 61 | defaults.worker = ffWorker 62 | } 63 | 64 | return ffWorker 65 | } 66 | -------------------------------------------------------------------------------- /src/ts/message.ts: -------------------------------------------------------------------------------- 1 | // import { Worker } from 'worker_threads' 2 | 3 | import { InferredFormatInfo } from "./types/ffmpeg" 4 | import { Flags } from "./types/flags" 5 | import { ChunkData, FormatMetadata, GraphInstance, StreamMetadata, WriteChunkData } from "./types/graph" 6 | 7 | 8 | type MessageType = keyof Messages 9 | interface Messages { 10 | load: { 11 | send: { wasm: ArrayBuffer }, 12 | reply: {wasm: ArrayBuffer}, 13 | } 14 | getMetadata: { 15 | send: { fileSize: number }, 16 | reply: { container: FormatMetadata, streams: StreamMetadata[] } 17 | } 18 | inferFormatInfo: { 19 | send: { format: string, url: string } 20 | reply: InferredFormatInfo 21 | } 22 | buildGraph: { 23 | send: { graphInstance: GraphInstance, flags: Flags } 24 | reply: void 25 | } 26 | nextFrame: { 27 | send: void, 28 | reply: { outputs: {[nodeId in string]?: WriteChunkData[]}, endWriting: boolean, progress: number} 29 | } 30 | deleteGraph: { 31 | send: void 32 | reply: void 33 | } 34 | releaseWorkerBuffer: { 35 | send: { buffer: Uint8Array } 36 | reply: void 37 | } 38 | } 39 | 40 | type BackMessageType = keyof BackMessages 41 | interface BackMessages { 42 | read: { 43 | send: undefined 44 | reply: { inputs: ChunkData[] } 45 | } 46 | seek: { 47 | send: { pos: number } 48 | reply: void 49 | } 50 | releaseMainBuffer: { 51 | send: { buffers: Uint8Array[] } 52 | reply: void 53 | } 54 | } 55 | 56 | type AllMessageType = keyof AllMessages 57 | type AllPostMessage = {type: AllMessageType, data: any, id: string} 58 | interface AllMessages extends Messages, BackMessages {} 59 | type AllReplyCallback = 60 | (t: AllMessages[T]['send'], id: string, transferArr: TransferArray) => 61 | AllMessages[T]['reply'] | Promise 62 | 63 | type TransferArray = (Transferable | VideoFrame | AudioData)[] 64 | 65 | // close VideoFrame/AudioData (refCount--) 66 | const closeTransferArray = (arr: TransferArray) => arr.forEach(data => 'close' in data && data.close()) 67 | 68 | function sendMessage( 69 | sender: any, 70 | sendMsg: T, 71 | data: AllMessages[T]['send'], 72 | transferArray: TransferArray = [], 73 | sendId='' 74 | ) { 75 | const promise = new Promise((resolve, reject) => { 76 | const listener = (e: MessageEvent<{type: T, data: AllMessages[T]['reply'], id: string}>) => { 77 | const {type: replyMsg, data, id: replyId } = e.data 78 | if (sendMsg != replyMsg || sendId != replyId) return // ignore the different msgType / id 79 | // execute the callback, then delete the listener 80 | resolve(data) 81 | // delete event listener 82 | sender.removeEventListener('message', listener) 83 | } 84 | sender.addEventListener('message', listener) 85 | sender.addEventListener('messageerror', function errorListener() { 86 | reject() 87 | sender.removeEventListener('messageerror', errorListener) 88 | }) 89 | }) 90 | const msg: AllPostMessage = {type: sendMsg, data, id: sendId} // make sure id cannot missing 91 | sender.postMessage(msg, transferArray) 92 | closeTransferArray(transferArray) 93 | 94 | return promise 95 | } 96 | 97 | /** 98 | * 99 | * @param replyId if replyId was given, filter the request (sendId). Otherwise, use sendId as replyId 100 | */ 101 | function replyMessage( 102 | replier: any, 103 | msgType: T, 104 | callback: AllReplyCallback, 105 | replyId?: string 106 | ) { 107 | replier.addEventListener('message', (e: MessageEvent<{type: T, data: AllMessages[T]['send'], id: string}>) => { 108 | const { type, data, id: sendId } = e.data 109 | if (msgType != type || (replyId && replyId != sendId)) return; // ignore different sendMsg / id 110 | const id = replyId ?? sendId 111 | const transferArr: TransferArray = [] 112 | const replyData = callback(data, sendId, transferArr) 113 | if (replyData instanceof Promise) { 114 | replyData.then(data => { 115 | const msg: AllPostMessage = {type, data, id} // make sure id cannot missing 116 | replier.postMessage(msg, [...transferArr]) 117 | closeTransferArray(transferArr) 118 | }) 119 | } 120 | else { 121 | const msg: AllPostMessage = {type, data: replyData, id} // make sure id cannot missing 122 | replier.postMessage(msg, [...transferArr]) 123 | closeTransferArray(transferArr) 124 | } 125 | // dont delete callback, since it register once, always listen to main thread 126 | }) 127 | } 128 | 129 | /** 130 | * only allow one access, others wait in promise 131 | */ 132 | export class Lock { 133 | #promises: PromiseHandle[] = [] 134 | 135 | /** 136 | * 137 | * @returns unlock function 138 | */ 139 | async start() { 140 | const prevPromise: PromiseHandle | undefined = this.#promises[this.#promises.length - 1] 141 | const promisehandle = new PromiseHandle() 142 | this.#promises.push(promisehandle) 143 | if (prevPromise) { 144 | await prevPromise.promise 145 | } 146 | 147 | // unlock function 148 | return () => { 149 | this.#promises = this.#promises.filter(p => promisehandle != p) 150 | promisehandle.resolve() 151 | } 152 | } 153 | } 154 | 155 | class PromiseHandle { 156 | #promise: Promise 157 | #resolve?: (value: void | PromiseLike) => void 158 | constructor() { 159 | this.#promise = new Promise((resolve) => { 160 | this.#resolve = resolve 161 | }) 162 | } 163 | 164 | get promise() { return this.#promise } 165 | 166 | resolve() { 167 | this.#resolve?.() 168 | } 169 | } 170 | 171 | export class FFWorker { 172 | worker: Worker 173 | lock = new Lock() 174 | 175 | constructor(worker: Worker) { 176 | this.worker = worker 177 | } 178 | 179 | async send(sendMsg: T, data: Messages[T]['send'], transferArray: TransferArray, id?: string) { 180 | const unlock = await this.lock.start() 181 | const result = await sendMessage(this.worker, sendMsg, data, transferArray, id) 182 | unlock() 183 | return result 184 | } 185 | 186 | reply(msgType: T, callback: AllReplyCallback, id: string) { 187 | return replyMessage(this.worker, msgType, callback, id) 188 | } 189 | 190 | close() { 191 | /* hacky way to avoid slowing down wasm loading in next worker starter. 192 | * Several experiments show that if create worker and load wasm immediately after worker.close(), 193 | * it will become 10x slower, guess it is because of GC issue. 194 | */ 195 | setTimeout(() => this.worker.terminate(), 5000) 196 | } 197 | } 198 | 199 | 200 | /** 201 | * 202 | */ 203 | export class WorkerHandlers { 204 | 205 | reply(msgType: T, callback: AllReplyCallback) { 206 | return replyMessage(self, msgType, callback) 207 | } 208 | 209 | send(msgType: T, data: BackMessages[T]['send'], transferArray: TransferArray, id: string) { 210 | return sendMessage(self, msgType, data, transferArray, id) 211 | } 212 | } 213 | 214 | -------------------------------------------------------------------------------- /src/ts/metadata.ts: -------------------------------------------------------------------------------- 1 | import { StreamMetadata } from "./types/graph"; 2 | 3 | 4 | interface DataFormatMap { 5 | pixel: {ff: string, web: VideoPixelFormat}[] 6 | sample: {ff: string, web: AudioSampleFormat}[] 7 | } 8 | 9 | export const dataFormatMap: DataFormatMap = 10 | { 11 | pixel: [ 12 | {ff: 'yuv420p', web: 'I420'}, 13 | {ff: 'yuva420p', web: 'I420A'}, 14 | {ff: 'yuv422p', web: 'I422'}, 15 | {ff: 'yuv444p', web: 'I444'}, 16 | {ff: 'nv12', web: 'NV12'}, 17 | {ff: 'rgba', web: 'RGBA'}, // choose when ff2web 18 | {ff: 'rgba', web: 'RGBX'}, 19 | {ff: 'bgra', web: 'BGRA'}, // choose when ff2web 20 | {ff: 'bgra', web: 'BGRX'}, 21 | ], 22 | sample: [ 23 | {ff: 'u8', web: 'u8'}, 24 | {ff: 'u8p', web: 'u8-planar'}, 25 | {ff: 's16', web: 's16'}, 26 | {ff: 's16p', web: 's16-planar'}, 27 | {ff: 's32', web: 's32'}, 28 | {ff: 's32p', web: 's32-planar'}, 29 | {ff: 'flt', web: 'f32'}, 30 | {ff: 'fltp', web: 'f32-planar'}, 31 | ], 32 | } 33 | 34 | export function formatFF2Web(type: T, format: string): typeof dataFormatMap[T][0]['web'] { 35 | for (const {ff, web} of dataFormatMap[type]) 36 | if (ff == format) return web 37 | throw `Cannot find ${type} format: FF ${format}` 38 | } 39 | 40 | export function formatWeb2FF(type: T, format: typeof dataFormatMap[T][0]['web']): string { 41 | for (const {ff, web} of dataFormatMap[type]) 42 | if (web == format) return ff 43 | throw `Cannot find ${type} format: Web ${format}` 44 | } 45 | 46 | 47 | interface StreamArgs { 48 | frameRate?: number 49 | } 50 | export function webFrameToStreamMetadata(frame: VideoFrame | AudioData, args: StreamArgs): StreamMetadata { 51 | const commonInfo = { 52 | bitRate: 0, 53 | index: 0, 54 | startTime: 0, 55 | duration: 0, 56 | timeBase: {num: 1, den: 1_000_000}, // microseconds 57 | codecName: '', 58 | extraData: new Uint8Array(), 59 | } 60 | if (frame instanceof VideoFrame) { 61 | return { 62 | mediaType: 'video', 63 | height: frame.codedHeight, 64 | width: frame.codedWidth, 65 | pixelFormat: frame.format ? formatWeb2FF('pixel', frame.format) : '', 66 | sampleAspectRatio: {num: 0, den: 1}, 67 | frameRate: args.frameRate ?? 30, 68 | ...commonInfo 69 | } 70 | } 71 | else { 72 | return { 73 | mediaType: 'audio', 74 | volume: 1, 75 | sampleFormat: frame.format ? formatWeb2FF('sample', frame.format) : '', 76 | sampleRate: frame.sampleRate, 77 | channels: frame.numberOfChannels, 78 | channelLayout: '', // todo... 79 | ...commonInfo 80 | } 81 | } 82 | } -------------------------------------------------------------------------------- /src/ts/streamIO.ts: -------------------------------------------------------------------------------- 1 | import { v4 as uuid } from 'uuid' 2 | import { globalFlags, SourceStream } from './globals' 3 | import { buildGraphInstance } from './graph' 4 | import { createHLSStream, isHLSStream } from './hls' 5 | import { FFWorker } from "./message" 6 | import { BufferData, ChunkData, SourceNode, SourceType, TargetNode, WriteChunkData } from "./types/graph" 7 | import { BufferPool } from './utils' 8 | 9 | 10 | type SourceStreamCreator = (seekPos: number) => Promise> 11 | 12 | 13 | async function fetchSourceInfo(url: URL | RequestInfo): Promise<{size: number, url: string}> { 14 | let urlStr = '' 15 | if (typeof url == 'string') urlStr = url 16 | else if (url instanceof URL) urlStr = url.href 17 | else urlStr = url.url 18 | 19 | const { headers } = await fetch(url, { method: "HEAD" }) 20 | // if (!headers.get('Accept-Ranges')?.includes('bytes')) throw `cannot accept range fetch` 21 | // todo... check if accept-ranges 22 | return { size: parseInt(headers.get('Content-Length') ?? '0'), url: urlStr } 23 | } 24 | 25 | 26 | export async function getSourceInfo(src: SourceType): Promise<{size: number, url?: string, isHLS?: boolean}> { 27 | if (typeof src == 'string') { 28 | // Check if it's an HLS stream 29 | if (await isHLSStream(src)) { 30 | return { size: 0, url: src, isHLS: true } 31 | } 32 | // normal url 33 | return await fetchSourceInfo(src) 34 | } 35 | else if (src instanceof URL || src instanceof Request) { 36 | return await fetchSourceInfo(src) 37 | } 38 | else if (src instanceof Blob) { 39 | return { size: src.size, url: src instanceof File ? src.name : '' } 40 | } 41 | else if (src instanceof ReadableStream) { 42 | return { size: 0} 43 | } 44 | else if (src instanceof ArrayBuffer) { 45 | return { size: src.byteLength } 46 | } 47 | 48 | throw `cannot read source: "${src}", type: "${typeof src}, as stream input."` 49 | } 50 | 51 | 52 | async function fetchSourceData(url: RequestInfo | URL, startPos: number): Promise> { 53 | const { body, headers } = await fetch(url, { headers: { range: `bytes=${startPos}-` } }) 54 | if (!body) throw `cannot fetch source url: ${url}` 55 | return new SourceStream(body) 56 | } 57 | 58 | 59 | /* convert any quanlified src into creator of SourceStream */ 60 | export const sourceToStreamCreator = (src: SourceType): SourceStreamCreator => async (seekPos: number) => { 61 | if (typeof src == 'string') { 62 | // Check if it's an HLS stream 63 | if (await isHLSStream(src)) { 64 | return new SourceStream(await createHLSStream(src)) 65 | } 66 | // normal url 67 | return await fetchSourceData(src, seekPos) 68 | } 69 | else if (src instanceof URL || src instanceof Request) { 70 | return await fetchSourceData(src, seekPos) 71 | } 72 | else if (src instanceof Blob) { 73 | return new SourceStream(src.slice(seekPos).stream()) 74 | } 75 | else if (src instanceof ReadableStream) { // ignore seekPos 76 | return new SourceStream(src) 77 | } 78 | else if (src instanceof ArrayBuffer) { 79 | return new SourceStream(new ReadableStream({ 80 | start(s) { 81 | s.enqueue(new Uint8Array(src.slice(seekPos))) 82 | s.close() 83 | } 84 | })) 85 | } 86 | 87 | throw `cannot read source: "${src}", type: "${typeof src}, as stream input."` 88 | } 89 | 90 | 91 | 92 | export class FileReader { 93 | #id: string 94 | #url = '' // todo.. send empty name 95 | source: SourceType 96 | streamCreator: SourceStreamCreator 97 | stream: SourceStream | undefined = undefined 98 | worker: FFWorker 99 | // #dataReady: boolean = false 100 | // #ondataReady = () => {} // callback when new data available 101 | 102 | constructor(id: string, source: SourceType, worker: FFWorker) { 103 | this.#id = id 104 | this.worker = worker 105 | this.streamCreator = sourceToStreamCreator(source) 106 | this.source = source 107 | this.#enableReply() 108 | } 109 | 110 | #enableReply() { 111 | 112 | this.worker.reply('read', async (_, _2, transferArr) => { 113 | // create stream for the first time 114 | this.stream = this.stream ?? (await this.streamCreator(0)) 115 | const data = await this.stream.read() 116 | // this.#dataReady = false 117 | // call after sended data 118 | // setTimeout(() => { 119 | // this.#ondataReady() 120 | // this.#ondataReady = () => {} // only call once 121 | // }, 0) 122 | // this.#dataReady = true 123 | data && transferArr.push('buffer' in data ? data.buffer : data) 124 | return {inputs: data ? [data] : []} 125 | }, this.#id) 126 | 127 | this.worker.reply('seek', async ({pos}) => { 128 | this.stream = await this.streamCreator(pos) 129 | }, this.#id) 130 | } 131 | 132 | get url() { return this.#url ?? '' } 133 | 134 | get end() { return this.stream?.end ?? true } 135 | 136 | // /* worker has already had data */ 137 | // async dataReady() { 138 | // if (this.#dataReady) return 139 | // return new Promise((resolve) => { 140 | // this.#ondataReady = () => resolve() 141 | // }) 142 | // } 143 | } 144 | 145 | 146 | export class StreamReader { 147 | cacheData: ChunkData[] 148 | #id: string 149 | worker: FFWorker 150 | stream: SourceStream 151 | bufferPool = new BufferPool() 152 | 153 | constructor(id: string, cacheData: ChunkData[], stream: SourceStream, worker: FFWorker) { 154 | this.#id = id 155 | this.worker = worker 156 | this.cacheData = cacheData 157 | this.stream = stream 158 | this.#enableReply() 159 | } 160 | 161 | #enableReply() { 162 | this.worker.reply('read', async (_, _2, transferArr) => { 163 | const chunk = await this.read() 164 | chunk && transferArr.push('buffer' in chunk ? chunk.buffer : chunk ) 165 | return {inputs: chunk ? [chunk] : []} 166 | }, this.#id) 167 | 168 | this.worker.reply('seek', () => { 169 | throw `Stream input cannot be seeked.` 170 | }, this.#id) 171 | 172 | this.worker.reply('releaseMainBuffer', ({ buffers }) => { 173 | buffers.forEach(buffer => this.bufferPool.delete(buffer)) 174 | }, this.#id) 175 | } 176 | 177 | get end() { return this.stream.end } 178 | 179 | async probe() { 180 | const data = await this.stream.read() 181 | data && this.cacheData.push(data) 182 | return data 183 | } 184 | 185 | async read() { 186 | const chunk = this.cacheData.shift() ?? await this.stream.read() 187 | if (chunk && 'byteLength' in chunk) { 188 | const cloned = this.bufferPool.create(chunk.byteLength) 189 | cloned.set(chunk) 190 | return cloned 191 | } 192 | return chunk 193 | } 194 | 195 | close(source: SourceNode) { 196 | SourceCacheData.set(source, this.cacheData) 197 | } 198 | } 199 | 200 | type Reader = FileReader | StreamReader 201 | 202 | /** 203 | * cache Reader of a SourceNode, when source created, to may be used for exporting. 204 | **/ 205 | const SourceCacheData = new WeakMap() 206 | 207 | 208 | /** 209 | * Generic data type for all data buffer with unified API. 210 | */ 211 | export class Chunk { 212 | #data: ChunkData 213 | #offset = 0 214 | worker: FFWorker 215 | id: string 216 | constructor(data: ChunkData | WriteChunkData, worker: FFWorker, id: string) { 217 | this.worker = worker 218 | this.id = id 219 | if ('offset' in data) { 220 | this.#data = data.data 221 | this.#offset = data.offset 222 | } 223 | else 224 | this.#data = data 225 | } 226 | 227 | get data() { 228 | return ('buffer' in this.#data) ? this.#data : undefined 229 | } 230 | 231 | get videoFrame() { 232 | return (this.#data instanceof VideoFrame) ? this.#data : undefined 233 | } 234 | 235 | get audioData() { 236 | return (this.#data instanceof AudioData) ? this.#data : undefined 237 | } 238 | 239 | get offset() { 240 | return this.#offset 241 | } 242 | 243 | close() { 244 | if (this.#data instanceof VideoFrame) 245 | this.#data.close() 246 | else if (this.#data instanceof AudioData) 247 | this.#data.close() 248 | else if ('buffer' in this.#data) { 249 | this.worker.send('releaseWorkerBuffer', { buffer: this.#data }, [this.#data.buffer], this.id) 250 | } 251 | } 252 | } 253 | 254 | 255 | /** 256 | * stream output handler 257 | */ 258 | export async function newExporter(node: TargetNode, worker: FFWorker) { 259 | const [graphInstance, node2id] = buildGraphInstance(node) 260 | const id = uuid() 261 | const readers = [] 262 | // create readers from sources 263 | for (const [node, id] of node2id) { 264 | if (node.type != 'source') continue 265 | const reader = node.data.type == 'file' ? 266 | new FileReader(id, node.data.source, worker) : 267 | new StreamReader(id, SourceCacheData.get(node) ?? [], node.data.source, worker) 268 | readers.push(reader) 269 | } 270 | await worker.send('buildGraph', { graphInstance, flags: globalFlags.get() }, [], id) 271 | 272 | return new Exporter(id, worker, readers) 273 | } 274 | 275 | 276 | /** 277 | * Exporter is a class that exports data from a graph. 278 | */ 279 | export class Exporter { 280 | id: string 281 | worker: FFWorker 282 | readers: Reader[] // readers should be exists when exporting 283 | 284 | constructor(id: string, worker: FFWorker, readers: Reader[]) { 285 | this.id = id 286 | this.worker = worker 287 | this.readers = readers 288 | } 289 | 290 | /* end when return undefined */ 291 | async next() { 292 | // // make sure input reply ready 293 | // await Promise.all(this.readers.map(r => r.dataReady())) 294 | const {outputs, progress, endWriting} = await this.worker.send('nextFrame', undefined, [], this.id) 295 | // todo... temporarily only output one target 296 | if (Object.values(outputs).length > 1) throw `Currently only one target at a time allowed` 297 | const output = Object.values(outputs)[0] 298 | 299 | const chunks = (output??[]).map(d => new Chunk(d, this.worker, this.id)) 300 | 301 | return { output: chunks, progress, done: endWriting } 302 | } 303 | 304 | async close() { 305 | await this.worker.send('deleteGraph', undefined, [], this.id) 306 | } 307 | } 308 | -------------------------------------------------------------------------------- /src/ts/types/WebCodecs.d.ts: -------------------------------------------------------------------------------- 1 | 2 | interface VideoDecoder extends EventTarget {} 3 | interface AudioDecoder extends EventTarget {} 4 | interface VideoEncoder extends EventTarget {} 5 | interface AudioEncoder extends EventTarget {} 6 | -------------------------------------------------------------------------------- /src/ts/types/ffmpeg.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /** Above will import declarations from @types/emscripten, including Module etc. */ 3 | 4 | 5 | 6 | export interface AVRational { num: number, den: number } 7 | 8 | interface StdVector { 9 | size(): number 10 | get(i: number): T 11 | set(i: number, value: T): void 12 | push_back(t: T): void 13 | delete(): void 14 | } 15 | 16 | interface StdMap { 17 | size(): number 18 | get(key: T1): T2 19 | keys(): StdVector 20 | set(key: T1, val: T2): void 21 | } 22 | 23 | class CppClass { 24 | delete(): void 25 | clone(): this 26 | } 27 | 28 | // demuxer 29 | interface ReaderForDemuxer { 30 | size: number 31 | offset: number 32 | read: (buffer: Uint8Array) => Promise 33 | seek: (pos: number) => Promise 34 | } 35 | class Demuxer extends CppClass { 36 | constructor() 37 | build(reader: ReaderForDemuxer): Promise 38 | seek(t: number, streamIndex: number): Promise 39 | read(): Promise 40 | getTimeBase(streamIndex: number): AVRational 41 | getMetadata(): FormatInfo 42 | currentTime(streamIndex: number): number 43 | dump(): void 44 | } 45 | interface FormatInfo { 46 | formatName: string 47 | bitRate: number 48 | duration: number 49 | streamInfos: StdVector 50 | } 51 | 52 | // decode 53 | class Decoder extends CppClass { 54 | constructor(dexmuer: Demuxer, streamIndex: number, name: string) 55 | constructor(streamInfo: StreamInfo, name: string) 56 | name: number 57 | get timeBase(): AVRational 58 | get dataFormat(): DataFormat 59 | decode(packet: Packet): StdVector 60 | flush(): StdVector 61 | } 62 | 63 | // stream 64 | class Stream extends CppClass { 65 | } 66 | 67 | interface StreamInfo { 68 | index: number 69 | timeBase: AVRational 70 | bitRate: number 71 | startTime: number 72 | duration: number 73 | codecName: string 74 | mediaType: 'video' | 'audio' | undefined 75 | format: string // pixelFormat if codecType is 'video'; sampleFormat if codecType is 'audio' 76 | extraData: Uint8Array 77 | // video 78 | width: number 79 | height: number 80 | frameRate: number 81 | sampleAspectRatio: AVRational 82 | // audio 83 | channels: number 84 | channelLayout: string 85 | sampleRate: number 86 | } 87 | 88 | interface DataFormat { 89 | format: string // pixel_fmt / sample_fmt 90 | channels: number 91 | channelLayout: string 92 | sampleRate: number 93 | } 94 | 95 | // packet 96 | interface TimeInfo { 97 | pts: number, dts: number, duration: number 98 | } 99 | class Packet extends CppClass { 100 | constructor() 101 | constructor(bufSize: number, timeInfo: TimeInfo) 102 | size: number 103 | key: boolean 104 | get streamIndex(): number 105 | getData(): Uint8Array 106 | getTimeInfo(): TimeInfo 107 | setTimeInfo(timeInfo: TimeInfo): void 108 | dump():void 109 | } 110 | 111 | // frame 112 | interface FrameInfo { 113 | format: string 114 | height: number 115 | width: number 116 | sampleRate: number 117 | channels: number 118 | channelLayout: string; 119 | nbSamples: number; 120 | } 121 | 122 | class Frame extends CppClass { 123 | constructor(info: FrameInfo, pts: number, name: string); 124 | getFrameInfo(): FrameInfo 125 | static inferChannelLayout(channels: number): string 126 | getPlanes(): StdVector 127 | key: boolean 128 | pts: number 129 | dump():void 130 | name: string 131 | } 132 | 133 | // filter 134 | class Filterer extends CppClass { 135 | constructor(inStreams: StdMap, outStreams: StdMap, mediaTypes: StdMap, graphSpec: string) 136 | filter(frames: StdVector): StdVector 137 | flush(): StdVector 138 | delete(): void 139 | } 140 | // bitstream filter 141 | class BitstreamFilterer extends CppClass { 142 | constructor(filterName: string, demuxer: Demuxer, inStreamIndex: number, muxer: Muxer, outStreamIndex: number) 143 | filter(packet: Packet): void 144 | delete(): void 145 | } 146 | 147 | // encode 148 | class Encoder extends CppClass { 149 | constructor(params: StreamInfo) 150 | get timeBase(): AVRational 151 | get dataFormat(): DataFormat 152 | encode(f: Frame): StdVector 153 | flush(): StdVector 154 | delete(): void 155 | } 156 | 157 | // inferred info 158 | interface InferredStreamInfo { 159 | codecName: string 160 | format: string // pix_format / sample_format 161 | } 162 | 163 | interface InferredFormatInfo { 164 | format: string, 165 | video: InferredStreamInfo 166 | audio: InferredStreamInfo 167 | } 168 | 169 | interface WriterForMuxer { 170 | write(data: Uint8Array): void 171 | seek(pos: number): void 172 | } 173 | 174 | // muxer 175 | class Muxer extends CppClass { 176 | constructor(formatName: string, writer: WriterForMuxer) 177 | static inferFormatInfo(format: string, filename: string): InferredFormatInfo 178 | dump(): void 179 | newStreamWithDemuxer(demuxer: Demuxer, streamIndex: number): void 180 | newStreamWithEncoder(encoder: Encoder): void 181 | newStreamWithInfo(streamInfo: StreamInfo): void 182 | // openIO(): void 183 | writeHeader(): void 184 | writeTrailer(): void 185 | writeFrame(packet: Packet, streamIndex: number): void 186 | delete(): void 187 | } 188 | 189 | interface ModuleClass { 190 | Demuxer: typeof Demuxer 191 | Muxer: typeof Muxer 192 | Decoder: typeof Decoder 193 | Encoder: typeof Encoder 194 | Frame: typeof Frame 195 | Packet: typeof Packet 196 | Filterer: typeof Filterer 197 | BitstreamFilterer: typeof BitstreamFilterer 198 | } 199 | 200 | type ModuleInstance = {[k in keyof ModuleClass]: InstanceType} 201 | 202 | interface ModuleFunction { 203 | setConsoleLogger(verbose: boolean): void 204 | createFrameVector(): StdVector 205 | createStringStringMap(): StdMap 206 | } 207 | 208 | export interface FFmpegModule extends ModuleClass, ModuleFunction, EmscriptenModule {} 209 | export interface ModuleType extends ModuleInstance, ModuleFunction {} 210 | 211 | export default function createFFmpegModule(moduleOverrides?: Partial): Promise 212 | 213 | -------------------------------------------------------------------------------- /src/ts/types/flags.ts: -------------------------------------------------------------------------------- 1 | 2 | export interface Flags { 3 | webCodecs?: boolean | {video?: boolean, audio?: boolean} 4 | hardware?: boolean 5 | } -------------------------------------------------------------------------------- /src/ts/types/graph.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * definition of graphs: 3 | * UserGraph -> GraphInstance -> GraphRuntime 4 | */ 5 | 6 | import { SourceStream } from "../globals" 7 | 8 | export type BufferData = Uint8Array 9 | export type ChunkData = BufferData | VideoFrame | AudioData 10 | export interface WriteChunkData { data: ChunkData, offset: number } 11 | 12 | 13 | interface Rational {num: number, den: number} 14 | /** 15 | * all kinds of metadata infomation 16 | */ 17 | 18 | export interface FormatMetadata { 19 | formatName: string 20 | duration: number 21 | bitRate: number 22 | } 23 | 24 | interface CommonStreamMetadata { 25 | index: number, 26 | timeBase: Rational 27 | startTime: number, 28 | duration: number, 29 | bitRate: number, 30 | codecName: string, 31 | extraData: Uint8Array 32 | } 33 | 34 | /** 35 | * Video Track (stream) metadata 36 | */ 37 | export interface VideoStreamMetadata extends CommonStreamMetadata { 38 | /** 39 | * mediaType = 'video' or 'audio' 40 | */ 41 | mediaType: 'video' 42 | /** 43 | * height of video frame 44 | */ 45 | height: number, 46 | width: number, 47 | pixelFormat: string 48 | frameRate: number 49 | sampleAspectRatio: Rational 50 | } 51 | 52 | export interface AudioStreamMetadata extends CommonStreamMetadata { 53 | mediaType: 'audio' 54 | volume: number 55 | sampleFormat: string 56 | sampleRate: number 57 | channels: number 58 | channelLayout: string 59 | } 60 | 61 | export type StreamMetadata = AudioStreamMetadata | VideoStreamMetadata 62 | 63 | 64 | /** 65 | * user defined graph 66 | */ 67 | export type UserNode = SourceNode | FilterNode | TargetNode 68 | export type SourceType = ReadableStream | string | URL | RequestInfo | Blob | BufferData 69 | type FileSource = string | URL | RequestInfo | Blob | BufferData 70 | type StreamSource = SourceStream 71 | export interface StreamRef { from: SourceNode | FilterNode, index: number } 72 | export interface SourceNode { 73 | type: 'source', outStreams: StreamMetadata[], url?: string 74 | data: { type: 'file', container: FormatMetadata, fileSize: number, source: FileSource } | 75 | { type: 'stream', container?: FormatMetadata, elementType: 'frame' | 'chunk', source: StreamSource } 76 | } 77 | 78 | export interface FilterNode { 79 | type: 'filter', inStreams: StreamRef[], outStreams: StreamMetadata[], 80 | filter: { name: string, ffmpegArgs: string | {[k in string]?: string | number} } 81 | } 82 | 83 | export interface TargetNode { 84 | type: 'target', inStreams: StreamRef[], outStreams: StreamMetadata[], 85 | format: { type: 'frame' | 'video', container: FormatMetadata } 86 | } 87 | 88 | export type StreamInstanceRef = {from: string, index: number} 89 | export type SourceInstance = 90 | Omit & 91 | {id: string} & 92 | {data: Omit, "source"> | 93 | Omit, "source"> } 94 | export type FilterInstance = Omit & {inStreams: StreamInstanceRef[], id: string} 95 | export type TargetInstance = Omit & {inStreams: StreamInstanceRef[], id: string} 96 | 97 | /** 98 | * graph instance for execution 99 | */ 100 | export interface GraphInstance { 101 | nodes: {[id in string]?: SourceInstance | FilterInstance | TargetInstance} 102 | sources: string[] 103 | filterInstance?: { 104 | inputs: StreamInstanceRef[], 105 | outputs: StreamInstanceRef[] 106 | filters: string[], 107 | } 108 | targets: string[] 109 | } -------------------------------------------------------------------------------- /src/ts/types/wasm.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.wasm' { 2 | const value: string; 3 | export default value; 4 | } -------------------------------------------------------------------------------- /src/ts/types/worker-loader.d.ts: -------------------------------------------------------------------------------- 1 | declare module "*.worker.ts" { 2 | // You need to change `Worker`, if you specified a different value for the `workerType` option 3 | class WebpackWorker extends Worker { 4 | constructor(); 5 | } 6 | 7 | // Uncomment this if you set the `esModule` option to `false` 8 | // export = WebpackWorker; 9 | export default WebpackWorker; 10 | } -------------------------------------------------------------------------------- /src/ts/utils.ts: -------------------------------------------------------------------------------- 1 | export const isWebWorker = 2 | typeof self === "object" && 3 | self.constructor && 4 | self.constructor.name === "DedicatedWorkerGlobalScope"; 5 | 6 | export function Log(...msg: any[]) { 7 | console.log(...msg) 8 | } 9 | 10 | export class BufferPool { 11 | pool: ArrayBuffer[] 12 | max: number 13 | 14 | constructor(max = 20) { 15 | this.pool = []; 16 | this.max = max; 17 | } 18 | 19 | private getNextPowerOf2(size: number): number { 20 | return Math.pow(2, Math.ceil(Math.log2(size))); 21 | } 22 | 23 | create(size: number) { 24 | // Find smallest buffer >= requested size 25 | const bestBuffer = this.pool 26 | .filter(buffer => buffer.byteLength >= size) 27 | .sort((a, b) => a.byteLength - b.byteLength)[0]; 28 | 29 | if (bestBuffer) { 30 | this.pool = this.pool.filter(b => b !== bestBuffer); 31 | return new Uint8Array(bestBuffer, 0, size); 32 | } 33 | 34 | // Create new buffer with next power of 2 size 35 | return new Uint8Array(new ArrayBuffer(this.getNextPowerOf2(size)), 0, size); 36 | } 37 | 38 | delete(buffer: Uint8Array) { 39 | if (this.pool.length < this.max) { 40 | this.pool.push(buffer.buffer); 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const webpack = require('webpack'); 3 | const TerserPlugin = require("terser-webpack-plugin"); 4 | const HtmlWebpackPlugin = require('html-webpack-plugin'); 5 | 6 | 7 | const basicConfig = { 8 | entry: './src/ts/main.ts', 9 | module: { 10 | rules: [ 11 | { test: /\.ts?$/, use: 'ts-loader', exclude: /node_modules/ }, 12 | { 13 | test: /\.worker\.js$/, 14 | loader: "worker-loader", 15 | options: { inline: "no-fallback" } 16 | }, 17 | { 18 | test: /\.wasm$/, 19 | type: `asset/resource`, 20 | generator: { filename: '[name].wasm' } 21 | }, 22 | ], 23 | }, 24 | 25 | resolve: { 26 | extensions: ['.ts', '.d.ts', '...'], 27 | fallback: { events: false, fs: false, module: false, url: false, crypto: false, path: false, worker_threads: false } 28 | }, 29 | 30 | output: { 31 | filename: 'frameflow.min.js', 32 | library: { 33 | name: 'frameflow', 34 | type: 'umd', 35 | export: 'default', 36 | }, 37 | globalObject: 'this', 38 | path: path.resolve(__dirname, 'dist'), 39 | }, 40 | } 41 | 42 | 43 | // for development 44 | const devConfig = { 45 | mode: 'development', 46 | plugins: [ 47 | new HtmlWebpackPlugin({ 48 | title: 'FrameFlow dev', 49 | template: 'examples/browser/index.html', 50 | }), 51 | ], 52 | devServer: { 53 | static: path.join(__dirname, "examples"), 54 | }, 55 | devtool: 'eval-cheap-module-source-map', 56 | } 57 | 58 | 59 | // for production 60 | const prodConfig = { 61 | mode: 'production', 62 | optimization: { 63 | minimize: true, 64 | minimizer: [ 65 | new TerserPlugin() 66 | ], 67 | } 68 | } 69 | 70 | 71 | module.exports = (env, argv) => { 72 | const config = {...basicConfig} 73 | const plugins = [...config.plugins??[]] 74 | if (argv.mode === 'development') { 75 | plugins.push(...devConfig.plugins??[]) 76 | Object.assign(config, devConfig) 77 | } 78 | else if (argv.mode === 'production') { 79 | plugins.push(...prodConfig.plugins??[]) 80 | Object.assign(config, prodConfig) 81 | } 82 | else throw `specify env` 83 | 84 | 85 | return {...config, plugins: plugins.length > 0 ? plugins : undefined} 86 | } --------------------------------------------------------------------------------