├── .gitignore ├── .gitmodules ├── data ├── selfie_segmentation_landscape.tflite └── selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite ├── CMakeLists.txt ├── README.md ├── obs-backscrub.cpp └── LICENCE.md /.gitignore: -------------------------------------------------------------------------------- 1 | # build output 2 | build/ 3 | 4 | # IDE cruft 5 | .vscode/ 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "backscrub"] 2 | path = backscrub 3 | url = https://github.com/phlash/backscrub.git 4 | -------------------------------------------------------------------------------- /data/selfie_segmentation_landscape.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phlash/obs-backscrub/HEAD/data/selfie_segmentation_landscape.tflite -------------------------------------------------------------------------------- /data/selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phlash/obs-backscrub/HEAD/data/selfiesegmentation_mlkit-256x256-2021_01_19-v1215.f16.tflite -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Build OBS-plugin 2 | cmake_minimum_required(VERSION 3.16) 3 | 4 | project (obs-backscrub) 5 | 6 | find_package(LibObs REQUIRED) 7 | find_package(OpenCV REQUIRED) 8 | find_package(Threads REQUIRED) 9 | 10 | # Define BACKSCRUB to build against an external copy.. 11 | if(DEFINED BACKSCRUB) 12 | set(CMAKE_MODULE_PATH "${BACKSCRUB}" CACHE PATH "Location of FindBackscrub.cmake") 13 | find_package(Backscrub REQUIRED) 14 | if(NOT BACKSCRUB_FOUND) 15 | message(FATAL_ERROR "Missing Backscrub") 16 | endif() 17 | else() 18 | # assume backscrub is a sub-module in git 19 | find_package(Git REQUIRED) 20 | execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive) 21 | add_subdirectory(backscrub ${CMAKE_CURRENT_BINARY_DIR}/backscrub EXCLUDE_FROM_ALL) 22 | # TODO: export include paths from backscrub 23 | set(BACKSCRUB_INCLUDE ${CMAKE_CURRENT_LIST_DIR}/backscrub) 24 | set(BACKSCRUB_LIBS backscrub) 25 | endif() 26 | 27 | set(obs-backscrub_SOURCES 28 | obs-backscrub.cpp) 29 | 30 | add_library(obs-backscrub MODULE 31 | ${obs-backscrub_SOURCES}) 32 | # Remove the 'lib' prefix for this target 33 | set_target_properties(obs-backscrub PROPERTIES PREFIX "") 34 | 35 | target_include_directories(obs-backscrub 36 | PUBLIC ${BACKSCRUB_INCLUDE}) 37 | 38 | target_link_libraries(obs-backscrub 39 | ${LIBOBS_LIBRARIES} 40 | ${BACKSCRUB_LIBS} 41 | ${OpenCV_LIBS}) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # obs-backscrub 2 | 3 | Integration for [backscrub](https://github.com/floe/backscrub) project into OBS Studio 4 | 5 | ## What is this? 6 | 7 | It's a video filter plugin for OBS Studio (currently limited to video capture devices that produce YUY2 format streams), which uses the `backscrub` 8 | library to remove the background of a video source (replacing it with green), allowing subsequent chroma keying of alternate backgrounds. 9 | 10 | ## Neat! How do I build it? 11 | 12 | ### Linux 13 | 14 | * Install the dependencies (assuming you have OBS Studio already!): `build-essentials`, `libobs-dev`, `libopencv-dev` 15 | * Clone this project. 16 | * Do the CMake dance: 17 | ```bash 18 | % mkdir build; cmake -B build; cmake --build build -j 19 | ``` 20 | This will pull the submodules (backscrub and Tensorflow), configure them (mostly Tensorflow fetching several more dependencies) then compile everything. 21 | Expect to wait 10+ minutes for a full build, and lose ~2GB of disk space. 22 | 23 | ### Windows 24 | 25 | __NB: This is fiddly, fragile and poorly tested__ 26 | 27 | * Development environment: VS2019 community edition, on Server2019 / Windows10/11 28 | * Download and install dependencies (assuming you have OBS Studio already!): 29 | * OpenCV 3.4.14: https://sourceforge.net/projects/opencvlibrary/files/3.4.14/opencv-3.4.14-vc14_vc15.exe/download 30 | * Note installed path, eg: `C:\Users\phlash\Download\opencv` 31 | * Download OBS source code and bodge an import library (in lieu of a libobs-dev package): 32 | * Match your version, eg for 29.0.2: https://github.com/obsproject/obs-studio/archive/refs/tags/29.0.2.zip 33 | * Create a fresh `libobs` folder eg: `C:\Users\phlash\Downloads\libobs` 34 | * Save [LibObsConfig.zip](https://github.com/phlash/obs-backscrub/files/11357427/LibObsConfig.zip) to the `libobs` folder and unpack in place to get a CMake config script 35 | * From OBS source ZIP, unpack only the `libobs` folder _into folder above_, then _rename as `inc`_, eg: `C:\Users\phlash\Downloads\libobs\inc` 36 | * Create a `bin` folder next to `inc` eg: `C:\Users\phlash\Downloads\libobs\bin` 37 | * Copy the `OBS.DLL` library out of your installed OBS Studio eg: `C:\Program Files\obs-studio\bin\64bit\OBS.DLL` into `bin` folder. 38 | * In a `VS2019 Developer Command Prompt`, generate an import library from the DLL, instructions modified from: https://stackoverflow.com/questions/9946322/how-to-generate-an-import-library-lib-file-from-a-dll 39 | ```cmd 40 | C> cd 41 | C> echo LIBRARY OBS > obs.def 42 | C> echo EXPORTS >> obs.def 43 | C> for /f "skip=19 tokens=4" %A in ('dumpbin /exports OBS.DLL') do echo %A >> obs.def 44 | C> lib /def:obs.def /out:OBS.LIB /machine:x64 45 | ``` 46 | * Clone this project. 47 | * Checkout `windows-build` branch. 48 | * In a `VS2019 Developer Command Prompt` Do the CMake dance, informing it where OpenCV and libobs are: 49 | ```cmd 50 | C> mkdir build 51 | C> cmake -B build -D CMAKE_PREFIX_PATH="\build;" 52 | C> cmake --build build -j 53 | ``` 54 | * The build will fail, you need to fix up a bug in Tensorflow (https://github.com/tensorflow/tensorflow/issues/54323) 55 | * Edit: `tensorflow\tensorflow\core\lib\random\random_distributions_utils.h` and replace `M_PI` symbol with `3.1415928`. 56 | * Re-run build: 57 | ```cmd 58 | C> cmake --build build -j 59 | ``` 60 | 61 | ## It's built - how do I install it? 62 | 63 | ### Linux 64 | 65 | I choose to create sym-links for the built object and the data directory out of the obs-studio installation, this avoids install-to-test issues: 66 | ```bash 67 | % cd /lib/x86_64-linux-gnu/obs-plugins 68 | % sudo ln -s /home/phlash/obs-backscrub/build/obs-backscrub.so . 69 | % cd /usr/share/obs/obs-plugins 70 | % sudo ln -s /home/phlash/obs-backscrub/data obs-backscrub 71 | ``` 72 | 73 | ### Windows 74 | 75 | Not quite the same as Linux as Windows cannot do file symlinks, only folders, so we copy the built object and dependencies: 76 | ```cmd 77 | C> cd \Program Files\obs-studio\obs-plugin\64bit 78 | C> copy \Users\phlash\obs-backscrub\build\obs-backscrub.dll 79 | C> copy \build\x64\opencv-world3414d.dll 80 | C> cd ..\..\data\obs-plugin 81 | C> mklink /D obs-backscrub \Users\phlash\obs-backscrub\data 82 | ``` 83 | 84 | ## Using it? 85 | 86 | Fire up OBS Studio - _check the logs_ to ensure `obs-backscrub.dll` loads successfully. 87 | 88 | Add a video capture source, go to filters, and add an audio/video filter 'Background scrubber'. That's it. 89 | -------------------------------------------------------------------------------- /obs-backscrub.cpp: -------------------------------------------------------------------------------- 1 | // Simple OBS Studio plugin to use libbackscrub as a background removal filter 2 | // 3 | // 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "lib/libbackscrub.h" 10 | 11 | // Setting names & default values 12 | static const char MODEL_SETTING[] = "Segmentation model"; 13 | static const char MODEL_DEFAULT[] = "selfie_segmentation_landscape.tflite"; 14 | static const size_t BS_THREADS = 2; 15 | static const size_t BS_WIDTH = 640; 16 | static const size_t BS_HEIGHT = 480; 17 | 18 | // debugging 19 | static void obs_backscrub_dbg(void *ctx, const char *msg) { 20 | blog(LOG_INFO, "obs-backscrub(%p): %s", ctx, msg); 21 | } 22 | static void obs_printf(void *ctx, const char *fmt, ...) { 23 | va_list ap; 24 | va_start(ap, fmt); 25 | char *msg; 26 | if (vasprintf(&msg, fmt, ap)) { 27 | obs_backscrub_dbg(ctx, msg); 28 | free(msg); 29 | } 30 | va_end(ap); 31 | } 32 | 33 | OBS_DECLARE_MODULE() 34 | 35 | // A source, used as a filter.. 36 | struct obs_backscrub_filter_t { 37 | // internal filter state 38 | void *maskctx; 39 | char *modelname; 40 | size_t width; 41 | size_t height; 42 | cv::Mat input; 43 | cv::Mat mask; 44 | std::thread tid; 45 | std::mutex lock; 46 | std::condition_variable_any cond; 47 | bool new_frame; 48 | bool done; 49 | // additional blend settings 50 | }; 51 | static void obs_backscrub_mask_thread(obs_backscrub_filter_t *filter) { 52 | obs_printf(filter, "mask_thread: starting.."); 53 | while (!filter->done) { 54 | // wait for a fresh video frame 55 | cv::Mat frame; 56 | { 57 | std::lock_guard hold(filter->lock); 58 | while (!filter->new_frame) 59 | filter->cond.wait(filter->lock); 60 | filter->new_frame = false; 61 | frame = filter->input.clone(); 62 | } 63 | // check for empty frame (can happen if we are terminated before video starts) 64 | if (frame.empty()) 65 | continue; 66 | // run inference 67 | cv::Mat mask; 68 | bs_maskgen_process(filter->maskctx, frame, mask); 69 | // update mask 70 | { 71 | std::lock_guard hold(filter->lock); 72 | filter->mask = mask; 73 | } 74 | } 75 | obs_printf(filter, "mask_thread: done"); 76 | } 77 | static char *_obs_backscrub_get_model(obs_data_t *settings) { 78 | const char *settings_path = obs_data_get_string(settings, MODEL_SETTING); 79 | char *rv = nullptr; 80 | // relative paths, map through module location 81 | if (settings_path[0] == '/') 82 | rv = bstrdup(settings_path); 83 | else 84 | rv = obs_module_file(settings_path); 85 | if (!rv) 86 | obs_printf(nullptr, "_get_path: NULL file mapping, maybe missing module data folder?"); 87 | return rv; 88 | } 89 | static const char *obs_backscrub_get_name(void *type_data) { return "Background scrubber"; } 90 | static void *obs_backscrub_create(obs_data_t *settings, obs_source_t *source) { 91 | // here we instantiate a new filter, loading all required resources (eg: model file) 92 | // and setting initial values for filter settings 93 | auto *filter = new obs_backscrub_filter_t; 94 | obs_printf(filter, "create"); 95 | filter->modelname = _obs_backscrub_get_model(settings); 96 | filter->width = BS_WIDTH; 97 | filter->height = BS_HEIGHT; 98 | filter->maskctx = bs_maskgen_new(filter->modelname, BS_THREADS, filter->width, filter->height, 99 | obs_backscrub_dbg, nullptr, nullptr, nullptr, nullptr); 100 | if (!filter->maskctx) { 101 | obs_printf(filter, "oops initialising backscrub"); 102 | // if creation failed we still need to return a state, 103 | // otherwise the user won't be able to fix the config. 104 | return filter; 105 | } 106 | filter->new_frame = false; 107 | filter->done = false; 108 | filter->tid = std::thread(obs_backscrub_mask_thread, filter); 109 | obs_printf(filter, "create: done"); 110 | return filter; 111 | } 112 | static void obs_backscrub_get_defaults(obs_data_t *settings) { 113 | obs_printf(nullptr, "get_defaults"); 114 | obs_data_set_default_string(settings, MODEL_SETTING, MODEL_DEFAULT); 115 | } 116 | static obs_properties_t *obs_backscrub_get_properties(void *state) { 117 | obs_printf(nullptr, "get_properties"); 118 | obs_properties_t *props = obs_properties_create(); 119 | obs_properties_add_path(props, MODEL_SETTING, "Segmentation model file", OBS_PATH_FILE, 120 | "TFLite models (*.tflite)", MODEL_DEFAULT); 121 | return props; 122 | } 123 | static void obs_backscrub_update(void *state, obs_data_t *settings) { 124 | obs_backscrub_filter_t *filter = (obs_backscrub_filter_t *)state; 125 | char *model = _obs_backscrub_get_model(settings); 126 | obs_printf(filter, "update: model: %s=>%s", filter->modelname, model); 127 | // here we change any filter settings (eg: model used, feathering edges, bilateral smoothing) 128 | if (!filter->modelname && !model) return; // both null, no change required 129 | if (!filter->modelname || !model || strcmp(model, filter->modelname)) { 130 | // stop mask thread 131 | if (filter->tid.joinable()) { 132 | filter->done = true; 133 | filter->new_frame = true; 134 | filter->cond.notify_one(); 135 | filter->tid.join(); 136 | } 137 | // re-init backscrub and start thread again 138 | if (filter->maskctx) 139 | bs_maskgen_delete(filter->maskctx); 140 | if (filter->modelname) 141 | bfree(filter->modelname); 142 | filter->modelname = model; 143 | filter->maskctx = bs_maskgen_new(filter->modelname, BS_THREADS, filter->width, filter->height, 144 | obs_backscrub_dbg, nullptr, nullptr, nullptr, nullptr); 145 | if (!filter->maskctx) { 146 | obs_printf(filter, "oops re-initialising backscrub"); 147 | return; 148 | } 149 | filter->new_frame = false; 150 | filter->done = false; 151 | filter->tid = std::thread(obs_backscrub_mask_thread, filter); 152 | obs_printf(filter, "update: done"); 153 | } else { 154 | // if we get here, modelname and model are both non-null, so 155 | // we need to free model because we didn't put it into filter. 156 | bfree(model); 157 | } 158 | } 159 | static void obs_backscrub_destroy(void *state) { 160 | obs_backscrub_filter_t *filter = (obs_backscrub_filter_t *)state; 161 | obs_printf(filter, "destroy"); 162 | // stop mask thread 163 | if (filter->tid.joinable()) { 164 | filter->done = true; 165 | filter->new_frame = true; 166 | filter->cond.notify_one(); 167 | filter->tid.join(); 168 | } 169 | // free memory 170 | if (filter->maskctx) 171 | bs_maskgen_delete(filter->maskctx); 172 | if (filter->modelname) 173 | bfree(filter->modelname); 174 | delete filter; 175 | obs_printf(state, "destroy(%p): done"); 176 | } 177 | static void obs_backscrub_video_tick(void *state, float secs) { } 178 | static obs_source_frame *obs_backscrub_filter_video(void *state, obs_source_frame *frame) { 179 | obs_backscrub_filter_t *filter = (obs_backscrub_filter_t *)state; 180 | // here we do the video frame processing 181 | // First, map data into an OpenCV Mat object, then convert to BGR24 (default OCV format) 182 | // for calling libbackscrub 183 | cv::Mat out; 184 | switch (frame->format) { 185 | // TODO: more video formats! 186 | case VIDEO_FORMAT_YUY2: 187 | { 188 | // YUV2 (https://www.fourcc.org/pixel-format/yuv-yuy2/) as OpenCV array is 2x8bit channels 189 | // in OBS it arrives as a single plane of 16bits/pixel 190 | cv::Mat obs(frame->height, frame->width, CV_8UC2, frame->data[0], frame->linesize[0]); 191 | // Re-size to backscrub if required 192 | if (frame->width != filter->width || frame->height != filter->height) 193 | cv::resize(obs, obs, cv::Size(filter->width, filter->height)); 194 | // feed the mask thread 195 | std::lock_guard hold(filter->lock); 196 | cv::cvtColor(obs, filter->input, cv::COLOR_YUV2BGR_YUY2); 197 | filter->new_frame = true; 198 | filter->cond.notify_one(); 199 | // while we have the lock, grab current mask (if any) 200 | out = filter->mask.clone(); 201 | break; 202 | } 203 | default: 204 | obs_printf(filter, "filter_video: unsupported frame format: %s", get_video_format_name(frame->format)); 205 | return frame; 206 | } 207 | // No mask yet? 208 | if (out.empty()) 209 | return frame; 210 | // Re-size back to OBS video if required 211 | if (frame->width != filter->width || frame->height != filter->height) 212 | cv::resize(out, out, cv::Size(frame->width, frame->height)); 213 | // Mask the video image, leave the human, green screen the rest 214 | // blend the edges of the mask. 215 | for (int row=0; rowdata[0] + frame->linesize[0]*row; 217 | for (int col=0; col