├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── Win64 ├── DemoMatting │ ├── DemoMatting.vcxproj │ ├── DemoMatting.vcxproj.filters │ └── DemoMatting.vcxproj.user ├── ovmatting.sln └── ovmatting │ ├── dllmain.cpp │ ├── framework.h │ ├── ovmatting.vcxproj │ ├── ovmatting.vcxproj.filters │ ├── ovmatting.vcxproj.user │ ├── pch.cpp │ └── pch.h ├── demo.py ├── demo1.py ├── include ├── cnn.hpp ├── cnn_backgroundv2.hpp ├── cnn_modnet.hpp ├── common.hpp ├── ext_ops.hpp ├── ns_bmp.hpp ├── ns_register.hpp ├── ns_thread.hpp ├── ns_utils.hpp ├── ocv_common.hpp ├── ovmatter.h ├── ovmatter_base_impl.hpp ├── ovmatter_bgv2_impl.hpp ├── ovmatter_modnet_impl.hpp └── samples.hpp ├── libovmatting └── CMakeLists.txt ├── main.cpp ├── run.sh ├── samples ├── inference_camera.cpp ├── inference_demo1.cpp ├── inference_modnet.cpp └── inference_video.cpp ├── share ├── modnet.bin ├── modnet.xml ├── pytorch_mobilenetv2.bin ├── pytorch_mobilenetv2.xml ├── replace.jpg ├── src.mp4 └── src.png └── src ├── cnn.cpp ├── cnn_backgroundv2.cpp ├── cnn_modnet.cpp ├── ext_ops.cpp ├── ns_bmp.cpp ├── ns_register.cpp ├── ns_thread.cpp ├── ns_utils.cpp ├── ovmatter.cpp ├── ovmatter_base_impl.cpp ├── ovmatter_bgv2_impl.cpp └── ovmatter_modnet_impl.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | .svn 35 | .vscode 36 | **/build/ 37 | **/.vs 38 | **/x64 39 | **/dst.mp4 40 | **/bxg1.png 41 | **/bxg1.mp4 42 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.10) 3 | 4 | add_subdirectory(libovmatting) 5 | 6 | project(sample) 7 | 8 | set(CMAKE_CXX_STANDARD 11) 9 | #set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${PROJECT_SOURCE_DIR}/out/debug) 10 | #set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/out/release) 11 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/build) 12 | 13 | set(TARGET_NAME "sample") 14 | 15 | include_directories(${PROJECT_SOURCE_DIR}/include) 16 | message("Root Directory:${PROJECT_SOURCE_DIR}") 17 | 18 | find_package(ngraph REQUIRED OPTIONAL_COMPONENTS onnx_importer) 19 | find_package(InferenceEngine REQUIRED) 20 | find_package(OpenCV REQUIRED) 21 | 22 | LINK_DIRECTORIES(${PROJECT_SOURCE_DIR}/build) 23 | 24 | 25 | file(GLOB NATIVE_SRCS ${PROJECT_SOURCE_DIR}/main.cpp ${PROJECT_SOURCE_DIR}/samples/*.cpp) 26 | # set(SRC ${PROJECT_SOURCE_DIR}/src/*.cpp ) 27 | # file(GLOB NATIVE_SRCS ${PROJECT_SOURCE_DIR}/src/*.cpp ${PROJECT_SOURCE_DIR}/main.cpp) 28 | 29 | ADD_EXECUTABLE(${TARGET_NAME} ${NATIVE_SRCS} "include/ovmatter.h") 30 | 31 | target_link_libraries(${TARGET_NAME} PRIVATE ovmatting) 32 | 33 | target_compile_definitions(${TARGET_NAME} PRIVATE OPENCV_IMPORT_ENABLED) 34 | target_link_libraries(${TARGET_NAME} PRIVATE opencv_core) 35 | 36 | target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_INFERENCE_EXTENSION_API) 37 | target_link_libraries(${TARGET_NAME} PRIVATE IE::inference_engine ${NGRAPH_LIBRARIES}) 38 | target_link_libraries(${TARGET_NAME} PRIVATE ${ONNX_IMPORTER_LIBRARIES}) 39 | target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCV_LIBS}) 40 | target_compile_definitions(${TARGET_NAME} PRIVATE NGRAPH_ONNX_IMPORT_ENABLED) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 kingpeter2015 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libovmatting 2 | A C++ library for Background Matting using openvino and deep learning models.
3 | 4 | Now support models such as BackgroundMattingV2, MODNet.
5 | 6 | ## 1 Application scenes 7 |
8 | ...
9 | 10 | ## 2 License 11 | The code of libovmatting is released under the MIT License. 12 | 13 | ## 3 Development environment 14 | CentOS 7
15 | Ubuntu
16 | Windows 10
17 | 18 | ## 4 Usage 19 | ### 4.1 Install openvino
20 | Please refer https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html
21 | Use version: openvino_2021.2.185
22 |
23 | ### 4.2 Build libovmatting library and test program
24 | Clone the project
25 | $git clone https://github.com/kingpeter2015/libovmatting.git
26 | 27 | #### 4.2.1 Ubuntu/CentOS 28 | 29 | $cd libovmatting
30 | $mkdir build
31 | $cmake ..
32 | $make
33 |
34 | Run the test program
35 | $cd ./build/
36 | $LD_LIBRARY_PATH=./:$(LD_LIBRARY_PATH) ./sample
37 | 38 | #### 4.2.2 Windows 10 39 | Install Visual Studio 2019
40 | open Win64\ovmatting.sln
41 | build and run in the Visual Studio IDE
42 | 43 | ## 5 About Source Code 44 | 45 | ### 5.1 Directory Structure 46 | 47 | * Win64: Windows Solution and projects
48 | * include: contains header files. **ovmatter.h is api header file**
49 | * libovmatting: CMakefiles for compile libovmatting.so under linux
50 | * samples: four samples using libovmatting library
51 | * share: contains model files which are used by openvino inference engine.
52 | * src: contains source code files
53 | 54 | main.cpp: startup point of demos
55 | 56 | ### 5.2 Please Attention !!! 57 | 58 | * 'include/ovmatter.h' is one only sdk header file which exposes api to applications. 59 | * libovmatting library only supports two matting method: METHOD_BACKGROUND_MATTING_V2 and METHOD_MODNET. They are defined in include/ovmatter.h 60 | * METHOD_BACKGROUND_MATTING_V2 method uses cnn model defined 'pytorch_mobilenetv2.bin/pytorch_mobilenetv2.xml'. 61 | * METHOD_MODNET method uses cnn model defined in 'modnet.bin/modnet.xml' 62 | 63 | if you want to use your own model, you can do following steps: 64 | * Add a enum item in enum OV_MATTER_API MattingMethod which is defined in include/ovmatter.h 65 | * declare and implement a new ovmatter class, and inherit 'MatterBaseImpl', then write a new cnn that uses your own models 66 | 67 | That's all, have fun ! 68 | 69 | ## 6 Reference 70 | OpenVINO https://software.intel.com/content/www/us/en/develop/tools/openvino-toolkit.html
71 | BackgroundMattingV2 https://github.com/PeterL1n/BackgroundMattingV2
72 | MODNet https://github.com/ZHKKKe/MODNet
73 | -------------------------------------------------------------------------------- /Win64/DemoMatting/DemoMatting.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | x64 7 | 8 | 9 | Release 10 | x64 11 | 12 | 13 | 14 | 16.0 15 | Win32Proj 16 | {2faea063-7fb6-4695-8471-42c5353c965c} 17 | DemoMatting 18 | 10.0 19 | 20 | 21 | 22 | Application 23 | true 24 | v142 25 | MultiByte 26 | 27 | 28 | Application 29 | false 30 | v142 31 | true 32 | MultiByte 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | true 48 | 49 | 50 | false 51 | 52 | 53 | 54 | Level3 55 | true 56 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 57 | true 58 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\external\tbb\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include;C:\Program Files (x86)\Intel\openvino_2021\opencv\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include\ngraph\frontend;..\..\include 59 | 4244;4251;4267;4275;26451;26495;26812; 60 | 61 | 62 | Console 63 | true 64 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\lib\intel64\Debug;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\lib;C:\Program Files (x86)\Intel\openvino_2021\opencv\lib;C:\Program Files (x86)\Intel\openvino_2021\inference_engine\external\tbb\lib;..\x64\Debug 65 | ovmatting.lib;inference_engined.lib;inference_engine_c_apid.lib;inference_engine_ir_readerd.lib;inference_engine_legacyd.lib;inference_engine_lp_transformationsd.lib;inference_engine_onnx_readerd.lib;inference_engine_preprocd.lib;inference_engine_transformationsd.lib;ngraphd.lib;onnx_importerd.lib;opencv_calib3d451d.lib;opencv_core451d.lib;opencv_dnn451d.lib;opencv_features2d451d.lib;opencv_flann451d.lib;opencv_gapi451d.lib;opencv_highgui451d.lib;opencv_imgcodecs451d.lib;opencv_imgproc451d.lib;opencv_ml451d.lib;opencv_objdetect451d.lib;opencv_photo451d.lib;opencv_stitching451d.lib;opencv_video451d.lib;opencv_videoio451d.lib;tbb.lib;tbbbind.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;tbbproxy.lib;tbb_preview.lib;%(AdditionalDependencies) 66 | 67 | 68 | 69 | 70 | Level3 71 | true 72 | true 73 | true 74 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 75 | true 76 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\external\tbb\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include;C:\Program Files (x86)\Intel\openvino_2021\opencv\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include\ngraph\frontend;..\..\include; 77 | 4244;4251;4267;4275;26451;26495;26812; 78 | 79 | 80 | Console 81 | true 82 | true 83 | true 84 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\lib\intel64\Release;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\lib;C:\Program Files (x86)\Intel\openvino_2021\opencv\lib;C:\Program Files (x86)\Intel\openvino_2021\inference_engine\external\tbb\lib;..\x64\Release 85 | ovmatting.lib;inference_engine.lib;inference_engine_c_api.lib;inference_engine_ir_reader.lib;inference_engine_legacy.lib;inference_engine_lp_transformations.lib;inference_engine_onnx_reader.lib;inference_engine_preproc.lib;inference_engine_transformations.lib;ngraph.lib;onnx_importer.lib;opencv_calib3d451.lib;opencv_core451.lib;opencv_dnn451.lib;opencv_features2d451.lib;opencv_flann451.lib;opencv_gapi451.lib;opencv_highgui451.lib;opencv_imgcodecs451.lib;opencv_imgproc451.lib;opencv_ml451.lib;opencv_objdetect451.lib;opencv_photo451.lib;opencv_stitching451.lib;opencv_video451.lib;opencv_videoio451.lib;tbb.lib;tbbbind.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;tbbproxy.lib;tbb_preview.lib;%(AdditionalDependencies) 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /Win64/DemoMatting/DemoMatting.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | Source Files 23 | 24 | 25 | Source Files 26 | 27 | 28 | Source Files 29 | 30 | 31 | Source Files 32 | 33 | 34 | -------------------------------------------------------------------------------- /Win64/DemoMatting/DemoMatting.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | $(TargetDir) 5 | WindowsLocalDebugger 6 | OMP_NUM_THREADS=4 7 | $(LocalDebuggerEnvironment) 8 | -method 1 9 | 10 | 11 | $(TargetDir) 12 | WindowsLocalDebugger 13 | OMP_NUM_THREADS=4 14 | $(LocalDebuggerEnvironment) 15 | -method 1 16 | 17 | -------------------------------------------------------------------------------- /Win64/ovmatting.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.30804.86 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "DemoMatting", "DemoMatting\DemoMatting.vcxproj", "{2FAEA063-7FB6-4695-8471-42C5353C965C}" 7 | ProjectSection(ProjectDependencies) = postProject 8 | {01F0A09C-F8D2-418F-981B-0E2DAFB14A01} = {01F0A09C-F8D2-418F-981B-0E2DAFB14A01} 9 | EndProjectSection 10 | EndProject 11 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ovmatting", "ovmatting\ovmatting.vcxproj", "{01F0A09C-F8D2-418F-981B-0E2DAFB14A01}" 12 | EndProject 13 | Global 14 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 15 | Debug|x64 = Debug|x64 16 | Release|x64 = Release|x64 17 | EndGlobalSection 18 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 19 | {2FAEA063-7FB6-4695-8471-42C5353C965C}.Debug|x64.ActiveCfg = Debug|x64 20 | {2FAEA063-7FB6-4695-8471-42C5353C965C}.Debug|x64.Build.0 = Debug|x64 21 | {2FAEA063-7FB6-4695-8471-42C5353C965C}.Release|x64.ActiveCfg = Release|x64 22 | {2FAEA063-7FB6-4695-8471-42C5353C965C}.Release|x64.Build.0 = Release|x64 23 | {01F0A09C-F8D2-418F-981B-0E2DAFB14A01}.Debug|x64.ActiveCfg = Debug|x64 24 | {01F0A09C-F8D2-418F-981B-0E2DAFB14A01}.Debug|x64.Build.0 = Debug|x64 25 | {01F0A09C-F8D2-418F-981B-0E2DAFB14A01}.Release|x64.ActiveCfg = Release|x64 26 | {01F0A09C-F8D2-418F-981B-0E2DAFB14A01}.Release|x64.Build.0 = Release|x64 27 | EndGlobalSection 28 | GlobalSection(SolutionProperties) = preSolution 29 | HideSolutionNode = FALSE 30 | EndGlobalSection 31 | GlobalSection(ExtensibilityGlobals) = postSolution 32 | SolutionGuid = {94AE093A-B668-44B0-A122-CEE6079EBF6E} 33 | EndGlobalSection 34 | EndGlobal 35 | -------------------------------------------------------------------------------- /Win64/ovmatting/dllmain.cpp: -------------------------------------------------------------------------------- 1 | // dllmain.cpp : Defines the entry point for the DLL application. 2 | #include "pch.h" 3 | 4 | BOOL APIENTRY DllMain( HMODULE hModule, 5 | DWORD ul_reason_for_call, 6 | LPVOID lpReserved 7 | ) 8 | { 9 | switch (ul_reason_for_call) 10 | { 11 | case DLL_PROCESS_ATTACH: 12 | case DLL_THREAD_ATTACH: 13 | case DLL_THREAD_DETACH: 14 | case DLL_PROCESS_DETACH: 15 | break; 16 | } 17 | return TRUE; 18 | } 19 | 20 | -------------------------------------------------------------------------------- /Win64/ovmatting/framework.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers 4 | // Windows Header Files 5 | #include 6 | -------------------------------------------------------------------------------- /Win64/ovmatting/ovmatting.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | x64 7 | 8 | 9 | Release 10 | x64 11 | 12 | 13 | 14 | 16.0 15 | Win32Proj 16 | {01f0a09c-f8d2-418f-981b-0e2dafb14a01} 17 | ovmatting 18 | 10.0 19 | 20 | 21 | 22 | DynamicLibrary 23 | true 24 | v142 25 | MultiByte 26 | 27 | 28 | DynamicLibrary 29 | false 30 | v142 31 | true 32 | MultiByte 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | true 48 | 49 | 50 | false 51 | 52 | 53 | 54 | Level3 55 | true 56 | _DEBUG;OVMATTING_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) 57 | true 58 | NotUsing 59 | pch.h 60 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\external\tbb\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include;C:\Program Files (x86)\Intel\openvino_2021\opencv\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include\ngraph\frontend;..\..\include 61 | Default 62 | Default 63 | 4244;4251;4267;4275;26451;26495;26812;%(DisableSpecificWarnings) 64 | 65 | 66 | Windows 67 | true 68 | false 69 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\lib\intel64\Debug;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\lib;C:\Program Files (x86)\Intel\openvino_2021\opencv\lib;C:\Program Files (x86)\Intel\openvino_2021\inference_engine\external\tbb\lib;%(AdditionalLibraryDirectories) 70 | inference_engined.lib;inference_engine_c_apid.lib;inference_engine_ir_readerd.lib;inference_engine_legacyd.lib;inference_engine_lp_transformationsd.lib;inference_engine_onnx_readerd.lib;inference_engine_preprocd.lib;inference_engine_transformationsd.lib;ngraphd.lib;onnx_importerd.lib;opencv_calib3d451d.lib;opencv_core451d.lib;opencv_dnn451d.lib;opencv_features2d451d.lib;opencv_flann451d.lib;opencv_gapi451d.lib;opencv_highgui451d.lib;opencv_imgcodecs451d.lib;opencv_imgproc451d.lib;opencv_ml451d.lib;opencv_objdetect451d.lib;opencv_photo451d.lib;opencv_stitching451d.lib;opencv_video451d.lib;opencv_videoio451d.lib;tbb.lib;tbbbind.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;tbbproxy.lib;tbb_preview.lib;%(AdditionalDependencies) 71 | 72 | 73 | 74 | 75 | Level3 76 | true 77 | true 78 | true 79 | NDEBUG;OVMATTING_EXPORTS;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) 80 | true 81 | NotUsing 82 | pch.h 83 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\external\tbb\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include;C:\Program Files (x86)\Intel\openvino_2021\opencv\include;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\include\ngraph\frontend;..\..\include;%(AdditionalIncludeDirectories) 84 | 4244;4251;4267;4275;26451;26495;26812;%(DisableSpecificWarnings) 85 | 86 | 87 | Windows 88 | true 89 | true 90 | true 91 | false 92 | inference_engine.lib;inference_engine_c_api.lib;inference_engine_ir_reader.lib;inference_engine_legacy.lib;inference_engine_lp_transformations.lib;inference_engine_onnx_reader.lib;inference_engine_preproc.lib;inference_engine_transformations.lib;ngraph.lib;onnx_importer.lib;opencv_calib3d451.lib;opencv_core451.lib;opencv_dnn451.lib;opencv_features2d451.lib;opencv_flann451.lib;opencv_gapi451.lib;opencv_highgui451.lib;opencv_imgcodecs451.lib;opencv_imgproc451.lib;opencv_ml451.lib;opencv_objdetect451.lib;opencv_photo451.lib;opencv_stitching451.lib;opencv_video451.lib;opencv_videoio451.lib;tbb.lib;tbbbind.lib;tbbmalloc.lib;tbbmalloc_proxy.lib;tbbproxy.lib;tbb_preview.lib;%(AdditionalDependencies) 93 | C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\inference_engine\lib\intel64\Release;C:\Program Files (x86)\Intel\openvino_2021\deployment_tools\ngraph\lib;C:\Program Files (x86)\Intel\openvino_2021\opencv\lib;C:\Program Files (x86)\Intel\openvino_2021\inference_engine\external\tbb\lib 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | Create 128 | Create 129 | 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /Win64/ovmatting/ovmatting.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 10 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 11 | 12 | 13 | {556ed048-d048-4890-a5a6-4966691db9f5} 14 | 15 | 16 | {15544307-ea9d-45cd-b834-8f6f2e463d8d} 17 | 18 | 19 | 20 | 21 | headers 22 | 23 | 24 | headers 25 | 26 | 27 | headers 28 | 29 | 30 | headers 31 | 32 | 33 | headers 34 | 35 | 36 | headers 37 | 38 | 39 | headers 40 | 41 | 42 | Source Files 43 | 44 | 45 | Source Files 46 | 47 | 48 | headers 49 | 50 | 51 | headers 52 | 53 | 54 | headers 55 | 56 | 57 | headers 58 | 59 | 60 | headers 61 | 62 | 63 | headers 64 | 65 | 66 | 67 | 68 | Source Files 69 | 70 | 71 | Source Files 72 | 73 | 74 | source 75 | 76 | 77 | source 78 | 79 | 80 | source 81 | 82 | 83 | source 84 | 85 | 86 | source 87 | 88 | 89 | source 90 | 91 | 92 | source 93 | 94 | 95 | source 96 | 97 | 98 | source 99 | 100 | 101 | source 102 | 103 | 104 | source 105 | 106 | 107 | -------------------------------------------------------------------------------- /Win64/ovmatting/ovmatting.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /Win64/ovmatting/pch.cpp: -------------------------------------------------------------------------------- 1 | // pch.cpp: source file corresponding to the pre-compiled header 2 | 3 | #include "pch.h" 4 | 5 | // When you are using pre-compiled headers, this source file is necessary for compilation to succeed. 6 | -------------------------------------------------------------------------------- /Win64/ovmatting/pch.h: -------------------------------------------------------------------------------- 1 | // pch.h: This is a precompiled header file. 2 | // Files listed below are compiled only once, improving build performance for future builds. 3 | // This also affects IntelliSense performance, including code completion and many code browsing features. 4 | // However, files listed here are ALL re-compiled if any one of them is updated between builds. 5 | // Do not add files here that you will be updating frequently as this negates the performance advantage. 6 | 7 | #ifndef PCH_H 8 | #define PCH_H 9 | 10 | // add headers that you want to pre-compile here 11 | #include "framework.h" 12 | 13 | #endif //PCH_H 14 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Copyright (c) 2018 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | """ 17 | from __future__ import print_function 18 | import sys 19 | import os 20 | from argparse import ArgumentParser, SUPPRESS 21 | import cv2 22 | import numpy as np 23 | import logging as log 24 | from openvino.inference_engine import IECore 25 | from PIL import Image 26 | 27 | import ngraph as ng 28 | 29 | def build_argparser(): 30 | parser = ArgumentParser(add_help=False) 31 | args = parser.add_argument_group("Options") 32 | args.add_argument('-h', '--help', action='help', default=SUPPRESS, help='Show this help message and exit.') 33 | args.add_argument("-m", "--model", help="Required. Path to an .xml or .onnx file with a trained model.", 34 | required=False, type=str) 35 | args.add_argument("-i", "--input", help="Required. Path to image file.", 36 | required=False, type=str, nargs="+") 37 | args.add_argument("-l", "--cpu_extension", 38 | help="Optional. Required for CPU custom layers. " 39 | "Absolute path to a shared library with the kernels implementations.", 40 | type=str, default=None) 41 | args.add_argument("-d", "--device", 42 | help="Optional. Specify the target device to infer on; " 43 | "CPU, GPU, FPGA or MYRIAD is acceptable. " 44 | "Sample will look for a suitable plugin for device specified (CPU by default)", 45 | default="CPU", type=str) 46 | args.add_argument("--labels", help="Optional. Labels mapping file", default=None, type=str) 47 | args.add_argument("-nt", "--number_top", help="Optional. Number of top results", default=10, type=int) 48 | 49 | return parser 50 | 51 | 52 | def main(): 53 | log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) 54 | args = build_argparser().parse_args() 55 | log.info("Loading Inference Engine") 56 | ie = IECore() 57 | 58 | # ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format --- 59 | # model = args.model 60 | model_xml = "pytorch_mobilenetv2.xml" 61 | model_bin = "pytorch_mobilenetv2.bin" 62 | log.info(f"Loading network:\n\t{model}") 63 | 64 | net = ie.read_network(model=model_xml, weights=model_bin) 65 | print("read_network Successfully") 66 | 67 | # ----------------------------------------------------------------------------------------------------- 68 | 69 | # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- 70 | log.info("Device info:") 71 | versions = ie.get_versions(args.device) 72 | print("{}{}".format(" " * 8, args.device)) 73 | print("{}MKLDNNPlugin version ......... {}.{}".format(" " * 8, versions[args.device].major, 74 | versions[args.device].minor)) 75 | print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) 76 | 77 | if args.cpu_extension and "CPU" in args.device: 78 | ie.add_extension(args.cpu_extension, "CPU") 79 | log.info("CPU extension loaded: {}".format(args.cpu_extension)) 80 | # ----------------------------------------------------------------------------------------------------- 81 | 82 | # --------------------------- 3. Read and preprocess input -------------------------------------------- 83 | 84 | path_bgr = "./src.png" 85 | bgr = [Image.open(path_bgr).convert('RGB')] 86 | path_vid = "./src.mp4" 87 | 88 | # ----------------------------------------------------------------------------------------------------- 89 | 90 | # --------------------------- 4. Configure input & output --------------------------------------------- 91 | # --------------------------- Prepare input blobs ----------------------------------------------------- 92 | log.info("Preparing input blobs") 93 | assert (len(net.input_info.keys()) == 1 or len( 94 | net.input_info.keys()) == 2), "Sample supports topologies only with 1 or 2 inputs" 95 | out_blob = next(iter(net.outputs)) 96 | input_name, input_info_name = "", "" 97 | 98 | for input_key in net.input_info: 99 | if len(net.input_info[input_key].layout) == 4: 100 | input_name = input_key 101 | log.info("Batch size is {}".format(net.batch_size)) 102 | net.input_info[input_key].precision = 'U8' 103 | elif len(net.input_info[input_key].layout) == 2: 104 | input_info_name = input_key 105 | net.input_info[input_key].precision = 'FP32' 106 | if net.input_info[input_key].input_data.shape[1] != 3 and net.input_info[input_key].input_data.shape[1] != 6 or \ 107 | net.input_info[input_key].input_data.shape[0] != 1: 108 | log.error('Invalid input info. Should be 3 or 6 values length.') 109 | 110 | data = {} 111 | data[input_name] = images 112 | 113 | if input_info_name != "": 114 | infos = np.ndarray(shape=(n, c), dtype=float) 115 | for i in range(n): 116 | infos[i, 0] = h 117 | infos[i, 1] = w 118 | infos[i, 2] = 1.0 119 | data[input_info_name] = infos 120 | 121 | # --------------------------- Prepare output blobs ---------------------------------------------------- 122 | log.info('Preparing output blobs') 123 | 124 | output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))] 125 | output_ops = {op.friendly_name : op for op in ops \ 126 | if op.friendly_name in net.outputs and op.get_type_name() == "DetectionOutput"} 127 | if len(output_ops) != 0: 128 | output_name, output_info = output_ops.popitem() 129 | 130 | if output_name == "": 131 | log.error("Can't find a DetectionOutput layer in the topology") 132 | 133 | output_dims = output_info.shape 134 | if len(output_dims) != 4: 135 | log.error("Incorrect output dimensions for SSD model") 136 | max_proposal_count, object_size = output_dims[2], output_dims[3] 137 | 138 | if object_size != 7: 139 | log.error("Output item should have 7 as a last dimension") 140 | 141 | output_info.precision = "FP32" 142 | # ----------------------------------------------------------------------------------------------------- 143 | 144 | # --------------------------- Performing inference ---------------------------------------------------- 145 | log.info("Loading model to the device") 146 | exec_net = ie.load_network(network=net, device_name=args.device) 147 | log.info("Creating infer request and starting inference") 148 | res = exec_net.infer(inputs=data) 149 | # ----------------------------------------------------------------------------------------------------- 150 | 151 | # --------------------------- Read and postprocess output --------------------------------------------- 152 | log.info("Processing output blobs") 153 | res = res[out_blob] 154 | boxes, classes = {}, {} 155 | data = res[0][0] 156 | for number, proposal in enumerate(data): 157 | if proposal[2] > 0: 158 | imid = np.int(proposal[0]) 159 | ih, iw = images_hw[imid] 160 | label = np.int(proposal[1]) 161 | confidence = proposal[2] 162 | xmin = np.int(iw * proposal[3]) 163 | ymin = np.int(ih * proposal[4]) 164 | xmax = np.int(iw * proposal[5]) 165 | ymax = np.int(ih * proposal[6]) 166 | print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}" \ 167 | .format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="") 168 | if proposal[2] > 0.5: 169 | print(" WILL BE PRINTED!") 170 | if not imid in boxes.keys(): 171 | boxes[imid] = [] 172 | boxes[imid].append([xmin, ymin, xmax, ymax]) 173 | if not imid in classes.keys(): 174 | classes[imid] = [] 175 | classes[imid].append(label) 176 | else: 177 | print() 178 | 179 | for imid in classes: 180 | tmp_image = cv2.imread(args.input[imid]) 181 | for box in boxes[imid]: 182 | cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]), (232, 35, 244), 2) 183 | cv2.imwrite("out.bmp", tmp_image) 184 | log.info("Image out.bmp created!") 185 | # ----------------------------------------------------------------------------------------------------- 186 | 187 | log.info("Execution successful\n") 188 | log.info( 189 | "This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool") 190 | 191 | 192 | if __name__ == '__main__': 193 | sys.exit(main() or 0) 194 | -------------------------------------------------------------------------------- /demo1.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | cv2.namedWindow("frame") #, cv2.WINDOW_NORMAL or cv2.WINDOW_FREERATIO 4 | cap = cv2.VideoCapture(1) 5 | height, width = 720, 1280 6 | cap.set(cv2.CAP_PROP_FRAME_WIDTH ,width) 7 | cap.set(cv2.CAP_PROP_FRAME_HEIGHT,height) 8 | cap.set(cv2.CAP_PROP_FPS, 60) 9 | 10 | while(True): 11 | success, frame=cap.read() 12 | cv2.imshow("frame", frame) 13 | ckey = cv2.waitKey(10) 14 | print("press key:",ckey) 15 | if(ckey == ord('c')): 16 | break 17 | 18 | # 19 | -------------------------------------------------------------------------------- /include/cnn.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "ocv_common.hpp" 12 | 13 | #include 14 | #include 15 | 16 | 17 | struct MattingObject { 18 | cv::Mat pha; 19 | cv::Mat com; 20 | cv::Mat fgr; 21 | }; 22 | 23 | using MattingObjects = std::vector; 24 | 25 | /** 26 | * @brief Base class of config for network 27 | */ 28 | struct CNNConfig 29 | { 30 | explicit CNNConfig(const std::string &path_to_model, const std::string &bin, const cv::Size in_shape=cv::Size(256,144)) : path_to_model(path_to_model), path_to_bin(bin), input_shape(in_shape){} 31 | 32 | std::string path_to_model; 33 | std::string path_to_bin; 34 | float scale{0.25}; 35 | int max_batch_size{1}; 36 | 37 | InferenceEngine::Core ie; 38 | std::string deviceName{"CPU"}; 39 | 40 | bool is_async{false}; 41 | cv::Size input_shape; 42 | 43 | int cpu_threads_num; //default 0 44 | bool cpu_bind_thread; //default true 45 | int cpu_throughput_streams; //default 1 46 | int interval; 47 | float motion_threshold; 48 | }; 49 | 50 | /** 51 | * @brief Base class of network 52 | */ 53 | class CnnDLSDKBase 54 | { 55 | public: 56 | /** 57 | * @brief Constructor 58 | */ 59 | explicit CnnDLSDKBase(const CNNConfig& config); 60 | 61 | /** 62 | * @brief Descructor 63 | */ 64 | ~CnnDLSDKBase() {} 65 | 66 | /** 67 | * @brief Loads network 68 | */ 69 | void Load(); 70 | 71 | /** 72 | * @brief Prints performance report 73 | */ 74 | void PrintPerformanceCounts(std::string fullDeviceName) const; 75 | 76 | protected: 77 | 78 | /** @brief Config */ 79 | CNNConfig _config; 80 | 81 | /** @brief Net inputs info */ 82 | InferenceEngine::InputsDataMap _inInfo; 83 | /** @brief Name of the input blob input blob */ 84 | std::vector _input_blob_names; 85 | 86 | /** @brief Net outputs info */ 87 | InferenceEngine::OutputsDataMap _outInfo; 88 | /** @brief Names of output blobs */ 89 | std::vector _output_blobs_names; 90 | 91 | InferenceEngine::CNNNetwork _cnn_network_; 92 | 93 | /** @brief IE network */ 94 | InferenceEngine::ExecutableNetwork _executable_network_; 95 | /** @brief IE InferRequest */ 96 | mutable InferenceEngine::InferRequest _infer_request_; 97 | std::map _input_shapes; 98 | 99 | 100 | }; 101 | 102 | class MattingCNN : public CnnDLSDKBase 103 | { 104 | public: 105 | explicit MattingCNN(const CNNConfig& config); 106 | 107 | void Compute(const cv::Mat &image, cv::Mat &bgr, std::map *result, cv::Size& outp_shape) const; 108 | void Compute2(const cv::Mat &image, cv::Mat &bgr, cv::Mat &bgr2, std::map *result, cv::Size& outp_shape) const; 109 | 110 | void Compute_Alpha(const cv::Mat &image, cv::Mat &bgr, std::map *result, cv::Size& outp_shape) const; 111 | 112 | private: 113 | cv::Size _originShape; 114 | }; 115 | 116 | //异步算法 117 | template 118 | class AsyncAlgorithm 119 | { 120 | public: 121 | virtual ~AsyncAlgorithm() {} 122 | virtual void enqueue(const std::string& name, const cv::Mat& frame) = 0; 123 | virtual void submitRequest() = 0; 124 | virtual void wait() = 0; 125 | virtual void printPerformanceCounts(const std::string &fullDeviceName) = 0; 126 | virtual std::vector fetchResults() = 0; 127 | }; 128 | 129 | template 130 | class BaseAsyncCNN : public AsyncAlgorithm 131 | { 132 | protected: 133 | InferenceEngine::InferRequest::Ptr request; 134 | bool isAsync; 135 | std::string topoName; 136 | 137 | public: 138 | BaseAsyncCNN(){} 139 | 140 | void submitRequest() override 141 | { 142 | if (request == nullptr) 143 | { 144 | return; 145 | } 146 | 147 | if (isAsync) 148 | { 149 | request->StartAsync(); 150 | } 151 | else 152 | { 153 | request->Infer(); 154 | } 155 | } 156 | 157 | void wait() override 158 | { 159 | if (!request || !isAsync) 160 | { 161 | return; 162 | } 163 | request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); 164 | } 165 | 166 | void printPerformanceCounts(const std::string &fullDeviceName) override 167 | { 168 | std::cout << "BaseMatting Performance counts for " << topoName << std::endl << std::endl; 169 | ::printPerformanceCounts(*request, std::cout, fullDeviceName, false); 170 | } 171 | 172 | void setAsync(bool bAsync) 173 | { 174 | isAsync = bAsync; 175 | } 176 | 177 | bool getAsync() 178 | { 179 | return isAsync; 180 | } 181 | }; 182 | -------------------------------------------------------------------------------- /include/cnn_backgroundv2.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2019 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "cnn.hpp" 14 | 15 | 16 | class CNN_Background_V2 : public BaseAsyncCNN 17 | { 18 | private: 19 | CNNConfig _config; 20 | InferenceEngine::ExecutableNetwork net_; 21 | InferenceEngine::CNNNetwork cnn_network_; 22 | 23 | bool _bBgr; 24 | 25 | private: 26 | void load(); 27 | 28 | public: 29 | explicit CNN_Background_V2(const CNNConfig& config); 30 | void reshape(cv::Size input_shape); 31 | 32 | void submitRequest() override; 33 | void enqueueAll(const cv::Mat& frame, const cv::Mat& bgr); 34 | void enqueue(const std::string& name, const cv::Mat& frame) override; 35 | void wait() override { BaseAsyncCNN::wait(); } 36 | void printPerformanceCounts(const std::string &fullDeviceName) override 37 | { 38 | BaseAsyncCNN::printPerformanceCounts(fullDeviceName); 39 | } 40 | 41 | MattingObjects fetchResults() override; 42 | bool isBgrEnqueued(); 43 | }; 44 | -------------------------------------------------------------------------------- /include/cnn_modnet.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2019 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "cnn.hpp" 14 | 15 | 16 | class CNN_Modnet : public BaseAsyncCNN 17 | { 18 | private: 19 | CNNConfig _config; 20 | InferenceEngine::ExecutableNetwork net_; 21 | InferenceEngine::CNNNetwork cnn_network_; 22 | 23 | bool _bBgr; 24 | 25 | private: 26 | void load(); 27 | 28 | public: 29 | explicit CNN_Modnet(const CNNConfig& config); 30 | void reshape(cv::Size input_shape); 31 | 32 | void submitRequest() override; 33 | void enqueueAll(const cv::Mat& frame, const cv::Mat& bgr); 34 | void enqueue(const std::string& name, const cv::Mat& frame) override; 35 | void wait() override { BaseAsyncCNN::wait(); } 36 | void printPerformanceCounts(const std::string &fullDeviceName) override 37 | { 38 | BaseAsyncCNN::printPerformanceCounts(fullDeviceName); 39 | } 40 | 41 | MattingObjects fetchResults() override; 42 | bool isBgrEnqueued(); 43 | }; 44 | -------------------------------------------------------------------------------- /include/ext_ops.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __EXT_OPS__HPP__ 2 | #define __EXT_OPS__HPP__ 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | #include // onnx_import/onnx_utils.hpp provides ngraph::onnx_import::register_operator function, that registers operator in ONNX importer's set. 9 | 10 | #include // ngraph/opsets/opset5.hpp provides the declaration of predefined nGraph operator set 11 | #include 12 | //#include
13 | 14 | 15 | void registerRoiAlign(); 16 | 17 | #endif -------------------------------------------------------------------------------- /include/ns_bmp.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __NS__BMP__HPP___ 2 | #define __NS__BMP__HPP___ 3 | 4 | #include 5 | #include 6 | 7 | #include "ns_register.hpp" 8 | 9 | 10 | namespace ovlib { 11 | /** 12 | * \class BitMap 13 | * \brief Reader for bmp files 14 | */ 15 | class BitMap : public Reader { 16 | private: 17 | static Register reg; 18 | 19 | typedef struct _bmpHeader_{ 20 | unsigned short type = 0u; /* Magic identifier */ 21 | unsigned int size = 0u; /* File size in bytes */ 22 | unsigned int reserved = 0u; 23 | unsigned int offset = 0u; /* Offset to image data, bytes */ 24 | } BmpHeader; 25 | 26 | typedef struct _bmpInfoHeader_{ 27 | unsigned int size = 0u; /* Header size in bytes */ 28 | int width = 0, height = 0; /* Width and height of image */ 29 | unsigned short planes = 0u; /* Number of colour planes */ 30 | unsigned short bits = 0u; /* Bits per pixel */ 31 | unsigned int compression = 0u; /* Compression type */ 32 | unsigned int imagesize = 0u; /* Image size in bytes */ 33 | int xresolution = 0, yresolution = 0; /* Pixels per meter */ 34 | unsigned int ncolours = 0u; /* Number of colours */ 35 | unsigned int importantcolours = 0u; /* Important colours */ 36 | } BmpInfoHeader; 37 | 38 | public: 39 | /** 40 | * \brief Constructor of BMP reader 41 | * @param filename - path to input data 42 | * @return BitMap reader object 43 | */ 44 | explicit BitMap(const std::string &filename); 45 | virtual ~BitMap() { 46 | } 47 | 48 | /** 49 | * \brief Get size 50 | * @return size 51 | */ 52 | size_t size() const override { 53 | return _width * _height * 3; 54 | } 55 | 56 | void Release() noexcept override { 57 | delete this; 58 | } 59 | 60 | std::shared_ptr getData(size_t width, size_t height) override { 61 | if ((width * height != 0) && (_width * _height != width * height)) { 62 | std::cout << "[ WARNING ] Image won't be resized! Please use OpenCV.\n"; 63 | return nullptr; 64 | } 65 | return _data; 66 | } 67 | }; 68 | } // namespace ovlib 69 | 70 | #endif -------------------------------------------------------------------------------- /include/ns_register.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __NS__REGISTER__HPP___ 2 | #define __NS__REGISTER__HPP___ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #if defined(_WIN32) 13 | #ifdef IMPLEMENT_FORMAT_READER 14 | #define FORMAT_READER_API(type) extern "C" __declspec(dllexport) type 15 | #else 16 | #define FORMAT_READER_API(type) extern "C" type 17 | #endif 18 | #elif (__GNUC__ >= 4) 19 | #ifdef IMPLEMENT_FORMAT_READER 20 | #define FORMAT_READER_API(type) extern "C" __attribute__((visibility("default"))) type 21 | #else 22 | #define FORMAT_READER_API(type) extern "C" type 23 | #endif 24 | #else 25 | #define FORMAT_READER_API(TYPE) extern "C" TYPE 26 | #endif 27 | 28 | #define NS_USE_OPENCV 1 29 | 30 | namespace ovlib 31 | { 32 | 33 | /** 34 | * \class FormatReader 35 | * \brief This is an abstract class for reading input data 36 | */ 37 | class Reader 38 | { 39 | protected: 40 | /// \brief height 41 | size_t _height = 0; 42 | /// \brief width 43 | size_t _width = 0; 44 | /// \brief data 45 | std::shared_ptr _data; 46 | 47 | public: 48 | /** 49 | * \brief Get width 50 | * @return width 51 | */ 52 | size_t width() const { return _width; } 53 | 54 | /** 55 | * \brief Get height 56 | * @return height 57 | */ 58 | size_t height() const { return _height; } 59 | 60 | /** 61 | * \brief Get input data ptr 62 | * @return shared pointer with input data 63 | * @In case of using OpenCV, parameters width and height will be used for image resizing 64 | */ 65 | virtual std::shared_ptr getData(size_t width = 0, size_t height = 0) = 0; 66 | 67 | /** 68 | * \brief Get size 69 | * @return size 70 | */ 71 | virtual size_t size() const = 0; 72 | 73 | virtual void Release() noexcept = 0; 74 | }; 75 | 76 | class ReaderPtr 77 | { 78 | public: 79 | explicit ReaderPtr(const char *imageName); 80 | 81 | /** 82 | * @brief dereference operator overload 83 | * @return Reader 84 | */ 85 | Reader *operator->() const noexcept 86 | { 87 | return reader.get(); 88 | } 89 | 90 | /** 91 | * @brief dereference operator overload 92 | * @return Reader 93 | */ 94 | Reader *operator*() const noexcept 95 | { 96 | return reader.get(); 97 | } 98 | 99 | Reader *get() 100 | { 101 | return reader.get(); 102 | } 103 | 104 | protected: 105 | std::unique_ptr> reader; 106 | }; 107 | 108 | /** 109 | * \class Registry 110 | * \brief Create reader from fabric 111 | */ 112 | class Registry 113 | { 114 | private: 115 | typedef std::function CreatorFunction; 116 | static std::vector _data; 117 | 118 | public: 119 | /** 120 | * \brief Create reader 121 | * @param filename - path to input data 122 | * @return Reader for input data or nullptr 123 | */ 124 | static Reader *CreateReader(const char *filename); 125 | 126 | /** 127 | * \brief Registers reader in fabric 128 | * @param f - a creation function 129 | */ 130 | static void RegisterReader(CreatorFunction f); 131 | }; 132 | 133 | /** 134 | * \class Register 135 | * \brief Registers reader in fabric 136 | */ 137 | template 138 | class Register 139 | { 140 | public: 141 | /** 142 | * \brief Constructor creates creation function for fabric 143 | * @return Register object 144 | */ 145 | Register() 146 | { 147 | Registry::RegisterReader([](const std::string &filename) -> Reader * { 148 | return new To(filename); 149 | }); 150 | } 151 | }; 152 | 153 | /** 154 | * \class OCVMAT 155 | * \brief OpenCV Wraper 156 | */ 157 | class OCVReader : public Reader 158 | { 159 | private: 160 | cv::Mat img; 161 | size_t _size; 162 | static Register reg; 163 | 164 | public: 165 | /** 166 | * \brief Constructor of BMP reader 167 | * @param filename - path to input data 168 | * @return BitMap reader object 169 | */ 170 | explicit OCVReader(const std::string &filename); 171 | virtual ~OCVReader() 172 | { 173 | } 174 | 175 | /** 176 | * \brief Get size 177 | * @return size 178 | */ 179 | size_t size() const override 180 | { 181 | return _size; 182 | } 183 | 184 | void Release() noexcept override 185 | { 186 | delete this; 187 | } 188 | 189 | std::shared_ptr getData(size_t width, size_t height) override; 190 | }; 191 | 192 | /** 193 | * \class MnistUbyte 194 | * \brief Reader for mnist db files 195 | */ 196 | class MnistUbyte : public Reader 197 | { 198 | private: 199 | int reverseInt(int i); 200 | 201 | static Register reg; 202 | 203 | public: 204 | /** 205 | * \brief Constructor of Mnist reader 206 | * @param filename - path to input data 207 | * @return MnistUbyte reader object 208 | */ 209 | explicit MnistUbyte(const std::string &filename); 210 | virtual ~MnistUbyte() 211 | { 212 | } 213 | 214 | /** 215 | * \brief Get size 216 | * @return size 217 | */ 218 | size_t size() const override 219 | { 220 | return _width * _height * 1; 221 | } 222 | 223 | void Release() noexcept override 224 | { 225 | delete this; 226 | } 227 | 228 | std::shared_ptr getData(size_t width, size_t height) override 229 | { 230 | if ((width * height != 0) && (_width * _height != width * height)) 231 | { 232 | std::cout << "[ WARNING ] Image won't be resized! Please use OpenCV.\n"; 233 | return nullptr; 234 | } 235 | return _data; 236 | } 237 | }; 238 | 239 | } // namespace ovlib 240 | 241 | 242 | /** 243 | * \brief Function for create reader 244 | * @return FormatReader pointer 245 | */ 246 | FORMAT_READER_API(ovlib::Reader *) 247 | CreateFormatReader(const char *filename); 248 | 249 | #endif -------------------------------------------------------------------------------- /include/ns_thread.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __NS__THREAD__HPP___ 2 | #define __NS__THREAD__HPP___ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | class ov_simple_thread 16 | { 17 | public: 18 | ov_simple_thread(); 19 | virtual ~ov_simple_thread(); 20 | 21 | protected: 22 | void start(); 23 | std::thread::id getId(); 24 | void interrupt(); 25 | bool isInterrupted(); 26 | void join(); 27 | virtual void run(); 28 | 29 | private: 30 | std::atomic _interript; 31 | std::thread _thread; 32 | }; 33 | 34 | template 35 | class ConcurrentQueue 36 | { 37 | std::queue queue_; 38 | mutable std::mutex mutex_; 39 | 40 | // Moved out of public interface to prevent races between this 41 | // and pop(). 42 | bool empty() const 43 | { 44 | return queue_.empty(); 45 | } 46 | 47 | public: 48 | ConcurrentQueue() = default; 49 | ConcurrentQueue(const ConcurrentQueue&) = delete; 50 | virtual ~ConcurrentQueue() {} 51 | ConcurrentQueue(ConcurrentQueue&& other) 52 | { 53 | std::lock_guard lock(mutex_); 54 | queue_ = std::move(other.queue_); 55 | } 56 | 57 | ConcurrentQueue& operator=(const ConcurrentQueue&) = delete; 58 | 59 | unsigned long size() const 60 | { 61 | std::lock_guard lock(mutex_); 62 | return queue_.size(); 63 | } 64 | 65 | T pop() 66 | { 67 | std::lock_guard lock(mutex_); 68 | if (queue_.empty()) 69 | { 70 | return {}; 71 | } 72 | T tmp = queue_.front(); 73 | queue_.pop(); 74 | return tmp; 75 | } 76 | 77 | void pop(T& item) 78 | { 79 | std::lock_guard lock(mutex_); 80 | if (queue_.empty()) 81 | { 82 | return; 83 | } 84 | item = queue_.front(); 85 | queue_.pop(); 86 | } 87 | 88 | void front(T& item) 89 | { 90 | std::lock_guard lock(mutex_); 91 | if (queue_.empty()) 92 | { 93 | return; 94 | } 95 | item = queue_.front(); 96 | } 97 | 98 | void popAll(std::vector& items) 99 | { 100 | std::lock_guard lock(mutex_); 101 | while (!queue_.empty()) 102 | { 103 | T item = queue_.front(); 104 | queue_.pop(); 105 | items.push_back(item); 106 | } 107 | } 108 | 109 | void push(const T& item) 110 | { 111 | std::lock_guard lock(mutex_); 112 | queue_.push(item); 113 | } 114 | 115 | void clear() 116 | { 117 | std::lock_guard lock(mutex_); 118 | while (!queue_.empty()) 119 | { 120 | queue_.pop(); 121 | } 122 | } 123 | 124 | }; 125 | 126 | 127 | #endif -------------------------------------------------------------------------------- /include/ns_utils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __NS__UTILS__HPP___ 2 | #define __NS__UTILS__HPP___ 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "ovmatter.h" 17 | 18 | namespace ovlib 19 | { 20 | class OV_MATTER_API Utils_String 21 | { 22 | public: 23 | static std::vector split(const std::string &s, char delim); 24 | }; 25 | 26 | 27 | class OV_MATTER_API TimerCounter 28 | { 29 | public: 30 | TimerCounter(std::string name) 31 | { 32 | _name = name; 33 | _start = std::chrono::high_resolution_clock::now(); 34 | } 35 | ~TimerCounter() 36 | { 37 | auto elapsed = std::chrono::high_resolution_clock::now() - _start; 38 | #if (_MSC_VER) 39 | _elapse = std::chrono::duration_cast(elapsed).count(); 40 | #else 41 | _elapse = std::chrono::duration_cast(elapsed).count(); 42 | #endif // _WIN 43 | 44 | 45 | std::cout << "Benchmarking " << _name << "\t\t| elapse miliseconds: " << _elapse / 1.0 << ",\t estimate fps: " << 1000.0 / _elapse << std::endl; 46 | } 47 | private: 48 | std::string _name; 49 | #if (_MSC_VER) 50 | std::chrono::time_point _start; 51 | #else 52 | std::chrono::_V2::system_clock::time_point _start; 53 | #endif // _WIN 54 | 55 | int64_t _elapse; 56 | bool _started = false; 57 | }; 58 | 59 | class OV_MATTER_API MatterBencher 60 | { 61 | public: 62 | MatterBencher(); 63 | virtual ~MatterBencher(); 64 | void Start(); 65 | int64_t Elapse(); 66 | float Get(); 67 | 68 | private: 69 | #if (_MSC_VER) 70 | std::chrono::time_point _start; 71 | #else 72 | std::chrono::_V2::system_clock::time_point _start; 73 | #endif // _WIN 74 | int64_t _elapse; 75 | bool _started = false; 76 | float _avg_fps; 77 | float _ratio; 78 | }; 79 | 80 | namespace slog 81 | { 82 | 83 | /** 84 | * @class LogStreamEndLine 85 | * @brief The LogStreamEndLine class implements an end line marker for a log stream 86 | */ 87 | class OV_MATTER_API LogStreamEndLine 88 | { 89 | }; 90 | 91 | static constexpr LogStreamEndLine endl; 92 | 93 | /** 94 | * @class LogStreamBoolAlpha 95 | * @brief The LogStreamBoolAlpha class implements bool printing for a log stream 96 | */ 97 | class OV_MATTER_API LogStreamBoolAlpha 98 | { 99 | }; 100 | 101 | static constexpr LogStreamBoolAlpha boolalpha; 102 | 103 | /** 104 | * @class LogStream 105 | * @brief The LogStream class implements a stream for sample logging 106 | */ 107 | class OV_MATTER_API LogStream 108 | { 109 | std::string _prefix; 110 | std::ostream *_log_stream; 111 | bool _new_line; 112 | 113 | public: 114 | /** 115 | * @brief A constructor. Creates a LogStream object 116 | * @param prefix The prefix to print 117 | */ 118 | LogStream(const std::string &prefix, std::ostream &log_stream) 119 | : _prefix(prefix), _new_line(true) 120 | { 121 | _log_stream = &log_stream; 122 | } 123 | 124 | /** 125 | * @brief A stream output operator to be used within the logger 126 | * @param arg Object for serialization in the logger message 127 | */ 128 | template 129 | LogStream &operator<<(const T &arg) 130 | { 131 | if (_new_line) 132 | { 133 | (*_log_stream) << "[ " << _prefix << " ] "; 134 | _new_line = false; 135 | } 136 | 137 | (*_log_stream) << arg; 138 | return *this; 139 | } 140 | 141 | // Specializing for LogStreamEndLine to support slog::endl 142 | LogStream &operator<<(const LogStreamEndLine & /*arg*/) 143 | { 144 | _new_line = true; 145 | 146 | (*_log_stream) << std::endl; 147 | return *this; 148 | } 149 | 150 | // Specializing for LogStreamBoolAlpha to support slog::boolalpha 151 | LogStream &operator<<(const LogStreamBoolAlpha & /*arg*/) 152 | { 153 | (*_log_stream) << std::boolalpha; 154 | return *this; 155 | } 156 | }; 157 | 158 | static LogStream info("INFO", std::cout); 159 | static LogStream warn("WARNING", std::cout); 160 | static LogStream err("ERROR", std::cerr); 161 | } // namespace slog 162 | 163 | class OV_MATTER_API Utils_Ov 164 | { 165 | public: 166 | static std::string getMatType(const cv::Mat& img, bool more_info=true); 167 | static void showImage(cv::Mat& img, std::string& title); 168 | static void frameData2Mat(matter::FrameData& frameData, cv::Mat& outMat); 169 | static void mat2FrameData(cv::Mat& mat, matter::FrameData& frameData); 170 | 171 | static void sleep(long milliseconds); 172 | static double getSceneScore(cv::Mat prev_frame, cv::Mat frame, double& prev_mafd); 173 | static double getSceneScore_V2(cv::Mat prev_frame, cv::Mat frame, double& prev_mafd); 174 | 175 | static std::string getRealPath(std::string relPath); 176 | }; 177 | } // namespace ovlib 178 | 179 | #endif -------------------------------------------------------------------------------- /include/ocv_common.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2020 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | /** 6 | * @brief a header file with common samples functionality using OpenCV 7 | * @file ocv_common.hpp 8 | */ 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | 15 | /** 16 | * @brief Sets image data stored in cv::Mat object to a given Blob object. 17 | * @param orig_image - given cv::Mat object with an image data. 18 | * @param blob - Blob object which to be filled by an image data. 19 | * @param batchIndex - batch index of an image inside of the blob. 20 | */ 21 | template 22 | void matU8ToBlob(const cv::Mat &orig_image, InferenceEngine::Blob::Ptr &blob, int batchIndex = 0) 23 | { 24 | InferenceEngine::SizeVector blobSize = blob->getTensorDesc().getDims(); 25 | const size_t width = blobSize[3]; 26 | const size_t height = blobSize[2]; 27 | const size_t channels = blobSize[1]; 28 | InferenceEngine::MemoryBlob::Ptr mblob = InferenceEngine::as(blob); 29 | if (!mblob) 30 | { 31 | THROW_IE_EXCEPTION << "We expect blob to be inherited from MemoryBlob in matU8ToBlob, " 32 | << "but by fact we were not able to cast inputBlob to MemoryBlob"; 33 | } 34 | // locked memory holder should be alive all time while access to its buffer happens 35 | auto mblobHolder = mblob->wmap(); 36 | 37 | T *blob_data = mblobHolder.as(); 38 | 39 | cv::Mat resized_image(orig_image); 40 | if (static_cast(width) != orig_image.size().width || 41 | static_cast(height) != orig_image.size().height) 42 | { 43 | cv::resize(orig_image, resized_image, cv::Size(width, height)); 44 | } 45 | 46 | int batchOffset = batchIndex * width * height * channels; 47 | 48 | for (size_t c = 0; c < channels; c++) 49 | { 50 | for (size_t h = 0; h < height; h++) 51 | { 52 | for (size_t w = 0; w < width; w++) 53 | { 54 | blob_data[batchOffset + c * width * height + h * width + w] = 55 | resized_image.at(h, w)[c]; 56 | } 57 | } 58 | } 59 | } 60 | 61 | /** 62 | * @brief Wraps data stored inside of a passed cv::Mat object by new Blob pointer. 63 | * @note: No memory allocation is happened. The blob just points to already existing 64 | * cv::Mat data. 65 | * @param mat - given cv::Mat object with an image data. 66 | * @return resulting Blob pointer. 67 | */ 68 | static UNUSED InferenceEngine::Blob::Ptr wrapMat2Blob(const cv::Mat &mat) 69 | { 70 | size_t channels = mat.channels(); 71 | size_t height = mat.size().height; 72 | size_t width = mat.size().width; 73 | 74 | size_t strideH = mat.step.buf[0]; 75 | size_t strideW = mat.step.buf[1]; 76 | 77 | bool is_dense = strideW == channels && strideH == channels * width; 78 | 79 | if (!is_dense) 80 | { 81 | THROW_IE_EXCEPTION << "Doesn't support conversion from not dense cv::Mat"; 82 | } 83 | 84 | InferenceEngine::TensorDesc tDesc(InferenceEngine::Precision::U8, {1, channels, height, width}, InferenceEngine::Layout::NHWC); 85 | 86 | return InferenceEngine::make_shared_blob(tDesc, mat.data); 87 | } 88 | 89 | static UNUSED InferenceEngine::Blob::Ptr wrapMat2Blob2(const cv::Mat &mat) 90 | { 91 | cv::cvtColor(mat, mat, cv::COLOR_BGR2RGB); 92 | cv::Mat fMat; 93 | mat.convertTo(fMat, CV_32FC3, 1.0/255.0); 94 | size_t channels = fMat.channels(); 95 | size_t channelsize = channels * sizeof(float); 96 | size_t height = fMat.size().height; 97 | size_t width = fMat.size().width; 98 | 99 | size_t strideH = fMat.step.buf[0]; 100 | size_t strideW = fMat.step.buf[1]; 101 | 102 | size_t number_of_data = height * width * channels * sizeof(float); 103 | 104 | bool is_dense = strideW == channelsize && strideH == channelsize * width; 105 | 106 | if (!is_dense) 107 | { 108 | THROW_IE_EXCEPTION << "Doesn't support conversion from not dense cv::Mat"; 109 | } 110 | 111 | InferenceEngine::TensorDesc tDesc(InferenceEngine::Precision::FP32, {1, channels, height, width}, InferenceEngine::Layout::NHWC); 112 | 113 | return InferenceEngine::make_shared_blob(tDesc, (float*)fMat.data, number_of_data); 114 | } 115 | -------------------------------------------------------------------------------- /include/ovmatter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ovmatter.h 3 | * 4 | */ 5 | 6 | #ifndef __OV_MATTER__H__ 7 | #define __OV_MATTER__H__ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #ifdef OVMATTING_EXPORTS 17 | #define OV_MATTER_API __declspec(dllexport) 18 | #else 19 | #define OV_MATTER_API 20 | #endif 21 | 22 | #define OV_MATTER_VERSION "0.0.1" 23 | 24 | namespace ovlib 25 | { 26 | namespace matter 27 | { 28 | //Methods of Background matting 29 | enum OV_MATTER_API MattingMethod 30 | { 31 | METHOD_BACKGROUND_MATTING_V2 = 0x00, 32 | METHOD_MODNET = 0x01, 33 | }; 34 | 35 | enum OV_MATTER_API MATTER_EFFECT 36 | { 37 | EFFECT_NONE, 38 | EFFECT_BLUR 39 | }; 40 | 41 | struct OV_MATTER_API Rect 42 | { 43 | int left; 44 | int top; 45 | int right; 46 | int bottom; 47 | }; 48 | 49 | struct OV_MATTER_API Shape 50 | { 51 | unsigned int width; 52 | unsigned int height; 53 | }; 54 | 55 | enum OV_MATTER_API FRAME_FORMAT 56 | { 57 | FRAME_FOMAT_ERROR = 0x00, 58 | FRAME_FOMAT_I420 = 0x01, 59 | FRAME_FOMAT_BGR = 0x02, 60 | FRAME_FOMAT_RGB = 0x03, 61 | FRAME_FOMAT_GRAY = 0x04 62 | }; 63 | 64 | struct OV_MATTER_API FrameData 65 | { 66 | unsigned char* frame=0; 67 | unsigned int height=0; 68 | unsigned int width=0; 69 | FRAME_FORMAT format= FRAME_FOMAT_ERROR; 70 | void* tag=0; 71 | }; 72 | 73 | const FrameData NullFrame; 74 | 75 | struct OV_MATTER_API MatterParams 76 | { 77 | MattingMethod method; 78 | MATTER_EFFECT effect; 79 | 80 | std::string device; 81 | std::string path_to_model; 82 | std::string path_to_bin; 83 | 84 | float scale; 85 | int max_batch_size; 86 | 87 | int cpu_threads_num; //default 0 88 | bool cpu_bind_thread; //default true 89 | int cpu_throughput_streams; //default 1 90 | 91 | bool is_async; 92 | int interval; 93 | float threshold_motion; 94 | 95 | Shape input_shape; 96 | 97 | }; 98 | 99 | class OV_MATTER_API MatterChannel 100 | { 101 | public: 102 | static MatterChannel* create(const MatterParams& params); 103 | static int getDefMatterParams(MatterParams& params); 104 | static void destroyed(MatterChannel* pChan); 105 | 106 | virtual ~MatterChannel() {}; 107 | 108 | virtual int process(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& out_shape, std::map* pResults = 0) = 0; 109 | 110 | virtual int getInferCount() = 0; 111 | virtual double getAttributeValue(std::string attName) = 0; 112 | virtual MatterChannel* setAttributeValue(std::string attrName, double dblValue) = 0; 113 | 114 | virtual void setStrategy_async(bool bAuto = true, int interval = 0, const Shape& input_shape = { 0,0 }, const Shape& out_shape = {0,0}) = 0; 115 | virtual void setBackground_async(FrameData& bgrReplace, MATTER_EFFECT = EFFECT_NONE, const FrameData& bgr = NullFrame) = 0; 116 | virtual int process_async(FrameData& frame, FrameData& frameCom, FrameData& frameAlpha, const ovlib::matter::Shape& out_shape = { 0,0 }) = 0; 117 | }; 118 | }; // namespace matter 119 | }; // namespace ovlib 120 | 121 | #endif 122 | -------------------------------------------------------------------------------- /include/ovmatter_base_impl.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __OVMATTER__BASE_IMPL_HPP__ 2 | #define __OVMATTER__BASE_IMPL_HPP__ 3 | 4 | /** 5 | * MIT License 6 | 7 | Copyright (c) 2021 kingpeter2015 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the "Software"), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all 17 | copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE. 26 | 27 | * this is base class of all the matters 28 | **/ 29 | 30 | #include "ovmatter.h" 31 | 32 | namespace ovlib 33 | { 34 | namespace matter 35 | { 36 | class MatterBaseImpl : public MatterChannel 37 | { 38 | public: 39 | MatterBaseImpl() {}; 40 | virtual ~MatterBaseImpl() {}; 41 | virtual bool init(const MatterParams& params) = 0; 42 | virtual int getInferCount(){ return m_nInferCount; } 43 | virtual double getAttributeValue(std::string attName); 44 | virtual MatterChannel* setAttributeValue(std::string attrName, double dblValue); 45 | 46 | MatterParams _params; 47 | 48 | protected: 49 | int m_nInterval; 50 | float m_fMotionThreshold = -1.0f; 51 | int m_nInferCount = 0; 52 | double m_preDiff = 0.0; 53 | int m_nForceInferLimit = 30; 54 | }; 55 | 56 | typedef MatterBaseImpl* (*NewInstancePt)(); 57 | class CMatterFactory 58 | { 59 | public: 60 | static MatterBaseImpl* CreateMatter(const char* className) 61 | { 62 | std::map::const_iterator it = dynCreateMap.find(className); 63 | if (it == dynCreateMap.end()) 64 | { 65 | return NULL; 66 | } 67 | else 68 | { 69 | NewInstancePt np = it->second; 70 | return np(); 71 | } 72 | } 73 | 74 | static MatterBaseImpl* CreateMatter(const int code) 75 | { 76 | std::map::const_iterator it = dynCreateMapType.find(code); 77 | if (it == dynCreateMapType.end()) 78 | { 79 | return NULL; 80 | } 81 | else 82 | { 83 | NewInstancePt np = it->second; 84 | return np(); 85 | } 86 | } 87 | 88 | static void RegisterClass(int code, const char* className, NewInstancePt np) 89 | { 90 | dynCreateMap[className] = np; 91 | dynCreateMapType[code] = np; 92 | } 93 | 94 | private: 95 | static std::map dynCreateMap; 96 | static std::map dynCreateMapType; 97 | }; 98 | 99 | class MatterRegister 100 | { 101 | public: 102 | MatterRegister(int code, const char* className, NewInstancePt np) 103 | { 104 | CMatterFactory::RegisterClass(code, className, np); 105 | } 106 | }; 107 | 108 | #define REGISTER_MATTER_CLASS(code, class_name) \ 109 | class class_name##MatterRegister \ 110 | { \ 111 | public: \ 112 | static MatterBaseImpl* NewInstance() \ 113 | { \ 114 | return new class_name(); \ 115 | } \ 116 | private: \ 117 | static MatterRegister reg; \ 118 | }; \ 119 | MatterRegister class_name##MatterRegister::reg(code, #class_name, class_name##MatterRegister::NewInstance); 120 | 121 | 122 | }; // namespace matter 123 | }; // namespace ovlib 124 | 125 | 126 | #endif -------------------------------------------------------------------------------- /include/ovmatter_bgv2_impl.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __OVMATTER__BACKGROUND_MATTING_V2_IMPL_HPP__ 2 | #define __OVMATTER__BACKGROUND_MATTING_V2_IMPL_HPP__ 3 | 4 | /** 5 | * MIT License 6 | 7 | Copyright (c) 2021 kingpeter2015 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the "Software"), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all 17 | copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE. 26 | 27 | * Please Ref: https://github.com/PeterL1n/BackgroundMattingV2 28 | **/ 29 | 30 | #include "ovmatter.h" 31 | #include "ovmatter_base_impl.hpp" 32 | #include "cnn_backgroundv2.hpp" 33 | #include "ns_thread.hpp" 34 | #include "ns_utils.hpp" 35 | 36 | namespace ovlib 37 | { 38 | namespace matter 39 | { 40 | class MatterBackgroundV2Impl : public MatterBaseImpl, public ov_simple_thread 41 | { 42 | public: 43 | MatterBackgroundV2Impl(); 44 | virtual ~MatterBackgroundV2Impl(); 45 | 46 | /***for MatterBaseImpl*****/ 47 | virtual bool init(const MatterParams& params) override; 48 | virtual int process(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults = 0) override; 49 | virtual void setStrategy_async(bool bAuto = true, int interval = 0, const Shape& input_shape = { 0,0 }, const Shape& out_shape = { 0,0 }) override; 50 | virtual void setBackground_async(FrameData& bgrReplace, MATTER_EFFECT = EFFECT_NONE, const FrameData& bgr = NullFrame) override; 51 | virtual int process_async(FrameData& frame, FrameData& frameCom, FrameData& frameAlpha, const ovlib::matter::Shape& out_shape) override; 52 | 53 | /***for ov_simple_thread***/ 54 | virtual void run() override; 55 | 56 | private: 57 | int doWork_sync(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults); 58 | int doWork_sync_V2(cv::Mat& frame, cv::Mat& bgr, cv::Mat& bgrReplace, cv::Size& out_shape, cv::Mat& matCom, cv::Mat& matAlpha); 59 | void compose(cv::Mat& src, cv::Mat& replace, cv::Mat& alpha, cv::Mat& com, cv::Size& out_shape); 60 | 61 | private: 62 | std::unique_ptr _pCnn; 63 | 64 | bool _bInit; 65 | 66 | cv::Mat _prevFrame; 67 | MattingObjects _matResult; 68 | 69 | private: /****for asynchronous*****/ 70 | ovlib::MatterBencher _bencher; 71 | long long _elapse; 72 | mutable std::mutex mutex_; 73 | ConcurrentQueue _queue_input; 74 | ConcurrentQueue _queue_output; 75 | cv::Mat _frame_bgr; 76 | cv::Mat _frame_replace; 77 | MATTER_EFFECT _effect; 78 | cv::Size _shape_input; 79 | cv::Size _shape_output; 80 | std::map _preResult; 81 | 82 | }; 83 | }; // namespace matter 84 | }; // namespace ovlib 85 | 86 | 87 | #endif -------------------------------------------------------------------------------- /include/ovmatter_modnet_impl.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __OVMATTER__MODNET_IMPL_HPP__ 2 | #define __OVMATTER__MODNET_IMPL_HPP__ 3 | 4 | /** 5 | * MIT License 6 | 7 | Copyright (c) 2021 kingpeter2015 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the "Software"), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all 17 | copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE. 26 | 27 | * Please Ref: https://github.com/PeterL1n/BackgroundMattingV2 28 | **/ 29 | 30 | #include "ovmatter.h" 31 | #include "ovmatter_base_impl.hpp" 32 | #include "cnn_modnet.hpp" 33 | #include "ns_thread.hpp" 34 | #include "ns_utils.hpp" 35 | 36 | namespace ovlib 37 | { 38 | namespace matter 39 | { 40 | class MatterModnetImpl : public MatterBaseImpl, public ov_simple_thread 41 | { 42 | public: 43 | MatterModnetImpl(); 44 | virtual ~MatterModnetImpl(); 45 | 46 | /***for MatterBaseImpl*****/ 47 | virtual bool init(const MatterParams& params) override; 48 | virtual int process(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults = 0) override; 49 | virtual void setStrategy_async(bool bAuto = true, int interval = 0, const Shape& input_shape = { 0,0 }, const Shape& out_shape = { 0,0 }) override; 50 | virtual void setBackground_async(FrameData& bgrReplace, MATTER_EFFECT = EFFECT_NONE, const FrameData& bgr = NullFrame) override; 51 | virtual int process_async(FrameData& frame, FrameData& frameCom, FrameData& frameAlpha, const ovlib::matter::Shape& out_shape) override; 52 | 53 | /***for ov_simple_thread***/ 54 | virtual void run() override; 55 | 56 | private: 57 | int doWork_sync(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults); 58 | void compose(cv::Mat& src, cv::Mat& replace, cv::Mat& alpha, cv::Mat& com, cv::Size& out_shape); 59 | 60 | private: 61 | std::unique_ptr _pCnn; 62 | 63 | bool _bInit; 64 | 65 | cv::Mat _prevFrame; 66 | MattingObjects _matResult; 67 | 68 | private: /****for asynchronous*****/ 69 | ovlib::MatterBencher _bencher; 70 | long long _elapse; 71 | mutable std::mutex mutex_; 72 | ConcurrentQueue _queue_input; 73 | ConcurrentQueue _queue_output; 74 | cv::Mat _frame_bgr; 75 | cv::Mat _frame_replace; 76 | MATTER_EFFECT _effect; 77 | cv::Size _shape_input; 78 | cv::Size _shape_output; 79 | std::map _preResult; 80 | 81 | }; 82 | }; // namespace matter 83 | }; // namespace ovlib 84 | 85 | 86 | #endif -------------------------------------------------------------------------------- /include/samples.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __SAMPLES__HPP___ 2 | #define __SAMPLES__HPP___ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | void Inference_Video(); 9 | 10 | void Inference_Camera(); 11 | void Inference_Modnet(); 12 | void Inference_demo1(int argc, char* argv[]); 13 | 14 | #endif -------------------------------------------------------------------------------- /libovmatting/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.10) 3 | project(libovmatting) 4 | 5 | set(CMAKE_CXX_STANDARD 11) 6 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) #-fPIC 7 | set(LIB_OVMATTER_EXPORTS 1) 8 | 9 | #set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${PROJECT_SOURCE_DIR}/../out/debug) 10 | #set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/../out/release) 11 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/../build) 12 | 13 | set(TARGET_NAME "ovmatting") 14 | 15 | include_directories(${PROJECT_SOURCE_DIR}/../include) 16 | message("Root Directory:${PROJECT_SOURCE_DIR}") 17 | 18 | find_package(ngraph REQUIRED OPTIONAL_COMPONENTS onnx_importer) 19 | find_package(InferenceEngine REQUIRED) 20 | find_package(OpenCV REQUIRED) 21 | 22 | file(GLOB NATIVE_SRCS "${PROJECT_SOURCE_DIR}/../src/*.cpp") 23 | message("Source Code Files:${NATIVE_SRCS}") 24 | # set(SRC ${PROJECT_SOURCE_DIR}/src/*.cpp ) 25 | 26 | ADD_LIBRARY (${TARGET_NAME} SHARED ${NATIVE_SRCS}) 27 | 28 | target_compile_definitions(${TARGET_NAME} PRIVATE OPENCV_IMPORT_ENABLED) 29 | target_link_libraries(${TARGET_NAME} PRIVATE opencv_core) 30 | 31 | target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_INFERENCE_EXTENSION_API) 32 | target_link_libraries(${TARGET_NAME} PRIVATE IE::inference_engine ${NGRAPH_LIBRARIES}) 33 | target_link_libraries(${TARGET_NAME} PRIVATE ${ONNX_IMPORTER_LIBRARIES}) 34 | target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCV_LIBS}) 35 | target_compile_definitions(${TARGET_NAME} PRIVATE NGRAPH_ONNX_IMPORT_ENABLED) -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include "samples.hpp" 2 | #include 3 | 4 | int main(int argc, char *argv[]) 5 | { 6 | try 7 | { 8 | //Inference_Camera(); 9 | //Inference_Video(); 10 | //Inference_Modnet(); 11 | Inference_demo1(argc, argv); 12 | } 13 | catch (const std::exception &ex) 14 | { 15 | std::cerr << "main(int argc, char *argv[]):" << ex.what() << std::endl; 16 | return EXIT_FAILURE; 17 | } 18 | std::cout << "This sample is an API example, for any performance measurements " 19 | "please use the dedicated benchmark_app tool" 20 | << std::endl; 21 | return EXIT_SUCCESS; 22 | } 23 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #/bin/bash 2 | 3 | export LD_LIBRARY_PATH=./:./build/:/opt/intel/openvino_2021/inference_engine/lib/intel64/:/opt/intel/openvino_2021/opencv/lib/:/opt/intel/openvino_2021/deployment_tools/ngraph/lib/:/opt/intel/openvino_2021/inference_engine/external/tbb/lib/:$LD_LIBRARY_PATH 4 | 5 | # echo $LD_LIBRARY_PATH 6 | 7 | ./build/sample -model ./share/pytorch_mobilenetv2.xml \ 8 | -bin ./share/pytorch_mobilenetv2.bin \ 9 | -dev CPU \ 10 | -src ./share/src.mp4 \ 11 | -sbgr ./share/src.png \ 12 | -dst ./share/dst.mp4 \ 13 | -dbgr ./share/replace.jpg \ 14 | -in_width 320 \ 15 | -in_height 180 \ 16 | -method 0 \ 17 | -interval 3 \ 18 | -cpu_thread 0 \ 19 | -cpu_stream 1 -------------------------------------------------------------------------------- /samples/inference_camera.cpp: -------------------------------------------------------------------------------- 1 | #include "samples.hpp" 2 | #include "ns_utils.hpp" 3 | #include 4 | 5 | using namespace ovlib::matter; 6 | 7 | static void InitWindows() 8 | { 9 | int width = 640; 10 | int height = 360; 11 | cv::namedWindow("com", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 12 | cv::resizeWindow("com", 1280, 720); 13 | cv::moveWindow("com", 0, 0); 14 | cv::namedWindow("pha", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 15 | cv::resizeWindow("pha", width, height); 16 | cv::moveWindow("pha", 650, 0); 17 | cv::namedWindow("frame", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 18 | cv::resizeWindow("frame", width, height); 19 | cv::moveWindow("pha", 0, 400); 20 | } 21 | 22 | void Inference_Camera() 23 | { 24 | #if (_MSC_VER) 25 | std::string model = ".\\share\\pytorch_mobilenetv2.xml"; 26 | std::string bin = ".\\share\\pytorch_mobilenetv2.bin"; 27 | std::string src = ".\\share\\src.mp4"; 28 | std::string bgr = ".\\share\\src.png"; 29 | std::string bgr2 = ".\\share\\replace1.jpg"; 30 | #else 31 | std::string model = "../share/pytorch_mobilenetv2.xml"; 32 | std::string bin = "../share/pytorch_mobilenetv2.bin"; 33 | std::string src = "../share/src.mp4"; 34 | std::string bgr = "../share/src.png"; 35 | std::string bgr2 = "../share/replace.jpg"; 36 | #endif // WINDOWS 37 | ovlib::matter::Shape in_shape, out_shape; 38 | in_shape.width = 320; 39 | in_shape.height = 180; 40 | out_shape.width = 1280; 41 | out_shape.height = 720; 42 | 43 | ovlib::matter::MatterParams params; 44 | ovlib::matter::MatterChannel::getDefMatterParams(params); 45 | params.input_shape = in_shape; 46 | params.path_to_model = model; 47 | params.path_to_bin = bin; 48 | params.method = ovlib::matter::METHOD_BACKGROUND_MATTING_V2; 49 | params.is_async = true; 50 | params.effect = ovlib::matter::EFFECT_BLUR; 51 | params.device = "CPU"; 52 | MatterChannel* pChan = MatterChannel::create(params); 53 | if (!pChan) 54 | { 55 | std::cout << "Can not create Matter Channel." << std::endl; 56 | 57 | return; 58 | } 59 | InitWindows(); 60 | 61 | cv::VideoCapture capture0(src); 62 | capture0.set(cv::CAP_PROP_FRAME_WIDTH, 640); 63 | capture0.set(cv::CAP_PROP_FRAME_HEIGHT, 360); 64 | capture0.set(cv::CAP_PROP_FPS, 200); 65 | 66 | int framecnt = 0; 67 | int nDelay = 10; 68 | cv::Mat frame, bgrFrame, bgrFrame2; 69 | cv::Mat matCom, matPha; 70 | bgrFrame = cv::imread(bgr); 71 | bgrFrame2 = cv::imread(bgr2); 72 | std::map output; 73 | ovlib::matter::FrameData frame_com, frame_pha; 74 | ovlib::MatterBencher timercounter; 75 | timercounter.Start(); 76 | double lElapse = 0; 77 | 78 | 79 | FrameData frame_bgr; 80 | ovlib::Utils_Ov::mat2FrameData(bgrFrame, frame_bgr); 81 | FrameData frame_bgr_replace; 82 | //ovlib::Utils_Ov::mat2FrameData(bgrFrame2, frame_bgr_replace); 83 | ovlib::Utils_Ov::mat2FrameData(bgrFrame2, frame_bgr_replace); 84 | pChan->setBackground_async(frame_bgr_replace, EFFECT_BLUR, frame_bgr); 85 | 86 | while (1) 87 | { 88 | if (!capture0.isOpened()) 89 | { 90 | break; 91 | } 92 | capture0 >> frame; 93 | 94 | if (frame.empty()) 95 | { 96 | break; 97 | } 98 | 99 | framecnt++; 100 | { 101 | std::stringstream ss; 102 | ss << "Phase:pChan->process_async()" << ", frame:(" << framecnt << ")"; 103 | ovlib::TimerCounter estimate(ss.str()); 104 | FrameData frame_main; 105 | ovlib::Utils_Ov::mat2FrameData(frame, frame_main); 106 | 107 | //pChan->process(frame_main, frame_bgr, frame_bgr_replace, out_shape, &output); 108 | pChan->process_async(frame_main, frame_com, frame_pha); 109 | lElapse += timercounter.Elapse(); 110 | std::cout << "Elapse:" << lElapse / 1000.0 << " seconds" << std::endl; 111 | } 112 | 113 | { 114 | ovlib::Utils_Ov::frameData2Mat(frame_com, matCom); 115 | ovlib::Utils_Ov::frameData2Mat(frame_pha, matPha); 116 | 117 | cv::imshow("com", matCom); 118 | if (!matPha.empty()) 119 | { 120 | cv::imshow("pha", matPha); 121 | } 122 | cv::imshow("frame", frame); 123 | delete[] frame_com.frame; 124 | delete[] frame_pha.frame; 125 | char c = cv::waitKey(nDelay); 126 | 127 | 128 | if (c == 'c') 129 | { 130 | break; 131 | } 132 | 133 | } 134 | } 135 | std::cout << "Speed:" << framecnt * 1000 / (lElapse) << " FPS" << std::endl; 136 | capture0.release(); 137 | cv::destroyAllWindows(); 138 | } -------------------------------------------------------------------------------- /samples/inference_demo1.cpp: -------------------------------------------------------------------------------- 1 | #include "samples.hpp" 2 | #include "ns_utils.hpp" 3 | 4 | #include 5 | 6 | #if (_MSC_VER) 7 | #include 8 | #else 9 | #include 10 | #endif 11 | 12 | 13 | 14 | using namespace InferenceEngine; 15 | 16 | using namespace ovlib::matter; 17 | 18 | std::string getRealPath(std::string relPath) 19 | { 20 | char dir[1024] = {0}; 21 | #if (_MSC_VER) 22 | _fullpath(dir, relPath.c_str(), 1024); 23 | #else 24 | realpath(relPath.c_str(), dir); 25 | #endif 26 | std::string result_str = dir; 27 | return result_str; 28 | } 29 | 30 | static void showUsage() 31 | { 32 | std::cout << std::endl; 33 | std::cout << "ovmatter [OPTION]" << std::endl; 34 | std::cout << "Options:" << std::endl; 35 | std::cout << std::endl; 36 | std::cout << " -h " << "Print a usage message." << std::endl; 37 | std::cout << " -src '' " << "Required. Path to a video or image file" << std::endl; 38 | std::cout << " -sbgr '' " << "Path to source video background image file." << std::endl; 39 | std::cout << " -dst '' " << "Required. Path to destination video or image file" << std::endl; 40 | std::cout << " -dbgr '' " << "Path to destination background image file for replacing." << std::endl; 41 | std::cout << " -model '' " << "Required. Path to the model (.xml) file." << std::endl; 42 | std::cout << " -dev '' " << "Optional. Specify the target device for Inference System (the list of available devices is shown below).Default value is CPU. The application looks for a suitable plugin for the specified device." << std::endl; 43 | std::cout << " -bin " << "Required. Path to the model (.bin) file." << std::endl; 44 | std::cout << " -in_width " << "Optional. Width of input shape. Default: 320" << std::endl; 45 | std::cout << " -in_height " << "Optional. Height of input shape. Default: 180" << std::endl; 46 | std::cout << " -method " << "Required. Matting Method, 0-background v2;1-modnet" << std::endl; 47 | std::cout << " -interval " << "Optional. Frame Skip count." << std::endl; 48 | std::cout << " -cpu_thread " << "Optional. CPU Thread Number." << std::endl; 49 | std::cout << " -cpu_stream " << "Optional. CPU Streams Throughput." << std::endl; 50 | std::cout << " -motion_f " << "Optional. Motion threshold." << std::endl; 51 | } 52 | 53 | void Inference_demo1(int argc, char* argv[]) 54 | { 55 | //1.get matter parameters 56 | ovlib::matter::MatterParams params; 57 | ovlib::matter::MatterChannel::getDefMatterParams(params); 58 | 59 | #if (_MSC_VER) 60 | std::string model = ".\\share\\pytorch_mobilenetv2.xml"; 61 | std::string bin = ".\\share\\pytorch_mobilenetv2.bin"; 62 | std::string src = ".\\share\\src.mp4"; 63 | std::string dst = ".\\share\\dst.mp4"; 64 | std::string bgr = ".\\share\\src.png"; 65 | std::string bgr2 = ".\\share\\replace.jpg"; 66 | #else 67 | std::string model = "../share/pytorch_mobilenetv2.xml"; 68 | std::string bin = "../share/pytorch_mobilenetv2.bin"; 69 | std::string src = "../share/src.mp4"; 70 | std::string dst = "../share/dst.mp4"; 71 | std::string bgr = "../share/src.png"; 72 | std::string bgr2 = "../share/replace.jpg"; 73 | #endif // WINDOWS 74 | ovlib::matter::Shape in_shape, out_shape; 75 | in_shape.width = 320; 76 | in_shape.height = 180; 77 | out_shape.width = 1280; 78 | out_shape.height = 720; 79 | 80 | for (int i = 1; i < argc; i++) 81 | { 82 | const char* pc = argv[i]; 83 | if (pc[0] == '-' && pc[1]) 84 | { 85 | if (!::strncmp(pc, "-dev", 4)) 86 | { 87 | params.device = argv[++i]; 88 | } 89 | else if (!::strncmp(pc, "-model", 6)) 90 | { 91 | model = argv[++i]; 92 | } 93 | else if (!::strncmp(pc, "-bin", 4)) 94 | { 95 | bin = argv[++i]; 96 | } 97 | else if (!::strncmp(pc, "-src", 4)) 98 | { 99 | src = argv[++i]; 100 | } 101 | else if (!::strncmp(pc, "-sbgr", 5)) 102 | { 103 | bgr = argv[++i]; 104 | } 105 | else if (!::strncmp(pc, "-dbgr", 5)) 106 | { 107 | bgr2 = argv[++i]; 108 | } 109 | else if (!::strncmp(pc, "-dst", 4)) 110 | { 111 | dst = argv[++i]; 112 | } 113 | else if (!::strncmp(pc, "-in_width", 9)) 114 | { 115 | in_shape.width = atoi(argv[++i]); 116 | } 117 | else if (!::strncmp(pc, "-in_height", 10)) 118 | { 119 | in_shape.height = atoi(argv[++i]); 120 | } 121 | else if (!::strncmp(pc, "-interval", 9)) 122 | { 123 | params.interval = atoi(argv[++i]); 124 | } 125 | else if (!::strncmp(pc, "-cpu_thread", 11)) 126 | { 127 | params.cpu_threads_num = atoi(argv[++i]); 128 | } 129 | else if (!::strncmp(pc, "-cpu_stream", 11)) 130 | { 131 | params.cpu_throughput_streams = atoi(argv[++i]); 132 | } 133 | else if (!::strncmp(pc, "-motion_f", 9)) 134 | { 135 | params.threshold_motion = atof(argv[++i]); 136 | } 137 | else if (!::strncmp(pc, "-method", 7)) 138 | { 139 | int nMethod = atoi(argv[++i]); 140 | if (nMethod == 0) 141 | { 142 | params.method = ovlib::matter::METHOD_BACKGROUND_MATTING_V2; 143 | } 144 | else 145 | { 146 | params.method = ovlib::matter::METHOD_MODNET; 147 | } 148 | } 149 | else if (!::strncmp(pc, "-h", 2)) 150 | { 151 | showUsage(); 152 | return; 153 | } 154 | } 155 | } 156 | 157 | model = getRealPath(model); 158 | bin = getRealPath(bin); 159 | src = getRealPath(src); 160 | dst = getRealPath(dst); 161 | bgr = getRealPath(bgr); 162 | bgr2 = getRealPath(bgr2); 163 | std::cout << "(model,bin,src,dst,bgr,bgr2):\n" << model << "\n" << bin << "\n" << src << "\n" << dst << "\n" << bgr << "\n" << bgr2 << std::endl; 164 | 165 | params.input_shape = in_shape; 166 | params.path_to_model = model; 167 | params.path_to_bin = bin; 168 | params.is_async = false; 169 | 170 | if (params.method == ovlib::matter::METHOD_MODNET) 171 | { 172 | #if (_MSC_VER) 173 | params.path_to_model = ".\\share\\modnet.xml"; 174 | params.path_to_bin = ".\\share\\modnet.bin"; 175 | #else 176 | params.path_to_model = "./share/modnet.xml"; 177 | params.path_to_bin = "./share/modnet.bin"; 178 | #endif 179 | //params.input_shape.width = 512; 180 | //params.input_shape.height = 512; 181 | params.input_shape.height = params.input_shape.width; 182 | } 183 | MatterChannel* pChan = MatterChannel::create(params); 184 | if (!pChan) 185 | { 186 | std::cout << "Can not create Matter Channel." << std::endl; 187 | 188 | return; 189 | } 190 | 191 | //2. src capture and dst writer 192 | cv::VideoCapture capture0(src); 193 | out_shape.width = capture0.get(cv::CAP_PROP_FRAME_WIDTH); 194 | out_shape.height = capture0.get(cv::CAP_PROP_FRAME_HEIGHT); 195 | cv::VideoWriter writer; 196 | bool bRet = writer.open(dst, cv::VideoWriter::fourcc('X', '2', '6', '4'), 30, cv::Size(out_shape.width, out_shape.height)); 197 | if (!bRet) 198 | { 199 | std::cout << "cv::VideoWriter Initializes failed!" << std::endl; 200 | } 201 | 202 | int framecnt = 0; 203 | int nDelay = 1; 204 | cv::Mat frame, bgrFrame, bgrFrame2; 205 | cv::Mat matCom, matPha; 206 | bgrFrame = cv::imread(bgr); 207 | bgrFrame2 = cv::imread(bgr2); 208 | std::map output; 209 | ovlib::matter::FrameData frame_com, frame_pha; 210 | ovlib::MatterBencher timercounter; 211 | 212 | double lElapse = 0; 213 | 214 | FrameData frame_bgr; 215 | ovlib::Utils_Ov::mat2FrameData(bgrFrame, frame_bgr); 216 | FrameData frame_bgr_replace; 217 | ovlib::Utils_Ov::mat2FrameData(bgrFrame2, frame_bgr_replace); 218 | 219 | //3.write 220 | while (1) 221 | { 222 | if (!capture0.isOpened()) 223 | { 224 | break; 225 | } 226 | 227 | capture0 >> frame; 228 | if (frame.empty()) 229 | { 230 | break; 231 | } 232 | 233 | framecnt++; 234 | { 235 | std::stringstream ss; 236 | ss << "Frame No." << framecnt << "..."; 237 | ovlib::TimerCounter estimate(ss.str()); 238 | FrameData frame_main; 239 | ovlib::Utils_Ov::mat2FrameData(frame, frame_main); 240 | 241 | timercounter.Start(); 242 | pChan->process(frame_main, frame_bgr, frame_bgr_replace, out_shape, &output); 243 | lElapse += timercounter.Elapse(); 244 | std::cout << "Elapse:" << lElapse / 1000.0 << " S" << std::endl; 245 | delete[] frame_main.frame; 246 | } 247 | 248 | frame_com = output["com"]; 249 | frame_pha = output["pha"]; 250 | ovlib::Utils_Ov::frameData2Mat(frame_com, matCom); 251 | writer.write(matCom); 252 | 253 | delete[] frame_com.frame; 254 | delete[] frame_pha.frame; 255 | 256 | } 257 | writer.release(); 258 | 259 | delete[] frame_bgr.frame; 260 | delete[] frame_bgr_replace.frame; 261 | 262 | int nInfer = pChan->getInferCount(); 263 | double dblInfer = pChan->getAttributeValue("InferCount"); 264 | std::cout << "Infer Count:" << nInfer << std::endl; 265 | std::cout << "dblInfer Count:" << dblInfer << std::endl; 266 | 267 | 268 | std::cout << "Speed:" << framecnt * 1000 / (lElapse) << " FPS" << std::endl; 269 | 270 | MatterChannel::destroyed(pChan); 271 | 272 | capture0.release(); 273 | 274 | } -------------------------------------------------------------------------------- /samples/inference_modnet.cpp: -------------------------------------------------------------------------------- 1 | #include "samples.hpp" 2 | #include "ns_utils.hpp" 3 | 4 | #include 5 | 6 | using namespace InferenceEngine; 7 | 8 | using namespace ovlib::matter; 9 | 10 | static void InitWindows() 11 | { 12 | int width = 1280; 13 | int height = 720; 14 | cv::namedWindow("com", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 15 | cv::resizeWindow("com", width, height); 16 | cv::moveWindow("com", 0, 0); 17 | cv::namedWindow("pha", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 18 | cv::resizeWindow("pha", width, height); 19 | cv::moveWindow("pha", 650, 0); 20 | } 21 | 22 | void Inference_Modnet() 23 | { 24 | #if (_MSC_VER) 25 | std::string model = ".\\share\\modnet.xml"; 26 | std::string bin = ".\\share\\modnet.bin"; 27 | std::string src = ".\\share\\src.mp4"; 28 | std::string bgr = ".\\share\\src.png"; 29 | std::string bgr2 = ".\\share\\replace.jpg"; 30 | #else 31 | std::string model = "../share/modnet.xml"; 32 | std::string bin = "../share/modnet.bin"; 33 | std::string src = "../share/src.mp4"; 34 | std::string bgr = "../share/src.png"; 35 | std::string bgr2 = "../share/replace.jpg"; 36 | #endif // WINDOWS 37 | ovlib::matter::Shape in_shape, out_shape; 38 | in_shape.width = 512; 39 | in_shape.height = 512; 40 | out_shape.width = 1280; 41 | out_shape.height = 720; 42 | 43 | ovlib::matter::MatterParams params; 44 | ovlib::matter::MatterChannel::getDefMatterParams(params); 45 | params.input_shape = in_shape; 46 | params.path_to_model = model; 47 | params.path_to_bin = bin; 48 | params.method = ovlib::matter::METHOD_MODNET; 49 | params.is_async = false; 50 | params.interval = 3; 51 | //params.effect = ovlib::matter::EFFECT_BLUR; 52 | MatterChannel* pChan = MatterChannel::create(params); 53 | if (!pChan) 54 | { 55 | std::cout << "Can not create Matter Channel." << std::endl; 56 | 57 | return; 58 | } 59 | 60 | InitWindows(); 61 | 62 | cv::VideoCapture capture0(src); 63 | 64 | int framecnt = 0; 65 | int nDelay = 5; 66 | cv::Mat frame, bgrFrame, bgrFrame2; 67 | 68 | bgrFrame = cv::imread(bgr); 69 | bgrFrame2 = cv::imread(bgr2); 70 | std::map output; 71 | ovlib::matter::FrameData frame_com, frame_pha; 72 | ovlib::MatterBencher timercounter; 73 | timercounter.Start(); 74 | double lElapse = 0; 75 | 76 | while (1) 77 | { 78 | if (!capture0.isOpened()) 79 | { 80 | break; 81 | } 82 | 83 | capture0 >> frame; 84 | if (frame.empty()) 85 | { 86 | break; 87 | } 88 | 89 | framecnt++; 90 | { 91 | ovlib::TimerCounter estimate("Phase..."); 92 | 93 | FrameData frame_main, frame_bgr, frame_bgr_replace; 94 | frame_main.format = FRAME_FOMAT_BGR; 95 | frame_main.width = frame.cols; 96 | frame_main.height = frame.rows; 97 | frame_main.frame = frame.data; 98 | frame_bgr_replace.format = FRAME_FOMAT_BGR; 99 | frame_bgr_replace.width = bgrFrame2.cols; 100 | frame_bgr_replace.height = bgrFrame2.rows; 101 | frame_bgr_replace.frame = bgrFrame2.data; 102 | 103 | 104 | pChan->process(frame_main, frame_bgr, frame_bgr_replace, out_shape, &output); 105 | lElapse += timercounter.Elapse(); 106 | std::cout << "Elapse:" << lElapse / 1000.0 << " S" << std::endl; 107 | } 108 | 109 | frame_com = output["com"]; 110 | frame_pha = output["pha"]; 111 | cv::Mat matCom, matPha; 112 | ovlib::Utils_Ov::frameData2Mat(frame_com, matCom); 113 | ovlib::Utils_Ov::frameData2Mat(frame_pha, matPha); 114 | 115 | cv::imshow("com", matCom); 116 | cv::imshow("pha", matPha); 117 | 118 | char c = cv::waitKey(nDelay); 119 | if (c == 'c') 120 | { 121 | break; 122 | } 123 | delete [] frame_com.frame; 124 | delete[] frame_pha.frame; 125 | } 126 | std::cout << "Speed:" << framecnt * 1000 / (lElapse) << " FPS" << std::endl; 127 | capture0.release(); 128 | cv::destroyAllWindows(); 129 | } -------------------------------------------------------------------------------- /samples/inference_video.cpp: -------------------------------------------------------------------------------- 1 | #include "samples.hpp" 2 | #include "ns_utils.hpp" 3 | 4 | #include 5 | 6 | using namespace InferenceEngine; 7 | 8 | using namespace ovlib::matter; 9 | 10 | static void InitWindows() 11 | { 12 | int width = 1280; 13 | int height = 720; 14 | cv::namedWindow("com", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 15 | cv::resizeWindow("com", width, height); 16 | cv::moveWindow("com", 0, 0); 17 | cv::namedWindow("pha", cv::WindowFlags::WINDOW_NORMAL | cv::WindowFlags::WINDOW_FREERATIO); 18 | cv::resizeWindow("pha", width, height); 19 | cv::moveWindow("pha", 650, 0); 20 | } 21 | 22 | void Inference_Video() 23 | { 24 | #if (_MSC_VER) 25 | std::string model = ".\\share\\pytorch_mobilenetv2.xml"; 26 | std::string bin = ".\\share\\pytorch_mobilenetv2.bin"; 27 | std::string src = ".\\share\\src.mp4"; 28 | std::string bgr = ".\\share\\src.png"; 29 | std::string bgr2 = ".\\share\\replace.jpg"; 30 | #else 31 | std::string model = "../share/pytorch_mobilenetv2.xml"; 32 | std::string bin = "../share/pytorch_mobilenetv2.bin"; 33 | std::string src = "../share/src.mp4"; 34 | std::string bgr = "../share/src.png"; 35 | std::string bgr2 = "../share/replace.jpg"; 36 | #endif // WINDOWS 37 | ovlib::matter::Shape in_shape, out_shape; 38 | in_shape.width = 320; 39 | in_shape.height = 180; 40 | out_shape.width = 1280; 41 | out_shape.height = 720; 42 | 43 | ovlib::matter::MatterParams params; 44 | ovlib::matter::MatterChannel::getDefMatterParams(params); 45 | params.input_shape = in_shape; 46 | params.path_to_model = model; 47 | params.path_to_bin = bin; 48 | params.method = ovlib::matter::METHOD_BACKGROUND_MATTING_V2; 49 | params.is_async = false; 50 | //params.effect = ovlib::matter::EFFECT_BLUR; 51 | params.interval = 2; 52 | MatterChannel* pChan = MatterChannel::create(params); 53 | if (!pChan) 54 | { 55 | std::cout << "Can not create Matter Channel." << std::endl; 56 | 57 | return; 58 | } 59 | 60 | InitWindows(); 61 | 62 | cv::VideoCapture capture0(src); 63 | 64 | int framecnt = 0; 65 | int nDelay = 1; 66 | cv::Mat frame, bgrFrame, bgrFrame2; 67 | cv::Mat matCom, matPha; 68 | bgrFrame = cv::imread(bgr); 69 | bgrFrame2 = cv::imread(bgr2); 70 | std::map output; 71 | ovlib::matter::FrameData frame_com, frame_pha; 72 | ovlib::MatterBencher timercounter; 73 | timercounter.Start(); 74 | double lElapse = 0; 75 | 76 | FrameData frame_bgr; 77 | ovlib::Utils_Ov::mat2FrameData(bgrFrame, frame_bgr); 78 | FrameData frame_bgr_replace; 79 | ovlib::Utils_Ov::mat2FrameData(bgrFrame2, frame_bgr_replace); 80 | 81 | while (1) 82 | { 83 | if (!capture0.isOpened()) 84 | { 85 | break; 86 | } 87 | 88 | capture0 >> frame; 89 | if (frame.empty()) 90 | { 91 | break; 92 | } 93 | 94 | framecnt++; 95 | { 96 | ovlib::TimerCounter estimate("Phase..."); 97 | FrameData frame_main; 98 | ovlib::Utils_Ov::mat2FrameData(frame, frame_main); 99 | 100 | pChan->process(frame_main, frame_bgr, frame_bgr_replace, out_shape, &output); 101 | lElapse += timercounter.Elapse(); 102 | std::cout << "Elapse:" << lElapse / 1000.0 << " S" << std::endl; 103 | delete[] frame_main.frame; 104 | } 105 | 106 | frame_com = output["com"]; 107 | frame_pha = output["pha"]; 108 | ovlib::Utils_Ov::frameData2Mat(frame_com, matCom); 109 | ovlib::Utils_Ov::frameData2Mat(frame_pha, matPha); 110 | 111 | cv::imshow("com", matCom); 112 | cv::imshow("pha", matPha); 113 | delete[] frame_com.frame; 114 | delete[] frame_pha.frame; 115 | char c = cv::waitKey(nDelay); 116 | if (c == 'c') 117 | { 118 | break; 119 | } 120 | 121 | } 122 | 123 | delete[] frame_bgr.frame; 124 | delete[] frame_bgr_replace.frame; 125 | 126 | std::cout << "Speed:" << framecnt * 1000 / (lElapse) << " FPS" << std::endl; 127 | capture0.release(); 128 | cv::destroyAllWindows(); 129 | } -------------------------------------------------------------------------------- /share/modnet.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingpeter2015/libovmatting/8d2a6b855f7d3aa7a53b6b107072a8619b26cf67/share/modnet.bin -------------------------------------------------------------------------------- /share/pytorch_mobilenetv2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingpeter2015/libovmatting/8d2a6b855f7d3aa7a53b6b107072a8619b26cf67/share/pytorch_mobilenetv2.bin -------------------------------------------------------------------------------- /share/replace.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingpeter2015/libovmatting/8d2a6b855f7d3aa7a53b6b107072a8619b26cf67/share/replace.jpg -------------------------------------------------------------------------------- /share/src.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingpeter2015/libovmatting/8d2a6b855f7d3aa7a53b6b107072a8619b26cf67/share/src.mp4 -------------------------------------------------------------------------------- /share/src.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kingpeter2015/libovmatting/8d2a6b855f7d3aa7a53b6b107072a8619b26cf67/share/src.png -------------------------------------------------------------------------------- /src/cnn.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2019 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | #include "cnn.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | #include "ns_utils.hpp" 16 | 17 | /* 18 | const float meanValues[] = {static_cast(FLAGS_mean_val_r), 19 | static_cast(FLAGS_mean_val_g), 20 | static_cast(FLAGS_mean_val_b)}; 21 | */ 22 | 23 | 24 | 25 | using namespace InferenceEngine; 26 | 27 | CnnDLSDKBase::CnnDLSDKBase(const CNNConfig&config) : _config(config) {} 28 | 29 | void CnnDLSDKBase::Load() 30 | { 31 | _cnn_network_ = _config.ie.ReadNetwork(_config.path_to_model, _config.path_to_bin); 32 | 33 | auto input_shapes = _cnn_network_.getInputShapes(); 34 | 35 | const int currentBatchSize = _cnn_network_.getBatchSize(); 36 | if (currentBatchSize != _config.max_batch_size) 37 | _cnn_network_.setBatchSize(_config.max_batch_size); 38 | 39 | _inInfo = _cnn_network_.getInputsInfo(); 40 | _input_blob_names.clear(); 41 | _input_shapes.clear(); 42 | for (auto &item : _inInfo) 43 | { 44 | auto input_data = item.second; 45 | _input_blob_names.push_back(item.first); 46 | input_data->setPrecision(Precision::FP32); 47 | Layout layout = input_data->getLayout(); 48 | std::cout << "Input Name:" << item.first << ", Layout Type:" << layout << std::endl; 49 | 50 | input_data->setLayout(Layout::NCHW); 51 | input_data->getPreProcess().setResizeAlgorithm(RESIZE_BILINEAR); 52 | input_data->getPreProcess().setColorFormat(ColorFormat::RGB); 53 | 54 | SizeVector input_dims = input_data->getInputData()->getTensorDesc().getDims(); 55 | input_dims[3] = _config.input_shape.width; 56 | input_dims[2] = _config.input_shape.height; 57 | _input_shapes[item.first] = input_dims; 58 | } 59 | 60 | _cnn_network_.reshape(_input_shapes); 61 | 62 | _outInfo = _cnn_network_.getOutputsInfo(); 63 | _output_blobs_names.clear(); 64 | for (auto &item : _outInfo) 65 | { 66 | item.second->setPrecision(Precision::FP32); 67 | _output_blobs_names.push_back(item.first); 68 | 69 | Layout layout = item.second->getLayout(); 70 | std::cout << "Output Name:" << item.first << ", Layout Type:" << layout << std::endl; 71 | if(layout != Layout::NCHW) 72 | { 73 | item.second->setLayout(Layout::NC); 74 | } 75 | else 76 | { 77 | item.second->setLayout(Layout::NCHW); 78 | } 79 | } 80 | 81 | if (_config.cpu_threads_num > 0) 82 | { 83 | std::map loadParams; 84 | loadParams[PluginConfigParams::KEY_CPU_THREADS_NUM] = std::to_string(_config.cpu_threads_num); 85 | loadParams[PluginConfigParams::KEY_CPU_BIND_THREAD] = _config.cpu_bind_thread ? PluginConfigParams::YES : PluginConfigParams::NO; 86 | loadParams[PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS] = std::to_string(_config.cpu_throughput_streams); 87 | _executable_network_ = _config.ie.LoadNetwork(_cnn_network_, _config.deviceName, loadParams); 88 | } 89 | else 90 | { 91 | _executable_network_ = _config.ie.LoadNetwork(_cnn_network_, _config.deviceName); 92 | } 93 | 94 | _infer_request_ = _executable_network_.CreateInferRequest(); 95 | } 96 | 97 | MattingCNN::MattingCNN(const CNNConfig&config) : CnnDLSDKBase(config) 98 | { 99 | Load(); 100 | } 101 | 102 | void MattingCNN::Compute2(const cv::Mat &frame, cv::Mat &bgr, cv::Mat &bgr2, std::map *result, cv::Size& outp_shape) const 103 | { 104 | try 105 | { 106 | //1.Input 107 | cv::Mat iBgr, iFrame, matBgr, matFrame, matFrame1; 108 | iBgr = bgr.clone(); 109 | iFrame = frame.clone(); 110 | matFrame1 = frame.clone(); 111 | unsigned char *dataMatFrame1 = matFrame1.data; 112 | 113 | if (frame.rows != _config.input_shape.height || frame.cols != _config.input_shape.width) 114 | { 115 | cv::resize(frame, iFrame, _config.input_shape); 116 | } 117 | 118 | if (bgr.rows != _config.input_shape.height || bgr.cols != _config.input_shape.width) 119 | { 120 | cv::resize(bgr, iBgr, _config.input_shape); 121 | } 122 | 123 | matBgr = iBgr.clone(); 124 | matFrame = iFrame.clone(); 125 | std::vector vecBlockNames; 126 | vecBlockNames.emplace_back("src"); 127 | vecBlockNames.emplace_back("bgr"); 128 | 129 | unsigned char *dataSrc = (matFrame.data); 130 | unsigned char *dataBgr = (matBgr.data); 131 | 132 | //Benchmarking Input 133 | { 134 | ovlib::TimerCounter tCount("Phase1-Inputting"); 135 | int count = -1; 136 | for (auto &blockName : vecBlockNames) 137 | { 138 | count++; 139 | 140 | Blob::Ptr blob = _infer_request_.GetBlob(blockName); 141 | auto data = blob->buffer().as::value_type *>(); 142 | if (data == nullptr) 143 | { 144 | throw std::runtime_error("Input blob has not allocated buffer"); 145 | } 146 | size_t num_channels = blob->getTensorDesc().getDims()[1]; 147 | size_t image_width = blob->getTensorDesc().getDims()[3]; 148 | size_t image_height = blob->getTensorDesc().getDims()[2]; 149 | size_t image_size = image_width * image_height; 150 | 151 | /** Iterate over all input images **/ 152 | unsigned char *imagesData = count == 0 ? dataSrc : dataBgr; 153 | 154 | /** Iterate over all pixel in image (b,g,r) **/ 155 | for (size_t pid = 0; pid < image_size; pid++) 156 | { 157 | /** Iterate over all channels **/ 158 | int numC = pid * num_channels; 159 | *(data + pid) = *(imagesData + numC + 2) / 255.0f; 160 | *(data + image_size + pid) = *(imagesData + numC + 1) / 255.0f; 161 | *(data + image_size + image_size + pid) = *(imagesData + numC) / 255.0f; 162 | } 163 | } 164 | } 165 | 166 | //2.Infer 167 | { 168 | ovlib::TimerCounter tCount("Phase2-Infer()"); 169 | _infer_request_.Infer(); 170 | } 171 | 172 | //3.Output 173 | { 174 | ovlib::TimerCounter tCount("Phase3-Outputting"); 175 | 176 | //3.1 get Alpha 177 | Blob::Ptr blobPha = _infer_request_.GetBlob("pha"); 178 | auto dataPha = blobPha->buffer().as::value_type *>(); 179 | 180 | size_t num_channels_pha = blobPha->getTensorDesc().getDims()[1]; 181 | size_t image_width = blobPha->getTensorDesc().getDims()[3]; 182 | size_t image_height = blobPha->getTensorDesc().getDims()[2]; 183 | unsigned long image_size = image_width * image_height; 184 | 185 | cv::Mat matPha = cv::Mat::zeros(_config.input_shape, CV_8UC1); 186 | cv::Mat matCom = cv::Mat::zeros(outp_shape, CV_8UC3); 187 | unsigned char *dataMatPha = matPha.data; 188 | unsigned char *dataMatCom = matCom.data; 189 | for (size_t pid = 0; pid < image_size; pid++) 190 | { 191 | float alpha = *(dataPha + pid); 192 | *(dataMatPha + pid) = alpha * 255.0f; 193 | } 194 | 195 | if(matPha.rows != outp_shape.height || matPha.cols != outp_shape.width) 196 | { 197 | cv::resize(matPha, matPha, outp_shape,0,0,cv::INTER_CUBIC); 198 | } 199 | dataMatPha = matPha.data; 200 | image_size = matPha.rows * matPha.cols; 201 | 202 | //3.2 get bgr2 203 | cv::Mat matBgr2 = bgr2.clone(); 204 | if(matBgr2.rows != outp_shape.height || matBgr2.cols != outp_shape.width) 205 | { 206 | cv::resize(matBgr2, matBgr2, outp_shape); 207 | } 208 | unsigned char *dataMatBgr2 = matBgr2.data; 209 | 210 | if(matFrame1.rows != outp_shape.height || matFrame1.cols != outp_shape.width) 211 | { 212 | cv::resize(matFrame1, matFrame1, outp_shape); 213 | } 214 | dataMatFrame1 = matFrame1.data; 215 | 216 | //3.3 get composite 217 | int num_channels = matCom.channels(); 218 | for (size_t pid = 0; pid < image_size; pid++) 219 | { 220 | int nAlpha = *(dataMatPha + pid); 221 | int rowC = pid * num_channels; 222 | if(nAlpha == 0) 223 | { 224 | *(dataMatCom + rowC + 2) = *(dataMatBgr2 + rowC + 2); 225 | *(dataMatCom + rowC + 1) = *(dataMatBgr2 + rowC + 1); 226 | *(dataMatCom + rowC) = *(dataMatBgr2 + rowC); 227 | } 228 | else if(nAlpha == 255) 229 | { 230 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2); 231 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1); 232 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC); 233 | } 234 | else 235 | { 236 | float alpha = nAlpha / 255.0; 237 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2) * alpha + *(dataMatBgr2 + rowC + 2) * (1 - alpha); 238 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1) * alpha + *(dataMatBgr2 + rowC + 1) * (1 - alpha); 239 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC) * alpha + *(dataMatBgr2 + rowC) * (1 - alpha); 240 | } 241 | } 242 | 243 | 244 | (*result)["com"] = matCom; 245 | (*result)["pha"] = matPha; 246 | } 247 | } 248 | catch (const std::exception &e) 249 | { 250 | std::cerr << "MattingCNN::Compute2():" << e.what() << '\n'; 251 | } 252 | } 253 | 254 | void MattingCNN::Compute(const cv::Mat &frame, cv::Mat &bgr, std::map *result, cv::Size &outp_shape) const 255 | { 256 | try 257 | { 258 | //1.Input 259 | cv::Mat iBgr, iFrame, matBgr, matFrame, matFrame1; 260 | iBgr = bgr.clone(); 261 | iFrame = frame.clone(); 262 | matFrame1 = frame.clone(); 263 | unsigned char *dataMatFrame1 = matFrame1.data; 264 | 265 | if (frame.rows != _config.input_shape.height || frame.cols != _config.input_shape.width) 266 | { 267 | cv::resize(frame, iFrame, _config.input_shape); 268 | } 269 | 270 | if (bgr.rows != _config.input_shape.height || bgr.cols != _config.input_shape.width) 271 | { 272 | cv::resize(bgr, iBgr, _config.input_shape); 273 | } 274 | 275 | matBgr = iBgr.clone(); 276 | matFrame = iFrame.clone(); 277 | std::vector vecBlockNames; 278 | vecBlockNames.emplace_back("src"); 279 | vecBlockNames.emplace_back("bgr"); 280 | 281 | unsigned char *dataSrc = (matFrame.data); 282 | unsigned char *dataBgr = (matBgr.data); 283 | 284 | //Benchmarking Input 285 | { 286 | ovlib::TimerCounter tCount("Phase1-Inputting"); 287 | int count = -1; 288 | for (auto &blockName : vecBlockNames) 289 | { 290 | count++; 291 | 292 | Blob::Ptr blob = _infer_request_.GetBlob(blockName); 293 | auto data = blob->buffer().as::value_type *>(); 294 | if (data == nullptr) 295 | { 296 | throw std::runtime_error("Input blob has not allocated buffer"); 297 | } 298 | size_t num_channels = blob->getTensorDesc().getDims()[1]; 299 | size_t image_width = blob->getTensorDesc().getDims()[3]; 300 | size_t image_height = blob->getTensorDesc().getDims()[2]; 301 | size_t image_size = image_width * image_height; 302 | /** Iterate over all input images **/ 303 | unsigned char *imagesData = count == 0 ? dataSrc : dataBgr; 304 | /** Iterate over all pixel in image (b,g,r) **/ 305 | for (size_t pid = 0; pid < image_size; pid++) 306 | { 307 | /** Iterate over all channels **/ 308 | data[0 * image_size + pid] = imagesData[pid * num_channels + 2] / 255.0; 309 | data[1 * image_size + pid] = imagesData[pid * num_channels + 1] / 255.0; 310 | data[2 * image_size + pid] = imagesData[pid * num_channels + 0] / 255.0; 311 | } 312 | } 313 | } 314 | 315 | //2.Infer 316 | { 317 | ovlib::TimerCounter tCount("Phase2-Infer()"); 318 | _infer_request_.Infer(); 319 | } 320 | 321 | //3.Output 322 | 323 | { 324 | ovlib::TimerCounter tCount("Phase3-Outputting"); 325 | 326 | //blobPha 327 | Blob::Ptr blobPha = _infer_request_.GetBlob("pha"); 328 | auto dataPha = blobPha->buffer().as::value_type *>(); 329 | 330 | Blob::Ptr blobFgr = _infer_request_.GetBlob("fgr"); 331 | auto dataFgr = blobFgr->buffer().as::value_type *>(); 332 | size_t num_channels = blobFgr->getTensorDesc().getDims()[1]; 333 | 334 | size_t num_channels_pha = blobPha->getTensorDesc().getDims()[1]; 335 | size_t image_width = blobPha->getTensorDesc().getDims()[3]; 336 | size_t image_height = blobPha->getTensorDesc().getDims()[2]; 337 | unsigned long image_size = image_width * image_height; 338 | 339 | cv::Mat matPha = cv::Mat::zeros(_config.input_shape, CV_8UC1); 340 | cv::Mat matFgr = cv::Mat::zeros(_config.input_shape, CV_8UC3); 341 | cv::Mat matCom = cv::Mat::zeros(outp_shape, CV_8UC3); 342 | cv::Mat matGreen(outp_shape, CV_8UC3, cv::Scalar(120, 255, 155)); 343 | unsigned char *dataMatPha = matPha.data; 344 | unsigned char *dataMatFgr = matFgr.data; 345 | unsigned char *dataMatCom = matCom.data; 346 | unsigned char *dataMatGreen = matGreen.data; 347 | for (size_t pid = 0; pid < image_size; pid++) 348 | { 349 | /** Iterate over all channels **/ 350 | float alpha = dataPha[pid]; 351 | dataMatPha[pid] = dataPha[pid] * 255.0; 352 | 353 | dataMatFgr[pid * num_channels + 2] = dataFgr[0 * image_size + pid] * 255.0; 354 | dataMatFgr[pid * num_channels + 1] = dataFgr[1 * image_size + pid] * 255.0; 355 | dataMatFgr[pid * num_channels + 0] = dataFgr[2 * image_size + pid] * 255.0; 356 | 357 | 358 | } 359 | 360 | cv::resize(matPha, matPha, outp_shape); 361 | cv::resize(matFgr, matFgr, outp_shape); 362 | dataMatPha = matPha.data; 363 | dataMatFgr = matFgr.data; 364 | image_size = matPha.rows * matPha.cols; 365 | 366 | for (size_t pid = 0; pid < image_size; pid++) 367 | { 368 | float alpha = dataMatPha[pid] / 255.0f; 369 | dataMatCom[pid * num_channels + 2] = (dataMatFrame1[pid * num_channels + 2] * alpha) + (dataMatGreen[pid * num_channels + 2] * (1 - alpha)); 370 | dataMatCom[pid * num_channels + 1] = (dataMatFrame1[pid * num_channels + 1] * alpha) + (dataMatGreen[pid * num_channels + 1] * (1 - alpha)); 371 | dataMatCom[pid * num_channels + 0] = (dataMatFrame1[pid * num_channels + 0] * alpha) + (dataMatGreen[pid * num_channels + 0] * (1 - alpha)); 372 | } 373 | 374 | 375 | (*result)["com"] = matCom; 376 | (*result)["fgr"] = matFgr; 377 | (*result)["pha"] = matPha; 378 | } 379 | } 380 | catch (const std::exception &e) 381 | { 382 | std::cerr << "MattingCNN::Compute():" << e.what() << '\n'; 383 | } 384 | } 385 | 386 | void MattingCNN::Compute_Alpha(const cv::Mat &frame, cv::Mat &bgr, std::map *result, cv::Size &outp_shape) const 387 | { 388 | cv::Mat iBgr, iFrame, matBgr, matFrame; 389 | iBgr = bgr.clone(); 390 | iFrame = frame.clone(); 391 | 392 | cv::Mat matPha = cv::Mat::zeros(frame.size(), CV_8UC3); 393 | cv::Mat matFgr = cv::Mat::zeros(frame.size(), CV_8UC3); 394 | cv::Mat matCom = cv::Mat::zeros(frame.size(), CV_8UC3); 395 | cv::Mat matGreen(frame.size(), CV_8UC3, cv::Scalar(120, 255, 155)); 396 | unsigned char *dataMatPha = matPha.data; 397 | unsigned char *dataMatFgr = matFgr.data; 398 | unsigned char *dataMatCom = matCom.data; 399 | unsigned char *dataMatGreen = matGreen.data; 400 | 401 | size_t image_width = frame.size().width; 402 | size_t image_height = frame.size().height; 403 | size_t image_size = image_width * image_height; 404 | 405 | try 406 | { 407 | 408 | 409 | matBgr = iBgr.clone(); 410 | matFrame = iFrame.clone(); 411 | 412 | unsigned char *dataSrc = (matFrame.data); 413 | unsigned char *dataBgr = (matBgr.data); 414 | 415 | //Benchmarking Input 416 | { 417 | ovlib::TimerCounter tCount("Phase1-Getting Alpha"); 418 | cv::bitwise_xor(frame, bgr, matPha); 419 | cv::bitwise_and(frame, matPha, matFgr); 420 | } 421 | 422 | 423 | //3.Output 424 | 425 | { 426 | ovlib::TimerCounter tCount("Phase3-Outputting"); 427 | 428 | (*result)["com"] = matCom; 429 | (*result)["fgr"] = matFgr; 430 | (*result)["pha"] = matPha; 431 | } 432 | } 433 | catch (const std::exception &e) 434 | { 435 | std::cerr << "MattingCNN::Compute():" << e.what() << '\n'; 436 | } 437 | } -------------------------------------------------------------------------------- /src/cnn_backgroundv2.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2019 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | #include "cnn_backgroundv2.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "ovmatter.h" 15 | 16 | using namespace InferenceEngine; 17 | 18 | CNN_Background_V2::CNN_Background_V2(const CNNConfig& config) : _config(config), _bBgr(false) 19 | { 20 | topoName = "CNN_Background_V2"; 21 | isAsync = config.is_async; 22 | 23 | cnn_network_ = _config.ie.ReadNetwork(_config.path_to_model, _config.path_to_bin); 24 | 25 | load(); 26 | } 27 | 28 | bool CNN_Background_V2::isBgrEnqueued() 29 | { 30 | return _bBgr; 31 | } 32 | 33 | void CNN_Background_V2::load() 34 | { 35 | InferenceEngine::InputsDataMap inputInfo = cnn_network_.getInputsInfo(); 36 | const int currentBatchSize = cnn_network_.getBatchSize(); 37 | if (currentBatchSize != _config.max_batch_size) 38 | { 39 | cnn_network_.setBatchSize(_config.max_batch_size); 40 | } 41 | 42 | int nInputInfoSize = inputInfo.size(); 43 | auto inInfo = cnn_network_.getInputsInfo(); 44 | std::map input_shapes; 45 | for (auto& item : inInfo) 46 | { 47 | auto input_data = item.second; 48 | input_data->setPrecision(Precision::FP32); 49 | input_data->setLayout(Layout::NCHW); 50 | input_data->getPreProcess().setResizeAlgorithm(RESIZE_BILINEAR); 51 | input_data->getPreProcess().setColorFormat(ColorFormat::RGB); 52 | 53 | SizeVector input_dims = input_data->getInputData()->getTensorDesc().getDims(); 54 | input_dims[3] = _config.input_shape.width; 55 | input_dims[2] = _config.input_shape.height; 56 | input_shapes[item.first] = input_dims; 57 | } 58 | cnn_network_.reshape(input_shapes); 59 | 60 | InferenceEngine::OutputsDataMap outputInfo = cnn_network_.getOutputsInfo(); 61 | for (auto& item : outputInfo) 62 | { 63 | item.second->setPrecision(Precision::FP32); 64 | Layout layout = item.second->getLayout(); 65 | if (layout != Layout::NCHW) 66 | { 67 | item.second->setLayout(Layout::NC); 68 | } 69 | else 70 | { 71 | item.second->setLayout(Layout::NCHW); 72 | } 73 | } 74 | 75 | if (_config.cpu_threads_num > 0) 76 | { 77 | std::map loadParams; 78 | loadParams[PluginConfigParams::KEY_CPU_THREADS_NUM] = std::to_string(_config.cpu_threads_num); 79 | loadParams[PluginConfigParams::KEY_CPU_BIND_THREAD] = _config.cpu_bind_thread ? PluginConfigParams::YES : PluginConfigParams::NO; 80 | loadParams[PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS] = std::to_string(_config.cpu_throughput_streams); 81 | net_ = _config.ie.LoadNetwork(cnn_network_, _config.deviceName, loadParams); 82 | } 83 | else 84 | { 85 | net_ = _config.ie.LoadNetwork(cnn_network_, _config.deviceName); 86 | } 87 | 88 | request = net_.CreateInferRequestPtr(); 89 | 90 | _bBgr = false; 91 | } 92 | 93 | void CNN_Background_V2::reshape(cv::Size input_shape) 94 | { 95 | _config.input_shape = input_shape; 96 | load(); 97 | } 98 | 99 | void CNN_Background_V2::submitRequest() 100 | { 101 | if (!request) 102 | { 103 | return; 104 | } 105 | 106 | BaseAsyncCNN::submitRequest(); 107 | } 108 | 109 | void CNN_Background_V2::enqueue(const std::string& name, const cv::Mat& frame) 110 | { 111 | if (!request) 112 | { 113 | return; 114 | } 115 | 116 | cv::Mat matFrame = frame.clone(); 117 | if (matFrame.rows != _config.input_shape.height || matFrame.cols != _config.input_shape.width) 118 | { 119 | cv::resize(matFrame, matFrame, _config.input_shape); 120 | } 121 | unsigned char* dataMatFrame = matFrame.data; 122 | 123 | Blob::Ptr blob = request->GetBlob(name); 124 | auto data = blob->buffer().as::value_type*>(); 125 | if (data == nullptr) 126 | { 127 | std::string errorName = "Input blob (" + name; 128 | errorName += ") has not allocated buffer"; 129 | throw std::runtime_error(errorName); 130 | //throw std::runtime_error("Input blob has not allocated buffer"); 131 | } 132 | size_t num_channels = blob->getTensorDesc().getDims()[1]; 133 | size_t image_width = blob->getTensorDesc().getDims()[3]; 134 | size_t image_height = blob->getTensorDesc().getDims()[2]; 135 | size_t image_size = image_width * image_height; 136 | 137 | unsigned char* imagesData = matFrame.data; 138 | 139 | /** Iterate over all pixel in image (b,g,r) **/ 140 | for (size_t pid = 0; pid < image_size; pid++) 141 | { 142 | /** Iterate over all channels **/ 143 | int numC = pid * num_channels; 144 | *(data + pid) = *(imagesData + numC + 2) / 255.0f; 145 | *(data + image_size + pid) = *(imagesData + numC + 1) / 255.0f; 146 | *(data + image_size + image_size + pid) = *(imagesData + numC) / 255.0f; 147 | } 148 | 149 | if (name == "bgr") 150 | { 151 | _bBgr = true; 152 | } 153 | } 154 | 155 | void CNN_Background_V2::enqueueAll(const cv::Mat& frame, const cv::Mat& bgr) 156 | { 157 | try 158 | { 159 | //1.init fields 160 | enqueue("src", frame); 161 | enqueue("bgr", bgr); 162 | } 163 | catch (const std::exception& e) 164 | { 165 | std::cerr << "CNN_Background_V2::enqueue():" << e.what() << '\n'; 166 | } 167 | } 168 | 169 | MattingObjects CNN_Background_V2::fetchResults() 170 | { 171 | MattingObjects results; 172 | 173 | if (!request) 174 | { 175 | return results; 176 | } 177 | 178 | try 179 | { 180 | cv::Mat matPha = cv::Mat::zeros(_config.input_shape, CV_8UC1); 181 | unsigned char* dataMatPha = matPha.data; 182 | 183 | //3.1. get alpha mat 184 | Blob::Ptr blobPha = request->GetBlob("pha"); 185 | auto dataPha = blobPha->buffer().as::value_type*>(); 186 | 187 | size_t num_channels_pha = blobPha->getTensorDesc().getDims()[1]; 188 | size_t image_width = blobPha->getTensorDesc().getDims()[3]; 189 | size_t image_height = blobPha->getTensorDesc().getDims()[2]; 190 | unsigned long image_size = image_width * image_height; 191 | for (size_t pid = 0; pid < image_size; pid++) 192 | { 193 | float alpha = *(dataPha + pid); 194 | *(dataMatPha + pid) = alpha * 255.0f; 195 | } 196 | 197 | MattingObject objMatter; 198 | objMatter.pha = matPha.clone(); 199 | results.push_back(objMatter); 200 | 201 | return results; 202 | } 203 | catch (const std::exception& e) 204 | { 205 | std::cerr << "CNN_Background_V2::fetchResults():" << e.what() << '\n'; 206 | return results; 207 | } 208 | } 209 | 210 | -------------------------------------------------------------------------------- /src/cnn_modnet.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2018-2019 Intel Corporation 2 | // SPDX-License-Identifier: Apache-2.0 3 | // 4 | 5 | #include "cnn_modnet.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "ovmatter.h" 15 | 16 | using namespace InferenceEngine; 17 | 18 | CNN_Modnet::CNN_Modnet(const CNNConfig& config) : _config(config), _bBgr(false) 19 | { 20 | topoName = "CNN_Modnet"; 21 | isAsync = config.is_async; 22 | 23 | cnn_network_ = _config.ie.ReadNetwork(_config.path_to_model, _config.path_to_bin); 24 | 25 | load(); 26 | } 27 | 28 | bool CNN_Modnet::isBgrEnqueued() 29 | { 30 | return _bBgr; 31 | } 32 | 33 | void CNN_Modnet::load() 34 | { 35 | InferenceEngine::InputsDataMap inputInfo = cnn_network_.getInputsInfo(); 36 | const int currentBatchSize = cnn_network_.getBatchSize(); 37 | if (currentBatchSize != _config.max_batch_size) 38 | { 39 | cnn_network_.setBatchSize(_config.max_batch_size); 40 | } 41 | 42 | int nInputInfoSize = inputInfo.size(); 43 | auto inInfo = cnn_network_.getInputsInfo(); 44 | std::map input_shapes; 45 | for (auto& item : inInfo) 46 | { 47 | std::cout << "input name:" << item.first << std::endl; 48 | auto input_data = item.second; 49 | input_data->setPrecision(Precision::FP32); 50 | input_data->setLayout(Layout::NCHW); 51 | input_data->getPreProcess().setResizeAlgorithm(RESIZE_BILINEAR); 52 | input_data->getPreProcess().setColorFormat(ColorFormat::RGB); 53 | 54 | SizeVector input_dims = input_data->getInputData()->getTensorDesc().getDims(); 55 | input_dims[3] = _config.input_shape.width; 56 | input_dims[2] = _config.input_shape.height; 57 | input_shapes[item.first] = input_dims; 58 | } 59 | cnn_network_.reshape(input_shapes); 60 | 61 | InferenceEngine::OutputsDataMap outputInfo = cnn_network_.getOutputsInfo(); 62 | for (auto& item : outputInfo) 63 | { 64 | std::cout << "output name:" << item.first << std::endl; 65 | item.second->setPrecision(Precision::FP32); 66 | Layout layout = item.second->getLayout(); 67 | if (layout != Layout::NCHW) 68 | { 69 | item.second->setLayout(Layout::NC); 70 | } 71 | else 72 | { 73 | item.second->setLayout(Layout::NCHW); 74 | } 75 | } 76 | 77 | if (_config.cpu_threads_num > 0) 78 | { 79 | std::map loadParams; 80 | loadParams[PluginConfigParams::KEY_CPU_THREADS_NUM] = std::to_string(_config.cpu_threads_num); 81 | loadParams[PluginConfigParams::KEY_CPU_BIND_THREAD] = _config.cpu_bind_thread ? PluginConfigParams::YES : PluginConfigParams::NO; 82 | loadParams[PluginConfigParams::KEY_CPU_THROUGHPUT_STREAMS] = std::to_string(_config.cpu_throughput_streams); 83 | net_ = _config.ie.LoadNetwork(cnn_network_, _config.deviceName, loadParams); 84 | } 85 | else 86 | { 87 | net_ = _config.ie.LoadNetwork(cnn_network_, _config.deviceName); 88 | } 89 | 90 | request = net_.CreateInferRequestPtr(); 91 | 92 | _bBgr = false; 93 | } 94 | 95 | void CNN_Modnet::reshape(cv::Size input_shape) 96 | { 97 | _config.input_shape = input_shape; 98 | load(); 99 | } 100 | 101 | void CNN_Modnet::submitRequest() 102 | { 103 | if (!request) 104 | { 105 | return; 106 | } 107 | 108 | BaseAsyncCNN::submitRequest(); 109 | } 110 | 111 | void CNN_Modnet::enqueue(const std::string& name, const cv::Mat& frame) 112 | { 113 | if (!request) 114 | { 115 | return; 116 | } 117 | 118 | cv::Mat matFrame = frame.clone(); 119 | if (matFrame.rows != _config.input_shape.height || matFrame.cols != _config.input_shape.width) 120 | { 121 | cv::resize(matFrame, matFrame, _config.input_shape); 122 | } 123 | unsigned char* dataMatFrame = matFrame.data; 124 | 125 | Blob::Ptr blob = request->GetBlob(name); 126 | auto data = blob->buffer().as::value_type*>(); 127 | if (data == nullptr) 128 | { 129 | std::string errorName = "Input blob (" + name; 130 | errorName += ") has not allocated buffer"; 131 | throw std::runtime_error(errorName); 132 | //throw std::runtime_error("Input blob has not allocated buffer"); 133 | } 134 | size_t num_channels = blob->getTensorDesc().getDims()[1]; 135 | size_t image_width = blob->getTensorDesc().getDims()[3]; 136 | size_t image_height = blob->getTensorDesc().getDims()[2]; 137 | size_t image_size = image_width * image_height; 138 | 139 | unsigned char* imagesData = matFrame.data; 140 | 141 | /** Iterate over all pixel in image (b,g,r) **/ 142 | for (size_t pid = 0; pid < image_size; pid++) 143 | { 144 | /** Iterate over all channels **/ 145 | int numC = pid * num_channels; 146 | *(data + pid) = *(imagesData + numC + 2) / 255.0f; 147 | *(data + image_size + pid) = *(imagesData + numC + 1) / 255.0f; 148 | *(data + image_size + image_size + pid) = *(imagesData + numC) / 255.0f; 149 | } 150 | 151 | if (name == "bgr") 152 | { 153 | _bBgr = true; 154 | } 155 | } 156 | 157 | void CNN_Modnet::enqueueAll(const cv::Mat& frame, const cv::Mat& bgr) 158 | { 159 | try 160 | { 161 | //1.init fields 162 | enqueue("src", frame); 163 | enqueue("bgr", bgr); 164 | } 165 | catch (const std::exception& e) 166 | { 167 | std::cerr << "CNN_Modnet::enqueue():" << e.what() << '\n'; 168 | } 169 | } 170 | 171 | MattingObjects CNN_Modnet::fetchResults() 172 | { 173 | MattingObjects results; 174 | 175 | if (!request) 176 | { 177 | return results; 178 | } 179 | 180 | try 181 | { 182 | cv::Mat matPha = cv::Mat::zeros(_config.input_shape, CV_8UC1); 183 | unsigned char* dataMatPha = matPha.data; 184 | 185 | //3.1. get alpha mat 186 | Blob::Ptr blobPha = request->GetBlob("1011"); 187 | auto dataPha = blobPha->buffer().as::value_type*>(); 188 | 189 | size_t num_channels_pha = blobPha->getTensorDesc().getDims()[1]; 190 | size_t image_width = blobPha->getTensorDesc().getDims()[3]; 191 | size_t image_height = blobPha->getTensorDesc().getDims()[2]; 192 | unsigned long image_size = image_width * image_height; 193 | for (size_t pid = 0; pid < image_size; pid++) 194 | { 195 | float alpha = *(dataPha + pid); 196 | *(dataMatPha + pid) = alpha * 255.0f; 197 | } 198 | 199 | MattingObject objMatter; 200 | objMatter.pha = matPha.clone(); 201 | results.push_back(objMatter); 202 | 203 | return results; 204 | } 205 | catch (const std::exception& e) 206 | { 207 | std::cerr << "CNN_Modnet::fetchResults():" << e.what() << '\n'; 208 | return results; 209 | } 210 | } 211 | 212 | -------------------------------------------------------------------------------- /src/ext_ops.cpp: -------------------------------------------------------------------------------- 1 | #include "ext_ops.hpp" 2 | 3 | #define REGISTER_OPERATOR_CUSOTM(name_, fn_) \ 4 | ngraph::onnx_import::register_operator(name_, 1, "", fn_) 5 | 6 | ngraph::OutputVector op_example(const ngraph::onnx_import::Node &onnx_node) 7 | { 8 | namespace opset = ngraph::opset5; 9 | 10 | ngraph::OutputVector ng_inputs{onnx_node.get_ng_inputs()}; 11 | const ngraph::Output &data = ng_inputs.at(0); 12 | // create constant node with a single element that's equal to zero 13 | std::shared_ptr zero_node = opset::Constant::create(data.get_element_type(), ngraph::Shape{}, {0}); 14 | // create a negative map for 'data' node, 1 for negative values , 0 for positive values or zero 15 | // then convert it from boolean type to `data.get_element_type()` 16 | std::shared_ptr negative_map = std::make_shared( 17 | std::make_shared(data, zero_node), data.get_element_type()); 18 | // create a positive map for 'data' node, 0 for negative values , 1 for positive values or zero 19 | // then convert it from boolean type to `data.get_element_type()` 20 | std::shared_ptr positive_map = std::make_shared( 21 | std::make_shared(data, zero_node), data.get_element_type()); 22 | 23 | // fetch alpha and beta attributes from ONNX node 24 | float alpha = onnx_node.get_attribute_value("alpha", 1); // if 'alpha' attribute is not provided in the model, then the default value is 1 25 | float beta = onnx_node.get_attribute_value("beta"); 26 | // create constant node with a single element 'alpha' with type f32 27 | std::shared_ptr alpha_node = opset::Constant::create(ngraph::element::f32, ngraph::Shape{}, {alpha}); 28 | // create constant node with a single element 'beta' with type f32 29 | std::shared_ptr beta_node = opset::Constant::create(ngraph::element::f32, ngraph::Shape{}, {beta}); 30 | 31 | return { 32 | std::make_shared( 33 | std::make_shared(alpha_node, std::make_shared(data, positive_map)), 34 | std::make_shared(beta_node, std::make_shared(data, negative_map)))}; 35 | } 36 | 37 | void registerRoiAlign() 38 | { 39 | 40 | REGISTER_OPERATOR_CUSOTM("OpExampl", op_example); 41 | } -------------------------------------------------------------------------------- /src/ns_bmp.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "ns_bmp.hpp" 3 | #include 4 | #include 5 | 6 | using namespace std; 7 | using namespace ovlib; 8 | 9 | BitMap::BitMap(const string &filename) 10 | { 11 | BmpHeader header; 12 | BmpInfoHeader infoHeader; 13 | 14 | ifstream input(filename, ios::binary); 15 | if (!input) 16 | { 17 | return; 18 | } 19 | 20 | input.read(reinterpret_cast(&header.type), 2); 21 | 22 | if (header.type != 'M' * 256 + 'B') 23 | { 24 | std::cerr << "[BMP] file is not bmp type\n"; 25 | return; 26 | } 27 | 28 | input.read(reinterpret_cast(&header.size), 4); 29 | input.read(reinterpret_cast(&header.reserved), 4); 30 | input.read(reinterpret_cast(&header.offset), 4); 31 | 32 | input.read(reinterpret_cast(&infoHeader), sizeof(BmpInfoHeader)); 33 | 34 | bool rowsReversed = infoHeader.height < 0; 35 | _width = infoHeader.width; 36 | _height = abs(infoHeader.height); 37 | 38 | if (infoHeader.bits != 24) 39 | { 40 | cerr << "[BMP] 24bpp only supported. But input has:" << infoHeader.bits << "\n"; 41 | return; 42 | } 43 | 44 | if (infoHeader.compression != 0) 45 | { 46 | cerr << "[BMP] compression not supported\n"; 47 | } 48 | 49 | int padSize = _width & 3; 50 | char pad[3]; 51 | size_t size = _width * _height * 3; 52 | 53 | _data.reset(new unsigned char[size], std::default_delete()); 54 | 55 | input.seekg(header.offset, ios::beg); 56 | 57 | // reading by rows in invert vertically 58 | for (uint32_t i = 0; i < _height; i++) 59 | { 60 | uint32_t storeAt = rowsReversed ? i : (uint32_t)_height - 1 - i; 61 | input.read(reinterpret_cast(_data.get()) + _width * 3 * storeAt, _width * 3); 62 | input.read(pad, padSize); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ns_register.cpp: -------------------------------------------------------------------------------- 1 | #include "ns_register.hpp" 2 | #include "ns_utils.hpp" 3 | #include "ns_bmp.hpp" 4 | 5 | using namespace std; 6 | using namespace ovlib; 7 | 8 | /***************************************MnistUbyte*******************************************************************/ 9 | int MnistUbyte::reverseInt(int i) 10 | { 11 | unsigned char ch1, ch2, ch3, ch4; 12 | ch1 = (unsigned char)(i & 255); 13 | ch2 = (unsigned char)((i >> 8) & 255); 14 | ch3 = (unsigned char)((i >> 16) & 255); 15 | ch4 = (unsigned char)((i >> 24) & 255); 16 | return (static_cast(ch1) << 24) + (static_cast(ch2) << 16) + (static_cast(ch3) << 8) + ch4; 17 | } 18 | 19 | MnistUbyte::MnistUbyte(const std::string &filename) 20 | { 21 | std::ifstream file(filename, std::ios::binary); 22 | if (!file.is_open()) 23 | { 24 | return; 25 | } 26 | int magic_number = 0; 27 | int number_of_images = 0; 28 | int n_rows = 0; 29 | int n_cols = 0; 30 | file.read(reinterpret_cast(&magic_number), sizeof(magic_number)); 31 | magic_number = reverseInt(magic_number); 32 | if (magic_number != 2051) 33 | { 34 | return; 35 | } 36 | file.read(reinterpret_cast(&number_of_images), sizeof(number_of_images)); 37 | number_of_images = reverseInt(number_of_images); 38 | file.read(reinterpret_cast(&n_rows), sizeof(n_rows)); 39 | n_rows = reverseInt(n_rows); 40 | _height = (size_t)n_rows; 41 | file.read(reinterpret_cast(&n_cols), sizeof(n_cols)); 42 | n_cols = reverseInt(n_cols); 43 | _width = (size_t)n_cols; 44 | if (number_of_images > 1) 45 | { 46 | std::cout << "[MNIST] Warning: number_of_images in mnist file equals " << number_of_images 47 | << ". Only a first image will be read." << std::endl; 48 | } 49 | 50 | size_t size = _width * _height * 1; 51 | 52 | _data.reset(new unsigned char[size], std::default_delete()); 53 | size_t count = 0; 54 | if (0 < number_of_images) 55 | { 56 | for (int r = 0; r < n_rows; ++r) 57 | { 58 | for (int c = 0; c < n_cols; ++c) 59 | { 60 | unsigned char temp = 0; 61 | file.read(reinterpret_cast(&temp), sizeof(temp)); 62 | _data.get()[count++] = temp; 63 | } 64 | } 65 | } 66 | 67 | file.close(); 68 | } 69 | 70 | /*************************************************OCVReader**********************************************************/ 71 | 72 | OCVReader::OCVReader(const string &filename) 73 | { 74 | img = cv::imread(filename); 75 | _size = 0; 76 | 77 | if (img.empty()) 78 | { 79 | return; 80 | } 81 | 82 | _size = img.size().width * img.size().height * img.channels(); 83 | _width = img.size().width; 84 | _height = img.size().height; 85 | } 86 | 87 | std::shared_ptr OCVReader::getData(size_t width = 0, size_t height = 0) 88 | { 89 | cv::Mat resized(img); 90 | if (width != 0 && height != 0) 91 | { 92 | size_t iw = img.size().width; 93 | size_t ih = img.size().height; 94 | if (width != iw || height != ih) 95 | { 96 | slog::warn << "Image is resized from (" << iw << ", " << ih << ") to (" << width << ", " << height << ")" << slog::endl; 97 | } 98 | cv::resize(img, resized, cv::Size(width, height)); 99 | } 100 | 101 | size_t size = resized.size().width * resized.size().height * resized.channels(); 102 | _data.reset(new unsigned char[size], std::default_delete()); 103 | for (size_t id = 0; id < size; ++id) 104 | { 105 | _data.get()[id] = resized.data[id]; 106 | } 107 | return _data; 108 | } 109 | 110 | /********************************************Registry************************************************************/ 111 | std::vector Registry::_data; 112 | 113 | Register MnistUbyte::reg; 114 | #ifdef NS_USE_OPENCV 115 | Register OCVReader::reg; 116 | #else 117 | Register BitMap::reg; 118 | #endif 119 | 120 | Reader *Registry::CreateReader(const char *filename) 121 | { 122 | for (auto maker : _data) 123 | { 124 | Reader *ol = maker(filename); 125 | if (ol != nullptr && ol->size() != 0) 126 | return ol; 127 | if (ol != nullptr) 128 | ol->Release(); 129 | } 130 | return nullptr; 131 | } 132 | 133 | void Registry::RegisterReader(CreatorFunction f) 134 | { 135 | _data.push_back(f); 136 | } 137 | 138 | ReaderPtr::ReaderPtr(const char *imageName) 139 | : reader(CreateFormatReader(imageName), [](Reader *p) { p->Release(); }) 140 | {} 141 | 142 | FORMAT_READER_API(Reader *) 143 | CreateFormatReader(const char *filename) 144 | { 145 | return Registry::CreateReader(filename); 146 | } 147 | -------------------------------------------------------------------------------- /src/ns_thread.cpp: -------------------------------------------------------------------------------- 1 | #include "ns_thread.hpp" 2 | 3 | 4 | ov_simple_thread::ov_simple_thread() 5 | { 6 | _interript = false; 7 | } 8 | 9 | 10 | ov_simple_thread::~ov_simple_thread() 11 | { 12 | if (!this->isInterrupted()) 13 | { 14 | this->interrupt(); 15 | } 16 | 17 | if (_thread.joinable()) { 18 | _thread.join(); 19 | } 20 | } 21 | 22 | void ov_simple_thread::start() 23 | { 24 | _interript = false; 25 | std::thread thr(std::bind(&ov_simple_thread::run, this)); 26 | _thread = std::move(thr); 27 | 28 | } 29 | 30 | std::thread::id ov_simple_thread::getId() 31 | { 32 | return _thread.get_id(); 33 | } 34 | 35 | void ov_simple_thread::interrupt() 36 | { 37 | _interript = true; 38 | } 39 | 40 | bool ov_simple_thread::isInterrupted() 41 | { 42 | return _interript; 43 | } 44 | 45 | void ov_simple_thread::join() 46 | { 47 | _thread.join(); 48 | } 49 | 50 | void ov_simple_thread::run() 51 | { 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/ns_utils.cpp: -------------------------------------------------------------------------------- 1 | #include "ns_utils.hpp" 2 | #include 3 | 4 | using namespace ovlib; 5 | 6 | MatterBencher::MatterBencher() 7 | { 8 | _avg_fps = -1.0f; 9 | _ratio = 0.5f; 10 | } 11 | 12 | MatterBencher::~MatterBencher() 13 | { 14 | } 15 | 16 | void MatterBencher::Start() 17 | { 18 | #if 0 19 | if (_started) 20 | { 21 | return; 22 | } 23 | #endif 24 | _elapse = 0; 25 | _start = std::chrono::high_resolution_clock::now(); 26 | _started = true; 27 | } 28 | 29 | int64_t MatterBencher::Elapse() 30 | { 31 | auto ckpnt = std::chrono::high_resolution_clock::now(); 32 | auto elapsed = ckpnt - _start; 33 | _elapse = std::chrono::duration_cast(elapsed).count(); 34 | _start = ckpnt; 35 | 36 | float fps_sample = 1000.0 / (_elapse); 37 | _avg_fps = _avg_fps < 0.0f ? fps_sample : (_ratio * fps_sample + (1 - _ratio) * _avg_fps); 38 | 39 | return _elapse; 40 | } 41 | 42 | float MatterBencher::Get() 43 | { 44 | return _avg_fps; 45 | } 46 | 47 | std::vector Utils_String::split(const std::string &s, char delim) 48 | { 49 | std::vector result; 50 | std::stringstream ss(s); 51 | std::string item; 52 | 53 | while (getline(ss, item, delim)) 54 | { 55 | result.push_back(item); 56 | } 57 | return result; 58 | } 59 | 60 | std::string Utils_Ov::getMatType(const cv::Mat &img, bool more_info) 61 | { 62 | std::string r; 63 | int type = img.type(); 64 | uchar depth = type & CV_MAT_DEPTH_MASK; 65 | uchar chans = 1 + (type >> CV_CN_SHIFT); 66 | 67 | switch (depth) 68 | { 69 | case CV_8U: 70 | r = "8U"; 71 | break; 72 | case CV_8S: 73 | r = "8S"; 74 | break; 75 | case CV_16U: 76 | r = "16U"; 77 | break; 78 | case CV_16S: 79 | r = "16S"; 80 | break; 81 | case CV_32S: 82 | r = "32S"; 83 | break; 84 | case CV_32F: 85 | r = "32F"; 86 | break; 87 | case CV_64F: 88 | r = "64F"; 89 | break; 90 | default: 91 | r = "User"; 92 | break; 93 | } 94 | 95 | r += "C"; 96 | r += (chans + '0'); 97 | 98 | if (more_info) 99 | { 100 | std::cout << "depth: " << img.depth() << " channels: " << img.channels() << std::endl; 101 | } 102 | 103 | return r; 104 | } 105 | 106 | void Utils_Ov::showImage(cv::Mat& img, std::string& title) 107 | { 108 | std::string image_type = getMatType(img); 109 | cv::namedWindow(title + " type:" + image_type, cv::WINDOW_NORMAL); // Create a window for display. 110 | cv::imshow(title, img); 111 | cv::waitKey(0); 112 | } 113 | 114 | 115 | void Utils_Ov::mat2FrameData(cv::Mat& mat, matter::FrameData& frameData) 116 | { 117 | if (mat.empty()) 118 | { 119 | frameData.width = 0; 120 | frameData.height = 0; 121 | frameData.frame = 0; 122 | return; 123 | } 124 | cv::Mat frame = mat.clone(); 125 | frameData.width = mat.cols; 126 | frameData.height = mat.rows; 127 | frameData.frame = frame.data; 128 | int type = mat.type(); 129 | if (type == CV_8UC3) 130 | { 131 | frameData.format = matter::FRAME_FOMAT_BGR; 132 | size_t size = 3 * frameData.width * frameData.height; 133 | 134 | frameData.frame = new unsigned char[size]; 135 | memcpy(frameData.frame, mat.data, size); 136 | //frameData.frame = mat.data; 137 | } 138 | else if (type == CV_8UC1) 139 | { 140 | frameData.format = matter::FRAME_FOMAT_GRAY; 141 | size_t size = frameData.width * frameData.height; 142 | 143 | frameData.frame = new unsigned char[size]; 144 | memcpy(frameData.frame, mat.data, size); 145 | //frameData.frame = mat.data; 146 | } 147 | } 148 | 149 | 150 | void Utils_Ov::frameData2Mat(matter::FrameData& frameData, cv::Mat& outMat) 151 | { 152 | if (frameData.format == 0 || frameData.width <= 0 || frameData.height <= 0) 153 | { 154 | outMat = cv::Mat(); 155 | } 156 | 157 | if (frameData.format == matter::FRAME_FOMAT_I420) 158 | { 159 | size_t size = frameData.height * frameData.width * 3 / 2; 160 | outMat = cv::Mat(frameData.height + frameData.height / 2, frameData.width, CV_8UC1, frameData.frame); 161 | cv::cvtColor(outMat, outMat, cv::COLOR_YUV2BGR_I420); 162 | } 163 | else if (frameData.format == matter::FRAME_FOMAT_RGB) 164 | { 165 | size_t size = frameData.height * frameData.width * 3; 166 | outMat = cv::Mat(frameData.height, frameData.width, CV_8UC3, frameData.frame); 167 | cv::cvtColor(outMat, outMat, cv::COLOR_RGB2BGR); 168 | } 169 | else if (frameData.format == matter::FRAME_FOMAT_BGR) 170 | { 171 | size_t size = frameData.height * frameData.width * 3; 172 | outMat = cv::Mat(frameData.height, frameData.width, CV_8UC3, frameData.frame); 173 | } 174 | else if (frameData.format == matter::FRAME_FOMAT_GRAY) 175 | { 176 | size_t size = frameData.height * frameData.width; 177 | outMat = cv::Mat(frameData.height, frameData.width, CV_8UC1, frameData.frame); 178 | } 179 | } 180 | 181 | void Utils_Ov::sleep(long milliseconds) 182 | { 183 | std::chrono::milliseconds dura(milliseconds); 184 | std::this_thread::sleep_for(dura); 185 | } 186 | 187 | #define OVMIN(a, b) ((a) > (b) ? (b) : (a)) 188 | 189 | inline const float av_clipf(float a, float amin, float amax) 190 | { 191 | if (a < amin) 192 | return amin; 193 | else if (a > amax) 194 | return amax; 195 | else 196 | return a; 197 | } 198 | 199 | double Utils_Ov::getSceneScore(cv::Mat prev_frame, cv::Mat frame, double& prev_mafd) 200 | { 201 | double ret = 0.0f; 202 | int w0 = prev_frame.cols; 203 | int h0 = prev_frame.rows; 204 | int w1 = frame.cols; 205 | int h1 = frame.rows; 206 | if (w0 == w1 && h0 == h1) 207 | { 208 | float mafd, diff; 209 | uint64 sad = 0; 210 | int nb_sad = 0; 211 | cv::Mat gray0(h0, w0, CV_8UC1); 212 | cv::Mat gray1(h0, w0, CV_8UC1); 213 | cv::cvtColor(prev_frame, gray0, cv::COLOR_BGR2GRAY); 214 | cv::cvtColor(frame, gray1, cv::COLOR_BGR2GRAY); 215 | for (int i = 0; i < h0; i++) 216 | { 217 | for (int j = 0; j < w0; j++) 218 | { 219 | sad += abs(gray1.at(i, j) - gray0.at(i, j)); 220 | nb_sad++; 221 | } 222 | } 223 | mafd = nb_sad ? (float)sad / nb_sad : 0; 224 | diff = fabs(mafd - prev_mafd); 225 | ret = av_clipf(OVMIN(mafd, diff) / 100., 0, 1); 226 | prev_mafd = mafd; 227 | } 228 | 229 | return ret; 230 | } 231 | 232 | double Utils_Ov::getSceneScore_V2(cv::Mat prev_frame, cv::Mat frame, double& prev_mafd) 233 | { 234 | double ret = 0.0f; 235 | int w0 = prev_frame.cols; 236 | int h0 = prev_frame.rows; 237 | int w1 = frame.cols; 238 | int h1 = frame.rows; 239 | if (w0 == w1 && h0 == h1) 240 | { 241 | float mafd, diff; 242 | uint64 sad = 0; 243 | int nb_sad = 0; 244 | cv::Mat gray0(h0, w0, CV_8UC1); 245 | cv::Mat gray1(h0, w0, CV_8UC1); 246 | cv::cvtColor(prev_frame, gray0, cv::COLOR_BGR2GRAY); 247 | cv::threshold(gray0, gray0,100,255, cv::THRESH_BINARY); 248 | 249 | cv::cvtColor(frame, gray1, cv::COLOR_BGR2GRAY); 250 | cv::threshold(gray1, gray1,100,255, cv::THRESH_BINARY); 251 | for (int i = 0; i < h0; i++) 252 | { 253 | for (int j = 0; j < w0; j++) 254 | { 255 | sad += abs(gray1.at(i, j) - gray0.at(i, j)); 256 | nb_sad++; 257 | } 258 | } 259 | mafd = nb_sad ? (float)sad / nb_sad : 0; 260 | diff = fabs(mafd - prev_mafd); 261 | ret = av_clipf(OVMIN(mafd, diff) / 100., 0, 1); 262 | prev_mafd = mafd; 263 | } 264 | 265 | return ret; 266 | } 267 | 268 | std::string Utils_Ov::getRealPath(std::string relPath) 269 | { 270 | char dir[1024] = {0}; 271 | #if (_MSC_VER) 272 | _fullpath(dir, relPath.c_str(), 1024); 273 | #else 274 | realpath(relPath.c_str(), dir); 275 | #endif 276 | std::string result_str = dir; 277 | return result_str; 278 | } -------------------------------------------------------------------------------- /src/ovmatter.cpp: -------------------------------------------------------------------------------- 1 | #include "ovmatter.h" 2 | #include "ovmatter_base_impl.hpp" 3 | #include "ovmatter_bgv2_impl.hpp" 4 | 5 | using namespace ovlib::matter; 6 | 7 | MatterChannel* MatterChannel::create(const MatterParams& params) 8 | { 9 | try 10 | { 11 | MatterBaseImpl* pChannel = CMatterFactory::CreateMatter(params.method); 12 | if (pChannel) 13 | { 14 | pChannel->init(params); 15 | } 16 | 17 | return pChannel; 18 | } 19 | catch (const std::exception& e) 20 | { 21 | std::string msgError = e.what(); 22 | std::cerr << "MatterChannel::create():" << msgError << '\n'; 23 | return NULL; 24 | } 25 | } 26 | 27 | int MatterChannel::getDefMatterParams(MatterParams& params) 28 | { 29 | params.method = METHOD_BACKGROUND_MATTING_V2; 30 | params.effect = EFFECT_NONE; 31 | params.device = "CPU"; 32 | params.scale = 0.25; 33 | params.max_batch_size = 1; 34 | #if (LINUX) 35 | params.path_to_model = "./share/pytorch_mobilenetv2.xml"; 36 | params.path_to_bin = "./share/pytorch_mobilenetv2.bin"; 37 | #else 38 | params.path_to_model = ".\\share\\pytorch_mobilenetv2.xml"; 39 | params.path_to_bin = ".\\share\\pytorch_mobilenetv2.bin"; 40 | #endif // _WIN 41 | params.cpu_threads_num = 0; 42 | params.cpu_bind_thread = true; 43 | params.cpu_throughput_streams = 1; 44 | 45 | params.input_shape.width = 256; 46 | params.input_shape.height = 144; 47 | 48 | params.is_async = false; 49 | params.interval = 1; 50 | params.threshold_motion = -0.1f; 51 | 52 | return 0; 53 | } 54 | 55 | void MatterChannel::destroyed(MatterChannel* pChan) 56 | { 57 | std::cout << "MatterChannel::destroy " << pChan << std::endl; 58 | if (pChan) 59 | { 60 | MatterChannel* tmp = pChan; 61 | pChan = nullptr; 62 | delete tmp; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ovmatter_base_impl.cpp: -------------------------------------------------------------------------------- 1 | #include "ovmatter_base_impl.hpp" 2 | 3 | using namespace ovlib::matter; 4 | 5 | std::map CMatterFactory::dynCreateMap; 6 | std::map CMatterFactory::dynCreateMapType; 7 | 8 | double MatterBaseImpl::getAttributeValue(std::string attName) 9 | { 10 | if(attName == "InferCount") 11 | { 12 | return m_nInferCount; 13 | } 14 | else if(attName == "Interval") 15 | { 16 | return m_nInterval; 17 | } 18 | else if(attName == "MotionThreshold") 19 | { 20 | return m_fMotionThreshold; 21 | } 22 | else if(attName == "ForceInferLimit") 23 | { 24 | return m_nForceInferLimit; 25 | } 26 | else 27 | { 28 | return -1.0; 29 | } 30 | } 31 | 32 | MatterChannel* MatterBaseImpl::setAttributeValue(std::string attrName, double dblValue) 33 | { 34 | if(attrName == "InferCount") 35 | { 36 | m_nInferCount = (int)dblValue; 37 | } 38 | else if(attrName == "Interval") 39 | { 40 | m_nInterval = (int)dblValue; 41 | } 42 | else if(attrName == "MotionThreshold") 43 | { 44 | m_fMotionThreshold = dblValue; 45 | } 46 | else if(attrName == "ForceInferLimit") 47 | { 48 | m_nForceInferLimit = (int)dblValue; 49 | } 50 | 51 | return this; 52 | } -------------------------------------------------------------------------------- /src/ovmatter_bgv2_impl.cpp: -------------------------------------------------------------------------------- 1 | #include "ovmatter_bgv2_impl.hpp" 2 | #include "ns_utils.hpp" 3 | #include 4 | 5 | #if (_MSC_VER) 6 | #include 7 | #else 8 | #include 9 | #endif 10 | 11 | using namespace ovlib::matter; 12 | using namespace InferenceEngine; 13 | 14 | 15 | static void compose_op(cv::Mat &src, cv::Mat &replace, cv::Mat &alpha, cv::Mat &com, cv::Size &out_shape) 16 | { 17 | 18 | if (out_shape.height == 0 || out_shape.width == 0) 19 | { 20 | out_shape.width = 1280; 21 | out_shape.height = 720; 22 | } 23 | cv::Size l_shape = out_shape; 24 | com = cv::Mat::zeros(l_shape, CV_8UC3); 25 | cv::Mat matSrc, matReplace; 26 | matSrc = src.clone(); 27 | matReplace = replace.clone(); 28 | if (matSrc.rows != l_shape.height || matSrc.cols != l_shape.width) 29 | { 30 | cv::resize(matSrc, matSrc, l_shape); 31 | } 32 | if (matReplace.rows != l_shape.height || matReplace.cols != l_shape.width) 33 | { 34 | cv::resize(matReplace, matReplace, l_shape); 35 | } 36 | if (alpha.rows != l_shape.height || alpha.cols != l_shape.width) 37 | { 38 | cv::resize(alpha, alpha, l_shape, 0, 0, cv::INTER_CUBIC); 39 | } 40 | cv::Mat matPha; 41 | cv::cvtColor(alpha, matPha, cv::COLOR_GRAY2BGR); 42 | com = matSrc.mul(matPha / 255.0) + matReplace.mul((255 - matPha)/255.0); 43 | } 44 | 45 | 46 | REGISTER_MATTER_CLASS(METHOD_BACKGROUND_MATTING_V2, MatterBackgroundV2Impl) 47 | 48 | 49 | 50 | MatterBackgroundV2Impl::MatterBackgroundV2Impl() : _bInit(false), _elapse(0), _shape_output(cv::Size(1280, 720)) 51 | { 52 | m_nInterval = 1; 53 | } 54 | 55 | MatterBackgroundV2Impl::~MatterBackgroundV2Impl() 56 | { 57 | interrupt(); 58 | _bInit = false; 59 | while (_queue_input.size() > 0) 60 | { 61 | _queue_input.clear(); 62 | } 63 | while (_queue_output.size() > 0) 64 | { 65 | _queue_output.clear(); 66 | } 67 | } 68 | 69 | bool MatterBackgroundV2Impl::init(const MatterParams ¶m) 70 | { 71 | if (_bInit) 72 | { 73 | return true; 74 | } 75 | 76 | //1 Loading Inference engine 77 | std::cout << "Loading Inference Engine" << std::endl; 78 | std::string device = param.device == "" ? "CPU" : param.device; 79 | Core ie; 80 | std::set loadedDevices; 81 | std::cout << "Device info: " << device << std::endl; 82 | std::cout << ie.GetVersions(device) << std::endl; 83 | if (device.find("CPU") != std::string::npos) 84 | { 85 | ie.SetConfig({{PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES}}, "CPU"); 86 | } 87 | else if (device.find("GPU") != std::string::npos) 88 | { 89 | ie.SetConfig({{PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES}}, "GPU"); 90 | } 91 | loadedDevices.insert(device); 92 | 93 | //2 init CNNConfig 94 | if (!param.path_to_model.empty()) 95 | { 96 | cv::Size shape(param.input_shape.width, param.input_shape.height); 97 | 98 | std::string sModelXml, sModelBin, sCwdPath; 99 | #if (_MSC_VER) 100 | sCwdPath = _getcwd(NULL, 0); 101 | #else 102 | sCwdPath = getcwd(NULL, 0); 103 | #endif 104 | 105 | std::cout << "current working path:" << sCwdPath << std::endl; 106 | 107 | sModelXml = Utils_Ov::getRealPath(param.path_to_model); 108 | sModelBin = Utils_Ov::getRealPath(param.path_to_bin); 109 | std::cout << "xml path:" << sModelXml << ", bin path:" << sModelBin << std::endl; 110 | 111 | CNNConfig config(sModelXml, sModelBin, shape); 112 | config.deviceName = device; 113 | config.is_async = param.is_async; 114 | config.scale = param.scale; 115 | config.max_batch_size = param.max_batch_size; 116 | config.cpu_bind_thread = param.cpu_bind_thread; 117 | config.cpu_threads_num = param.cpu_threads_num; 118 | config.cpu_throughput_streams = param.cpu_throughput_streams; 119 | config.path_to_model = param.path_to_model; 120 | config.path_to_bin = param.path_to_bin; 121 | config.interval = param.interval; 122 | config.motion_threshold = param.threshold_motion; 123 | m_nInterval = param.interval; 124 | m_fMotionThreshold = param.threshold_motion / 1000.0f; 125 | _pCnn.reset(new CNN_Background_V2(config)); 126 | 127 | start(); //�����߳� 128 | 129 | _bInit = true; 130 | } 131 | else 132 | { 133 | _bInit = false; 134 | } 135 | return _bInit; 136 | } 137 | 138 | int MatterBackgroundV2Impl::process(FrameData &frame, FrameData &bgr, FrameData &bgrReplace, const ovlib::matter::Shape &shape, std::map *pResults) 139 | { 140 | if (!_pCnn) 141 | { 142 | return -1; 143 | } 144 | if (_pCnn->getAsync()) 145 | { 146 | _pCnn->setAsync(false); 147 | } 148 | 149 | return doWork_sync(frame, bgr, bgrReplace, shape, pResults); 150 | } 151 | 152 | int MatterBackgroundV2Impl::doWork_sync_V2(cv::Mat &frame, cv::Mat &bgr, cv::Mat &bgrReplace, cv::Size &out_shape, cv::Mat &matCom, cv::Mat &matAlpha) 153 | { 154 | if (frame.empty()) 155 | { 156 | matCom = bgrReplace; 157 | return -1; 158 | } 159 | 160 | { 161 | std::lock_guard lock(mutex_); 162 | _bencher.Start(); 163 | _pCnn->enqueue("src", frame); 164 | _pCnn->enqueue("bgr", bgr); 165 | _pCnn->submitRequest(); 166 | _pCnn->wait(); 167 | _matResult = _pCnn->fetchResults(); 168 | m_nInferCount++; 169 | if (_matResult.size() <= 0) 170 | { 171 | return -1; 172 | } 173 | 174 | matAlpha = _matResult[0].pha; 175 | compose(frame, bgrReplace, matAlpha, matCom, out_shape); 176 | _elapse = _bencher.Elapse(); 177 | } 178 | 179 | return 0; 180 | } 181 | 182 | /// 183 | /// Infer Synchronously 184 | /// 185 | /// 186 | /// 187 | /// 188 | /// 189 | /// 190 | /// 191 | int MatterBackgroundV2Impl::doWork_sync(FrameData &frame, FrameData &bgr, FrameData &bgrReplace, const ovlib::matter::Shape &shape, std::map *pResults) 192 | { 193 | int ret = -1; 194 | if (frame.frame == 0 || bgr.frame == 0 || bgrReplace.frame == 0 || frame.width == 0 || frame.height == 0 || bgr.width == 0 || bgr.height == 0 || bgrReplace.width == 0 || bgrReplace.height == 0) 195 | { 196 | return ret; 197 | } 198 | 199 | if (!pResults) 200 | { 201 | return -1; 202 | } 203 | static int l_frame_count_sync = 0; 204 | static int l_no_infer_count = 0; 205 | l_frame_count_sync++; 206 | 207 | cv::Mat matFrame; 208 | ovlib::Utils_Ov::frameData2Mat(frame, matFrame); 209 | cv::Mat matBgr; 210 | ovlib::Utils_Ov::frameData2Mat(bgr, matBgr); 211 | cv::Mat matBgrReplace; 212 | ovlib::Utils_Ov::frameData2Mat(bgrReplace, matBgrReplace); 213 | cv::Size out_shape(shape.width, shape.height); 214 | cv::Mat matCom; 215 | cv::Mat matPha; 216 | 217 | //1 skip Frame intervally 218 | if (m_nInterval <= 0) 219 | { 220 | m_nInterval = 1; 221 | } 222 | l_frame_count_sync = l_frame_count_sync % m_nInterval; 223 | bool bExist = (_preResult.find("pha") != _preResult.end()); 224 | if (l_frame_count_sync != 0 && bExist && l_no_infer_count < m_nForceInferLimit) 225 | { 226 | l_no_infer_count++; 227 | matPha = _preResult["pha"]; 228 | matCom = _preResult["com"]; 229 | //compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 230 | FrameData frameCom; 231 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 232 | FrameData frameAlpha; 233 | ovlib::Utils_Ov::mat2FrameData(matPha, frameAlpha); 234 | (*pResults)["com"] = frameCom; 235 | (*pResults)["pha"] = frameAlpha; 236 | _prevFrame = matFrame.clone(); 237 | return 0; 238 | } 239 | 240 | //2.check if frame changes 241 | if (m_fMotionThreshold > 0.0f) 242 | { 243 | 244 | double dblDiff = Utils_Ov::getSceneScore(_prevFrame, matFrame, m_preDiff); 245 | //double dblDiff = Utils_Ov::getSceneScore_V2(_prevFrame, matFrame, m_preDiff); 246 | 247 | bool bExist = (_preResult.find("pha") != _preResult.end()); 248 | if (dblDiff < m_fMotionThreshold && bExist && l_no_infer_count < m_nForceInferLimit) 249 | { 250 | //ovlib::TimerCounter estimate("compose_op()..."); 251 | l_no_infer_count++; 252 | matPha = _preResult["pha"]; 253 | matCom = _preResult["com"]; 254 | //compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 255 | //compose_op(matFrame, matBgrReplace, matPha, matCom, out_shape); 256 | FrameData frameCom; 257 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 258 | FrameData frameAlpha; 259 | ovlib::Utils_Ov::mat2FrameData(matPha, frameAlpha); 260 | (*pResults)["com"] = frameCom; 261 | (*pResults)["pha"] = frameAlpha; 262 | _prevFrame = matFrame.clone(); 263 | return 0; 264 | } 265 | } 266 | _prevFrame = matFrame.clone(); 267 | 268 | //3. Infer result 269 | { 270 | //ovlib::TimerCounter estimate("Infer..."); 271 | l_no_infer_count = 0; 272 | _pCnn->enqueue("src", matFrame); 273 | _pCnn->enqueue("bgr", matBgr); 274 | _pCnn->submitRequest(); 275 | _pCnn->wait(); 276 | _matResult = _pCnn->fetchResults(); 277 | m_nInferCount++; 278 | if (_matResult.size() <= 0) 279 | { 280 | return -1; 281 | } 282 | 283 | matPha = _matResult[0].pha; 284 | compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 285 | //compose_op(matFrame, matBgrReplace, matPha, matCom, out_shape); 286 | 287 | FrameData frameCom; 288 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 289 | FrameData frameAlpha; 290 | ovlib::Utils_Ov::mat2FrameData(_matResult[0].pha, frameAlpha); 291 | (*pResults)["com"] = frameCom; 292 | (*pResults)["pha"] = frameAlpha; 293 | 294 | _preResult["com"] = matCom; 295 | _preResult["pha"] = _matResult[0].pha; 296 | } 297 | 298 | return 0; 299 | } 300 | 301 | void MatterBackgroundV2Impl::compose(cv::Mat &src, cv::Mat &replace, cv::Mat &alpha, cv::Mat &com, cv::Size &out_shape) 302 | { 303 | 304 | if (out_shape.height == 0 || out_shape.width == 0) 305 | { 306 | out_shape.width = 1280; 307 | out_shape.height = 720; 308 | } 309 | cv::Size l_shape = out_shape; 310 | com = cv::Mat::zeros(l_shape, CV_8UC3); 311 | cv::Mat matSrc, matReplace; 312 | matSrc = src.clone(); 313 | matReplace = replace.clone(); 314 | if (matSrc.rows != l_shape.height || matSrc.cols != l_shape.width) 315 | { 316 | cv::resize(matSrc, matSrc, l_shape); 317 | } 318 | if (matReplace.rows != l_shape.height || matReplace.cols != l_shape.width) 319 | { 320 | cv::resize(matReplace, matReplace, l_shape); 321 | } 322 | if (alpha.rows != l_shape.height || alpha.cols != l_shape.width) 323 | { 324 | cv::resize(alpha, alpha, l_shape, 0, 0, cv::INTER_CUBIC); 325 | } 326 | unsigned char *dataMatPha = alpha.data; 327 | unsigned char *dataMatCom = com.data; 328 | unsigned char *dataMatFrame1 = matSrc.data; 329 | unsigned char *dataMatBgr2 = matReplace.data; 330 | int num_channels = 3; 331 | int image_size = alpha.rows * alpha.cols; 332 | for (size_t pid = 0; pid < image_size; pid++) 333 | { 334 | int nAlpha = *(dataMatPha + pid); 335 | int rowC = pid * num_channels; 336 | if (nAlpha == 0) 337 | { 338 | *(dataMatCom + rowC + 2) = *(dataMatBgr2 + rowC + 2); 339 | *(dataMatCom + rowC + 1) = *(dataMatBgr2 + rowC + 1); 340 | *(dataMatCom + rowC) = *(dataMatBgr2 + rowC); 341 | } 342 | else if (nAlpha == 255) 343 | { 344 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2); 345 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1); 346 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC); 347 | } 348 | else 349 | { 350 | float falpha = nAlpha / 255.0; 351 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2) * falpha + *(dataMatBgr2 + rowC + 2) * (1 - falpha); 352 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1) * falpha + *(dataMatBgr2 + rowC + 1) * (1 - falpha); 353 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC) * falpha + *(dataMatBgr2 + rowC) * (1 - falpha); 354 | } 355 | } 356 | } 357 | 358 | 359 | /****************************************Asynchronous Process*******************************************************/ 360 | void MatterBackgroundV2Impl::setStrategy_async(bool bAuto, int interval, const Shape &input_shape, const Shape &out_shape) 361 | { 362 | } 363 | 364 | void MatterBackgroundV2Impl::setBackground_async(FrameData &bgrReplace, MATTER_EFFECT effect, const FrameData &bgr) 365 | { 366 | cv::Mat matReplace, matBgr; 367 | ovlib::Utils_Ov::frameData2Mat(bgrReplace, matReplace); 368 | FrameData bgrFd = bgr; 369 | ovlib::Utils_Ov::frameData2Mat(bgrFd, matBgr); 370 | 371 | std::lock_guard lock(mutex_); 372 | if (!matReplace.empty()) 373 | { 374 | _frame_replace = matReplace.clone(); 375 | } 376 | if (!matBgr.empty()) 377 | { 378 | _frame_bgr = matBgr.clone(); 379 | } 380 | 381 | if (effect == ovlib::matter::EFFECT_BLUR) 382 | { 383 | //cv::bilateralFilter(_frame_replace, _frame_replace, 30, 500.0, 10.0); 384 | cv::GaussianBlur(_frame_replace, _frame_replace, cv::Size(3, 3), 11.0, 11.0); 385 | } 386 | } 387 | 388 | int MatterBackgroundV2Impl::process_async(FrameData &frame, FrameData &frameCom, FrameData &frameAlpha, const ovlib::matter::Shape &out_shape) 389 | { 390 | static int l_frame_count = 0; 391 | //1.put input frame into input_queue 392 | if (frame.frame == 0 || frame.height == 0 || frame.width == 0) 393 | { 394 | //do nothing 395 | } 396 | else 397 | { 398 | l_frame_count++; 399 | if (m_nInterval <= 0) 400 | { 401 | m_nInterval = 1; 402 | } 403 | l_frame_count = l_frame_count % m_nInterval; 404 | if (l_frame_count != 0) 405 | { 406 | cv::Mat matFrame; 407 | Utils_Ov::frameData2Mat(frame, matFrame); 408 | _queue_input.push(matFrame); 409 | } 410 | } 411 | 412 | //1.2 input shape 413 | if (out_shape.height > 0 && out_shape.width > 0) 414 | { 415 | _shape_output.width = out_shape.width; 416 | _shape_output.height = out_shape.height; 417 | } 418 | 419 | // 2.return raw frame if out put shape is empty or replace bgr is empty 420 | if (_shape_output.empty() || _frame_replace.empty()) 421 | { 422 | frameCom = frame; 423 | return -1; 424 | } 425 | 426 | if (_frame_replace.rows != _shape_output.height || _frame_replace.cols != _shape_output.width) 427 | { 428 | cv::resize(_frame_replace, _frame_replace, _shape_output); 429 | } 430 | 431 | // 3.if output queue is empty, return replace bgr 432 | if (_queue_output.size() <= 0) 433 | { 434 | Utils_Ov::mat2FrameData(_frame_replace, frameCom); 435 | frameAlpha.frame = 0; 436 | return 1; 437 | } 438 | 439 | /* 440 | else if (_queue_output.size() == 1) 441 | { 442 | MattingObject f; 443 | _queue_output.front(f); 444 | Utils_Ov::mat2FrameData(f.com, frameCom); 445 | Utils_Ov::mat2FrameData(f.pha, frameAlpha); 446 | return 1; 447 | }*/ 448 | 449 | while (_queue_output.size() > 1) 450 | { 451 | MattingObject f; 452 | _queue_output.pop(f); 453 | } 454 | // 4. if output queue is not empty, return first item in the output queue 455 | MattingObject result; 456 | _queue_output.front(result); 457 | Utils_Ov::mat2FrameData(result.com, frameCom); 458 | Utils_Ov::mat2FrameData(result.pha, frameAlpha); 459 | 460 | return 0; 461 | } 462 | 463 | void MatterBackgroundV2Impl::run() 464 | { 465 | long lSleep = 5; 466 | while (!isInterrupted()) 467 | { 468 | Utils_Ov::sleep(lSleep); 469 | if (_queue_input.size() <= 0) 470 | { 471 | continue; 472 | } 473 | 474 | cv::Mat frame, matAlpha, matCom; 475 | _queue_input.pop(frame); 476 | { 477 | std::lock_guard lock(mutex_); 478 | _bencher.Start(); 479 | if (!_pCnn->getAsync()) 480 | { 481 | _pCnn->setAsync(true); 482 | } 483 | 484 | _pCnn->enqueue("src", frame); 485 | if (!_pCnn->isBgrEnqueued()) 486 | { 487 | _pCnn->enqueue("bgr", _frame_bgr); 488 | } 489 | _pCnn->submitRequest(); 490 | _pCnn->wait(); 491 | _matResult = _pCnn->fetchResults(); 492 | if (_matResult.size() <= 0) 493 | { 494 | continue; 495 | } 496 | 497 | matAlpha = _matResult[0].pha; 498 | compose(frame, _frame_replace, matAlpha, matCom, _shape_output); 499 | if (matAlpha.empty() || matCom.empty()) 500 | { 501 | continue; 502 | } 503 | MattingObject obj; 504 | //obj.com = matCom.clone(); 505 | //obj.pha = matAlpha.clone(); 506 | obj.com = matCom; 507 | obj.pha = matAlpha; 508 | _queue_output.push(obj); 509 | _elapse = _bencher.Elapse(); 510 | } 511 | } 512 | } -------------------------------------------------------------------------------- /src/ovmatter_modnet_impl.cpp: -------------------------------------------------------------------------------- 1 | #include "ovmatter_modnet_impl.hpp" 2 | #include "ns_utils.hpp" 3 | 4 | #if (_MSC_VER) 5 | #include 6 | #else 7 | #include 8 | #endif 9 | 10 | using namespace ovlib::matter; 11 | using namespace InferenceEngine; 12 | 13 | REGISTER_MATTER_CLASS(METHOD_MODNET, MatterModnetImpl) 14 | 15 | MatterModnetImpl::MatterModnetImpl() : _bInit(false), _elapse(0), _shape_output(cv::Size(1280,720)) 16 | { 17 | m_nInterval = 1; 18 | } 19 | 20 | MatterModnetImpl::~MatterModnetImpl() 21 | { 22 | interrupt(); 23 | _bInit = false; 24 | } 25 | 26 | bool MatterModnetImpl::init(const MatterParams& param) 27 | { 28 | if (_bInit) 29 | { 30 | return true; 31 | } 32 | 33 | //1 Loading Inference engine 34 | std::cout << "Loading Inference Engine" << std::endl; 35 | std::string device = param.device == "" ? "CPU" : param.device; 36 | Core ie; 37 | std::set loadedDevices; 38 | std::cout << "Device info: " << device << std::endl; 39 | std::cout << ie.GetVersions(device) << std::endl; 40 | if (device.find("CPU") != std::string::npos) 41 | { 42 | ie.SetConfig({ {PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES} }, "CPU"); 43 | } 44 | else if (device.find("GPU") != std::string::npos) 45 | { 46 | ie.SetConfig({ {PluginConfigParams::KEY_DYN_BATCH_ENABLED, PluginConfigParams::YES} }, "GPU"); 47 | } 48 | loadedDevices.insert(device); 49 | 50 | //2 init CNNConfig 51 | if (!param.path_to_model.empty()) 52 | { 53 | cv::Size shape(param.input_shape.width, param.input_shape.height); 54 | 55 | std::string sModelXml, sModelBin, sCwdPath; 56 | #if (_MSC_VER) 57 | sCwdPath = _getcwd(NULL, 0); 58 | #else 59 | sCwdPath = getcwd(NULL, 0); 60 | #endif 61 | std::cout << "current working path:" << sCwdPath << std::endl; 62 | 63 | sModelXml = Utils_Ov::getRealPath(param.path_to_model); 64 | sModelBin = Utils_Ov::getRealPath(param.path_to_bin); 65 | std::cout << "xml path:" << sModelXml << ", bin path:" << sModelBin << std::endl; 66 | 67 | CNNConfig config(sModelXml, sModelBin, shape); 68 | config.deviceName = device; 69 | config.is_async = param.is_async; 70 | config.scale = param.scale; 71 | config.max_batch_size = param.max_batch_size; 72 | config.cpu_bind_thread = param.cpu_bind_thread; 73 | config.cpu_threads_num = param.cpu_threads_num; 74 | config.cpu_throughput_streams = param.cpu_throughput_streams; 75 | config.path_to_model = param.path_to_model; 76 | config.path_to_bin = param.path_to_bin; 77 | config.interval = param.interval; 78 | config.motion_threshold = param.threshold_motion; 79 | m_nInterval = param.interval; 80 | m_fMotionThreshold = param.threshold_motion / 1000.0f; 81 | _pCnn.reset(new CNN_Modnet(config)); 82 | 83 | start(); //�����߳� 84 | 85 | _bInit = true; 86 | } 87 | else 88 | { 89 | _bInit = false; 90 | } 91 | return _bInit; 92 | } 93 | 94 | int MatterModnetImpl::process(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults) 95 | { 96 | if (!_pCnn) 97 | { 98 | return -1; 99 | } 100 | if (!_pCnn->getAsync()) 101 | { 102 | _pCnn->setAsync(true); 103 | } 104 | 105 | return doWork_sync(frame, bgr, bgrReplace, shape, pResults); 106 | } 107 | 108 | /// 109 | /// Infer Synchronously 110 | /// 111 | /// 112 | /// 113 | /// 114 | /// 115 | /// 116 | /// 117 | int MatterModnetImpl::doWork_sync(FrameData& frame, FrameData& bgr, FrameData& bgrReplace, const ovlib::matter::Shape& shape, std::map* pResults) 118 | { 119 | int ret = -1; 120 | if (frame.frame == 0 || bgrReplace.frame == 0 121 | || frame.width == 0 || frame.height == 0 122 | || bgrReplace.width == 0 || bgrReplace.height == 0) 123 | { 124 | return ret; 125 | } 126 | 127 | if (!pResults) 128 | { 129 | return -1; 130 | } 131 | 132 | static int l_frame_count_sync = 0; 133 | static int l_no_infer_count = 0; 134 | l_frame_count_sync++; 135 | 136 | 137 | cv::Mat matFrame; 138 | ovlib::Utils_Ov::frameData2Mat(frame, matFrame); 139 | cv::Mat matBgr; 140 | ovlib::Utils_Ov::frameData2Mat(bgr, matBgr); 141 | cv::Mat matBgrReplace; 142 | ovlib::Utils_Ov::frameData2Mat(bgrReplace, matBgrReplace); 143 | cv::Size out_shape(shape.width, shape.height); 144 | cv::Mat matCom; 145 | cv::Mat matPha; 146 | 147 | //1 skip Frame intervally 148 | if (m_nInterval <= 0) 149 | { 150 | m_nInterval = 1; 151 | } 152 | l_frame_count_sync = l_frame_count_sync % m_nInterval; 153 | bool bExist = (_preResult.find("pha") != _preResult.end()); 154 | if (l_frame_count_sync != 0 && bExist && l_no_infer_count < m_nForceInferLimit) 155 | { 156 | l_no_infer_count++; 157 | matPha = _preResult["pha"]; 158 | matCom = _preResult["com"]; 159 | //compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 160 | FrameData frameCom; 161 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 162 | FrameData frameAlpha; 163 | ovlib::Utils_Ov::mat2FrameData(matPha, frameAlpha); 164 | (*pResults)["com"] = frameCom; 165 | (*pResults)["pha"] = frameAlpha; 166 | return 0; 167 | } 168 | 169 | //2.check if frame changes 170 | if (m_fMotionThreshold > 0.0f) 171 | { 172 | double dblDiff = Utils_Ov::getSceneScore(_prevFrame, matFrame, m_preDiff); 173 | 174 | bool bExist = (_preResult.find("pha") != _preResult.end()); 175 | if (dblDiff < m_fMotionThreshold && bExist && l_no_infer_count < m_nForceInferLimit) 176 | { 177 | l_no_infer_count++; 178 | matPha = _preResult["pha"]; 179 | compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 180 | FrameData frameCom; 181 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 182 | FrameData frameAlpha; 183 | ovlib::Utils_Ov::mat2FrameData(matPha, frameAlpha); 184 | (*pResults)["com"] = frameCom; 185 | (*pResults)["pha"] = frameAlpha; 186 | return 0; 187 | } 188 | } 189 | _prevFrame = matFrame.clone(); 190 | 191 | //3. Infer result 192 | { 193 | l_no_infer_count = 0; 194 | _pCnn->enqueue("input.1", matFrame); 195 | _pCnn->submitRequest(); 196 | _pCnn->wait(); 197 | _matResult = _pCnn->fetchResults(); 198 | m_nInferCount++; 199 | if (_matResult.size() <= 0) 200 | { 201 | return -1; 202 | } 203 | 204 | matPha = _matResult[0].pha; 205 | 206 | compose(matFrame, matBgrReplace, matPha, matCom, out_shape); 207 | 208 | FrameData frameCom; 209 | ovlib::Utils_Ov::mat2FrameData(matCom, frameCom); 210 | FrameData frameAlpha; 211 | ovlib::Utils_Ov::mat2FrameData(_matResult[0].pha, frameAlpha); 212 | (*pResults)["com"] = frameCom; 213 | (*pResults)["pha"] = frameAlpha; 214 | 215 | _preResult["com"] = matCom; 216 | _preResult["pha"] = _matResult[0].pha; 217 | } 218 | 219 | return 0; 220 | } 221 | 222 | void MatterModnetImpl::compose(cv::Mat& src, cv::Mat& replace, cv::Mat& alpha, cv::Mat& com, cv::Size& out_shape) 223 | { 224 | 225 | if (out_shape.height == 0 || out_shape.width == 0) 226 | { 227 | out_shape.width = 1280; 228 | out_shape.height = 720; 229 | } 230 | cv::Size l_shape = out_shape; 231 | com = cv::Mat::zeros(l_shape, CV_8UC3); 232 | cv::Mat matSrc, matReplace; 233 | matSrc = src.clone(); 234 | matReplace = replace.clone(); 235 | if (matSrc.rows != l_shape.height || matSrc.cols != l_shape.width) 236 | { 237 | cv::resize(matSrc, matSrc, l_shape); 238 | } 239 | if (matReplace.rows != l_shape.height || matReplace.cols != l_shape.width) 240 | { 241 | cv::resize(matReplace, matReplace, l_shape); 242 | } 243 | if (alpha.rows != l_shape.height || alpha.cols != l_shape.width) 244 | { 245 | cv::resize(alpha, alpha, l_shape, 0, 0, cv::INTER_CUBIC); 246 | } 247 | unsigned char* dataMatPha = alpha.data; 248 | unsigned char* dataMatCom = com.data; 249 | unsigned char* dataMatFrame1 = matSrc.data; 250 | unsigned char* dataMatBgr2 = matReplace.data; 251 | int num_channels = 3; 252 | int image_size = alpha.rows * alpha.cols; 253 | for (size_t pid = 0; pid < image_size; pid++) 254 | { 255 | int nAlpha = *(dataMatPha + pid); 256 | int rowC = pid * num_channels; 257 | if (nAlpha == 0) 258 | { 259 | *(dataMatCom + rowC + 2) = *(dataMatBgr2 + rowC + 2); 260 | *(dataMatCom + rowC + 1) = *(dataMatBgr2 + rowC + 1); 261 | *(dataMatCom + rowC) = *(dataMatBgr2 + rowC); 262 | } 263 | else if (nAlpha == 255) 264 | { 265 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2); 266 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1); 267 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC); 268 | } 269 | else 270 | { 271 | float falpha = nAlpha / 255.0; 272 | *(dataMatCom + rowC + 2) = *(dataMatFrame1 + rowC + 2) * falpha + *(dataMatBgr2 + rowC + 2) * (1 - falpha); 273 | *(dataMatCom + rowC + 1) = *(dataMatFrame1 + rowC + 1) * falpha + *(dataMatBgr2 + rowC + 1) * (1 - falpha); 274 | *(dataMatCom + rowC) = *(dataMatFrame1 + rowC) * falpha + *(dataMatBgr2 + rowC) * (1 - falpha); 275 | } 276 | } 277 | } 278 | 279 | /****************************************Asynchronous Process*******************************************************/ 280 | void MatterModnetImpl::setStrategy_async(bool bAuto, int interval, const Shape& input_shape, const Shape& out_shape) 281 | { 282 | 283 | } 284 | 285 | void MatterModnetImpl::setBackground_async(FrameData& bgrReplace, MATTER_EFFECT effect, const FrameData& bgr) 286 | { 287 | cv::Mat matReplace, matBgr; 288 | ovlib::Utils_Ov::frameData2Mat(bgrReplace, matReplace); 289 | FrameData bgrFd = bgr; 290 | ovlib::Utils_Ov::frameData2Mat(bgrFd, matBgr); 291 | 292 | std::lock_guard lock(mutex_); 293 | if (!matReplace.empty()) 294 | { 295 | _frame_replace = matReplace.clone(); 296 | } 297 | if (!matBgr.empty()) 298 | { 299 | _frame_bgr = matBgr.clone(); 300 | } 301 | 302 | if (effect == ovlib::matter::EFFECT_BLUR) 303 | { 304 | //cv::bilateralFilter(_frame_replace, _frame_replace, 30, 500.0, 10.0); 305 | cv::GaussianBlur(_frame_replace, _frame_replace, cv::Size(3, 3), 11.0, 11.0); 306 | } 307 | } 308 | 309 | int MatterModnetImpl::process_async(FrameData& frame, FrameData& frameCom, FrameData& frameAlpha, const ovlib::matter::Shape& out_shape) 310 | { 311 | static int l_frame_count = 0; 312 | //1.put input frame into input_queue 313 | if (frame.frame == 0 || frame.height == 0 || frame.width == 0) 314 | { 315 | //do nothing 316 | } 317 | else 318 | { 319 | l_frame_count++; 320 | if (m_nInterval <= 0) 321 | { 322 | m_nInterval = 1; 323 | } 324 | l_frame_count = l_frame_count % m_nInterval; 325 | if (l_frame_count == 0) 326 | { 327 | cv::Mat matFrame; 328 | Utils_Ov::frameData2Mat(frame, matFrame); 329 | _queue_input.push(matFrame); 330 | } 331 | } 332 | 333 | //1.2 input shape 334 | if (out_shape.height > 0 && out_shape.width > 0) 335 | { 336 | _shape_output.width = out_shape.width; 337 | _shape_output.height = out_shape.height; 338 | } 339 | 340 | // 2.return raw frame if out put shape is empty or replace bgr is empty 341 | if (_shape_output.empty() || _frame_replace.empty()) 342 | { 343 | frameCom = frame; 344 | return -1; 345 | } 346 | 347 | if (_frame_replace.rows != _shape_output.height || _frame_replace.cols != _shape_output.width) 348 | { 349 | cv::resize(_frame_replace, _frame_replace, _shape_output); 350 | } 351 | 352 | // 3.if output queue is empty, return replace bgr 353 | if (_queue_output.size() <= 0) 354 | { 355 | Utils_Ov::mat2FrameData(_frame_replace, frameCom); 356 | frameAlpha.frame = 0; 357 | return 1; 358 | } 359 | /* 360 | else if (_queue_output.size() == 1) 361 | { 362 | MattingObject f; 363 | _queue_output.front(f); 364 | Utils_Ov::mat2FrameData(f.com, frameCom); 365 | Utils_Ov::mat2FrameData(f.pha, frameAlpha); 366 | return 1; 367 | }*/ 368 | 369 | while (_queue_output.size() > 1) 370 | { 371 | MattingObject f; 372 | _queue_output.pop(f); 373 | } 374 | // 4. if output queue is not empty, return first item in the output queue 375 | MattingObject result; 376 | _queue_output.front(result); 377 | Utils_Ov::mat2FrameData(result.com, frameCom); 378 | Utils_Ov::mat2FrameData(result.pha, frameAlpha); 379 | 380 | return 0; 381 | } 382 | 383 | void MatterModnetImpl::run() 384 | { 385 | long lSleep = 5; 386 | while (!isInterrupted()) 387 | { 388 | Utils_Ov::sleep(lSleep); 389 | if (_queue_input.size() <= 0) 390 | { 391 | continue; 392 | } 393 | 394 | cv::Mat frame, matAlpha, matCom; 395 | _queue_input.pop(frame); 396 | { 397 | std::lock_guard lock(mutex_); 398 | _bencher.Start(); 399 | if (!_pCnn->getAsync()) 400 | { 401 | _pCnn->setAsync(true); 402 | } 403 | 404 | _pCnn->enqueue("input.1", frame); 405 | _pCnn->submitRequest(); 406 | _pCnn->wait(); 407 | _matResult = _pCnn->fetchResults(); 408 | if (_matResult.size() <= 0) 409 | { 410 | continue; 411 | } 412 | 413 | matAlpha = _matResult[0].pha; 414 | compose(frame, _frame_replace, matAlpha, matCom, _shape_output); 415 | if (matAlpha.empty() || matCom.empty()) 416 | { 417 | continue; 418 | } 419 | MattingObject obj; 420 | obj.com = matCom.clone(); 421 | obj.pha = matAlpha.clone(); 422 | _queue_output.push(obj); 423 | _elapse = _bencher.Elapse(); 424 | } 425 | 426 | } 427 | } --------------------------------------------------------------------------------