├── README.md ├── demo_cpp ├── demo_cpp.sln └── demo_cpp │ ├── demo_cpp.cpp │ ├── demo_cpp.vcxproj │ ├── demo_cpp.vcxproj.filters │ ├── demo_cpp.vcxproj.user │ └── timer.h ├── demo_cs ├── demo_cs.sln └── demo_cs │ ├── App.config │ ├── Program.cs │ ├── Properties │ └── AssemblyInfo.cs │ └── demo_cs.csproj ├── logger.h ├── model_infer.cpp ├── model_infer.h └── thread_pool.h /README.md: -------------------------------------------------------------------------------- 1 | # model_infer_multiThreads 2 | 3 | 该repo基于PaddleX模型推理动态链接库的接口代码进行修改，支持多线程并行访问。大部分代码均来自paddleX的model_infer.cpp 4 | #### 最近更新： 5 | - 2021-12-15 增加激活tensorRT接口的参数设置 6 | - 2021-11-24 增加demo_cs, demo_cpp工程 7 | - 2021-10-28 增加了原生的所有api接口，支持clas/det/seg/mask) 8 | 9 | ## 修改部分： 10 | - 把模型作为对象从初始化函数返回，便于进行多线程控制 11 | - 对模型和线程池进行二次封装，避免了多线程调用时推理失败 12 | 13 | ## 使用方法： 14 | 替换原有的model_infer项目中的部分文件，实现model_infer.dll的多线程访问的支持，具体如下: 15 | 16 | - 基于paddleX的model_infer动态库代码，确保model_infer.dll能够正确生成，并能够整理依赖的dll，让model_infer.dll在单线程下运行正常 17 | - 把该repo中的model_infer.cpp替代原来model_infer动态库代码中的model_infer.cpp，同时把model_infer.h/thread_pool.h/logger.h都拷贝到model_deploy/common/include文件夹中 18 | ![files_position](https://user-images.githubusercontent.com/24242483/139017800-78736d89-f2cc-452b-9d99-82361fa8be6e.png) 19 | 20 | - 在model_infer动态库代码项目中右键/属性/C/C++/常规/附件包含目录，添加PaddleX-release-2.0.0\deploy\cpp文件夹，从而保证整个model_deploy文件夹都能被model_infer项目找到，确保可以成功生成model_infer.dll 21 | ![add_head_folder](https://user-images.githubusercontent.com/24242483/139017936-44a5399f-c203-4842-9a58-4ff4ffcbfd7f.png) 22 | 23 | - 使用该repo的demo_cs代码验证用c#多线程访问模型 24 | 该工程下载后，需要添加对opencvsharp的引用，同时还需要把前几步生成好的多线程model_infer.dll，以及其他paddle_inference相关的dll都拷贝到exe目录即可运行 25 | ![image](https://user-images.githubusercontent.com/24242483/143397023-f046a327-4446-4f2c-bae2-0d89fb2653d5.png) 26 | 27 | - 使用该repo的deomo_cpp代码验证用cpp多线程访问模型 28 | 该工程下载后，需要在项目/属性中，分别增加对model_infer, opencv, yaml-cpp这几个库的头文件夹、库文件夹、库文件名的引入，以及model_infer.dll和其他图paddle_inference相关的dll拷贝到exe目录即可运行 29 | 30 | ![image](https://user-images.githubusercontent.com/24242483/143397092-0bc0f774-92fe-487d-a5c8-3420943c759b.png) 31 | 32 | ##### 注意 33 | - 模型/配置文件的名称被在代码里边固定了，如果不匹配会报错，如果需要可修改model_infer.cpp代码 34 | - 默认只能在gpu下跑，cpu被在代码里边关掉了，如果需要可修改model_infer.cpp代码 35 | 36 | ![result](https://user-images.githubusercontent.com/24242483/139020183-f0b997c1-c293-4de9-bb72-e3ca8b9185ef.png) 37 | 38 | ## 注意： 39 | - paddle的模型不支持多线程并行推理，虽然通过改造后，模型可以被多线程访问，但本质上底层是被线程池锁掉了，每次实际上只有单线程访问模型。如果强行让多线程并行访问模型，必然导致推理报错。 40 | - paddle的模型在多线程分时访问时，也不支持不同线程反复创建、销毁，同样会导致底层cuda报错。 41 | - 该repo中所有原生api接口都已导出，但仅简单验证了clas/det/seg部分的功能，部分代码可能存在一些疏漏和错误，仅作为一个抛砖引玉，大家可在上面自行修改。后续项目稳定后我会尽快把完整测试版本更新上面 42 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.31729.503 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "demo_cpp", "demo_cpp\demo_cpp.vcxproj", "{C8F26176-5301-4EFA-83AB-D188498CDDD9}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|x64 = Debug|x64 11 | Debug|x86 = Debug|x86 12 | Release|x64 = Release|x64 13 | Release|x86 = Release|x86 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Debug|x64.ActiveCfg = Debug|x64 17 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Debug|x64.Build.0 = Debug|x64 18 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Debug|x86.ActiveCfg = Debug|Win32 19 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Debug|x86.Build.0 = Debug|Win32 20 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Release|x64.ActiveCfg = Release|x64 21 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Release|x64.Build.0 = Release|x64 22 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Release|x86.ActiveCfg = Release|Win32 23 | {C8F26176-5301-4EFA-83AB-D188498CDDD9}.Release|x86.Build.0 = Release|Win32 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {8A860304-2B7C-44DB-837B-E8808162D26C} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp/demo_cpp.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "timer.h" 5 | #include "model_deploy/common/include/model_infer.h" // 因为引入了paddle_inference的头文件，所以还要增加yaml-cpp引用 6 | 7 | // 因logger.h的引入，还要增加"属性/C/C++/预处理器"里边的_CRT_SECURE_NO_WARNINGS 8 | 9 | void TestSingleModelMultiThreads() 10 | { 11 | const char* model_dir = "D:\\suliang\\My_Lib11\\models\\2"; 12 | int gpu_id = 0; 13 | bool use_trt = false; 14 | const char* model_type = "seg"; 15 | ModelWrapper* model = ModelObjInit(model_type, model_dir, gpu_id, use_trt); 16 | ModelWrapper* model2 = ModelObjInit(model_type, model_dir, gpu_id, use_trt); 17 | 18 | cv::Mat src1 = cv::imread("D:\\suliang\\My_Dataset\\stain1k_test\\31042232357036_down_2k_lb.bmp", cv::IMREAD_GRAYSCALE); 19 | cv::Mat src2 = cv::imread("D:\\suliang\\My_Dataset\\stain1k_test\\31050024054020_up_2k_lt.bmp", cv::IMREAD_GRAYSCALE); 20 | 21 | int width = src1.cols; 22 | int height = src1.rows; 23 | int channels = src1.channels(); 24 | printf("image width=%d, height=%d, channels=%d", width, height, channels); 25 | unsigned char* result_map1 = (unsigned char*)malloc(width * height * sizeof(unsigned char)); 26 | unsigned char* result_map2 = (unsigned char*)malloc(width * height * sizeof(unsigned char)); 27 | 28 | Timer timer; 29 | 30 | // 启动多线程预热 31 | timer.start(); 32 | std::thread t1(ModelObjPredict_Seg, model, src1.data, width, height, channels, result_map1); 33 | std::thread t2(ModelObjPredict_Seg, model2, src2.data, width, height, channels, result_map2); 34 | t1.join(); 35 | t2.join(); 36 | timer.stop_and_show("one model warmup time:"); 37 | std::cout << "finished warmup" << std::endl; 38 | 39 | // 单模型跑1次的时间 40 | timer.start(); 41 | std::thread t0(ModelObjPredict_Seg, model, src1.data, width, height, channels, result_map1); 42 | t0.join(); 43 | timer.stop_and_show("1 model with 1 thread to predict 1 pic:"); 44 | 45 | // 单模型跑两次的时间 46 | timer.start(); 47 | std::thread t3(ModelObjPredict_Seg, model, src1.data, width, height, channels, result_map1); 48 | std::thread t4(ModelObjPredict_Seg, model, src2.data, width, height, channels, result_map2); 49 | t3.join(); 50 | t4.join(); 51 | timer.stop_and_show("1 model with 2 thread to predict 2 pics:"); 52 | 53 | // 两个模型分别预测单图的时间 54 | timer.start(); 55 | std::thread t5(ModelObjPredict_Seg, model, src1.data, width, height, channels, result_map1); 56 | std::thread t6(ModelObjPredict_Seg, model2, src2.data, width, height, channels, result_map2); 57 | t5.join(); 58 | t6.join(); 59 | timer.stop_and_show("2 model with 2 thread to predict 2 pics:"); 60 | 61 | //ModelObjPredict_Seg(model, src1.data, width, height, channels, result_map1); 62 | cv::Mat dst1 = cv::Mat(height, width, CV_8UC1, result_map1); 63 | cv::Mat dst2 = cv::Mat(height, width, CV_8UC1, result_map2); 64 | cv::imwrite("D:\\result_map1.bmp", dst1); 65 | cv::imwrite("D:\\result_map2.bmp", dst2); 66 | 67 | ModelObjDestruct(model); 68 | ModelObjDestruct(model2); 69 | free(result_map1); 70 | free(result_map2); 71 | } 72 | 73 | int main() 74 | { 75 | TestSingleModelMultiThreads(); 76 | } 77 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp/demo_cpp.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 16.0 23 | Win32Proj 24 | {c8f26176-5301-4efa-83ab-d188498cddd9} 25 | democpp 26 | 10.0 27 | 28 | 29 | 30 | Application 31 | true 32 | v142 33 | Unicode 34 | 35 | 36 | Application 37 | false 38 | v142 39 | true 40 | Unicode 41 | 42 | 43 | Application 44 | true 45 | v142 46 | Unicode 47 | 48 | 49 | Application 50 | false 51 | v142 52 | true 53 | Unicode 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | true 75 | 76 | 77 | false 78 | 79 | 80 | true 81 | 82 | 83 | false 84 | 85 | 86 | 87 | Level3 88 | true 89 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 90 | true 91 | 92 | 93 | Console 94 | true 95 | 96 | 97 | 98 | 99 | Level3 100 | true 101 | true 102 | true 103 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 104 | true 105 | 106 | 107 | Console 108 | true 109 | true 110 | true 111 | 112 | 113 | 114 | 115 | Level3 116 | true 117 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 118 | true 119 | D:\suliang\model_infer_multiThreads-main\lib\opencv\include;%(AdditionalIncludeDirectories) 120 | 121 | 122 | Console 123 | true 124 | D:\suliang\model_infer_multiThreads-main\lib\opencv\lib;%(AdditionalLibraryDirectories) 125 | 126 | 127 | 128 | 129 | Level3 130 | true 131 | true 132 | true 133 | _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 134 | true 135 | D:\suliang\model_infer_multiThreads-main\lib\model_infer_v2\include;D:\suliang\model_infer_multiThreads-main\lib\opencv\include;D:\suliang\model_infer_multiThreads-main\lib\yaml-cpp\src\ext-yaml-cpp\include;%(AdditionalIncludeDirectories) 136 | 137 | 138 | Console 139 | true 140 | true 141 | true 142 | D:\suliang\model_infer_multiThreads-main\lib\opencv\lib;D:\suliang\model_infer_multiThreads-main\lib\model_infer_v2\lib;D:\suliang\model_infer_multiThreads-main\lib\yaml-cpp\lib\Release;%(AdditionalLibraryDirectories) 143 | libyaml-cppmt.lib;model_infer.lib;opencv_core453.lib;opencv_highgui453.lib;opencv_imgcodecs453.lib;opencv_imgproc453.lib;opencv_videoio453.lib;%(AdditionalDependencies) 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp/demo_cpp.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 源文件 20 | 21 | 22 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp/demo_cpp.vcxproj.user: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /demo_cpp/demo_cpp/timer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ximitiejiang/model_infer_multiThreads/66e4fefe6fc17cda08c3012a5618d8ed0feeeff7/demo_cpp/demo_cpp/timer.h -------------------------------------------------------------------------------- /demo_cs/demo_cs.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.31729.503 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "demo_cs", "demo_cs\demo_cs.csproj", "{78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Debug|x64 = Debug|x64 12 | Release|Any CPU = Release|Any CPU 13 | Release|x64 = Release|x64 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Debug|x64.ActiveCfg = Debug|x64 19 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Debug|x64.Build.0 = Debug|x64 20 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Release|Any CPU.Build.0 = Release|Any CPU 22 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Release|x64.ActiveCfg = Release|x64 23 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB}.Release|x64.Build.0 = Release|x64 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {2CAB856C-F026-4122-A5AC-90B0D248E174} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /demo_cs/demo_cs/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /demo_cs/demo_cs/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | using OpenCvSharp; 5 | using System.Runtime.InteropServices; // Marshal 6 | using System.IO; // Directory 7 | 8 | 9 | namespace mycstest_framework 10 | { 11 | class AlgoTest 12 | { 13 | // 尝试直接调用model_infer.dll中的多线程api 14 | // 模型初始化 15 | [DllImport("model_infer.dll", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi)] 16 | public extern static IntPtr ModelObjInit(string model_type, string model_dir, int gpu_id, bool use_trt); 17 | // 模型推理 18 | [DllImport("model_infer.dll", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi)] 19 | public extern static void ModelObjPredict_Seg(IntPtr segObj, IntPtr imageData, int width, int height, int channels, [In, Out] IntPtr resultMap); 20 | // 模型资源回收 21 | [DllImport("model_infer.dll", CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Ansi)] 22 | public extern static void ModelObjDestruct(IntPtr segObj); 23 | 24 | // 单模型多线程调用(一般为单显卡) 25 | public static void TestSingleModelMultiThreads() 26 | { 27 | string[] model_type_maps = { "det", "seg", "clas" }; // 支持的模型类型集合 28 | string model_dir = "D:\\suliang\\My_Lib11\\models\\2"; 29 | int gpu_id = 0; 30 | bool use_trt = false; 31 | string imgdir = "D:\\suliang\\My_Dataset\\stain1k_test"; 32 | string[] imgfiles = Directory.GetFiles(imgdir, "*.bmp"); 33 | byte[] paddlex_model_type = new byte[10]; 34 | // 模型 35 | IntPtr model = ModelObjInit(model_type_maps[1], model_dir, gpu_id, use_trt); 36 | int idx = 0; 37 | 38 | // 图像尺寸 39 | Mat _src = Cv2.ImRead(imgfiles[idx], ImreadModes.Grayscale); // 每个线程负责一张图 40 | int _h = _src.Rows; 41 | int _w = _src.Cols; 42 | IntPtr result_map1 = Marshal.AllocHGlobal(_w * _h); 43 | IntPtr result_map2 = Marshal.AllocHGlobal(_w * _h); 44 | while (idx < (imgfiles.Length / 2 * 2)) 45 | { 46 | // 获取数据 47 | Mat src1 = Cv2.ImRead(imgfiles[idx], ImreadModes.Grayscale); // 每个线程负责一张图 48 | Mat src2 = Cv2.ImRead(imgfiles[idx + 1], ImreadModes.Grayscale); // 每个线程负责一张图 49 | int h = src1.Rows; 50 | int w = src1.Cols; 51 | int c = src1.Channels(); 52 | 53 | // 启动2个线程同时访问模型也不会冲突 54 | Task task1 = Task.Run(() => 55 | { 56 | Console.WriteLine($"pred image id={idx}"); 57 | ModelObjPredict_Seg(model, src1.Data, w, h, c, result_map1); 58 | }); 59 | Task task2 = Task.Run(() => 60 | { 61 | Console.WriteLine($"pred image id={idx + 1}"); 62 | ModelObjPredict_Seg(model, src2.Data, w, h, c, result_map2); 63 | }); 64 | // 等待两个线程结束 65 | task1.Wait(); 66 | task2.Wait(); 67 | // 结果处理 68 | List imgSize = new List(); 69 | imgSize.Add(w); imgSize.Add(h); 70 | Mat dst1 = new Mat(imgSize, MatType.CV_8UC1, result_map1); 71 | Mat dst2 = new Mat(imgSize, MatType.CV_8UC1, result_map2); 72 | Cv2.ImWrite("D:\\result_map1.bmp", dst1); 73 | Cv2.ImWrite("D:\\result_map2.bmp", dst2); 74 | // 每次两张图 75 | idx += 2; 76 | Console.WriteLine("finish 2 image pred."); 77 | } 78 | ModelObjDestruct(model); 79 | Marshal.FreeHGlobal(result_map1); 80 | Marshal.FreeHGlobal(result_map2); 81 | } 82 | 83 | // 多模型多线程(可以单显卡或者多显卡) 84 | public static void TestMultiModelMultiThreads() 85 | { 86 | string[] model_type_maps = { "det", "seg", "clas" }; // 支持的模型类型集合 87 | string model_dir = "D:\\suliang\\My_Lib11\\models\\2"; 88 | int gpu_id = 0; 89 | bool use_trt = false; 90 | string imgdir = "D:\\suliang\\My_Dataset\\stain1k_test"; 91 | string[] imgfiles = Directory.GetFiles(imgdir, "*.bmp"); 92 | byte[] paddlex_model_type = new byte[10]; 93 | // 模型 94 | IntPtr model1 = ModelObjInit(model_type_maps[1], model_dir, gpu_id, use_trt); 95 | IntPtr model2 = ModelObjInit(model_type_maps[1], model_dir, gpu_id, use_trt); 96 | int idx = 0; 97 | 98 | // 图像尺寸 99 | Mat _src = Cv2.ImRead(imgfiles[idx], ImreadModes.Grayscale); 100 | int _h = _src.Rows; 101 | int _w = _src.Cols; 102 | IntPtr result_map1 = Marshal.AllocHGlobal(_w * _h); 103 | IntPtr result_map2 = Marshal.AllocHGlobal(_w * _h); 104 | IntPtr result_map3 = Marshal.AllocHGlobal(_w * _h); 105 | IntPtr result_map4 = Marshal.AllocHGlobal(_w * _h); 106 | while (idx < (imgfiles.Length / 4 * 4)) 107 | { 108 | // 获取数据 109 | Mat src1 = Cv2.ImRead(imgfiles[idx], ImreadModes.Grayscale); // 每个线程负责一张图 110 | Mat src2 = Cv2.ImRead(imgfiles[idx + 1], ImreadModes.Grayscale); // 每个线程负责一张图 111 | Mat src3 = Cv2.ImRead(imgfiles[idx + 2], ImreadModes.Grayscale); // 每个线程负责一张图 112 | Mat src4 = Cv2.ImRead(imgfiles[idx + 3], ImreadModes.Grayscale); // 每个线程负责一张图 113 | int h = src1.Rows; 114 | int w = src1.Cols; 115 | int c = src1.Channels(); 116 | 117 | // 模型1的多线程访问 118 | Task task1 = Task.Run(() => 119 | { 120 | Console.WriteLine($"pred image id={idx}"); 121 | ModelObjPredict_Seg(model1, src1.Data, w, h, c, result_map1); 122 | }); 123 | Task task2 = Task.Run(() => 124 | { 125 | Console.WriteLine($"pred image id={idx + 1}"); 126 | ModelObjPredict_Seg(model1, src2.Data, w, h, c, result_map2); 127 | }); 128 | // 模型2的多线程访问 129 | Task task3 = Task.Run(() => 130 | { 131 | Console.WriteLine($"pred image id={idx + 2}"); 132 | ModelObjPredict_Seg(model2, src3.Data, w, h, c, result_map3); 133 | }); 134 | Task task4 = Task.Run(() => 135 | { 136 | Console.WriteLine($"pred image id={idx + 3}"); 137 | ModelObjPredict_Seg(model2, src4.Data, w, h, c, result_map4); 138 | }); 139 | // 等待两个线程结束 140 | task1.Wait(); 141 | task2.Wait(); 142 | task3.Wait(); 143 | task4.Wait(); 144 | // 结果处理 145 | List imgSize = new List(); 146 | imgSize.Add(w); imgSize.Add(h); 147 | Mat dst1 = new Mat(imgSize, MatType.CV_8UC1, result_map1); 148 | Mat dst2 = new Mat(imgSize, MatType.CV_8UC1, result_map2); 149 | Mat dst3 = new Mat(imgSize, MatType.CV_8UC1, result_map3); 150 | Mat dst4 = new Mat(imgSize, MatType.CV_8UC1, result_map4); 151 | Cv2.ImWrite("D:\\result_map1.bmp", dst1); 152 | Cv2.ImWrite("D:\\result_map2.bmp", dst2); 153 | Cv2.ImWrite("D:\\result_map3.bmp", dst3); 154 | Cv2.ImWrite("D:\\result_map4.bmp", dst4); 155 | // 每次两张图 156 | idx += 4; 157 | Console.WriteLine("finish 4 image pred."); 158 | } 159 | ModelObjDestruct(model1); 160 | ModelObjDestruct(model2); 161 | Marshal.FreeHGlobal(result_map1); 162 | Marshal.FreeHGlobal(result_map2); 163 | Marshal.FreeHGlobal(result_map3); 164 | Marshal.FreeHGlobal(result_map4); 165 | } 166 | 167 | } 168 | class Program 169 | { 170 | static void Main(string[] args) 171 | { 172 | //AlgoTest.TestSingleModelMultiThreads(); 173 | 174 | AlgoTest.TestMultiModelMultiThreads(); 175 | 176 | Console.WriteLine("finished pred"); 177 | Console.ReadLine(); 178 | } 179 | } 180 | } -------------------------------------------------------------------------------- /demo_cs/demo_cs/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // 有关程序集的一般信息由以下 6 | // 控制。更改这些特性值可修改 7 | // 与程序集关联的信息。 8 | [assembly: AssemblyTitle("demo_cs")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("demo_cs")] 13 | [assembly: AssemblyCopyright("Copyright © 2021")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // 将 ComVisible 设置为 false 会使此程序集中的类型 18 | //对 COM 组件不可见。如果需要从 COM 访问此程序集中的类型 19 | //请将此类型的 ComVisible 特性设置为 true。 20 | [assembly: ComVisible(false)] 21 | 22 | // 如果此项目向 COM 公开，则下列 GUID 用于类型库的 ID 23 | [assembly: Guid("78e9af85-a9e9-4395-9d65-6f57deab77db")] 24 | 25 | // 程序集的版本信息由下列四个值组成: 26 | // 27 | // 主版本 28 | // 次版本 29 | // 生成号 30 | // 修订号 31 | // 32 | //可以指定所有这些值，也可以使用“生成号”和“修订号”的默认值 33 | //通过使用 "*"，如下所示: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /demo_cs/demo_cs/demo_cs.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {78E9AF85-A9E9-4395-9D65-6F57DEAB77DB} 8 | Exe 9 | demo_cs 10 | demo_cs 11 | v4.7.2 12 | 512 13 | true 14 | true 15 | 16 | 17 | AnyCPU 18 | true 19 | full 20 | false 21 | bin\Debug\ 22 | DEBUG;TRACE 23 | prompt 24 | 4 25 | 26 | 27 | AnyCPU 28 | pdbonly 29 | true 30 | bin\Release\ 31 | TRACE 32 | prompt 33 | 4 34 | 35 | 36 | true 37 | bin\x64\Debug\ 38 | DEBUG;TRACE 39 | full 40 | x64 41 | 7.3 42 | prompt 43 | true 44 | 45 | 46 | bin\x64\Release\ 47 | TRACE 48 | true 49 | pdbonly 50 | x64 51 | 7.3 52 | prompt 53 | true 54 | 55 | 56 | 57 | ..\..\lib\opencvsharp453\ManagedLib\net461\OpenCvSharp.dll 58 | 59 | 60 | ..\..\lib\opencvsharp453\ManagedLib\net461\OpenCvSharp.Extensions.dll 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /logger.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ximitiejiang/model_infer_multiThreads/66e4fefe6fc17cda08c3012a5618d8ed0feeeff7/logger.h -------------------------------------------------------------------------------- /model_infer.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include "model_deploy/common/include/paddle_deploy.h" 19 | #include "model_deploy/common/include/model_infer.h" 20 | #include // GetCurrentThreadId() 21 | 22 | 23 | /* 24 | * 模型初始化/注册接口 25 | * 26 | * model_type: 初始化模型类型: det,seg,clas,paddlex 27 | * 28 | * model_filename: 模型文件路径 29 | * 30 | * params_filename: 参数文件路径 31 | * 32 | * cfg_file: 配置文件路径 33 | * 34 | * use_gpu: 是否使用GPU 35 | * 36 | * gpu_id: 指定第x号GPU 37 | * 38 | * paddlex_model_type: model_type为paddlx时，返回的实际paddlex模型的类型: det, seg, clas 39 | */ 40 | extern "C" __declspec(dllexport) PaddleDeploy::Model * InitModel(const char* model_type, const char* model_filename, const char* params_filename, const char* cfg_file, bool use_gpu, int gpu_id, char* paddlex_model_type) 41 | { 42 | // create model 43 | PaddleDeploy::Model* model = PaddleDeploy::CreateModel(model_type); //FLAGS_model_type 44 | 45 | // model init 46 | model->Init(cfg_file); 47 | 48 | // inference engine init 49 | PaddleDeploy::PaddleEngineConfig engine_config; 50 | engine_config.model_filename = model_filename; 51 | engine_config.params_filename = params_filename; 52 | engine_config.use_gpu = use_gpu; 53 | engine_config.gpu_id = gpu_id; 54 | bool init = model->PaddleEngineInit(engine_config); 55 | if (!init) 56 | { 57 | LOGC("ERR", "init model failed"); 58 | } 59 | else 60 | { 61 | LOGC("INFO", "init model successfully: use_gpu=%d, gpu_id=%d, model path=%s", use_gpu, gpu_id, model_filename); 62 | } 63 | 64 | // det, seg, clas, paddlex 65 | if (strcmp(model_type, "paddlex") == 0) // 是paddlex模型，则返回具体支持的模型类型: det, seg, clas 66 | { 67 | // detector 68 | if (model->yaml_config_["model_type"].as() == std::string("detector")) 69 | { 70 | strcpy(paddlex_model_type, "det"); 71 | } 72 | else if (model->yaml_config_["model_type"].as() == std::string("segmenter")) 73 | { 74 | strcpy(paddlex_model_type, "seg"); 75 | } 76 | else if (model->yaml_config_["model_type"].as() == std::string("classifier")) 77 | { 78 | strcpy(paddlex_model_type, "clas"); 79 | } 80 | } 81 | return model; 82 | } 83 | 84 | // 初始化模型带tensorRT加速 85 | // [suliang] 2021-12-15 增加5个输入参数：min_input_shape, max_input_shape, optim_input_shape分别代表输入尺寸的输入范围， precision代表计算精度(0=fp32,1=fp16,2=int8),min_subgraph_size代表最小优化子图 86 | extern "C" __declspec(dllexport) PaddleDeploy::Model * InitModel_TRT(const char* model_type, const char* model_filename, const char* params_filename, const char* cfg_file, bool use_gpu, int gpu_id, char* paddlex_model_type, 87 | std::vectormin_input_shape, std::vectormax_input_shape, std::vectoroptim_input_shape, int precision, int min_subgraph_size) 88 | { 89 | // create model 90 | PaddleDeploy::Model* model = PaddleDeploy::CreateModel(model_type); //FLAGS_model_type 91 | 92 | // model init 93 | model->Init(cfg_file); 94 | 95 | // inference engine init 96 | PaddleDeploy::PaddleEngineConfig engine_config; 97 | engine_config.model_filename = model_filename; 98 | engine_config.params_filename = params_filename; 99 | engine_config.use_gpu = use_gpu; 100 | engine_config.gpu_id = gpu_id; 101 | 102 | // 使用tensorRT则强制打开gpu 103 | engine_config.use_gpu = true; 104 | engine_config.use_trt = true; 105 | 106 | // 注意：根据优化目标需要手动调整 107 | engine_config.precision = precision; // 精度选择，默认fp32,还有fp16,int8 108 | engine_config.min_subgraph_size = min_subgraph_size;// 最小子图，越大则优化度越低，越大越可能忽略动态图: 设置40+不报错但也没啥优化 109 | engine_config.max_workspace_size = 1 << 30; 110 | 111 | // 注意：根据模型和输入图像大小，需要手动调整如下变量 112 | //std::vector min_input_shape = { 1, 3, 512, 512 }; 113 | //std::vector max_input_shape = { 1, 3, 1024, 1024 }; 114 | //std::vector optim_input_shape = { 1, 3, 1024, 1024 }; 115 | 116 | // 分别定义最小、最大、最优输入尺寸：需要根据模型输入尺寸调整 117 | // 这里三种模型输入的关键字不同(clas对应inputs, det对应image, seg对应x)，可通过netron查看INPUTS.name，比如seg模型INPUTS.name=x 118 | // 另外如果有动态输入尺寸不匹配的节点，需要手动定义 119 | if (strcmp("clas", model_type) == 0) { 120 | // Adjust shape according to the actual model 121 | engine_config.min_input_shape["inputs"] = min_input_shape; 122 | engine_config.max_input_shape["inputs"] = max_input_shape; 123 | engine_config.optim_input_shape["inputs"] = optim_input_shape; 124 | } 125 | else if (strcmp("det", model_type) == 0) { 126 | // Adjust shape according to the actual model 127 | engine_config.min_input_shape["image"] = min_input_shape; 128 | engine_config.max_input_shape["image"] = max_input_shape; 129 | engine_config.optim_input_shape["image"] = optim_input_shape; 130 | } 131 | else if (strcmp("seg", model_type) == 0) { 132 | // Additional nodes need to be added, pay attention to the output prompt 133 | engine_config.min_input_shape["x"] = min_input_shape; 134 | engine_config.max_input_shape["x"] = max_input_shape; 135 | engine_config.optim_input_shape["x"] = optim_input_shape; 136 | } 137 | bool init = model->PaddleEngineInit(engine_config); 138 | if (!init) 139 | { 140 | LOGC("INFO", "init model failed"); 141 | } 142 | // det, seg, clas, paddlex 143 | if (strcmp(model_type, "paddlex") == 0) // 是paddlex模型，则返回具体支持的模型类型: det, seg, clas 144 | { 145 | // detector 146 | if (model->yaml_config_["model_type"].as() == std::string("detector")) 147 | { 148 | strcpy(paddlex_model_type, "det"); 149 | } 150 | else if (model->yaml_config_["model_type"].as() == std::string("segmenter")) 151 | { 152 | strcpy(paddlex_model_type, "seg"); 153 | } 154 | else if (model->yaml_config_["model_type"].as() == std::string("classifier")) 155 | { 156 | strcpy(paddlex_model_type, "clas"); 157 | } 158 | } 159 | return model; 160 | } 161 | 162 | 163 | /* 164 | * 检测推理接口 165 | * 166 | * img: input for predicting. 167 | * 168 | * nWidth: width of img. 169 | * 170 | * nHeight: height of img. 171 | * 172 | * nChannel: channel of img. 173 | * 174 | * output: result of pridict ,include category_id£¬score£¬coordinate¡£ 175 | * 176 | * nBoxesNum£º number of box 177 | * 178 | * LabelList: label list of result 179 | */ 180 | extern "C" __declspec(dllexport) void Det_ModelPredict(PaddleDeploy::Model * model, const unsigned char* img, int nWidth, int nHeight, int nChannel, float* output, int* nBoxesNum, char* LabelList) 181 | { 182 | // prepare data 183 | std::vector imgs; 184 | 185 | int nType = 0; 186 | if (nChannel == 3) 187 | { 188 | nType = CV_8UC3; 189 | } 190 | else 191 | { 192 | std::cout << "Only support 3 channel image." << std::endl; 193 | return; 194 | } 195 | 196 | cv::Mat input = cv::Mat::zeros(cv::Size(nWidth, nHeight), nType); 197 | memcpy(input.data, img, nHeight * nWidth * nChannel * sizeof(uchar)); 198 | //cv::imwrite("./1.png", input); 199 | imgs.push_back(std::move(input)); 200 | 201 | // predict 202 | std::vector results; 203 | model->Predict(imgs, &results, 1); 204 | 205 | // nBoxesNum[0] = results.size(); // results.size()得到的是batch_size 206 | nBoxesNum[0] = results[0].det_result->boxes.size(); // 得到单张图片预测的bounding box数 207 | std::string label = ""; 208 | //std::cout << "res: " << results[num] << std::endl; 209 | for (int i = 0; i < results[0].det_result->boxes.size(); i++) // 得到所有框的数据 210 | { 211 | //std::cout << "category: " << results[num].det_result->boxes[i].category << std::endl; 212 | label = label + results[0].det_result->boxes[i].category + " "; 213 | // labelindex 214 | output[i * 6 + 0] = results[0].det_result->boxes[i].category_id; // 类别的id 215 | // score 216 | output[i * 6 + 1] = results[0].det_result->boxes[i].score; // 得分 217 | //// box 218 | output[i * 6 + 2] = results[0].det_result->boxes[i].coordinate[0]; // x1, y1, x2, y2 219 | output[i * 6 + 3] = results[0].det_result->boxes[i].coordinate[1]; // 左上、右下的顶点 220 | output[i * 6 + 4] = results[0].det_result->boxes[i].coordinate[2]; 221 | output[i * 6 + 5] = results[0].det_result->boxes[i].coordinate[3]; 222 | } 223 | memcpy(LabelList, label.c_str(), strlen(label.c_str())); 224 | } 225 | 226 | 227 | /* 228 | * 分割推理接口 229 | * 230 | * img: input for predicting. 231 | * 232 | * nWidth: width of img. 233 | * 234 | * nHeight: height of img. 235 | * 236 | * nChannel: channel of img. 237 | * 238 | * output: result of pridict ,include label_map 239 | */ 240 | extern "C" __declspec(dllexport) void Seg_ModelPredict(PaddleDeploy::Model * model, const unsigned char* img, int nWidth, int nHeight, int nChannel, unsigned char* output) 241 | { 242 | //LOGC("INFO", "seg in thread id [%d]", GetCurrentThreadId()); 243 | // prepare data 244 | std::vector imgs; 245 | 246 | int nType = 0; 247 | if (nChannel == 3) 248 | { 249 | nType = CV_8UC3; 250 | //LOGC("INFO", "infer input img w=%d, h=%d, c=%d", nWidth, nHeight, nChannel); 251 | } 252 | else 253 | { 254 | //std::cout << "Only support 3 channel image." << std::endl; 255 | LOGC("ERR", "Only support 3 channel images, but got channels ", nChannel); 256 | return; 257 | } 258 | 259 | cv::Mat input = cv::Mat::zeros(cv::Size(nWidth, nHeight), nType); 260 | memcpy(input.data, img, nHeight * nWidth * nChannel * sizeof(uchar)); 261 | //cv::imwrite("D://modelinfercpp_275.bmp", input); 262 | imgs.push_back(std::move(input)); 263 | 264 | // predict 265 | std::vector results; 266 | model->Predict(imgs, &results, 1); 267 | // batch修改：这里应该会得到返回的每张图的label_map，那么下面就应该分别处理results中每张图对应的label_map 268 | std::vector result_map = results[0].seg_result->label_map.data; // vector -- 结果map 269 | //LOGC("INFO", "finish infer, with result_map length=%d", result_map.size()); 270 | // 拷贝输出结果到输出上返回 -- 将vector转成unsigned char * 271 | memcpy(output, &result_map[0], result_map.size() * sizeof(uchar)); 272 | } 273 | 274 | 275 | /* 276 | * 分割推理接口batch predict 277 | * 278 | * img: input for predicting. 279 | * 280 | * nWidth: width of img. 281 | * 282 | * nHeight: height of img. 283 | * 284 | * nChannel: channel of img. 285 | * 286 | * output: result of pridict ,include label_map 287 | */ 288 | extern "C" __declspec(dllexport) void Seg_ModelBatchPredict(PaddleDeploy::Model * model, const std::vector imgs, int nWidth, int nHeight, int nChannel, std::vector output) 289 | { 290 | std::vector results; 291 | if (imgs.size() != output.size()) { 292 | LOGC("ERR", "image batch size(%d) not match with results size(%d)", imgs.size(), output.size()); 293 | } 294 | // Read image 295 | int im_vec_size = imgs.size(); 296 | std::vector im_vec; 297 | 298 | int nType = 0; 299 | if (nChannel == 3) 300 | { 301 | nType = CV_8UC3; 302 | } 303 | else 304 | { 305 | LOGC("ERR", "Only support 3 channel images, but got channels ", nChannel); 306 | return; 307 | } 308 | for (int i = 0; i < im_vec_size; i++) { 309 | cv::Mat input = cv::Mat::zeros(cv::Size(nWidth, nHeight), nType); 310 | memcpy(input.data, imgs[i], nHeight * nWidth * nChannel * sizeof(uchar)); 311 | im_vec.emplace_back(std::move(input)); 312 | } 313 | if (!model->Predict(im_vec, &results, 1)) { 314 | LOGC("ERR", "predict batch images failed"); 315 | } 316 | // batch修改：这里应该会得到返回的每张图的label_map，那么下面就应该分别处理results中每张图对应的label_map 317 | for (int i = 0; i < im_vec_size; i++) { 318 | std::vector result_map = results[i].seg_result->label_map.data; // vector -- 结果map 319 | // 拷贝输出结果到输出上返回 -- 将vector转成unsigned char * 320 | memcpy(output[i], &result_map[0], result_map.size() * sizeof(uchar)); 321 | } 322 | } 323 | 324 | 325 | 326 | 327 | /* 328 | * 识别推理接口 329 | * 330 | * img: input for predicting. 331 | * 332 | * nWidth: width of img. 333 | * 334 | * nHeight: height of img. 335 | * 336 | * nChannel: channel of img. 337 | * 338 | * score: result of pridict ,include score 339 | * 340 | * category: result of pridict ,include category_string 341 | * 342 | * category_id: result of pridict ,include category_id 343 | */ 344 | extern "C" __declspec(dllexport) void Cls_ModelPredict(PaddleDeploy::Model * model, const unsigned char* img, int nWidth, int nHeight, int nChannel, float* score, char* category, int* category_id) 345 | { 346 | // prepare data 347 | std::vector imgs; 348 | 349 | int nType = 0; 350 | if (nChannel == 3) 351 | { 352 | nType = CV_8UC3; 353 | } 354 | else 355 | { 356 | std::cout << "Only support 3 channel image." << std::endl; 357 | return; 358 | } 359 | 360 | cv::Mat input = cv::Mat::zeros(cv::Size(nWidth, nHeight), nType); 361 | memcpy(input.data, img, nHeight * nWidth * nChannel * sizeof(uchar)); 362 | cv::imwrite("D:\\1.png", input); 363 | imgs.push_back(std::move(input)); 364 | 365 | // predict 366 | std::vector results; 367 | //LOGC("INFO", "begin predict"); 368 | model->Predict(imgs, &results, 1); 369 | //LOGC("INFO", "got pred result: score=%f", results[0].clas_result->score); 370 | //LOGC("INFO", "got pred result: category_id=%d", results[0].clas_result->category_id); 371 | //LOGC("INFO", "got pred result: category=%s", results[0].clas_result->category); 372 | *category_id = results[0].clas_result->category_id; 373 | // 拷贝输出类别结果到输出上返回 -- string --> char* 374 | memcpy(category, results[0].clas_result->category.c_str(), strlen(results[0].clas_result->category.c_str())); 375 | // 拷贝输出概率值返回 376 | *score = results[0].clas_result->score; 377 | } 378 | 379 | 380 | /* 381 | * MaskRCNN推理接口 382 | * 383 | * img: input for predicting. 384 | * 385 | * nWidth: width of img. 386 | * 387 | * nHeight: height of img. 388 | * 389 | * nChannel: channel of img. 390 | * 391 | * box_output: result of pridict ,include label+score+bbox 392 | * 393 | * mask_output: result of pridict ,include label_map 394 | * 395 | * nBoxesNum: result of pridict ,include BoxesNum 396 | * 397 | * LabelList: result of pridict ,include LabelList 398 | */ 399 | extern "C" __declspec(dllexport) void Mask_ModelPredict(PaddleDeploy::Model * model, const unsigned char* img, int nWidth, int nHeight, int nChannel, float* box_output, unsigned char* mask_output, int* nBoxesNum, char* LabelList) 400 | { 401 | // prepare data 402 | std::vector imgs; 403 | 404 | int nType = 0; 405 | if (nChannel == 3) 406 | { 407 | nType = CV_8UC3; 408 | } 409 | else 410 | { 411 | std::cout << "Only support 3 channel image." << std::endl; 412 | return; 413 | } 414 | 415 | cv::Mat input = cv::Mat::zeros(cv::Size(nWidth, nHeight), nType); 416 | memcpy(input.data, img, nHeight * nWidth * nChannel * sizeof(uchar)); 417 | imgs.push_back(std::move(input)); 418 | 419 | // predict -- 多次点击单张推理时会出错 420 | std::vector results; 421 | model->Predict(imgs, &results, 1); // 在Infer处发生错误 422 | 423 | nBoxesNum[0] = results[0].det_result->boxes.size(); // 得到单张图片预测的bounding box数 424 | std::string label = ""; 425 | 426 | for (int i = 0; i < results[0].det_result->boxes.size(); i++) // 得到所有框的数据 427 | { 428 | // 边界框预测结果 429 | label = label + results[0].det_result->boxes[i].category + " "; 430 | // labelindex 431 | box_output[i * 6 + 0] = results[0].det_result->boxes[i].category_id; // 类别的id 432 | // score 433 | box_output[i * 6 + 1] = results[0].det_result->boxes[i].score; // 得分 434 | //// box 435 | box_output[i * 6 + 2] = results[0].det_result->boxes[i].coordinate[0]; // x1, y1, x2, y2 436 | box_output[i * 6 + 3] = results[0].det_result->boxes[i].coordinate[1]; // 左上、右下的顶点 437 | box_output[i * 6 + 4] = results[0].det_result->boxes[i].coordinate[2]; 438 | box_output[i * 6 + 5] = results[0].det_result->boxes[i].coordinate[3]; 439 | 440 | //Mask预测结果 441 | for (int j = 0; j < results[0].det_result->boxes[i].mask.data.size(); j++) 442 | { 443 | if (mask_output[j] == 0) 444 | { 445 | mask_output[j] = results[0].det_result->boxes[i].mask.data[j]; 446 | } 447 | } 448 | } 449 | memcpy(LabelList, label.c_str(), strlen(label.c_str())); 450 | } 451 | 452 | 453 | /* 454 | * 模型销毁/注销接口 455 | */ 456 | extern "C" __declspec(dllexport) void DestructModel(PaddleDeploy::Model * model) 457 | { 458 | if (model != NULL) { 459 | delete model; 460 | model = NULL; 461 | } 462 | if (model == NULL) LOGC("INFO", "destruct model success"); 463 | else LOGC("ERR", "delete model failed"); 464 | } 465 | 466 | 467 | // 新增二次封装：初始化 468 | void ModelWrapper::InitModelEnter(const char* model_type, const char* model_dir, int gpu_id, bool use_trt, 469 | const std::vectormin_input_shape, const std::vectormax_input_shape, const std::vectoroptim_input_shape, int precision, int min_subgraph_size) 470 | { 471 | // 初始化线程池：创建指定个数线程，每个线程指定到线程池的一个线程号 472 | pool = new ThreadPool(num_threads); 473 | pool->init(); 474 | 475 | std::string model_filename = std::string(model_dir) + "\\model.pdmodel"; 476 | std::string params_filename = std::string(model_dir) + "\\model.pdiparams"; 477 | std::string cfg_file = std::string(model_dir) + "\\deploy.yaml"; 478 | 479 | bool use_gpu = true; 480 | char* paddle_model_type = NULL; 481 | if (!use_trt) { 482 | _model = InitModel(model_type, 483 | model_filename.c_str(), // *.pdmodel 484 | params_filename.c_str(), // *.pdiparams 485 | cfg_file.c_str(), // *.yaml 486 | use_gpu, 487 | gpu_id, 488 | paddle_model_type); 489 | } 490 | else 491 | { 492 | _model = InitModel_TRT(model_type, 493 | model_filename.c_str(), // *.pdmodel 494 | params_filename.c_str(), // *.pdiparams 495 | cfg_file.c_str(), // *.yaml 496 | use_gpu, 497 | gpu_id, 498 | paddle_model_type, 499 | min_input_shape, max_input_shape, optim_input_shape, precision, min_subgraph_size); 500 | } 501 | } 502 | 503 | // 新增二次封装：单图推理 504 | void ModelWrapper::SegPredictEnter(unsigned char* imageData, int width, int height, int channels, unsigned char* result_map) 505 | { 506 | cv::Mat src; 507 | if (channels == 1) { 508 | src = cv::Mat(height, width, CV_8UC1, imageData); 509 | cv::cvtColor(src, src, cv::COLOR_GRAY2BGR); 510 | } 511 | else 512 | { 513 | src = cv::Mat(height, width, CV_8UC3, imageData); 514 | } 515 | int predChannels = src.channels(); 516 | UCHAR* _imageData = src.data; 517 | auto future1 = pool->submit(Seg_ModelPredict, _model, _imageData, width, height, predChannels, result_map); 518 | future1.get(); 519 | } 520 | 521 | // 检测模型 522 | void ModelWrapper::DetPredictEnter(unsigned char* imageData, int width, int height, int channels, float* output, int* nBoxesNum, char* LabelList) 523 | { 524 | cv::Mat src; 525 | if (channels == 1) { 526 | src = cv::Mat(height, width, CV_8UC1, imageData); 527 | cv::cvtColor(src, src, cv::COLOR_GRAY2BGR); 528 | } 529 | else 530 | { 531 | src = cv::Mat(height, width, CV_8UC3, imageData); 532 | } 533 | int predChannels = src.channels(); 534 | UCHAR* _imageData = src.data; 535 | auto future1 = pool->submit(Det_ModelPredict, _model, _imageData, width, height, predChannels, output, nBoxesNum, LabelList); 536 | future1.get(); 537 | } 538 | 539 | // 分类模型 540 | void ModelWrapper::ClsPredictEnter(unsigned char* imageData, int width, int height, int channels, float* score, char* category, int* category_id) 541 | { 542 | cv::Mat src; 543 | if (channels == 1) { 544 | src = cv::Mat(height, width, CV_8UC1, imageData); 545 | cv::cvtColor(src, src, cv::COLOR_GRAY2BGR); 546 | } 547 | else 548 | { 549 | src = cv::Mat(height, width, CV_8UC3, imageData); 550 | } 551 | int predChannels = src.channels(); 552 | UCHAR* _imageData = src.data; 553 | auto future1 = pool->submit(Cls_ModelPredict, _model, _imageData, width, height, predChannels, score, category, category_id); 554 | future1.get(); 555 | } 556 | 557 | // Mask模型 558 | void ModelWrapper::MaskPredictEnter(unsigned char* imageData, int width, int height, int channels, float* box_output, unsigned char* mask_output, int* nBoxesNum, char* LabelList) 559 | { 560 | cv::Mat src; 561 | if (channels == 1) { 562 | src = cv::Mat(height, width, CV_8UC1, imageData); 563 | cv::cvtColor(src, src, cv::COLOR_GRAY2BGR); 564 | } 565 | else 566 | { 567 | src = cv::Mat(height, width, CV_8UC3, imageData); 568 | } 569 | int predChannels = src.channels(); 570 | UCHAR* _imageData = src.data; 571 | auto future1 = pool->submit(Mask_ModelPredict, _model, _imageData, width, height, predChannels, box_output, mask_output, nBoxesNum, LabelList); 572 | future1.get(); 573 | } 574 | 575 | 576 | // 新增二次封装：模型资源释放 577 | void ModelWrapper::DestructModelEnter() 578 | { 579 | // 释放线程池中所有线程 580 | pool->shutdown(); 581 | if (pool != NULL) { 582 | delete pool; 583 | pool = NULL; 584 | } 585 | // 释放模型资源 586 | if (_model != NULL) { 587 | DestructModel(_model); 588 | } 589 | } 590 | 591 | 592 | // 新增二次封装接口api 593 | extern "C" __declspec(dllexport) ModelWrapper * ModelObjInit(const char* model_type, const char* model_dir, int gpu_id, bool use_trt, 594 | const std::vectormin_input_shape, const std::vectormax_input_shape, const std::vectoroptim_input_shape, int precision, int min_subgraph_size) 595 | { 596 | ModelWrapper* modelObj = new ModelWrapper(); 597 | modelObj->InitModelEnter(model_type, model_dir, gpu_id, use_trt, min_input_shape, max_input_shape, optim_input_shape, precision, min_subgraph_size); 598 | return modelObj; 599 | } 600 | 601 | 602 | extern "C" __declspec(dllexport) void ModelObjDestruct(ModelWrapper * modelObj) 603 | { 604 | // 先释放模型内部的资源 605 | modelObj->DestructModelEnter(); 606 | // 再释放堆区模型资源 607 | delete modelObj; 608 | } 609 | 610 | extern "C" __declspec(dllexport) void ModelObjPredict_Seg(ModelWrapper * modelObj, unsigned char* imageData, int width, int height, int channels, unsigned char* resultMap) 611 | { 612 | modelObj->SegPredictEnter(imageData, width, height, channels, resultMap); 613 | } 614 | 615 | extern "C" __declspec(dllexport) void ModelObjPredict_Det(ModelWrapper * modelObj, unsigned char* imageData, int width, int height, int channels, float* output, int* nBoxesNum, char* LabelList) 616 | { 617 | modelObj->DetPredictEnter(imageData, width, height, channels, output, nBoxesNum, LabelList); 618 | } 619 | 620 | extern "C" __declspec(dllexport) void ModelObjPredict_Cls(ModelWrapper * modelObj, unsigned char* imageData, int width, int height, int channels, float* score, char* category, int* category_id) 621 | { 622 | modelObj->ClsPredictEnter(imageData, width, height, channels, score, category, category_id); 623 | } 624 | 625 | extern "C" __declspec(dllexport) void ModelObjPredict_Mask(ModelWrapper * modelObj, unsigned char* imageData, int width, int height, int channels, float* box_output, unsigned char* mask_output, int* nBoxesNum, char* LabelList) 626 | { 627 | modelObj->MaskPredictEnter(imageData, width, height, channels, box_output, mask_output, nBoxesNum, LabelList); 628 | } -------------------------------------------------------------------------------- /model_infer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ximitiejiang/model_infer_multiThreads/66e4fefe6fc17cda08c3012a5618d8ed0feeeff7/model_infer.h -------------------------------------------------------------------------------- /thread_pool.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | template 25 | class SafeQueue { 26 | private: 27 | std::queue m_queue; 28 | std::mutex m_mutex; 29 | 30 | public: 31 | SafeQueue() {} 32 | 33 | SafeQueue(SafeQueue& other) {} 34 | 35 | ~SafeQueue() {} 36 | 37 | bool empty() { 38 | std::unique_lock lock(m_mutex); 39 | return m_queue.empty(); 40 | } 41 | 42 | int size() { 43 | std::unique_lock lock(m_mutex); 44 | return m_queue.size(); 45 | } 46 | 47 | void enqueue(T& t) { 48 | std::unique_lock lock(m_mutex); 49 | m_queue.push(t); 50 | } 51 | 52 | bool dequeue(T& t) { 53 | std::unique_lock lock(m_mutex); 54 | if (m_queue.empty()) { 55 | return false; 56 | } 57 | t = std::move(m_queue.front()); 58 | m_queue.pop(); 59 | return true; 60 | } 61 | }; 62 | 63 | class ThreadPool { 64 | private: 65 | class ThreadWorker { 66 | 67 | private: 68 | int m_id; 69 | ThreadPool* m_pool; 70 | 71 | public: 72 | ThreadWorker(ThreadPool* pool, const int id) 73 | : m_pool(pool), m_id(id) { 74 | } 75 | 76 | void operator()() { 77 | std::function func; 78 | bool dequeued; 79 | while (!m_pool->m_shutdown) { 80 | { 81 | std::unique_lock lock(m_pool->m_conditional_mutex); 82 | if (m_pool->m_queue.empty()) { 83 | m_pool->m_conditional_lock.wait(lock); 84 | } 85 | dequeued = m_pool->m_queue.dequeue(func); 86 | } 87 | if (dequeued) { 88 | func(); 89 | } 90 | } 91 | } 92 | }; 93 | 94 | bool m_shutdown; 95 | SafeQueue> m_queue; 96 | std::vector m_threads; 97 | std::mutex m_conditional_mutex; 98 | std::condition_variable m_conditional_lock; 99 | 100 | public: 101 | ThreadPool(const int n_threads) 102 | : m_threads(std::vector(n_threads)), m_shutdown(false) { 103 | } 104 | 105 | ThreadPool(const ThreadPool&) = delete; 106 | ThreadPool(ThreadPool&&) = delete; 107 | ThreadPool& operator=(const ThreadPool&) = delete; 108 | ThreadPool& operator=(ThreadPool&&) = delete; 109 | 110 | void init() { 111 | for (int i = 0; i < m_threads.size(); ++i) { 112 | m_threads[i] = std::thread(ThreadWorker(this, i)); 113 | } 114 | } 115 | 116 | void shutdown() { 117 | m_shutdown = true; 118 | m_conditional_lock.notify_all(); 119 | for (int i = 0; i < m_threads.size(); ++i) { 120 | if (m_threads[i].joinable()) { 121 | m_threads[i].join(); 122 | } 123 | } 124 | } 125 | 126 | // Submit a function to be executed asynchronously by the pool 127 | template 128 | auto submit(F&& f, Args&&... args) -> std::future { 129 | std::function func = std::bind(std::forward(f), std::forward(args)...); 130 | auto task_ptr = std::make_shared>(func); 131 | std::function wrapper_func = [task_ptr]() { 132 | (*task_ptr)(); 133 | }; 134 | m_queue.enqueue(wrapper_func); 135 | m_conditional_lock.notify_one(); 136 | return task_ptr->get_future(); 137 | } 138 | }; --------------------------------------------------------------------------------