label = {
15 | " ",
16 | "#",
17 | "京",
18 | "津",
19 | "冀",
20 | "晋",
21 | "蒙",
22 | "辽",
23 | "吉",
24 | "黑",
25 | "沪",
26 | "苏",
27 | "浙",
28 | "皖",
29 | "闽",
30 | "赣",
31 | "鲁",
32 | "豫",
33 | "鄂",
34 | "湘",
35 | "粤",
36 | "桂",
37 | "琼",
38 | "渝",
39 | "川",
40 | "贵",
41 | "云",
42 | "藏",
43 | "陕",
44 | "甘",
45 | "青",
46 | "宁",
47 | "新",
48 | "使",
49 | "0",
50 | "1",
51 | "2",
52 | "3",
53 | "4",
54 | "5",
55 | "6",
56 | "7",
57 | "8",
58 | "9",
59 | "A",
60 | "B",
61 | "C",
62 | "D",
63 | "E",
64 | "F",
65 | "G",
66 | "H",
67 | "J",
68 | "K",
69 | "L",
70 | "M",
71 | "N",
72 | "O",
73 | "P",
74 | "Q",
75 | "R",
76 | "S",
77 | "T",
78 | "U",
79 | "V",
80 | "W",
81 | "X",
82 | "Y",
83 | "Z",
84 | "警",
85 | "学",
86 | "挂",
87 | "港",
88 | "澳",
89 | "领",
90 | "民",
91 | "航",
92 | "应",
93 | "急"
94 | };
95 |
96 |
97 | #endif /* SRC_LABEL_HPP_ */
98 |
--------------------------------------------------------------------------------
/install.md:
--------------------------------------------------------------------------------
1 | ## 安装依赖
2 | cmake >= 3.10.0
3 | opencv >= 3.0.0
4 | openmp
5 |
6 | ## x86平台Linux安装指令
7 |
8 | git clone https://github.com/lqian/light-LPR
9 | cd light-LPR && mkdir build && cd build
10 | cmake ../
11 | make
12 |
13 |
14 | ## ARM平台Linux安装指令
15 |
16 | git clone https://github.com/lqian/light-LPR
17 | cd light-LPR && mkdir build && cd build
18 | cmake ../ -DLIGHT_LPR_ARCH=arm
19 | make
20 |
21 |
22 | ## Windows平台上安装指令
23 | - 下载cmake 3.10以上版本并安装
24 | - 首先下载Visual Studio 2017或者 Native Builder Tool for Visual Studio 2017，安装c++编译工具
25 | - 如果编译64位系统，下载64位[opencv-3.4.2-install-win64.zip](https://pan.baidu.com/s/1CtabojjfEK-bK_XwfG9HTA), 32位系统则下载[opencv-3.4.2-install-win32.zip](https://pan.baidu.com/s/1E7zhRsrrpc9JEhB_6gpehg)，解压到任意目录
26 | - 克隆[MNN](https://github.com/alibaba/MNN)的源码
27 | - 下载[flatc_windows_exe.zip](https://github.com/google/flatbuffers/releases/download/v1.11.0/flatc_windows_exe.zip)，把flatc.exe可执行文件复制到{MNN}/3rd_party/flatbuffers/tmp目录下
28 | - 以管理员权限打开powershell.exe，然后执行set-executionpolicy -executionpolicy unrestricted，提示选Y
29 | - 注释掉MNN的源码目录中的CMakelist.txt中的`COMMAND powershell ${CMAKE_CURRENT_SOURCE_DIR}/schema/generate.ps1 -lazy`这行，大约在461行
30 |
31 | > cd MNN
32 | > schema\enerate.ps1
33 | > mkdir build
34 | > cd build
35 | 按win键，根据需要，搜索x86 native tools command prompt for VS 2017 或者x64 native tools command prompt for VS 2017
36 | > cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ../
37 | > nmake
38 | 把编译成功的MNN.dll、MNN.lib文件复制到light-LPR项目的lib目录下
39 | > cd light-LPR && mkdir build && cd build
40 | > set OpenCV_DIR=/path/to/opencv-install/directory
41 | > cmake -G "NMake Makefiles" ..
42 | > nmake
43 |
44 |
45 | ## 运行测试
46 | `./examples/demo ../models/ [/path/to/a/image]`
47 | 本项目在Fedora 29，CentOS 7.6, Windows 10 64位家庭版，Ubuntu 18.04 mate for ARM平台测试通过
48 |
49 |
50 | ## 参考和引用
51 | - [Alibaba MNN](https://github.com/alibaba/MNN)
52 | - [License-Plate-Detect-Recognition-via-Deep-Neural-Networks-accuracy-up-to-99.9](https://github.com/zhubenfu/License-Plate-Detect-Recognition-via-Deep-Neural-Networks-accuracy-up-to-99.9)
53 | - [Caffe_OCR](https://github.com/senlinuc/caffe_ocr)
54 | - [MNN MTCNN CPU OPENCL](https://github.com/liushuan/MNN-MTCNN-CPU-OPENCL)
--------------------------------------------------------------------------------
/install_en.md:
--------------------------------------------------------------------------------
1 | ## requirements
2 | cmake >= 3.10.0
3 | opencv >= 3.0.0
4 | openmp
5 |
6 | ## installation commands for x86 Linux
7 |
8 | git clone https://github.com/lqian/light-LPR
9 | cd light-LPR && mkdir build && cd build
10 | cmake ../
11 | make
12 |
13 |
14 | ## installation commands for ARM Linux
15 |
16 | git clone https://github.com/lqian/light-LPR
17 | cd light-LPR && mkdir build && cd build
18 | cmake ../ -DLIGHT_LPR_ARCH=arm
19 | make
20 |
21 |
22 | ## installation commands for Windows
23 | - Download and install cmake 3.10 and above
24 | - Download Visual Studio 2017 or Native Builder Tool for Visual Studio 2017, install c ++ compilation tool
25 | - if you compile for x64, download [opencv-3.4.2-install-win64.zip](https://pan.baidu.com/s/1CtabojjfEK-bK_XwfG9HTA), x86 archictecture [opencv-3.4.2-install-win32.zip](https://pan.baidu.com/s/1E7zhRsrrpc9JEhB_6gpehg)，and unzip the file.
26 | - clone source code: [MNN](https://github.com/alibaba/MNN)
27 | - download [flatc_windows_exe.zip](https://github.com/google/flatbuffers/releases/download/v1.11.0/flatc_windows_exe.zip)，and put the file to {MNN}/3rd_party/flatbuffers/tmp directory
28 | - run powershell.exe as administrator，and then run the command: set-executionpolicy -executionpolicy unrestricted，select Y from prompt.
29 | - comment the line `COMMAND powershell ${CMAKE_CURRENT_SOURCE_DIR}/schema/generate.ps1 -lazy` in CMakeLists.txt of MNN project，the line number is about 461.
30 |
31 | > cd MNN
32 | > schema\enerate.ps1
33 | > mkdir build
34 | > cd build
35 | run x86 native tools command prompt for VS 2017 or x64 native tools command prompt for VS 2017
36 | > cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ../
37 | > nmake
38 | copy the MNN.dll、MNN.lib into light-LPR/lib directory
39 | > cd light-LPR && mkdir build && cd build
40 | > set OpenCV_DIR=/path/to/opencv-install/directory
41 | > cmake -G "NMake Makefiles" ..
42 | > nmake
43 |
44 |
45 | ## test
46 | `./examples/demo ../models/ [/path/to/a/image]`
47 | This project passed the test on Fedora 29, CentOS 7.6, Windows 10 64-bit Home Edition, Ubuntu 18.04 mate for ARM platform
48 |
49 |
50 | ## reference
51 | - [Alibaba MNN](https://github.com/alibaba/MNN)
52 | - [License-Plate-Detect-Recognition-via-Deep-Neural-Networks-accuracy-up-to-99.9](https://github.com/zhubenfu/License-Plate-Detect-Recognition-via-Deep-Neural-Networks-accuracy-up-to-99.9)
53 | - [Caffe_OCR](https://github.com/senlinuc/caffe_ocr)
54 | - [MNN MTCNN CPU OPENCL](https://github.com/liushuan/MNN-MTCNN-CPU-OPENCL)
--------------------------------------------------------------------------------
/kr.md:
--------------------------------------------------------------------------------
1 | Light-LPR은 임베디드 장치, 휴대폰 및 x86 플랫폼에서 실행할 수있는 번호판 인식을 목표로하는 오픈 소스 프로젝트로, 다양한 시나리오에서 번호판 인식을 지원하는 것을 목표로하며, 번호판 문자 인식의 정확도는 99.95 %를 초과하고 포괄적 인 인식 정확도는 99를 초과합니다. %, 다중 국가, 다중 언어 및 다중 유형 번호판 인식을 지원합니다. [testing video resources](https://www.bilibili.com/video/BV12L4y1Y7K1?spm_id_from=333.999.0.0)
2 |
3 |
4 | [ About ](README.md) | [ English ](en.md) | [ 中文 ](cn-zh.md) | [ 中文繁体 ](cn-tw.md)| [ 한국어 ](kr.md)
5 |
6 | ## 비즈니스 에디션 지원 한국 번호판 인식
7 | | 번호판 유형 | 1973 년 이전 | 1996년 이전 | 2004년 이전 | 2006공식 | 2013공식 | 2019공식 |
8 | | --------: | :-----: | :----: | :----: | :----: |:----: |:----: |
9 | | 전용차 |Y | Y | Y | Y | Y | Y |
10 | | 상용차 | - | Y | Y | Y | Y | |
11 | | 특수차 | E | Y| Y| -| | |
12 | | 렌트카 | | Y |Y| Y| Y | |
13 | | 군용 | -|E|E|E| | |
14 | | 엔지니어링 차량 | | E| E | | | |
15 | | 외교 | | | E |E | E | |
16 |
17 | 비고： Y 지원，- 지원되지 않거나 사용할 수 없음，E 평가 단계에서
18 |
19 | ## 1080P 이미지 인식 벤치 마크 성능
20 | | | CPU | Memory | average cost of community version (ms) | average cost of Commercial version(ms) |
21 | | :-------- | :----- | :----: | ----: | ----: |
22 | | X86 | i5-8265 | 8G | 451 | < 50 |
23 | | ARM | A53 | 1G | 1532| < 160 |
24 | | Huwei P20 pro| ... | 4G | - | < 100 |
25 | | 3519A100 | ... | | - | < 16 (NPU support) |
26 | | 3516CV500 | ... | | - | < 45 (NPU support) |
27 |
28 | ## 지원되는 프로그래밍 언어
29 | - C/C++
30 | - C#
31 | - JAVA
32 |
33 | ## 설치 방법은 프로젝트 파일을 참조하십시오 [install_en.md](install_en.md)
34 |
35 | ## 기타
36 | 사업 제휴 연락처 : link.com@yeah.net, 휴대폰 : +86 18010870244, Skype: +86 18010870244
--------------------------------------------------------------------------------
/lib-arm/libMNN.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/lib-arm/libMNN.so
--------------------------------------------------------------------------------
/lib-win32/MNN.dll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/lib-win32/MNN.dll
--------------------------------------------------------------------------------
/lib-win32/MNN.lib:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/lib-win32/MNN.lib
--------------------------------------------------------------------------------
/lib-win32/MNN_Express.dll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/lib-win32/MNN_Express.dll
--------------------------------------------------------------------------------
/lib/libMNN.so:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/lib/libMNN.so
--------------------------------------------------------------------------------
/light-LPR.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/light-LPR.png
--------------------------------------------------------------------------------
/models/det1.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det1.caffemodel
--------------------------------------------------------------------------------
/models/det1.mnn:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det1.mnn
--------------------------------------------------------------------------------
/models/det1.prototxt:
--------------------------------------------------------------------------------
1 | name: "PNet"
2 | input: "data"
3 | input_dim: 1
4 | input_dim: 3
5 | input_dim: 12
6 | input_dim: 30
7 |
8 | layer {
9 | name: "conv1"
10 | type: "Convolution"
11 | bottom: "data"
12 | top: "conv1"
13 | param {
14 | lr_mult: 1
15 | decay_mult: 1
16 | }
17 | param {
18 | lr_mult: 2
19 | decay_mult: 0
20 | }
21 | convolution_param {
22 | num_output: 10
23 | kernel_size: 3
24 | stride: 1
25 | weight_filler {
26 | type: "xavier"
27 | }
28 | bias_filler {
29 | type: "constant"
30 | value: 0
31 | }
32 | }
33 | }
34 | layer {
35 | name: "PReLU1"
36 | type: "PReLU"
37 | bottom: "conv1"
38 | top: "conv1"
39 | }
40 | layer {
41 | name: "pool1"
42 | type: "Pooling"
43 | bottom: "conv1"
44 | top: "pool1"
45 | pooling_param {
46 | pool: MAX
47 | kernel_size: 2
48 | stride: 2
49 | }
50 | }
51 |
52 | layer {
53 | name: "conv2"
54 | type: "Convolution"
55 | bottom: "pool1"
56 | top: "conv2"
57 | param {
58 | lr_mult: 1
59 | decay_mult: 1
60 | }
61 | param {
62 | lr_mult: 2
63 | decay_mult: 0
64 | }
65 | convolution_param {
66 | num_output: 16
67 | kernel_size: 3
68 | stride: 1
69 | weight_filler {
70 | type: "xavier"
71 | }
72 | bias_filler {
73 | type: "constant"
74 | value: 0
75 | }
76 | }
77 | }
78 | layer {
79 | name: "PReLU2"
80 | type: "PReLU"
81 | bottom: "conv2"
82 | top: "conv2"
83 | }
84 |
85 | layer {
86 | name: "conv3"
87 | type: "Convolution"
88 | bottom: "conv2"
89 | top: "conv3"
90 | param {
91 | lr_mult: 1
92 | decay_mult: 1
93 | }
94 | param {
95 | lr_mult: 2
96 | decay_mult: 0
97 | }
98 | convolution_param {
99 | num_output: 32
100 | kernel_size: 3
101 | stride: 1
102 | weight_filler {
103 | type: "xavier"
104 | }
105 | bias_filler {
106 | type: "constant"
107 | value: 0
108 | }
109 | }
110 | }
111 | layer {
112 | name: "PReLU3"
113 | type: "PReLU"
114 | bottom: "conv3"
115 | top: "conv3"
116 | }
117 |
118 |
119 |
120 | layer {
121 | name: "conv4j-1"
122 | type: "Convolution"
123 | bottom: "conv3"
124 | top: "conv4-1"
125 | param {
126 | lr_mult: 1
127 | decay_mult: 1
128 | }
129 | param {
130 | lr_mult: 2
131 | decay_mult: 0
132 | }
133 | convolution_param {
134 | num_output: 2
135 | kernel_h:1
136 | kernel_w:10
137 | stride:1
138 |
139 | weight_filler {
140 | type: "xavier"
141 | }
142 | bias_filler {
143 | type: "constant"
144 | value: 0
145 | }
146 | }
147 | }
148 |
149 |
150 |
151 |
152 | layer {
153 | name: "conv4j-2"
154 | type: "Convolution"
155 | bottom: "conv3"
156 | top: "conv4-2"
157 | param {
158 | lr_mult: 1
159 | decay_mult: 1
160 | }
161 | param {
162 | lr_mult: 2
163 | decay_mult: 0
164 | }
165 | convolution_param {
166 | num_output: 4
167 | kernel_h:1
168 | kernel_w:10
169 | stride: 1
170 |
171 | weight_filler {
172 | type: "xavier"
173 | }
174 | bias_filler {
175 | type: "constant"
176 | value: 0
177 | }
178 | }
179 | }
180 | layer {
181 | name: "prob1"
182 | type: "Softmax"
183 | bottom: "conv4-1"
184 | top: "prob1"
185 | }
--------------------------------------------------------------------------------
/models/det2.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det2.caffemodel
--------------------------------------------------------------------------------
/models/det2.mnn:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det2.mnn
--------------------------------------------------------------------------------
/models/det2.prototxt:
--------------------------------------------------------------------------------
1 | name: "RNet"
2 | input: "data"
3 | input_dim: 1
4 | input_dim: 3
5 | input_dim: 24
6 | input_dim: 60
7 | ################################################
8 | layer {
9 | name: "conv1"
10 | type: "Convolution"
11 | bottom: "data"
12 | top: "conv1"
13 | param {
14 | lr_mult: 1
15 | decay_mult: 1
16 | }
17 | param {
18 | lr_mult: 2
19 | decay_mult: 1
20 | }
21 | convolution_param {
22 | num_output: 28
23 | kernel_size: 3
24 | stride: 1
25 | weight_filler {
26 | type: "xavier"
27 | }
28 | bias_filler {
29 | type: "constant"
30 | value: 0
31 | }
32 | }
33 | }
34 | layer {
35 | name: "prelu1"
36 | type: "PReLU"
37 | bottom: "conv1"
38 | top: "conv1"
39 | }
40 | layer {
41 | name: "pool1"
42 | type: "Pooling"
43 | bottom: "conv1"
44 | top: "pool1"
45 | pooling_param {
46 | pool: MAX
47 | kernel_size: 3
48 | stride: 2
49 | }
50 | }
51 |
52 | layer {
53 | name: "conv2"
54 | type: "Convolution"
55 | bottom: "pool1"
56 | top: "conv2"
57 | param {
58 | lr_mult: 1
59 | decay_mult: 1
60 | }
61 | param {
62 | lr_mult: 2
63 | decay_mult: 1
64 | }
65 | convolution_param {
66 | num_output: 48
67 | kernel_size: 3
68 | stride: 1
69 | weight_filler {
70 | type: "xavier"
71 | }
72 | bias_filler {
73 | type: "constant"
74 | value: 0
75 | }
76 | }
77 | }
78 | layer {
79 | name: "prelu2"
80 | type: "PReLU"
81 | bottom: "conv2"
82 | top: "conv2"
83 | }
84 | layer {
85 | name: "pool2"
86 | type: "Pooling"
87 | bottom: "conv2"
88 | top: "pool2"
89 | pooling_param {
90 | pool: MAX
91 | kernel_size: 3
92 | stride: 2
93 | }
94 | }
95 | ####################################
96 |
97 | ##################################
98 | layer {
99 | name: "conv3"
100 | type: "Convolution"
101 | bottom: "pool2"
102 | top: "conv3"
103 | param {
104 | lr_mult: 1
105 | decay_mult: 1
106 | }
107 | param {
108 | lr_mult: 2
109 | decay_mult: 1
110 | }
111 | convolution_param {
112 | num_output: 64
113 | kernel_size: 2
114 | stride: 1
115 | weight_filler {
116 | type: "xavier"
117 | }
118 | bias_filler {
119 | type: "constant"
120 | value: 0
121 | }
122 | }
123 | }
124 | layer {
125 | name: "prelu3"
126 | type: "PReLU"
127 | bottom: "conv3"
128 | top: "conv3"
129 | }
130 | ###############################
131 |
132 | ###############################
133 |
134 | layer {
135 | name: "conv4i"
136 | type: "InnerProduct"
137 | bottom: "conv3"
138 | top: "conv4"
139 | param {
140 | lr_mult: 1
141 | decay_mult: 1
142 | }
143 | param {
144 | lr_mult: 2
145 | decay_mult: 1
146 | }
147 | inner_product_param {
148 | num_output: 128
149 | weight_filler {
150 | type: "xavier"
151 | }
152 | bias_filler {
153 | type: "constant"
154 | value: 0
155 | }
156 | }
157 | }
158 | layer {
159 | name: "prelu4"
160 | type: "PReLU"
161 | bottom: "conv4"
162 | top: "conv4"
163 | }
164 |
165 | layer {
166 | name: "conv5i-1"
167 | type: "InnerProduct"
168 | bottom: "conv4"
169 | top: "conv5-1"
170 | param {
171 | lr_mult: 1
172 | decay_mult: 1
173 | }
174 | param {
175 | lr_mult: 2
176 | decay_mult: 1
177 | }
178 | inner_product_param {
179 | num_output: 2
180 | #kernel_size: 1
181 | #stride: 1
182 | weight_filler {
183 | type: "xavier"
184 | }
185 | bias_filler {
186 | type: "constant"
187 | value: 0
188 | }
189 | }
190 | }
191 |
192 |
193 |
194 |
195 |
196 | layer {
197 | name: "conv5i-2"
198 | type: "InnerProduct"
199 | bottom: "conv4"
200 | top: "conv5-2"
201 | param {
202 | lr_mult: 1
203 | decay_mult: 1
204 | }
205 | param {
206 | lr_mult: 2
207 | decay_mult: 1
208 | }
209 | inner_product_param {
210 | num_output: 4
211 | #kernel_size: 1
212 | #stride: 1
213 | weight_filler {
214 | type: "xavier"
215 | }
216 | bias_filler {
217 | type: "constant"
218 | value: 0
219 | }
220 | }
221 | }
222 |
223 | layer {
224 | name: "prob1"
225 | type: "Softmax"
226 | bottom: "conv5-1"
227 | top: "prob1"
228 | }
--------------------------------------------------------------------------------
/models/det3.caffemodel:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det3.caffemodel
--------------------------------------------------------------------------------
/models/det3.mnn:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/det3.mnn
--------------------------------------------------------------------------------
/models/det3.prototxt:
--------------------------------------------------------------------------------
1 | name: "ONet"
2 | input: "data"
3 | input_dim: 1
4 | input_dim: 3
5 | input_dim: 48
6 | input_dim: 120
7 |
8 | ##################################
9 | layer {
10 | name: "conv1"
11 | type: "Convolution"
12 | bottom: "data"
13 | top: "conv1"
14 | param {
15 | lr_mult: 1
16 | decay_mult: 1
17 | }
18 | param {
19 | lr_mult: 2
20 | decay_mult: 1
21 | }
22 | convolution_param {
23 | num_output: 32
24 | kernel_size: 3
25 | stride: 1
26 | weight_filler {
27 | type: "xavier"
28 | }
29 | bias_filler {
30 | type: "constant"
31 | value: 0
32 | }
33 | }
34 | }
35 | layer {
36 | name: "prelu1"
37 | type: "PReLU"
38 | bottom: "conv1"
39 | top: "conv1"
40 | }
41 | layer {
42 | name: "pool1"
43 | type: "Pooling"
44 | bottom: "conv1"
45 | top: "pool1"
46 | pooling_param {
47 | pool: MAX
48 | kernel_size: 3
49 | stride: 2
50 | }
51 | }
52 | layer {
53 | name: "conv2"
54 | type: "Convolution"
55 | bottom: "pool1"
56 | top: "conv2"
57 | param {
58 | lr_mult: 1
59 | decay_mult: 1
60 | }
61 | param {
62 | lr_mult: 2
63 | decay_mult: 1
64 | }
65 | convolution_param {
66 | num_output: 64
67 | kernel_size: 3
68 | stride: 1
69 | weight_filler {
70 | type: "xavier"
71 | }
72 | bias_filler {
73 | type: "constant"
74 | value: 0
75 | }
76 | }
77 | }
78 |
79 | layer {
80 | name: "prelu2"
81 | type: "PReLU"
82 | bottom: "conv2"
83 | top: "conv2"
84 | }
85 | layer {
86 | name: "pool2"
87 | type: "Pooling"
88 | bottom: "conv2"
89 | top: "pool2"
90 | pooling_param {
91 | pool: MAX
92 | kernel_size: 3
93 | stride: 2
94 | }
95 | }
96 |
97 | layer {
98 | name: "conv3"
99 | type: "Convolution"
100 | bottom: "pool2"
101 | top: "conv3"
102 | param {
103 | lr_mult: 1
104 | decay_mult: 1
105 | }
106 | param {
107 | lr_mult: 2
108 | decay_mult: 1
109 | }
110 | convolution_param {
111 | num_output: 64
112 | kernel_size: 3
113 | weight_filler {
114 | type: "xavier"
115 | }
116 | bias_filler {
117 | type: "constant"
118 | value: 0
119 | }
120 | }
121 | }
122 | layer {
123 | name: "prelu3"
124 | type: "PReLU"
125 | bottom: "conv3"
126 | top: "conv3"
127 | }
128 | layer {
129 | name: "pool3"
130 | type: "Pooling"
131 | bottom: "conv3"
132 | top: "pool3"
133 | pooling_param {
134 | pool: MAX
135 | kernel_size: 2
136 | stride: 2
137 | }
138 | }
139 | layer {
140 | name: "conv4"
141 | type: "Convolution"
142 | bottom: "pool3"
143 | top: "conv4"
144 | param {
145 | lr_mult: 1
146 | decay_mult: 1
147 | }
148 | param {
149 | lr_mult: 2
150 | decay_mult: 1
151 | }
152 | convolution_param {
153 | num_output: 128
154 | kernel_size: 2
155 | weight_filler {
156 | type: "xavier"
157 | }
158 | bias_filler {
159 | type: "constant"
160 | value: 0
161 | }
162 | }
163 | }
164 | layer {
165 | name: "prelu4"
166 | type: "PReLU"
167 | bottom: "conv4"
168 | top: "conv4"
169 | }
170 |
171 |
172 | layer {
173 | name: "conv5i"
174 | type: "InnerProduct"
175 | bottom: "conv4"
176 | top: "conv5"
177 | param {
178 | lr_mult: 1
179 | decay_mult: 1
180 | }
181 | param {
182 | lr_mult: 2
183 | decay_mult: 1
184 | }
185 | inner_product_param {
186 | #kernel_size: 3
187 | num_output: 256
188 | weight_filler {
189 | type: "xavier"
190 | }
191 | bias_filler {
192 | type: "constant"
193 | value: 0
194 | }
195 | }
196 | }
197 |
198 | layer {
199 | name: "drop5i"
200 | type: "Dropout"
201 | bottom: "conv5"
202 | top: "conv5"
203 | dropout_param {
204 | dropout_ratio: 0.25
205 | }
206 | }
207 | layer {
208 | name: "prelu5"
209 | type: "PReLU"
210 | bottom: "conv5"
211 | top: "conv5"
212 | }
213 |
214 |
215 | layer {
216 | name: "conv6i-1"
217 | type: "InnerProduct"
218 | bottom: "conv5"
219 | top: "conv6-1"
220 | param {
221 | lr_mult: 1
222 | decay_mult: 1
223 | }
224 | param {
225 | lr_mult: 2
226 | decay_mult: 1
227 | }
228 | inner_product_param {
229 | #kernel_size: 1
230 | num_output: 2
231 | weight_filler {
232 | type: "xavier"
233 | }
234 | bias_filler {
235 | type: "constant"
236 | value: 0
237 | }
238 | }
239 | }
240 |
241 |
242 | layer {
243 | name: "conv6i-2"
244 | type: "InnerProduct"
245 | bottom: "conv5"
246 | top: "conv6-2"
247 | param {
248 | lr_mult: 1
249 | decay_mult: 1
250 | }
251 | param {
252 | lr_mult: 2
253 | decay_mult: 1
254 | }
255 | inner_product_param {
256 | #kernel_size: 1
257 | num_output: 4
258 | weight_filler {
259 | type: "xavier"
260 | }
261 | bias_filler {
262 | type: "constant"
263 | value: 0
264 | }
265 | }
266 | }
267 |
268 | layer {
269 | name: "conv6-3"
270 | type: "InnerProduct"
271 | bottom: "conv5"
272 | top: "conv6-3"
273 | param {
274 | lr_mult: 1
275 | decay_mult: 1
276 | }
277 | param {
278 | lr_mult: 2
279 | decay_mult: 1
280 | }
281 | inner_product_param {
282 | #kernel_size: 1
283 | num_output: 8
284 | weight_filler {
285 | type: "xavier"
286 | }
287 | bias_filler {
288 | type: "constant"
289 | value: 0
290 | }
291 | }
292 | }
293 |
294 |
295 | layer {
296 | name: "prob1"
297 | type: "Softmax"
298 | bottom: "conv6-1"
299 | top: "prob1"
300 | }
--------------------------------------------------------------------------------
/models/lpc.mnn:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/lpc.mnn
--------------------------------------------------------------------------------
/models/lpr.mnn:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lqian/light-LPR/6449d571739d164f3fd84d8862883f1f6dadb8ae/models/lpr.mnn
--------------------------------------------------------------------------------
/src/examples/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | find_package(OpenCV 3.4.0 REQUIRED COMPONENTS core imgcodecs imgproc highgui)
2 |
3 | file(GLOB_RECURSE test_srcs *.cpp)
4 |
5 | foreach(source_file ${test_srcs})
6 | # get file name
7 | get_filename_component(name ${source_file} NAME_WE)
8 | # get folder name
9 | get_filename_component(path ${source_file} PATH)
10 | get_filename_component(folder ${path} NAME_WE)
11 |
12 | add_executable(${name} ${source_file})
13 | target_link_libraries(${name} mlpdr ${OpenCV_LIBRARIES} ${OpenMP_CXX})
14 |
15 |
16 | # set back RUNTIME_OUTPUT_DIRECTORY
17 | set_target_properties(${name} PROPERTIES
18 | RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/${folder}")
19 |
20 | endforeach()
21 |
--------------------------------------------------------------------------------
/src/examples/demo.cpp:
--------------------------------------------------------------------------------
1 | /*
2 | * mtcnn_test.cpp
3 | *
4 | * Created on: Jun 25, 2019
5 | * Author: lqian
6 | */
7 |
8 | #include
9 | #include
10 | #include
11 |
12 | #include
13 | #include "mlpdr/MLPDR.h"
14 |
15 | using namespace std;
16 | using namespace mlpdr;
17 | using namespace cv;
18 | int main(int argc, char ** argv) {
19 | MLPDR detector(argv[1], 0.9f, 0.7f, 0.7f);
20 | Mat img = imread(argv[2]);
21 | assert(!img.empty());
22 | TickMeter tm;
23 | tm.start();
24 | // std::vector plateInfos = detector.Detect(img, 40, 3); // 608.23 ms
25 | vector plateInfos = detector.recognize(img);
26 | tm.stop();
27 | printf("detect cost: %f (ms)\n", tm.getTimeMilli());
28 |
29 | for (auto pi: plateInfos) {
30 | cout << "plateNo: " << pi.plateNo << endl;
31 | }
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/mlpdr/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | file(GLOB_RECURSE mlpdr_srcs *.cpp)
2 | file(GLOB mlpdr_api_srcs *_api.cpp)
3 |
4 | if(NOT BUILD_JAVA_API)
5 | list(REMOVE_ITEM mlpdr_srcs ${mlpdr_api_srcs} )
6 | endif()
7 |
8 | add_library(mlpdr ${mlpdr_srcs})
9 | target_link_libraries(mlpdr MNN ${OpenMP_CXX} ${OpenCV_LIBRARIES})
--------------------------------------------------------------------------------
/src/mlpdr/MLPDR.cpp:
--------------------------------------------------------------------------------
1 | /*
2 | * LPRDetector.cpp
3 | *
4 | * Created on: Jun 25, 2019
5 | * Author: lqian
6 | */
7 |
8 | #include "mlpdr/MLPDR.h"
9 | #include "mlpdr/label.hpp"
10 |
11 | #include
12 | #include
13 | #include
14 | #include
15 |
16 | namespace mlpdr {
17 |
18 | using namespace std;
19 | using namespace MNN;
20 | using namespace MNN::CV;
21 | using namespace cv;
22 |
23 | std::shared_ptr LPRNet = NULL;
24 | Session * ocr_session = nullptr;
25 | Tensor * ocr_input = nullptr;
26 | Tensor * ocr_output = nullptr;
27 |
28 | std::shared_ptr LPCNet_ = NULL;
29 | Session * lpc_session = nullptr;
30 | Tensor * lpc_input = nullptr;
31 | Tensor * lpc_output = nullptr;
32 |
33 |
34 | std::shared_ptr PNet_ = NULL;
35 | std::shared_ptr RNet_ = NULL;
36 | std::shared_ptr ONet_ = NULL;
37 |
38 |
39 | MNN::Session * sess_p = NULL;
40 | MNN::Session * sess_r = NULL;
41 | MNN::Session * sess_o = NULL;
42 |
43 | MNN::Tensor * p_input = nullptr;
44 | MNN::Tensor * p_out_pro = nullptr;
45 | MNN::Tensor * p_out_reg = nullptr;
46 |
47 | MNN::Tensor * r_input = nullptr;
48 | MNN::Tensor * r_out_pro = nullptr;
49 | MNN::Tensor * r_out_reg = nullptr;
50 |
51 | MNN::Tensor * o_input = nullptr;
52 | MNN::Tensor * o_out_pro = nullptr;
53 | MNN::Tensor * o_out_reg = nullptr;
54 | MNN::Tensor * o_out_lank = nullptr;
55 |
56 | std::shared_ptr pretreat_data;
57 | std::shared_ptr lpr_pretreat_data, lpc_pretreat_data;
58 |
59 | std::vector candidate_boxes_;
60 | std::vector total_boxes_;
61 |
62 | static float threhold_p = 0.7f;
63 | static float threhold_r = 0.8f;
64 | static float threhold_o = 0.8f;
65 | static float iou_threhold = 0.7f;
66 | static float factor = 0.709f;
67 | //static int min_face = 48;
68 |
69 | //pnet config
70 | static const float pnet_stride = 2;
71 | static const float pnet_cell_size_height = 12;
72 | static const float pnet_cell_size_width = 30;
73 | static const int pnet_max_detect_num = 5000;
74 | //mean & std
75 | static const float mean_val = 127.5f;
76 | static const float std_val = 0.0078125f;
77 |
78 | static bool CompareBBox(const PlateInfo & a, const PlateInfo & b) {
79 | return a.bbox.score > b.bbox.score;
80 | }
81 |
82 | static float IoU(float xmin, float ymin, float xmax, float ymax, float xmin_,
83 | float ymin_, float xmax_, float ymax_, bool is_iom) {
84 | float iw = std::min(xmax, xmax_) - std::max(xmin, xmin_) + 1;
85 | float ih = std::min(ymax, ymax_) - std::max(ymin, ymin_) + 1;
86 | if (iw <= 0 || ih <= 0)
87 | return 0;
88 | float s = iw * ih;
89 | if (is_iom) {
90 | float ov = s / std::min((xmax - xmin + 1) * (ymax - ymin + 1), (xmax_ - xmin_ + 1) * (ymax_ - ymin_ + 1));
91 | return ov;
92 | } else {
93 | float ov = s / ((xmax - xmin + 1) * (ymax - ymin + 1) + (xmax_ - xmin_ + 1) * (ymax_ - ymin_ + 1) - s);
94 | return ov;
95 | }
96 | }
97 |
98 | static std::vector NMS(std::vector& bboxes, float thresh,
99 | char methodType) {
100 | std::vector bboxes_nms;
101 | if (bboxes.size() == 0) {
102 | return bboxes_nms;
103 | }
104 | std::sort(bboxes.begin(), bboxes.end(), CompareBBox);
105 |
106 | int32_t select_idx = 0;
107 | int32_t num_bbox = static_cast(bboxes.size());
108 | std::vector mask_merged(num_bbox, 0);
109 | bool all_merged = false;
110 |
111 | while (!all_merged) {
112 | while (select_idx < num_bbox && mask_merged[select_idx] == 1)
113 | select_idx++;
114 | if (select_idx == num_bbox) {
115 | all_merged = true;
116 | continue;
117 | }
118 | bboxes_nms.push_back(bboxes[select_idx]);
119 | mask_merged[select_idx] = 1;
120 |
121 | FaceBox select_bbox = bboxes[select_idx].bbox;
122 | float area1 = static_cast((select_bbox.xmax - select_bbox.xmin
123 | + 1) * (select_bbox.ymax - select_bbox.ymin + 1));
124 | float x1 = static_cast(select_bbox.xmin);
125 | float y1 = static_cast(select_bbox.ymin);
126 | float x2 = static_cast(select_bbox.xmax);
127 | float y2 = static_cast(select_bbox.ymax);
128 |
129 | select_idx++;
130 | #ifdef _OPENMP
131 | #pragma omp parallel for num_threads(threads_num)
132 | #endif
133 | for (int32_t i = select_idx; i < num_bbox; i++) {
134 | if (mask_merged[i] == 1)
135 | continue;
136 |
137 | FaceBox & bbox_i = bboxes[i].bbox;
138 | float x = std::max(x1, static_cast(bbox_i.xmin));
139 | float y = std::max(y1, static_cast(bbox_i.ymin));
140 | float w = std::min(x2, static_cast(bbox_i.xmax)) - x + 1;
141 | float h = std::min(y2, static_cast(bbox_i.ymax)) - y + 1;
142 | if (w <= 0 || h <= 0)
143 | continue;
144 |
145 | float area2 = static_cast((bbox_i.xmax - bbox_i.xmin + 1)
146 | * (bbox_i.ymax - bbox_i.ymin + 1));
147 | float area_intersect = w * h;
148 |
149 | switch (methodType) {
150 | case 'u':
151 | if (static_cast(area_intersect)
152 | / (area1 + area2 - area_intersect) > thresh)
153 | mask_merged[i] = 1;
154 | break;
155 | case 'm':
156 | if (static_cast(area_intersect) / std::min(area1, area2)
157 | > thresh)
158 | mask_merged[i] = 1;
159 | break;
160 | default:
161 | break;
162 | }
163 | }
164 | }
165 | return bboxes_nms;
166 | }
167 | static void BBoxRegression(vector& bboxes) {
168 | #ifdef _OPENMP
169 | #pragma omp parallel for num_threads(threads_num)
170 | #endif
171 | for (int i = 0; i < bboxes.size(); ++i) {
172 | FaceBox &bbox = bboxes[i].bbox;
173 | float *bbox_reg = bboxes[i].bbox_reg;
174 | float w = bbox.xmax - bbox.xmin + 1;
175 | float h = bbox.ymax - bbox.ymin + 1;
176 | bbox.xmin += bbox_reg[0] * w;
177 | bbox.ymin += bbox_reg[1] * h;
178 | bbox.xmax += bbox_reg[2] * w;
179 | bbox.ymax += bbox_reg[3] * h;
180 |
181 | // bbox.xmax += bbox_reg[2] * w;
182 | // bbox.ymax += bbox_reg[3] * h;
183 | }
184 | }
185 | static void BBoxPad(vector& bboxes, int width, int height) {
186 | #ifdef _OPENMP
187 | #pragma omp parallel for num_threads(threads_num)
188 | #endif
189 | for (int i = 0; i < bboxes.size(); ++i) {
190 | FaceBox &bbox = bboxes[i].bbox;
191 | bbox.xmin = round(std::max(bbox.xmin, 1.0f));
192 | bbox.ymin = round(std::max(bbox.ymin, 1.0f));
193 | bbox.xmax = round(std::min(bbox.xmax, width - 1.f));
194 | bbox.ymax = round(std::min(bbox.ymax, height - 1.f));
195 | }
196 | }
197 | static void BBoxPadSquare(vector& bboxes, int width, int height) {
198 | #ifdef _OPENMP
199 | #pragma omp parallel for num_threads(threads_num)
200 | #endif
201 | for (int i = 0; i < bboxes.size(); ++i) {
202 | FaceBox &bbox = bboxes[i].bbox;
203 | float w = bbox.xmax - bbox.xmin + 1;
204 | float h = bbox.ymax - bbox.ymin + 1;
205 | float side = h > w ? h : w;
206 | float side_w = h>w ? h:w;
207 | float side_h = 12 / 30.0 * side_w; // 24,60, 48,120
208 |
209 | bbox.xmin = round(std::max(bbox.xmin + (w - side_w) * 0.5f, 0.f));
210 | bbox.ymin = round(std::max(bbox.ymin + (h - side_h) * 0.5f, 0.f));
211 | bbox.xmax = round(std::min(bbox.xmin + side_w - 1, width - 1.f));
212 | bbox.ymax = round(std::min(bbox.ymin + side_h - 1, height - 1.f));
213 | }
214 | }
215 | static void GenerateBBox(float * prob1_confidence, float *reg_box,
216 | int feature_map_w_, int feature_map_h_, float scale, float thresh) {
217 | candidate_boxes_.clear();
218 | int spatical_size = feature_map_w_ * feature_map_h_;
219 | for (int h=0; h thresh) {
222 | PlateInfo faceInfo;
223 | FaceBox & bbox = faceInfo.bbox;
224 | bbox.score = *prob1_confidence;
225 | bbox.xmin = w * pnet_stride * 2 / scale;
226 | bbox.ymin = h * pnet_stride / scale;
227 | bbox.xmax = ( w * pnet_stride + pnet_cell_size_width + 1 - 1.f) / scale;
228 | bbox.ymax = ( h * pnet_stride + pnet_cell_size_height + 1 - 1.f) / scale;
229 |
230 | // NxCxHxW format
231 | int index = h * feature_map_w_ + w;
232 | faceInfo.bbox_reg[0] = reg_box[index];
233 | faceInfo.bbox_reg[1] = reg_box[index + spatical_size];
234 | faceInfo.bbox_reg[2] = reg_box[index + spatical_size * 2];
235 | faceInfo.bbox_reg[3] = reg_box[index + spatical_size * 3];
236 | candidate_boxes_.push_back(faceInfo);
237 | }
238 | }
239 | }
240 |
241 | // float v_scale = 1 / scale;
242 | // for (int i = 0; i < spatical_size; ++i) {
243 | // int stride = i << 2;
244 | // if (confidence_data[stride + 1] >= thresh) {
245 | // int y = i / feature_map_w_;
246 | // int x = i - feature_map_w_ * y;
247 | // FaceInfo faceInfo;
248 | // FaceBox &faceBox = faceInfo.bbox;
249 | //
250 | // faceBox.xmin = (float) (x * pnet_stride) * v_scale;
251 | // faceBox.ymin = (float) (y * pnet_stride) * v_scale;
252 | // faceBox.xmax = (float) (x * pnet_stride + pnet_cell_size_width - 1.f) * v_scale;
253 | // faceBox.ymax = (float) (y * pnet_stride + pnet_cell_size_height - 1.f) * v_scale;
254 | //
255 | // faceInfo.bbox_reg[0] = reg_box[stride];
256 | // faceInfo.bbox_reg[1] = reg_box[stride + 1];
257 | // faceInfo.bbox_reg[2] = reg_box[stride + 2];
258 | // faceInfo.bbox_reg[3] = reg_box[stride + 3];
259 | //
260 | // faceBox.score = confidence_data[stride];
261 | // candidate_boxes_.push_back(faceInfo);
262 | // }
263 | // }
264 | }
265 |
266 | MLPDR::MLPDR(const string& proto_model_dir,
267 | float threhold_p_, float threhold_r_, float threhold_o_ , float factor_) {
268 | threhold_p = threhold_p_;
269 | threhold_r = threhold_r_;
270 | threhold_o = threhold_o_;
271 | factor = factor_;
272 | // threads_num = 2;
273 | PNet_ = std::shared_ptr(MNN::Interpreter::createFromFile((proto_model_dir + "/det1.mnn").c_str()));
274 | RNet_ = std::shared_ptr(MNN::Interpreter::createFromFile((proto_model_dir + "/det2.mnn").c_str()));
275 | ONet_ = std::shared_ptr(MNN::Interpreter::createFromFile((proto_model_dir + "/det3.mnn").c_str()));
276 |
277 | MNN::ScheduleConfig config;
278 | config.type = MNN_FORWARD_CPU;
279 | config.numThread = 4; // 1 faster
280 |
281 | // we dont use backend config for x86 testing
282 | // BackendConfig backendConfig;
283 | // backendConfig.precision = BackendConfig::Precision_Low;
284 | // backendConfig.power = BackendConfig::Power_High;
285 | // config.backendConfig = &backendConfig;
286 |
287 | sess_p = PNet_->createSession(config);
288 | sess_r = RNet_->createSession(config);
289 | sess_o = ONet_->createSession(config);
290 |
291 | p_input = PNet_->getSessionInput(sess_p, NULL);
292 | p_out_pro = PNet_->getSessionOutput(sess_p, "prob1");
293 | p_out_reg = PNet_->getSessionOutput(sess_p, "conv4-2");
294 |
295 | r_input = RNet_->getSessionInput(sess_r, NULL);
296 | r_out_pro = RNet_->getSessionOutput(sess_r, "prob1");
297 | r_out_reg = RNet_->getSessionOutput(sess_r, "conv5-2");
298 |
299 | // auto outputs = ONet_->getSessionOutputAll(sess_o);
300 | // for (auto output: outputs) {
301 | // printf("ONet output name: %s \n", output.first.c_str());
302 | // // ,output.second->shape()[1],
303 | // // output.second->shape()[2],
304 | // // output.second->shape()[3]
305 | //// );
306 | // }
307 |
308 | o_input = ONet_->getSessionInput(sess_o, NULL);
309 | o_out_pro = ONet_->getSessionOutput(sess_o, "prob1");
310 | o_out_reg = ONet_->getSessionOutput(sess_o, "conv6-2");
311 | o_out_lank = ONet_->getSessionOutput(sess_o, "conv6-3");
312 |
313 |
314 |
315 | LPRNet = shared_ptr( Interpreter::createFromFile((proto_model_dir + "/lpr.mnn").c_str()));
316 | ocr_session = LPRNet->createSession(config);
317 | ocr_input = LPRNet->getSessionInput(ocr_session, NULL);
318 | ocr_output = LPRNet->getSessionOutput(ocr_session, NULL);
319 | LPRNet->resizeTensor(ocr_input, ocr_input->shape());
320 | LPRNet->resizeSession(ocr_session);
321 |
322 | Matrix lpr_trans;
323 | lpr_trans.setScale(1.0f, 1.0f);
324 | ImageProcess::Config lpr_config;
325 | lpr_config.filterType = NEAREST;
326 | const float mean_vals[3] = { 116.407, 133.722, 124.187 };
327 | const float norm_vals[3] = { 1.0f, 1.0f, 1.0f };
328 | ::memcpy(lpr_config.mean, mean_vals, sizeof(mean_vals));
329 | ::memcpy(lpr_config.normal, norm_vals, sizeof(norm_vals));
330 | lpr_config.sourceFormat = RGBA;
331 | lpr_config.destFormat = BGR;
332 |
333 | lpr_pretreat_data = std::shared_ptr(ImageProcess::create(lpr_config));
334 | lpr_pretreat_data->setMatrix(lpr_trans);
335 |
336 | LPCNet_ = shared_ptr( Interpreter::createFromFile((proto_model_dir + "/lpc.mnn").c_str()));
337 | lpc_session = LPCNet_->createSession(config);
338 | lpc_input = LPCNet_->getSessionInput(lpc_session, NULL);
339 | lpc_output = LPCNet_->getSessionOutput(lpc_session, NULL);
340 |
341 | Matrix lpc_trans;
342 | lpc_trans.setScale(1.0f, 1.0f);
343 | ImageProcess::Config lpc_config;
344 | lpc_config.filterType = BICUBIC;
345 | const float lpc_mean_vals[3] = { 89.9372, 81.1989, 73.6352 };
346 | // norm_vals[3] = { 1.0f, 1.0f, 1.0f };
347 | ::memcpy(lpc_config.mean, mean_vals, sizeof(lpc_mean_vals));
348 | ::memcpy(lpc_config.normal, norm_vals, sizeof(norm_vals));
349 | lpc_config.sourceFormat = RGBA;
350 | lpc_config.destFormat = BGR;
351 | lpc_pretreat_data = std::shared_ptr(ImageProcess::create(lpc_config));
352 | lpc_pretreat_data->setMatrix(lpc_trans);
353 |
354 | plateColorDict.push_back("白");
355 | plateColorDict.push_back("黄");
356 | plateColorDict.push_back("蓝");
357 | plateColorDict.push_back("黑");
358 | plateColorDict.push_back("绿");
359 |
360 | printf("initialized!\n");
361 | }
362 |
363 | MLPDR::~MLPDR() {
364 | PNet_->releaseModel();
365 | RNet_->releaseModel();
366 | ONet_->releaseModel();
367 | LPRNet->releaseModel();
368 | }
369 |
370 | uint8_t* get_img(cv::Mat img) {
371 | uchar * colorData = new uchar[img.total() * 4];
372 | cv::Mat MatTemp(img.size(), CV_8UC4, colorData);
373 | cv::cvtColor(img, MatTemp, CV_BGR2RGBA, 4);
374 | return (uint8_t *) MatTemp.data;
375 | }
376 |
377 |
378 | void ctc_decode(Tensor * output, vector & codes) {
379 | Tensor outputHost(output, output->getDimensionType());
380 | output->copyToHostTensor(&outputHost);
381 | auto values = outputHost.host();
382 | int prev_class_idx = -1;
383 | for (int t=0; tbatch(); t++) {
384 | int max_class_idx = 0;
385 | float max_prob = *values;
386 | values++;
387 | for (int c=1; c < output->height(); c++, values++) {
388 | if (*values > max_prob) {
389 | max_prob = *values;
390 | max_class_idx = c;
391 | }
392 | }
393 |
394 | if (max_class_idx !=0 && max_class_idx != prev_class_idx) {
395 | codes.push_back(max_class_idx);
396 | }
397 | prev_class_idx = max_class_idx;
398 | }
399 | }
400 |
401 | std::string decode_plateNo(const vector & codes) {
402 | string plateNo = "";
403 | for( auto it=codes.begin(); it != codes.end(); ++it) {
404 | plateNo += label[*it];
405 | }
406 | return plateNo;
407 | }
408 |
409 | void MLPDR::recognize_plate_infos(const cv::Mat & img, vector & plateInfos) {
410 | for (auto & faceInfo: plateInfos) {
411 | vector codes = {};
412 | cv::Point2f srcPoints[4];
413 | cv::Point2f dstPoints[4];
414 |
415 | int x0 = 0; int y0 = 0;
416 | int x1 = 128; int y1 = 0;
417 | int x2 = 128; int y2 = 32;
418 | int x3 = 0; int y3 = 32;
419 | dstPoints[0] = cv::Point2f(x0, y0);
420 | dstPoints[1] = cv::Point2f(x1, y1);
421 | dstPoints[2] = cv::Point2f(x2, y2);
422 | dstPoints[3] = cv::Point2f(x3, y3);
423 |
424 | for (int i=0; i<4; i++) {
425 | int x = i*2;
426 | int y = x + 1;
427 | srcPoints[i] = cv::Point2f(faceInfo.landmark[x], faceInfo.landmark[y]);
428 | }
429 |
430 | cv::Mat plate = cv::Mat::zeros(32, 128, img.type());
431 | cv::Mat warp_mat = cv::getAffineTransform(srcPoints, dstPoints);
432 | cv::warpAffine(img, plate, warp_mat, plate.size(), cv::INTER_LINEAR);
433 |
434 | uint8_t *pImg = get_img(plate);
435 | lpr_pretreat_data->convert(pImg, plate.cols, plate.rows, 0, ocr_input);
436 | LPRNet->runSession(ocr_session);
437 | ctc_decode(ocr_output, codes);
438 | faceInfo.plateNo = decode_plateNo(codes);
439 | delete pImg;
440 |
441 | // predict plate color
442 | cv::resize(plate, plate, Size(110, 22));
443 | pImg = get_img(plate);
444 | lpc_pretreat_data->convert(pImg, plate.cols, plate.rows, 0, lpc_input);
445 | LPCNet_->runSession(lpc_session);
446 | Tensor lpc_output_host(lpc_output, lpc_output->getDimensionType());
447 | lpc_output->copyToHostTensor(&lpc_output_host);
448 | auto * probs = lpc_output_host.host();
449 | float max = probs[0];
450 | int clsId = 0;
451 | for (int i=1; i<5; i++) {
452 | if (probs[i] > max) {
453 | max = probs[i];
454 | clsId = i;
455 | }
456 | }
457 | faceInfo.plateColor = plateColorDict[clsId];
458 | delete pImg;
459 | }
460 | }
461 |
462 | void fillInput(const std::vector & dim, const cv::Mat & sample, Tensor* input) {
463 | int hs = dim[2];
464 | int ws = dim[3];
465 | auto inputHost = std::shared_ptr(MNN::Tensor::create({1, hs, ws, 3}));
466 | cv::Mat resized;
467 | if (sample.cols != ws || sample.rows != hs) {
468 | // printf("resize to %d x %d \n", hs, ws);
469 | cv::resize(sample, resized, cv::Size(ws, hs), 0, 0, cv::INTER_NEAREST);
470 | }
471 | else {
472 | resized = sample;
473 | }
474 |
475 | int index=0;
476 | for (int h=0; h < hs; h++) {
477 | for (int w=0; w < ws; w++) {
478 | cv::Vec3f pixel = resized.at(h, w);
479 | for (int c = 0 ; c < 3; c++, index++) {
480 | inputHost->host()[index] = pixel.val[c];
481 | }
482 | }
483 | }
484 | input->copyFromHostTensor(inputHost.get());
485 | }
486 |
487 |
488 | static vector ProposalNet(unsigned char * inputImage, int height, int width, int minSize,
489 | float threshold, float factor) {
490 |
491 | float scale = 12.0f / minSize;
492 | float minWH = std::min(height, width) * scale;
493 | std::vector scales;
494 | while (minWH >= minSize) {
495 | scales.push_back(scale);
496 | minWH *= factor;
497 | scale *= factor;
498 | }
499 | total_boxes_.clear();
500 |
501 | for (int i = 0; i < scales.size(); i++) {
502 | int ws = (int) std::ceil(width * scales[i]);
503 | int hs = (int) std::ceil(height * scales[i]);
504 | std::vector inputDims = { 1, 3, hs, ws };
505 | PNet_->resizeTensor(p_input, inputDims);
506 |
507 | PNet_->resizeSession(sess_p);
508 |
509 | MNN::CV::Matrix trans;
510 | trans.postScale(1.0f / ws, 1.0f / hs);
511 | trans.postScale(width, height);
512 | pretreat_data->setMatrix(trans);
513 | pretreat_data->convert(inputImage, width, height, 0, p_input);
514 |
515 | PNet_->runSession(sess_p);
516 |
517 | //onCopy to NCHW format
518 | Tensor prob_host(p_out_pro, p_out_pro->getDimensionType());
519 | Tensor reg_host(p_out_reg, p_out_reg->getDimensionType());
520 | p_out_pro->copyToHostTensor(&prob_host);
521 | p_out_reg->copyToHostTensor(®_host);
522 | auto * prob1_confidence = prob_host.host() + prob_host.stride(1);
523 | auto * reg = reg_host.host();
524 |
525 | int feature_w = p_out_pro->width();
526 | int feature_h = p_out_pro->height();
527 |
528 | GenerateBBox(prob1_confidence, reg, feature_w, feature_h, scales[i], threshold);
529 | std::vector bboxes_nms = NMS(candidate_boxes_, 0.5f, 'u');
530 | if (bboxes_nms.size() > 0) {
531 | total_boxes_.insert(total_boxes_.end(), bboxes_nms.begin(),
532 | bboxes_nms.end());
533 | }
534 | }
535 |
536 | int num_box = (int) total_boxes_.size();
537 | vector res_boxes;
538 | if (num_box != 0) {
539 | res_boxes = NMS(total_boxes_, 0.7f, 'u');
540 | BBoxRegression(res_boxes);
541 | BBoxPadSquare(res_boxes, width, height);
542 | }
543 | return res_boxes;
544 | }
545 |
546 | /**
547 | * @sample is a normalized Mat
548 | */
549 | static vector ProposalNet(const cv::Mat& sample, int minSize,
550 | float threshold, float factor) {
551 | int width = sample.cols;
552 | int height = sample.rows;
553 | float scale = factor;
554 | float minWH = std::min(height, width) * factor;
555 | std::vector scales;
556 | while (minWH >= minSize) {
557 | scales.push_back(scale);
558 | minWH *= factor;
559 | scale *= factor;
560 | }
561 | total_boxes_.clear();
562 |
563 | for (int i = 0; i < scales.size(); i++) {
564 | int ws = (int) std::ceil(width * scales[i]);
565 | int hs = (int) std::ceil(height * scales[i]);
566 | std::vector inputDims = { 1, 3, hs, ws };
567 | PNet_->resizeTensor(p_input, inputDims);
568 | PNet_->resizeSession(sess_p);
569 | fillInput(inputDims, sample, p_input);
570 | PNet_->runSession(sess_p);
571 | // MNN_PRINT("input tensor shape: %d %d %d %d\n", p_input->batch(), p_input->channel(), p_input->height(), p_input->width());
572 |
573 | Tensor prob_host(p_out_pro, p_out_pro->getDimensionType());
574 | Tensor reg_host(p_out_reg, p_out_reg->getDimensionType());
575 | p_out_pro->copyToHostTensor(&prob_host);
576 | p_out_reg->copyToHostTensor(®_host);
577 | auto * prob1_confidence = prob_host.host() + prob_host.stride(1);
578 | auto * reg = reg_host.host();
579 | int feature_w = p_out_pro->width();
580 | int feature_h = p_out_pro->height();
581 | // MNN_PRINT("output tensor shape: %d %d %d %d \n", p_out_reg->batch(), p_out_reg->channel(), p_out_reg->height(), p_out_reg->width());
582 | GenerateBBox(prob1_confidence, reg, feature_w, feature_h, scales[i], threshold);
583 | std::vector bboxes_nms = NMS(candidate_boxes_, 0.5f, 'u');
584 | if (bboxes_nms.size() > 0) {
585 | total_boxes_.insert(total_boxes_.end(), bboxes_nms.begin(),
586 | bboxes_nms.end());
587 | }
588 | }
589 | int num_box = (int) total_boxes_.size();
590 | vector res_boxes;
591 | if (num_box != 0) {
592 | res_boxes = NMS(total_boxes_, 0.7f, 'u');
593 | BBoxRegression(res_boxes);
594 | BBoxPadSquare(res_boxes, width, height);
595 | }
596 | return res_boxes;
597 | }
598 |
599 | static std::vector NextStage(const cv::Mat& sample,
600 | vector &pre_stage_res, int input_w, int input_h,
601 | int stage_num, const float threshold) {
602 | vector res;
603 | int batch_size = pre_stage_res.size();
604 | std::vector inputDims = {1, 3, input_h, input_w };
605 | switch (stage_num) {
606 | case 2: {
607 | for (int n = 0; n < batch_size; ++n) {
608 | FaceBox & box = pre_stage_res[n].bbox;
609 | cv::Rect rect(cv::Point((int) box.xmin, (int) box.ymin),
610 | cv::Point((int) box.xmax, (int) box.ymax));
611 | // cout << "regression bbox: " << rect << endl;
612 | cv::Mat roi(sample, rect);
613 | imshow("rnet sample", roi);
614 | waitKey(0);
615 | fillInput(inputDims, roi, r_input);
616 | RNet_->runSession(sess_r);
617 |
618 | Tensor r_out_pro_host(r_out_pro, r_out_pro->getDimensionType());
619 | Tensor r_out_reg_host(r_out_reg, r_out_reg->getDimensionType());
620 | r_out_pro->copyToHostTensor(&r_out_pro_host);
621 | r_out_reg->copyToHostTensor(&r_out_reg_host);
622 | MNN_PRINT("rnet run session and on copy to host\n");
623 | auto confidence = r_out_pro_host.host() + r_out_pro_host.stride(1);
624 | auto reg_box = r_out_reg_host.host();
625 |
626 | float conf = confidence[0];
627 | if (conf >= threshold) {
628 | PlateInfo info;
629 | info.bbox.score = conf;
630 | info.bbox.xmin = pre_stage_res[n].bbox.xmin;
631 | info.bbox.ymin = pre_stage_res[n].bbox.ymin;
632 | info.bbox.xmax = pre_stage_res[n].bbox.xmax;
633 | info.bbox.ymax = pre_stage_res[n].bbox.ymax;
634 | for (int i = 0; i < 4; ++i) {
635 | info.bbox_reg[i] = reg_box[i];
636 | }
637 | res.push_back(info);
638 | }
639 | }
640 | break;
641 | }
642 | case 3: {
643 | #ifdef _OPENMP
644 | #pragma omp parallel for num_threads(threads_num)
645 | #endif
646 | for (int n = 0; n < batch_size; ++n) {
647 | FaceBox &box = pre_stage_res[n].bbox;
648 | cv::Rect rect(cv::Point((int) box.xmin, (int) box.ymin),
649 | cv::Point((int) box.xmax, (int) box.ymax));
650 | cv::Mat roi(sample, rect);
651 | fillInput(inputDims, roi, o_input);
652 | ONet_->runSession(sess_o);
653 |
654 | Tensor o_out_pro_host(o_out_pro, o_out_pro->getDimensionType());
655 | Tensor o_out_reg_host(o_out_reg, o_out_reg->getDimensionType());
656 | Tensor o_out_lank_host(o_out_lank, o_out_lank->getDimensionType());
657 | o_out_pro->copyToHostTensor(&o_out_pro_host);
658 | o_out_reg->copyToHostTensor(&o_out_reg_host);
659 | o_out_lank->copyToHostTensor(&o_out_lank_host);
660 |
661 | auto confidence = o_out_pro_host.host() + o_out_pro_host.stride(1);
662 | auto reg_box = o_out_reg_host.host();
663 | auto reg_landmark = o_out_lank_host.host();
664 |
665 | float conf = confidence[0];
666 | if (*confidence >= threshold) {
667 | PlateInfo info;
668 | info.bbox.score = conf;
669 | info.bbox.xmin = pre_stage_res[n].bbox.xmin;
670 | info.bbox.ymin = pre_stage_res[n].bbox.ymin;
671 | info.bbox.xmax = pre_stage_res[n].bbox.xmax;
672 | info.bbox.ymax = pre_stage_res[n].bbox.ymax;
673 | for (int i = 0; i < 4; ++i) {
674 | info.bbox_reg[i] = reg_box[i];
675 | }
676 | float w = info.bbox.xmax - info.bbox.xmin + 1.f;
677 | float h = info.bbox.ymax - info.bbox.ymin + 1.f;
678 | // x x x x y y y y to x y x y x y x y
679 | for (int i = 0; i < 4; ++i) {
680 | info.landmark[2 * i] = reg_landmark[2 * i] * w + info.bbox.xmin;
681 | info.landmark[2 * i + 1] = reg_landmark[2 * i + 1] * h + info.bbox.ymin;
682 | }
683 | res.push_back(info);
684 | }
685 | }
686 | break;
687 | }
688 | default:
689 | return res;
690 | break;
691 | }
692 | return res;
693 | }
694 |
695 | vector MLPDR::detect(unsigned char * inputImage, int height, int width, const int min_face, const int stage) {
696 | vector pnet_res;
697 | vector rnet_res;
698 | vector onet_res;
699 |
700 | if (stage >= 1) {
701 | pnet_res = ProposalNet(inputImage, height, width, min_face, threhold_p, factor);
702 | }
703 |
704 | if (stage == 1) {
705 | return pnet_res;
706 | } else if (stage == 2) {
707 | return rnet_res;
708 | } else if (stage == 3) {
709 | return onet_res;
710 | } else {
711 | return onet_res;
712 | }
713 | }
714 |
715 | std::vector MLPDR::recognize(const cv::Mat & img) {
716 | vector faceInfos = Detect(img, 40, 3);
717 | recognize_plate_infos(img, faceInfos);
718 | return faceInfos;
719 | }
720 | vector MLPDR::Detect(const cv::Mat & image, int min_face,
721 | int stage) {
722 | vector pnet_res;
723 | vector rnet_res;
724 | vector onet_res;
725 |
726 | cv::Mat sample = image.clone();
727 | sample.convertTo(sample, CV_32FC3, 0.0078125, -127.5*0.0078125);
728 |
729 | if (stage >= 1) {
730 | pnet_res = ProposalNet(sample, min_face, threhold_p, factor);
731 | MNN_PRINT("done ProposalNet with %d proposal \n", pnet_res.size());
732 | }
733 |
734 | if (stage >= 2 && pnet_res.size() > 0) {
735 | if (pnet_max_detect_num < (int) pnet_res.size()) {
736 | pnet_res.resize(pnet_max_detect_num);
737 | }
738 | rnet_res = NextStage(sample, pnet_res, 60, 24, 2, threhold_r);
739 | rnet_res = NMS(rnet_res, iou_threhold, 'u');
740 | BBoxRegression(rnet_res);
741 | BBoxPadSquare(rnet_res, image.cols, image.rows);
742 | }
743 | if (stage >= 3 && rnet_res.size() > 0) {
744 | onet_res = NextStage(sample, rnet_res, 120, 48, 3, threhold_o);
745 | BBoxRegression(onet_res);
746 | onet_res = NMS(onet_res, iou_threhold, 'm');
747 | BBoxPad(onet_res, image.cols, image.rows);
748 | }
749 | if (stage == 1) {
750 | return pnet_res;
751 | } else if (stage == 2) {
752 | return rnet_res;
753 | } else if (stage == 3) {
754 | return onet_res;
755 | } else {
756 | return onet_res;
757 | }
758 | }
759 |
760 | static void extractMaxFace(vector& boundingBox_) {
761 | if (boundingBox_.empty()) {
762 | return;
763 | }
764 | sort(boundingBox_.begin(), boundingBox_.end(), CompareBBox);
765 | for (std::vector::iterator itx = boundingBox_.begin() + 1;
766 | itx != boundingBox_.end();) {
767 | itx = boundingBox_.erase(itx);
768 | }
769 | }
770 |
771 | std::vector MLPDR::Detect_MaxFace(const cv::Mat& img,
772 | const int min_face, const int stage) {
773 | vector pnet_res;
774 | vector rnet_res;
775 | vector onet_res;
776 |
777 | //total_boxes_.clear();
778 | //candidate_boxes_.clear();
779 |
780 | int width = img.cols;
781 | int height = img.rows;
782 | float scale = 12.0f / min_face;
783 | float minWH = std::min(height, width) * scale;
784 | std::vector scales;
785 | while (minWH >= 24) {
786 | scales.push_back(scale);
787 | minWH *= factor;
788 | scale *= factor;
789 | }
790 |
791 | std::reverse(scales.begin(), scales.end());
792 |
793 | uint8_t *pImg = get_img(img);
794 | for (int i = 0; i < scales.size(); i++) {
795 | int ws = (int) std::ceil(width * scales[i]);
796 | int hs = (int) std::ceil(height * scales[i]);
797 | std::vector inputDims = { 1, 3, hs, ws };
798 | PNet_->resizeTensor(p_input, inputDims);
799 | PNet_->resizeSession(sess_p);
800 |
801 | MNN::CV::Matrix trans;
802 | trans.postScale(1.0f / ws, 1.0f / hs);
803 | trans.postScale(width, height);
804 | pretreat_data->setMatrix(trans);
805 | pretreat_data->convert(pImg, width, height, 0, p_input);
806 |
807 | PNet_->runSession(sess_p);
808 | float * confidence = p_out_pro->host();
809 | float * reg = p_out_reg->host();
810 |
811 | int feature_w = p_out_pro->width();
812 | int feature_h = p_out_pro->height();
813 |
814 | GenerateBBox(confidence, reg, feature_w, feature_h, scales[i],
815 | threhold_p);
816 | std::vector bboxes_nms = NMS(candidate_boxes_, 0.5f, 'u');
817 |
818 | //nmsTwoBoxs(bboxes_nms, pnet_res, 0.5);
819 | if (bboxes_nms.size() > 0) {
820 | pnet_res.insert(pnet_res.end(), bboxes_nms.begin(),
821 | bboxes_nms.end());
822 | } else {
823 | continue;
824 | }
825 | BBoxRegression(pnet_res);
826 | BBoxPadSquare(pnet_res, width, height);
827 |
828 | bboxes_nms.clear();
829 | bboxes_nms = NextStage(img, pnet_res, 60, 24, 2, threhold_r);
830 | bboxes_nms = NMS(bboxes_nms, iou_threhold, 'u');
831 | //nmsTwoBoxs(bboxes_nms, rnet_res, 0.5)
832 | if (bboxes_nms.size() > 0) {
833 | rnet_res.insert(rnet_res.end(), bboxes_nms.begin(),
834 | bboxes_nms.end());
835 | } else {
836 | pnet_res.clear();
837 | continue;
838 | }
839 | BBoxRegression(rnet_res);
840 | BBoxPadSquare(rnet_res, img.cols, img.rows);
841 |
842 | onet_res = NextStage(img, rnet_res, 120, 48, 3, threhold_r);
843 |
844 | BBoxRegression(onet_res);
845 | onet_res = NMS(onet_res, iou_threhold, 'm');
846 | BBoxPad(onet_res, img.cols, img.rows);
847 |
848 | if (onet_res.size() < 1) {
849 | pnet_res.clear();
850 | rnet_res.clear();
851 | continue;
852 | } else {
853 | extractMaxFace(onet_res);
854 | delete pImg;
855 | return onet_res;
856 | }
857 | }
858 | delete pImg;
859 | return std::vector { };
860 | }
861 |
862 | } /* namespace mlpdr */
863 |
--------------------------------------------------------------------------------
/src/mlpdr/java_api.cpp:
--------------------------------------------------------------------------------
1 | /*
2 | * api.cpp
3 | *
4 | * Created on: Jul 14, 2019
5 | * Author: link
6 | */
7 |
8 |
9 | #include
10 | #include
11 | #include
12 | #include
13 | #include
14 | #include
15 |
16 | #include
17 |
18 | using namespace std;
19 | using namespace mlpdr;
20 | using namespace cv;
21 |
22 | mlpdr::MLPDR * ptr_mlpdr;
23 |
24 | #if (defined _WIN32 || defined WINCE || defined __CYGWIN__) && defined CVAPI_EXPORTS
25 | # define API_EXPORTS __declspec(dllexport)
26 | #elif defined __GNUC__ && __GNUC__ >= 4
27 | # define API_EXPORTS __attribute__ ((visibility ("default")))
28 | #else
29 | # define API_EXPORTS
30 | #endif
31 |
32 | #ifdef __cplusplus
33 | extern "C" {
34 | #endif
35 |
36 |
37 | API_EXPORTS void coreInitContext() {
38 | ptr_mlpdr = new MLPDR("/home/lqian/cpp-workspace/light-LPDR/models/");
39 | }
40 |
41 | API_EXPORTS void cleanContext() {
42 | delete ptr_mlpdr;
43 | }
44 |
45 | /**
46 | * @res resource URL represent a image to be recognized
47 | * @contentType char buffer, full filename, network resource etc
48 | * @output a char point that point output content buffer
49 | */
50 | API_EXPORTS int recogSingleJson(char * res, int contentType, char ** output) {
51 | Mat img = imread(res);
52 | if (img.empty()) {
53 | return -1;
54 | }
55 |
56 | vector plateInfos = ptr_mlpdr->recognize(img);
57 | if (plateInfos.size() == 0) return 0;
58 | string plateNo = plateInfos[0].plateNo; //only return 0 for highest confidence plate
59 | int len = plateNo.length();
60 | *output = (char *) calloc(len, sizeof(char));
61 | memcpy(*output, plateNo.c_str(), len);
62 | return len;
63 | }
64 |
65 | #ifdef __cplusplus
66 | }
67 | #endif
68 |
69 |
--------------------------------------------------------------------------------