├── .gitignore
├── README.md
├── caffe
    ├── ssh_detector
    │   ├── CMakeLists.txt
    │   └── ssh_detector.cpp
    └── ssh_test.cpp
├── include
    ├── anchors.h
    ├── ssh_detector.h
    ├── ssh_detector_mxnet.h
    └── tensor_utils.h
├── mxnet
    ├── CMakeLists.txt
    ├── main.cpp
    ├── ssh_bench.cpp
    ├── ssh_detector
    │   ├── CMakeLists.txt
    │   ├── mxnet_model.cpp
    │   ├── mxnet_model.h
    │   └── ssh_detector.cpp
    └── ssh_test.cpp
└── tvm
    ├── CMakeLists.txt
    ├── python
        ├── batch_converter.py
        └── mxnet2tvm.py
    ├── ssh_bench.cpp
    └── ssh_detector
        ├── CMakeLists.txt
        ├── ssh_detector.cpp
        └── tvm_runtime_pack.cc


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Prerequisites
 2 | *.d
 3 | 
 4 | # Compiled Object files
 5 | *.slo
 6 | *.lo
 7 | *.o
 8 | *.obj
 9 | 
10 | # Precompiled Headers
11 | *.gch
12 | *.pch
13 | 
14 | # Compiled Dynamic libraries
15 | *.so
16 | *.dylib
17 | *.dll
18 | 
19 | # Fortran module files
20 | *.mod
21 | *.smod
22 | 
23 | # Compiled Static libraries
24 | *.lai
25 | *.la
26 | *.a
27 | *.lib
28 | 
29 | # Executables
30 | *.exe
31 | *.out
32 | *.app
33 | 
34 | build/*
35 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # ssh_detector
 2 | c++ implementation for ssh detector for object detect.  something likes ssd
 3 | 
 4 | # benchmark
 5 | 
 6 |     | Software           |   HardWare      | Image Resolution |  Performance  |
 7 |     | -----------------  | --------------- | ---------------- |  -----------  |
 8 |     | MacOSX mxnet1.2.1  |    CPU 1 core   |   2688*1520      |  500ms/frame  |
 9 |     | MacOSX mxnet1.2.1  |    CPU 2 core   |   2688*1520      |  360ms/frame  |
10 | 
11 |   
12 | 


--------------------------------------------------------------------------------
/caffe/ssh_detector/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | file(GLOB ssh_detector_source "*.cpp")
2 | add_library( ssh_detector ${ssh_detector_source} )
3 | 


--------------------------------------------------------------------------------
/caffe/ssh_detector/ssh_detector.cpp:
--------------------------------------------------------------------------------
  1 | #include "ssh_detector.h"
  2 | 
  3 | #include <iostream>
  4 | #include <iomanip>
  5 | #include <sys/time.h>
  6 | 
  7 | #include "opencv2/dnn.hpp"
  8 | 
  9 | #include "anchors.h"
 10 | 
 11 | static float getElapse(struct timeval *tv1,struct timeval *tv2)
 12 | {
 13 |     float t = 0.0f;
 14 |     if (tv1->tv_sec == tv2->tv_sec)
 15 |         t = (tv2->tv_usec - tv1->tv_usec)/1000.0f;
 16 |     else
 17 |         t = ((tv2->tv_sec - tv1->tv_sec) * 1000 * 1000 + tv2->tv_usec - tv1->tv_usec)/1000.0f;
 18 |     return t;
 19 | }
 20 | 
 21 | SSH::SSH(const std::string& model_path, int w, int h) {
 22 | 
 23 |     generate_anchors_fpn(anchors_fpn, num_anchors);
 24 | 
 25 |     std::string model  = model_path + ".caffemodel";
 26 |     std::string config = model_path + ".prototxt";
 27 |     // Load model handle
 28 |     auto net = new cv::dnn::Net();
 29 | 
 30 |     *net= cv::dnn::readNetFromCaffe(config, model);
 31 | 
 32 |     std::cout << "opencv dnn read caffe net \n";
 33 | 
 34 |     net->setPreferableBackend(0);
 35 |     net->setPreferableTarget(0);
 36 | 
 37 |     handle = (void *) net;
 38 | }
 39 | 
 40 | 
 41 | SSH::~SSH(){
 42 |     // Free model handle
 43 |     delete (cv::dnn::Net *) handle;
 44 |     std::cout << "release opencv dnn inference handle!\n";
 45 | }
 46 | 
 47 | void SSH::detect(cv::Mat& im, std::vector<cv::Rect2f>  & target_boxes, 
 48 |                               std::vector<cv::Point2f> & target_landmarks) {
 49 | 
 50 | 
 51 |     cv::Mat blob;
 52 |     cv::dnn::blobFromImage(im, blob, 1.0, cv::Size(224,224) );
 53 | 
 54 |     cv::dnn::Net * net = ( cv::dnn::Net *) handle;
 55 | 
 56 |     net->setInput(blob);
 57 |     cv::Mat feature = net->forward("fc1000");
 58 | 
 59 |     std::cout << "cols: "<< feature.cols << " "
 60 |               << "rows: "<< feature.rows << " " 
 61 |               << "channels: "<<  feature.channels()
 62 |               << std::endl;
 63 | 
 64 |     /*
 65 |     struct timeval  tv1,tv2;
 66 |     float sum_time = 0;
 67 |     gettimeofday(&tv1,NULL);
 68 |     // Forward Inference 
 69 | 
 70 |     gettimeofday(&tv2,NULL);
 71 |     sum_time += getElapse(&tv1, &tv2);
 72 |     // Inference
 73 |     std::vector<float> scores;
 74 |     std::vector<cv::Rect2f> boxes;
 75 |     std::vector<cv::Point2f> landmarks;
 76 | 
 77 |     for(int i=0; i< 3; i++) {
 78 |         std::vector<int> shape;
 79 |         std::vector<float> scores1;
 80 |         int index;
 81 |         index = i*3;
 82 | 
 83 |         gettimeofday(&tv1,NULL);
 84 | 
 85 |         gettimeofday(&tv2,NULL);
 86 |         sum_time += getElapse(&tv1, &tv2);
 87 | 
 88 |         int hscore = shape[2];
 89 |         int wscore = shape[3];
 90 |         std::vector<float> scores2;
 91 |         int count = scores1.size()/2;
 92 |         scores2.resize(count);
 93 |         for(size_t i=0;i<scores2.size();i++){
 94 |             scores2[i] = scores1[i+count];
 95 |         }
 96 |         std::vector<float> scores3;
 97 |         tensor_reshape(scores2, scores3, hscore, wscore );
 98 | 
 99 |         index++;
100 |         std::vector<float> bbox_deltas;
101 | 
102 |         gettimeofday(&tv1,NULL);
103 | 
104 |         gettimeofday(&tv2,NULL);
105 |         sum_time += getElapse(&tv1, &tv2);
106 | 
107 |         int h = shape[2];
108 |         int w = shape[3];
109 | 
110 |         int stride = stride_fpn[i];
111 |         std::vector<cv::Rect2f> anchors;
112 |         anchor_plane(h,w, stride, anchors_fpn[stride], anchors);
113 | 
114 |         std::vector<cv::Rect2f> boxes1;
115 |         bbox_pred(anchors, boxes1, bbox_deltas, h, w);
116 |         clip_boxes(boxes1, im.rows, im.cols);
117 |         // for(size_t i=0; i<5; i++) std::cout <<  "boxes1: " << boxes1[i] << "\n";
118 |         index++;
119 |         std::vector<float> landmark_deltas;
120 | 
121 |         gettimeofday(&tv1,NULL);
122 | 
123 |         gettimeofday(&tv2,NULL);
124 |         sum_time += getElapse(&tv1, &tv2);
125 | 
126 |         std::vector<cv::Point2f> landmarks1;
127 |         landmark_pred(anchors, landmarks1, landmark_deltas, h, w);
128 |         // for(size_t i=0; i<20; i++) std::cout <<  "landmarks: " << landmarks1[i] << "\n";
129 | 
130 |         std::vector<bool> idx;
131 |         filter_threshold(idx, scores3, threshold);
132 |         // for(size_t i=0; i<idx.size(); i++) if(idx[i]==true)std::cout <<  "idx: " << i << "\n";
133 |         std::vector<float> scores4;
134 |         tensor_slice(scores3, scores4, idx, 1);
135 |         scores.insert(scores.end(), scores4.begin(), scores4.end());
136 |         std::vector<cv::Rect2f> boxes2;
137 |         tensor_slice(boxes1, boxes2, idx, 1);
138 |         boxes.insert(boxes.end(), boxes2.begin(), boxes2.end());
139 |         std::vector<cv::Point2f> landmarks2;
140 |         tensor_slice(landmarks1, landmarks2, idx, 5);
141 |         landmarks.insert(landmarks.end(), landmarks2.begin(), landmarks2.end());
142 |     }
143 |     
144 |     std::vector<int> order;
145 |     argsort(order, scores);
146 |     // for(size_t i=0;i<order.size();i++) std::cout << "order: " << order[i] << "\n";
147 | 
148 |     std::vector<float> order_scores;
149 |     std::vector<cv::Rect2f> order_boxes;
150 |     std::vector<cv::Point2f> order_landmarks;
151 | 
152 |     sort_with_idx(scores, order_scores, order, 1);
153 |     sort_with_idx(boxes,  order_boxes, order, 1);
154 |     sort_with_idx(landmarks, order_landmarks, order, 5);
155 | 
156 |     // for(auto & s: order_scores) std::cout << "scores: " << s << "\n";
157 |     // for(auto & b: order_boxes) std::cout  << "boxes: "  << b << "\n";
158 |     // for(auto & l: order_landmarks) std::cout << "landmarks: " << l << "\n";
159 | 
160 |     std::vector<bool> keep(order_scores.size(),false);
161 |     nms(order_scores, order_boxes, keep, nms_threshold);
162 |     // for(size_t i=0;i<keep.size();i++) if(keep[i]) std::cout << "keep: " << i << "\n";
163 | 
164 |     tensor_slice(order_boxes,     target_boxes,     keep, 1);
165 |     tensor_slice(order_landmarks, target_landmarks, keep, 5);
166 | 
167 |     std::cout << "mxnet infer, time eclipsed: " << sum_time  << " ms\n";
168 | 
169 |     */
170 | }


--------------------------------------------------------------------------------
/caffe/ssh_test.cpp:
--------------------------------------------------------------------------------
 1 | #include <thread>
 2 | #include <unistd.h>
 3 | 
 4 | #include <glog/logging.h>
 5 | 
 6 | #include "ssh_detector.h"
 7 | 
 8 | void process(SSH * det_pointer, cv::Mat im, std::string model_path, std::string mode, int count) {
 9 |     std::vector<cv::Rect2f> bboxes;
10 |     std::vector<cv::Point2f> landmarks;
11 |         if(mode=="single"){
12 |             for(int i=0; i<count; i++){
13 |                 det_pointer->detect(im,bboxes,landmarks);
14 |                 google::FlushLogFiles(google::GLOG_INFO);                
15 |             }
16 |                    
17 |         } else if(mode=="multi"){
18 |             SSH det(model_path, im.cols, im.rows);
19 |             for(int i=0; i<count; i++){
20 |                 det.detect(im,bboxes,landmarks);
21 |                 google::FlushLogFiles(google::GLOG_INFO);                
22 |             }
23 |         }
24 | }
25 | 
26 | int main(int argc, char* argv[]) {
27 | 
28 |   const std::string keys =
29 |       "{help h usage ? |                                | print this message   }"
30 |       "{model          |../../model/caffe/ResNet-50     | path to ssh model  }"
31 |       "{parallel       |4                               | thread num for detector }" 
32 |       "{count          |100                             | detect times for one thread }" 
33 |       "{mode           |single   | single : share one detector instance in multi thread; multi: one thread one instance }"       
34 |       "{@image         |../../1.png                     | input image          }"
35 |   ;
36 | 
37 |   cv::CommandLineParser parser(argc, argv, keys);
38 |   parser.about("ssh detector");
39 |   if (parser.has("help")) {
40 |       parser.printMessage();
41 |       return 0;
42 |   }
43 | 
44 |   if (!parser.check()) {
45 |       parser.printErrors();
46 |       return EXIT_FAILURE;
47 |   }
48 | 
49 |   google::InitGoogleLogging("multi-thread-ssh-detector");
50 |   FLAGS_log_dir = "./";
51 | 
52 |   std::string model_path = parser.get<std::string>("model");
53 |   int num = parser.get<int>("parallel");
54 |   int count = parser.get<int>("count");
55 |   std::string mode = parser.get<std::string>("mode");  
56 |   std::string image_path = parser.get<std::string>(0);
57 | 
58 |   cv::Mat im = cv::imread(image_path, cv::IMREAD_COLOR);
59 |   SSH det(model_path, im.cols, im.rows);
60 |   std::vector<cv::Rect2f> bboxes;
61 |   std::vector<cv::Point2f> landmarks;
62 |   det.detect(im,bboxes,landmarks);
63 | 
64 |   std::vector<std::thread> threads;
65 |   for(int i = 0; i<num; i++){
66 |       std::thread t(process, &det, im, model_path, mode, count );
67 |       threads.push_back(std::move(t));
68 |   }
69 |   for( size_t i=0;i<threads.size();i++){
70 |     if(threads[i].joinable()){
71 |         threads[i].join();
72 |     } 
73 |   }
74 |   std::cout<< "all thread proccess over!\n";
75 | 
76 |   /*
77 |   assert(bboxes.size()*5 == landmarks.size());
78 | 
79 |   for(auto & b: bboxes) std::cout << "b:" << b << "\n";
80 |   for(auto & l: landmarks) std::cout << "l:" << l << "\n";
81 | 
82 |   if(bboxes.size()==0) std::cout << "detect no face!\n";
83 |   */
84 | }


--------------------------------------------------------------------------------
/include/anchors.h:
--------------------------------------------------------------------------------
  1 | #ifndef __ANCHORS__
  2 | #define __ANCHORS__
  3 | #include <vector>
  4 | #include <string>
  5 | #include <map>
  6 | #include <math.h>
  7 | #include <cassert>
  8 | #include <opencv2/opencv.hpp>
  9 | 
 10 | #include <nlohmann/json.hpp>
 11 | using json = nlohmann::json;
 12 | 
 13 | #include "tensor_utils.h"
 14 | 
 15 | void generate_anchors(int base_size, std::vector<float> & ratios, std::vector<int> & scales,
 16 |                       std::vector<cv::Rect2f> & anchors){
 17 | 
 18 |     cv::Rect2f base_anchor(0,0,base_size,base_size);
 19 |     std::vector<cv::Rect2f> anchor_ratios;
 20 |     for(auto ratio: ratios){
 21 |         float size_ratios = base_anchor.area() / ratio;
 22 |         float ws = roundf(sqrt(size_ratios));
 23 |         float hs = roundf(ws * ratio);
 24 |         
 25 |         cv::Point2f center = base_anchor.tl() + 0.5 * cv::Point2f(base_anchor.width-1, base_anchor.height-1);
 26 |         cv::Point2f tl = center - 0.5 * cv::Point2f(ws-1, hs-1);
 27 |         anchor_ratios.push_back(cv::Rect2f(tl.x,tl.y,ws,hs));
 28 | 
 29 |     }
 30 | 
 31 |     for(auto scale: scales){
 32 |         for(auto anchor_ratio: anchor_ratios){
 33 |             float ws = anchor_ratio.width  * scale;
 34 |             float hs = anchor_ratio.height * scale;
 35 | 
 36 |             cv::Point2f center = anchor_ratio.tl() + 0.5 * cv::Point2f(anchor_ratio.width-1, anchor_ratio.height-1);
 37 |             cv::Point2f tl = center - 0.5 * cv::Point2f(ws-1, hs-1);
 38 |             anchors.push_back(cv::Rect2f(tl.x,tl.y,ws,hs));
 39 |         }
 40 |     }
 41 | 
 42 | }
 43 | 
 44 | const int stride_fpn[3] = {32, 16, 8};
 45 | 
 46 | std::string config = "  \
 47 | {   \
 48 |     \"32\": {\"SCALES\": [32,16], \"BASE_SIZE\": 16, \"RATIOS\": [1], \"ALLOWED_BORDER\": 9999},  \
 49 |     \"16\": {\"SCALES\": [8,4],   \"BASE_SIZE\": 16, \"RATIOS\": [1], \"ALLOWED_BORDER\": 9999},  \
 50 |     \"8\":  {\"SCALES\": [2,1],   \"BASE_SIZE\": 16, \"RATIOS\": [1], \"ALLOWED_BORDER\": 9999}   \
 51 | }   \
 52 | ";
 53 | 
 54 | void generate_anchors_fpn( std::map<int, std::vector<cv::Rect2f>> & anchors_fpn, 
 55 |                            std::map<int,int> & num_anchors ) {
 56 | 
 57 |     auto json_conf = json::parse(config);
 58 |     for(auto stride: stride_fpn){
 59 |         std::vector<cv::Rect2f> anchors;
 60 |         int base_size = json_conf[std::to_string(stride)]["BASE_SIZE"];
 61 |         auto scale_array = json_conf[std::to_string(stride)]["SCALES"];
 62 |         std::vector<int> scales;
 63 |         scales.resize(scale_array.size());
 64 |         for(size_t i = 0; i< scales.size(); i++){
 65 |             scales[i] = scale_array[i];
 66 |         }
 67 |         auto ratio_array = json_conf[std::to_string(stride)]["RATIOS"];
 68 |         std::vector<float> ratios;
 69 |         ratios.resize(ratio_array.size());
 70 |         for(size_t i = 0; i< ratios.size(); i++){
 71 |             ratios[i] = ratio_array[i];
 72 |         }
 73 | 
 74 |         generate_anchors(base_size,ratios,scales,anchors);
 75 | 
 76 |         anchors_fpn[stride] = anchors;
 77 |         num_anchors[stride] = anchors.size();
 78 |     }
 79 | 
 80 | }
 81 | 
 82 | void anchor_plane(int h, int w, int stride, 
 83 |                   std::vector<cv::Rect2f>& anchors, 
 84 |                   std::vector<cv::Rect2f>& anchor_plane){
 85 |   anchor_plane.clear();
 86 |   for(int j = 0; j< h; j++)
 87 |       for(int i = 0; i<w; i++)
 88 |         for(auto & anchor: anchors){
 89 |           cv::Point2f tl = anchor.tl() + cv::Point2f(i*stride,j*stride);
 90 |           anchor_plane.push_back(cv::Rect2f(tl.x, tl.y, anchor.width, anchor.height));
 91 |         }
 92 | }
 93 | 
 94 | void clip_pad(std::vector<float> & tensor, int h, int w, std::vector<float> & pad_tensor , int pad_h, int pad_w){
 95 |     pad_tensor.clear();
 96 |     int loops = tensor.size()/(h*w);
 97 |     for(size_t k=0;k<loops;k++)
 98 |         for(size_t i=0;i<h;i++)
 99 |             for(size_t j=0;j<w;j++)
100 |                 if(i<pad_h && j<pad_w)
101 |                     pad_tensor.push_back(tensor[k*(w*h) + i*w + j ]);
102 | }
103 | 
104 | 
105 | void clip_boxes(std::vector<cv::Rect2f> & boxes , int h, int w){
106 |     for(auto & box: boxes){
107 |         if(box.x < 0) {
108 |             box.width += box.x;
109 |             box.x =0;
110 |         } else if(box.x>w){
111 |             box.width -= box.x - w;
112 |             box.x = w;
113 |         }
114 |         if(box.y<0) {
115 |             box.height += box.y;
116 |             box.y =0;            
117 |         } else if(box.y>h){
118 |             box.height -= box.y - h;
119 |             box.y = h;
120 |         }
121 |     }
122 | }
123 | 
124 | void bbox_pred(std::vector<cv::Rect2f> & anchors, std::vector<cv::Rect2f> & boxes, 
125 |                std::vector<float> & box_deltas, int h, int w) {
126 | 
127 |     std::vector<float> box_deltas2;
128 |     tensor_reshape(box_deltas, box_deltas2, h, w);
129 | 
130 |     int count = box_deltas2.size() / 4;
131 |     assert(anchors.size() == count );
132 |     boxes.resize(anchors.size());
133 | 
134 |     for(size_t i=0; i < count; i++){
135 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width-1, anchors[i].height-1);
136 |         center = center + cv::Point2f(box_deltas2[i*4]* anchors[i].width, box_deltas2[i*4+1]*anchors[i].height);
137 |         boxes[i].width  = anchors[i].width  * exp(box_deltas2[i*4+2]);
138 |         boxes[i].height = anchors[i].height * exp(box_deltas2[i*4+3]);
139 |         boxes[i].x = center.x - 0.5 * (boxes[i].width  -1.0);
140 |         boxes[i].y = center.y - 0.5 * (boxes[i].height -1.0);;
141 |     }
142 | 
143 | }
144 | 
145 | void bbox_pred(std::vector<cv::Rect2f> & anchors, std::vector<cv::Rect2f> & boxes,
146 |                std::vector<float> & box_deltas, int H, int W, int c)
147 | {
148 |     std::vector<float> box_deltas2;
149 |     tensor_reshape(box_deltas, box_deltas2, H, W, c);
150 | 
151 |     int count = box_deltas2.size() / 4;
152 |     assert(anchors.size() == count );
153 |     boxes.resize(anchors.size());
154 | 
155 |     for(size_t i=0; i < count; i++)
156 |     {
157 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width-1, anchors[i].height-1);
158 |         center = center + cv::Point2f(box_deltas2[i*4]* anchors[i].width, box_deltas2[i*4+1]*anchors[i].height);
159 |         boxes[i].width  = anchors[i].width  * exp(box_deltas2[i*4+2]);
160 |         boxes[i].height = anchors[i].height * exp(box_deltas2[i*4+3]);
161 |         boxes[i].x = center.x - 0.5 * (boxes[i].width  -1.0);
162 |         boxes[i].y = center.y - 0.5 * (boxes[i].height -1.0);;
163 |     }
164 | 
165 | }
166 | 
167 | void bbox_pred_blur(std::vector<cv::Rect2f> & anchors, std::vector<cv::Rect2f> & boxes, std::vector<float> & blur_scores,
168 |                std::vector<float> & box_deltas, int pred_len , int h, int w) {
169 | 
170 |     std::vector<float> box_deltas2;
171 |     tensor_reshape(box_deltas, box_deltas2, h, w);
172 | 
173 |     int count = box_deltas2.size() / pred_len;
174 |     assert(anchors.size() == count );
175 |     boxes.resize(anchors.size());
176 | 
177 |     for(size_t i=0; i < count; i++){
178 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width-1, anchors[i].height-1);
179 |         center = center + cv::Point2f(box_deltas2[i*pred_len]* anchors[i].width, box_deltas2[i*pred_len+1]*anchors[i].height);
180 |         boxes[i].width  = anchors[i].width  * exp(box_deltas2[i*pred_len+2]);
181 |         boxes[i].height = anchors[i].height * exp(box_deltas2[i*pred_len+3]);
182 |         boxes[i].x = center.x - 0.5 * (boxes[i].width  -1.0);
183 |         boxes[i].y = center.y - 0.5 * (boxes[i].height -1.0);;
184 |     }
185 | 
186 |     blur_scores.clear();
187 |     for(size_t i=0; i < count; i++){
188 |         blur_scores.push_back(box_deltas2[i*pred_len+4]);
189 |     }
190 | }
191 | 
192 | void bbox_pred_blur(std::vector<cv::Rect2f> & anchors, std::vector<cv::Rect2f> & boxes, std::vector<float> & blur_scores,
193 |                std::vector<float> & box_deltas, int pred_len , int h, int w, int C) {
194 | 
195 |     std::vector<float> box_deltas2;
196 |     tensor_reshape(box_deltas, box_deltas2, h, w, C);
197 | 
198 |     int count = box_deltas2.size() / pred_len;
199 |     assert(anchors.size() == count );
200 |     boxes.resize(anchors.size());
201 | 
202 |     for(size_t i=0; i < count; i++){
203 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width-1, anchors[i].height-1);
204 |         center = center + cv::Point2f(box_deltas2[i*pred_len]* anchors[i].width, box_deltas2[i*pred_len+1]*anchors[i].height);
205 |         boxes[i].width  = anchors[i].width  * exp(box_deltas2[i*pred_len+2]);
206 |         boxes[i].height = anchors[i].height * exp(box_deltas2[i*pred_len+3]);
207 |         boxes[i].x = center.x - 0.5 * (boxes[i].width  -1.0);
208 |         boxes[i].y = center.y - 0.5 * (boxes[i].height -1.0);;
209 |     }
210 | 
211 |     blur_scores.clear();
212 |     for(size_t i=0; i < count; i++){
213 |         blur_scores.push_back(box_deltas2[i*pred_len+4]);
214 |     }
215 | }
216 | 
217 | void landmark_pred(std::vector<cv::Rect2f> & anchors, std::vector<cv::Point2f> & landmarks, 
218 |                    std::vector<float> & landmark_deltas, int h, int w) {
219 | 
220 |     std::vector<float> landmark_deltas2;
221 |     tensor_reshape(landmark_deltas, landmark_deltas2, h, w);
222 |     size_t count = landmark_deltas2.size() / 10;
223 |     assert(anchors.size() == count );
224 | 
225 |     landmarks.resize(count*5);
226 | 
227 |     for(size_t i=0; i < count; i++){
228 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width-1, anchors[i].height-1);
229 |         for(size_t j=0; j<5; j++){
230 |             landmarks[i*5+j].x = center.x + anchors[i].width  * landmark_deltas2[i*10+j*2];
231 |             landmarks[i*5+j].y = center.y + anchors[i].height * landmark_deltas2[i*10+j*2+1];
232 |         }
233 |     }
234 |     
235 | }
236 | 
237 | void landmark_pred(std::vector<cv::Rect2f> & anchors, std::vector<cv::Point2f> & landmarks,
238 |                    std::vector<float> & landmark_deltas, int H, int W, int c)
239 | {
240 |     std::vector<float> landmark_deltas2;
241 |     tensor_reshape(landmark_deltas, landmark_deltas2, H, W, c);
242 |     size_t count = landmark_deltas2.size() / 10;
243 |     assert(anchors.size() == count);
244 | 
245 |     landmarks.resize(count*5);
246 | 
247 |     for (size_t i = 0; i < count; i++)
248 |     {
249 |         cv::Point2f center = anchors[i].tl() + 0.5 * cv::Point2f(anchors[i].width - 1, anchors[i].height - 1);
250 |         for (size_t j = 0; j < 5; j++)
251 |         {
252 |             landmarks[i * 5 + j].x = center.x + anchors[i].width  * landmark_deltas2[i * 10 + j * 2];
253 |             landmarks[i * 5 + j].y = center.y + anchors[i].height * landmark_deltas2[i * 10 + j * 2 + 1];
254 |         }
255 |     }
256 | 
257 | }
258 | 
259 | void nms(std::vector<float> & scores, std::vector<cv::Rect2f> & boxes,
260 |          std::vector<bool> & keep, float thresh) {
261 |     std::vector<bool> suppressed( scores.size(), false);
262 |     for(int i=0;i<scores.size();i++){
263 |         if(suppressed[i] == true)
264 |             continue;
265 |         keep[i] = true;
266 |         float ix1 = boxes[i].tl().x;
267 |         float iy1 = boxes[i].tl().y;
268 |         float ix2 = boxes[i].br().x;
269 |         float iy2 = boxes[i].br().y;
270 |         float iarea = boxes[i].area();
271 |         for(int j=i+1;j<scores.size();j++){
272 |             if(suppressed[j]==true)
273 |                 continue;
274 |             float xx1 = fmax(ix1, boxes[j].tl().x);
275 |             float yy1 = fmax(iy1, boxes[j].tl().y);
276 |             float xx2 = fmin(ix2, boxes[j].br().x);
277 |             float yy2 = fmin(iy2, boxes[j].br().y);
278 |             float w = fmax(0.0, xx2 - xx1 + 1);
279 |             float h = fmax(0.0, yy2 - yy1 + 1);
280 |             float inter = w * h;
281 |             float overlap = inter / (iarea + boxes[j].area() - inter);
282 |             if(overlap > thresh)
283 |                 suppressed[j] = true;
284 |         }
285 |     }
286 | }
287 | 
288 | #endif


--------------------------------------------------------------------------------
/include/ssh_detector.h:
--------------------------------------------------------------------------------
 1 | #ifndef __SSH_DETECTOR__
 2 | #define __SSH_DETECTOR__
 3 | 
 4 | #include <vector>
 5 | #include <string>
 6 | 
 7 | #include <opencv2/opencv.hpp>
 8 | 
 9 | class SSH{
10 | public:
11 |     SSH(const std::string& model_path, float threshold = 0.95, float nms_threshold = 0.3, bool infer_blur_score=false);
12 |     SSH(const std::string& model_path, std::vector<float> means, std::vector<float> stds, float scale,
13 |                                        float threshold = 0.95, float nms_threshold = 0.3, bool infer_blur_score=false);
14 |     ~SSH();
15 |     void detect(cv::Mat& img, std::vector<cv::Rect2f>  & target_boxes,
16 |                               std::vector<cv::Point2f> & target_landmarks,
17 |                               std::vector<float>       & target_scores);
18 |     void detect(cv::Mat& img, std::vector<cv::Rect2f>  & target_boxes,
19 |                               std::vector<cv::Point2f> & target_landmarks,
20 |                               std::vector<float>       & target_scores,
21 |                               std::vector<float>       & target_blur_scores);
22 | private:
23 | 
24 |     float pixel_means[3] = {0.406, 0.456, 0.485};
25 |     float pixel_stds[3]  = {0.225, 0.224, 0.229};
26 |     float pixel_scale = 255.0;
27 | 
28 |     std::map<int, std::vector<cv::Rect2f>> anchors_fpn;
29 |     std::map<int,int>                      num_anchors;
30 | 
31 |     // const int   rpn_pre_nms_top_n = 1000;
32 |     float nms_threshold = 0.3;
33 |     float threshold = 0.95;
34 | 
35 |     bool infer_blur_score = false;
36 | 
37 |     void * handle;
38 |     void * infer_buff = nullptr;
39 |     
40 |     // int w;
41 |     // int h;
42 |     // nms();
43 | 
44 | };
45 | 
46 | #endif


--------------------------------------------------------------------------------
/include/ssh_detector_mxnet.h:
--------------------------------------------------------------------------------
 1 | #ifndef __SSH_DETECTOR__
 2 | #define __SSH_DETECTOR__
 3 | 
 4 | #include <vector>
 5 | #include <string>
 6 | 
 7 | #include <opencv2/opencv.hpp>
 8 | 
 9 | #include "../mxnet/ssh_detector/mxnet_model.h"
10 | 
11 | class SSH{
12 | public:
13 |     SSH(const std::string& model_path, float threshold = 0.95, float nms_threshold = 0.3);
14 |     SSH(const std::string& model_path, const std::string& model_name,
15 |                                        float threshold = 0.95, float nms_threshold = 0.3,
16 |                                        bool infer_blur_score=false);
17 |     SSH(const std::string& model_path, const std::string& model_name,
18 |                                        std::vector<float> means, std::vector<float> stds, float scale,
19 |                                        float threshold = 0.95, float nms_threshold = 0.3,
20 |                                        bool infer_blur_score=false);
21 |     // ~SSH();
22 |     void detect(cv::Mat& img, std::vector<cv::Rect2f>  & target_boxes,
23 |                               std::vector<cv::Point2f> & target_landmarks,
24 |                               std::vector<float>       & target_scores);
25 |     void detect(cv::Mat& img, std::vector<cv::Rect2f>  & target_boxes,
26 |                               std::vector<cv::Point2f> & target_landmarks,
27 |                               std::vector<float>       & target_scores,
28 |                               std::vector<float>       & target_blur_scores);
29 | private:
30 | 
31 |     float pixel_means[3] = {0.406, 0.456, 0.485};
32 |     float pixel_stds[3]  = {0.225, 0.224, 0.229};
33 |     float pixel_scale = 255.0;
34 | 
35 |     std::map<int, std::vector<cv::Rect2f>> anchors_fpn;
36 |     std::map<int,int>                      num_anchors;
37 | 
38 |     // const int   rpn_pre_nms_top_n = 1000;
39 |     float nms_threshold = 0.3;
40 |     float threshold = 0.95;
41 | 
42 |     bool infer_blur_score = false;
43 | 
44 |     // void * handle;
45 |     // int w;
46 |     // int h;
47 |     mxBufferFile json_data;
48 |     mxBufferFile param_data;
49 |     // nms();
50 | 
51 | };
52 | 
53 | #endif


--------------------------------------------------------------------------------
/include/tensor_utils.h:
--------------------------------------------------------------------------------
 1 | #include <vector>
 2 | #include <functional>
 3 | 
 4 | template<typename T>
 5 | inline void argsort( std::vector<int> & idx, const std::vector<T>& v){
 6 |     int Len = v.size();
 7 |     idx.resize(Len);
 8 |     for(int i = 0; i < Len; i++){
 9 |             idx[i] = i;
10 |     }
11 |     std::sort(idx.begin(), idx.end(), [&v](int i1, int i2){return v[i1] > v[i2];});
12 | }
13 | 
14 | template<typename T>
15 | inline void filter_threshold( std::vector<bool> & idx, const std::vector<T> & v, const T threshold){
16 |     int Len = v.size();
17 |     idx.resize(Len);
18 |     for(size_t i=0;i<Len;i++){
19 |         if( v[i] >= threshold ){
20 |             idx[i] = true;
21 |         } 
22 |         else 
23 |             idx[i] = false;
24 |     }
25 | }
26 | 
27 | template<typename T>
28 | inline void tensor_slice(std::vector<T> & tensor1, std::vector<T> & tensor2, 
29 |                          std::vector<bool> & idx, int stride) {
30 |     tensor2.clear();
31 |     for(size_t i=0; i<idx.size(); i++)
32 |         if(idx[i]==true)
33 |             for(int j=0;j<stride;j++)
34 |                 tensor2.push_back(tensor1[stride*i+j]);
35 | }
36 | 
37 | template<typename T>
38 | inline void sort_with_idx(std::vector<T> & tensor1, std::vector<T> & tensor2, 
39 |                           std::vector<int> & idx, int stride){
40 |     for(size_t i=0; i<idx.size(); i++)
41 |         for(int j=0;j<stride;j++)
42 |             tensor2.push_back(tensor1[idx[i]*stride +j]);
43 | }
44 | 
45 | inline void tensor_reshape(std::vector<float> & tensor1, std::vector<float> & tensor2, int h, int w){
46 | 
47 |     int d1 = h * w;
48 |     int d2 = tensor1.size()/d1;
49 |     tensor2.resize(tensor1.size());
50 | 
51 |     for(size_t i = 0; i< d2; i++)
52 |         for(size_t j = 0; j< d1; j++)
53 |             tensor2[i+j*d2] = tensor1[i*d1+j];
54 | }
55 | 
56 | inline void tensor_reshape(std::vector<float> & tensor1, std::vector<float> & tensor2, int H, int W, int c)
57 | {
58 | 
59 |     int d1 = H * W;
60 |     int d2 = tensor1.size() / d1 / c;
61 |     tensor2.resize(tensor1.size());
62 | 
63 |     if (c == 1)
64 |     {
65 |         for (int i = 0; i < d2; ++i)
66 |         {
67 |             for (int j = 0; j < d1; ++j)
68 |             {
69 |                 tensor2[i + j * d2] = tensor1[i * d1 + j];
70 |             }
71 |         }
72 |     }
73 |     else
74 |     {
75 |         for (int i = 0; i < d2; ++i)
76 |         {
77 |             for (int j = 0; j < d1; ++j)
78 |             {
79 |                 int base1 = (i * d1 + j) * c;
80 |                 int base2 = (i + j * d2) * c;
81 |                 for (int k = 0; k < c; ++k)
82 |                 {
83 |                     tensor2[base2 + k] = tensor1[base1 + k];
84 |                 }
85 |             }
86 |         }
87 |     }
88 | }
89 | 


--------------------------------------------------------------------------------
/mxnet/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8.10)
 2 | 
 3 | project(ssh_detector_mxnet)
 4 | 
 5 | add_definitions("-Wall -g -O2")
 6 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 7 | 
 8 | if(BENCH_SSH)
 9 |         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBENCH_SSH")
10 | endif()
11 | 
12 | find_package( OpenCV REQUIRED )
13 | 
14 | message(STATUS "CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
15 | message(STATUS "CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
16 | include_directories(${OpenCV_INCLUDE_DIRS})
17 | 
18 | include_directories(${PROJECT_SOURCE_DIR}/../include)
19 | 
20 | message(STATUS "mxnet include path: ${MXNET_PATH}/include")
21 | message(STATUS "mxnet link path: ${MXNET_PATH}/lib")
22 | include_directories(${MXNET_PATH}/include)
23 | link_directories(${MXNET_PATH}/lib/)
24 | 
25 | if (APPLE)
26 |     include_directories(/opt/local/include)
27 |     link_directories(/opt/local/lib)
28 | endif()
29 | 
30 | link_directories(
31 |   ${OpenCV_Install}/lib
32 |   /usr/local/lib
33 | )
34 | 
35 | add_subdirectory(${PROJECT_SOURCE_DIR}/ssh_detector)
36 | 
37 | add_executable(main main.cpp )
38 | target_link_libraries(main ssh_detector mxnet ${OpenCV_LIBS} )
39 | 
40 | add_executable(ssh_test ssh_test.cpp )
41 | target_link_libraries(ssh_test ssh_detector mxnet ${OpenCV_LIBS} )
42 | 
43 | add_executable(ssh_bench ssh_bench.cpp )
44 | target_link_libraries(ssh_bench ssh_detector mxnet ${OpenCV_LIBS} )


--------------------------------------------------------------------------------
/mxnet/main.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one
  3 |  * or more contributor license agreements.  See the NOTICE file
  4 |  * distributed with this work for additional information
  5 |  * regarding copyright ownership.  The ASF licenses this file
  6 |  * to you under the Apache License, Version 2.0 (the
  7 |  * "License"); you may not use this file except in compliance
  8 |  * with the License.  You may obtain a copy of the License at
  9 |  *
 10 |  *   http://www.apache.org/licenses/LICENSE-2.0
 11 |  *
 12 |  * Unless required by applicable law or agreed to in writing,
 13 |  * software distributed under the License is distributed on an
 14 |  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 15 |  * KIND, either express or implied.  See the License for the
 16 |  * specific language governing permissions and limitations
 17 |  * under the License.
 18 |  */
 19 | 
 20 | /*!
 21 |  * Copyright (c) 2015 by Xiao Liu, pertusa, caprice-j
 22 |  * \file image_classification-predict.cpp
 23 |  * \brief C++ predict example of mxnet
 24 |  *
 25 |  * This is a simple predictor which shows how to use c api for image classification. It uses
 26 |  * opencv for image reading.
 27 |  *
 28 |  * Created by liuxiao on 12/9/15.
 29 |  * Thanks to : pertusa, caprice-j, sofiawu, tqchen, piiswrong
 30 |  * Home Page: www.liuxiao.org
 31 |  * E-mail: liuxiao@foxmail.com
 32 | */
 33 | 
 34 | #include "ssh_detector/mxnet_model.h"
 35 | 
 36 | #include "anchors.h"
 37 | 
 38 | int main(int argc, char* argv[]) {
 39 | 
 40 |   const std::string keys =
 41 |       "{help h usage ? |                | print this message   }"
 42 |       "{model          |../../model     | path to ssh model    }"
 43 |       "{index          |0               | mxnet output index   }"      
 44 |       "{@image         |                | input image          }"
 45 |   ;
 46 | 
 47 |   cv::CommandLineParser parser(argc, argv, keys);
 48 |   parser.about("ssh detector");
 49 |   if (parser.has("help")) {
 50 |       parser.printMessage();
 51 |       return 0;
 52 |   }
 53 | 
 54 |   if (!parser.check()) {
 55 |       parser.printErrors();
 56 |       return EXIT_FAILURE;
 57 |   }
 58 | 
 59 |   std::string model_path = parser.get<std::string>("model");
 60 |   int index = parser.get<int>("index");
 61 |   std::string image_path = parser.get<std::string>(0);
 62 | 
 63 |   // Models path for your model, you have to modify it
 64 |   std::string json_file = model_path + "/mneti-symbol.json";
 65 |   std::string param_file = model_path + "/mneti-0000.params";
 66 | 
 67 |   // Image size and channels
 68 |   int width = 640;
 69 |   int height = 640;
 70 |   int channels = 3;
 71 |   mxInputShape input_shape (width, height, channels);
 72 |   
 73 |   // Load model
 74 |   PredictorHandle pred_hnd = nullptr;
 75 |   mxLoadMXNetModel(&pred_hnd, json_file, param_file, input_shape);
 76 | 
 77 |   // Read Image Data
 78 |   auto image_size = static_cast<std::size_t>(width * height * channels);
 79 |   std::vector<mx_float> image_data(image_size);
 80 |   mxGetImageFile(image_path, image_data);
 81 | 
 82 |   // Inference
 83 |   std::vector<float> data;
 84 |   mx_uint output_index = index;
 85 |   std::vector<int> shape;
 86 |   mxInfer(pred_hnd, image_data);
 87 |   mxOutputOfIndex(pred_hnd, data, shape, output_index);
 88 | 
 89 |   // normalize the output vector
 90 |   std::vector<float> output(data.size());
 91 |   cv::normalize(data, output);
 92 | 
 93 |   // Print Output Data
 94 |   // mxPrintOutputResult(output);
 95 | 
 96 |   // Release Predictor
 97 |   MXPredFree(pred_hnd);
 98 | 
 99 |   std::map<int, std::vector<cv::Rect2f>> anchors_fpn;
100 |   std::map<int,int> num_anchors;
101 | 
102 |   generate_anchors_fpn(anchors_fpn, num_anchors);
103 | 
104 |   for(auto & element: anchors_fpn){
105 |       std::cout << "k:" << element.first << std::endl;
106 |       for(auto & anckor: element.second)
107 |         std::cout << "anckor:" << anckor << std::endl;
108 |   }
109 |   for(auto & element: num_anchors){
110 |       std::cout << "k:" << element.first << " v:" << element.second<< std::endl;
111 |   }
112 | 
113 |   std::vector<float> tensor;
114 |   std::vector<float> tensor_pad;
115 |   tensor.push_back(1);
116 |   tensor.push_back(2);
117 |   tensor.push_back(3);
118 |   tensor.push_back(4);
119 |   tensor.push_back(5);
120 |   tensor.push_back(6);
121 |   tensor.push_back(7);
122 |   tensor.push_back(8);
123 |   tensor.push_back(9);
124 |   tensor.push_back(10);
125 |   tensor.push_back(11);
126 |   tensor.push_back(12);  
127 | 
128 |   clip_pad(tensor,2,3, tensor_pad, 1,1);
129 | 
130 |   for(auto & t: tensor_pad)
131 |     std::cout<< "t: " << t <<"\n";
132 | 
133 |   std::vector<float> tensor2;
134 |   tensor_reshape(tensor, tensor2, 2, 3);
135 | 
136 |   for(auto & t: tensor2)
137 |     std::cout<< "t2: " << t <<"\n";
138 | 
139 |   return EXIT_SUCCESS;
140 | }
141 | 


--------------------------------------------------------------------------------
/mxnet/ssh_bench.cpp:
--------------------------------------------------------------------------------
  1 | #include "ssh_detector_mxnet.h"
  2 | 
  3 | #include <stdlib.h>
  4 | #include <sys/time.h>
  5 | 
  6 | #include <stdio.h>
  7 | 
  8 | static float getElapse(struct timeval *tv1,struct timeval *tv2)
  9 | {
 10 |     float t = 0.0f;
 11 |     if (tv1->tv_sec == tv2->tv_sec)
 12 |         t = (tv2->tv_usec - tv1->tv_usec)/1000.0f;
 13 |     else
 14 |         t = ((tv2->tv_sec - tv1->tv_sec) * 1000 * 1000 + tv2->tv_usec - tv1->tv_usec)/1000.0f;
 15 |     return t;
 16 | }
 17 | 
 18 | double tenengrad(const cv::Mat& src, int ksize)
 19 | {
 20 |     cv::Mat Gx, Gy;
 21 |     cv::Sobel(src, Gx, CV_64F, 1, 0, ksize);
 22 |     cv::Sobel(src, Gy, CV_64F, 0, 1, ksize);
 23 | 
 24 |     cv::Mat FM = Gx.mul(Gx) + Gy.mul(Gy);
 25 | 
 26 |     double focusMeasure = cv::mean(FM).val[0];
 27 |     return focusMeasure;
 28 | }
 29 | 
 30 | bool getEyeRoi(cv::Mat img, cv::Point2f eye_point, cv::Mat & res) {
 31 |     cv::Point2f p1(eye_point.x-5,eye_point.y-5);
 32 |     cv::Rect2f r1(p1.x,p1.y,10,10);
 33 |     cv::Rect2f rect_all(0,0,img.cols,img.rows);
 34 |     cv::Rect2f r2 = r1 & rect_all;
 35 |     if(r2.area()<=0) return false;
 36 |     res = img(r2);
 37 |     return true;
 38 | }
 39 | 
 40 | int main(int argc, char* argv[]) {
 41 | 
 42 |   const std::string keys =
 43 |       "{help h usage ? |                | print this message   }"
 44 |       "{model_path     |../../model     | path to ssh model  }"
 45 |       "{model_name     |mneti           | model name }"
 46 |       "{blur           |false           | if use blur scores }" 
 47 |       "{peroid         |1              | detect peroid }"
 48 |       "{threshold      |0.95            | threshold for detect score }"
 49 |       "{nms_threshold  |0.3             | nms_threshold for ssh detector }"      
 50 |       "{output         |./output        | path to save detect output  }"
 51 |       "{input          |../../../video  | path to input video file  }"      
 52 |       "{dynamic        |false           | true if input size is dynamic  }"
 53 |       "{@video         |camera-244-crop-8p.mov     | input video file }"
 54 |   ;
 55 | 
 56 |   cv::CommandLineParser parser(argc, argv, keys);
 57 |   parser.about("ssh detector benchmark");
 58 |   if (parser.has("help")) {
 59 |       parser.printMessage();
 60 |       return 0;
 61 |   }
 62 | 
 63 |   if (!parser.check()) {
 64 |       parser.printErrors();
 65 |       return EXIT_FAILURE;
 66 |   }
 67 | 
 68 |   std::string model_path = parser.get<std::string>("model_path");
 69 |   std::string model_name = parser.get<std::string>("model_name");
 70 |   bool blur = parser.get<bool>("blur");
 71 |   int peroid              = parser.get<int>("peroid");
 72 |   float threshold         = parser.get<float>("threshold");
 73 |   float nms_threshold     = parser.get<float>("nms_threshold");
 74 |   std::string output_path = parser.get<std::string>("output");
 75 |   std::string input_path  = parser.get<std::string>("input");
 76 |   std::string video_file  = parser.get<std::string>(0);
 77 |   bool dynamic = parser.get<bool>("dynamic");
 78 |   std::string video_path  = input_path + "/" + video_file;
 79 | 
 80 |   std::string output_folder = output_path + "/" + video_file;
 81 |   std::string cmd = "rm -rf " + output_folder;
 82 |   system(cmd.c_str());
 83 |   cmd = "mkdir -p " + output_folder;
 84 |   system(cmd.c_str());
 85 | 
 86 |   cv::VideoCapture capture(video_path);
 87 |   cv::Mat frame_, frame;
 88 |   int frame_count = 1;
 89 |   capture >> frame;
 90 |   if(!frame.data) {
 91 |       std::cout<< "read first frame failed!";
 92 |       exit(1);
 93 |   }
 94 |   SSH * det = new SSH(model_path, model_name, threshold, nms_threshold, blur);
 95 | 
 96 |   std::cout << "frame resolution: " << frame.cols << "*" << frame.rows << "\n";
 97 | 
 98 |   std::vector<cv::Rect2f> boxes;
 99 |   std::vector<cv::Point2f> landmarks;
100 |   std::vector<float> scores;
101 |   std::vector<float> blur_scores;  
102 | 
103 |   struct timeval  tv1,tv2;
104 | 
105 |   while(1){
106 |       capture >> frame_;
107 |       /*
108 |       float resize_rate = 1;
109 |       resize_rate += (rand() % 10) / 10.0;
110 |       cv::Size s = frame_.size();
111 |       s.width  /= resize_rate;
112 |       s.height /= resize_rate;
113 |       cv::resize(frame_,frame,s);
114 |       */
115 |       if(!frame_.data)
116 |         break;
117 |       int w = frame_.cols;
118 |       int h = frame_.rows;
119 |       w *=  (rand() % 20 + 1 ) / 20.0;
120 |       h *=  (rand() % 20 + 1 ) / 20.0;
121 |       int x = rand() % frame_.cols;
122 |       int y = rand() % frame_.rows;
123 |       cv::Rect roi(x,y,w,h);
124 |       roi = roi & cv::Rect(0,0,frame_.cols,frame_.rows);
125 |       
126 |       if(dynamic)
127 |         frame = cv::Mat(frame_, roi);
128 |       else
129 |           frame = frame_;
130 | 
131 |       frame_count++;
132 |       if(frame_count%peroid!=0) continue;
133 |       
134 |       gettimeofday(&tv1,NULL);
135 |       if(blur)
136 |         det->detect(frame,boxes,landmarks,scores,blur_scores);
137 |       else
138 |         det->detect(frame,boxes,landmarks,scores);
139 |       gettimeofday(&tv2,NULL);
140 |       std::cout << "detected one frame, time eclipsed: " <<  getElapse(&tv1, &tv2) << " ms\n";
141 | 
142 |       for(auto & b: boxes)
143 |         cv::rectangle( frame, b, cv::Scalar( 255, 0, 0 ), 2, 1 );
144 |         
145 |       for(int i=0;i<boxes.size();i++){
146 |         cv::rectangle( frame, boxes[i], cv::Scalar( 255, 0, 0 ), 2, 1 );
147 | 
148 |         // cv::circle(frame,landmarks[i*5], 5, cv::Scalar( 255, 0, 0 ), 2, 1);
149 |         // cv::circle(frame,landmarks[i*5+1], 5, cv::Scalar( 255, 0, 0 ), 2, 1);
150 |         /*
151 |         cv::Mat e1,e2;
152 |         double s1,s2;
153 |         bool res = getEyeRoi(frame, landmarks[i*5], e1);
154 |         if(res) 
155 |             s1 = tenengrad(e1,3);
156 |         else 
157 |             s1 = 0;
158 |         res = getEyeRoi(frame, landmarks[i*5+1],e2);
159 |         if(res) 
160 |             s2 = tenengrad(e1,3);
161 |         else 
162 |             s2 =0;
163 |         
164 |         double s = 0;
165 |         cv::Rect2f rect_ = boxes[i] & cv::Rect2f(0,0,frame.cols,frame.rows);
166 |         if(rect_.area()>0)  {
167 |             cv::Mat box = frame(boxes[i]);
168 |             s = tenengrad(box,7);
169 |         }
170 |         */
171 |         // s = s / 10000;
172 |         cv::Point middleHighPoint = cv::Point(boxes[i].x+boxes[i].width/2, boxes[i].y);
173 |         char buf_text[64];
174 |         if(blur)
175 |             snprintf(buf_text, 64, "s:%.3f, b: %.3f", scores[i], blur_scores[i]);
176 |         else
177 |             snprintf(buf_text, 64, "s:%.3e", scores[i]);
178 |         std::string text(buf_text);
179 |         cv::putText(frame, text, middleHighPoint, cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2);
180 | 
181 |       }
182 | 
183 |       //for(auto & p: landmarks)
184 |       //  cv::drawMarker(frame, p,  cv::Scalar(0, 255, 0), cv::MARKER_CROSS, 10, 1);    
185 | 
186 |       if(boxes.size()>0)
187 |         cv::imwrite( output_folder + "/" + std::to_string(frame_count) + ".jpg", frame);
188 | 
189 |   }
190 | 
191 |   delete det;
192 | 
193 | }


--------------------------------------------------------------------------------
/mxnet/ssh_detector/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | file(GLOB ssh_detector_source "*.cpp")
2 | add_library( ssh_detector ${ssh_detector_source} )
3 | 


--------------------------------------------------------------------------------
/mxnet/ssh_detector/mxnet_model.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one
  3 |  * or more contributor license agreements.  See the NOTICE file
  4 |  * distributed with this work for additional information
  5 |  * regarding copyright ownership.  The ASF licenses this file
  6 |  * to you under the Apache License, Version 2.0 (the
  7 |  * "License"); you may not use this file except in compliance
  8 |  * with the License.  You may obtain a copy of the License at
  9 |  *
 10 |  *   http://www.apache.org/licenses/LICENSE-2.0
 11 |  *
 12 |  * Unless required by applicable law or agreed to in writing,
 13 |  * software distributed under the License is distributed on an
 14 |  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 15 |  * KIND, either express or implied.  See the License for the
 16 |  * specific language governing permissions and limitations
 17 |  * under the License.
 18 |  */
 19 | 
 20 | /*!
 21 |  * Copyright (c) 2015 by Xiao Liu, pertusa, caprice-j
 22 |  * \file image_classification-predict.cpp
 23 |  * \brief C++ predict example of mxnet
 24 |  *
 25 |  * This is a simple predictor which shows how to use c api for image classification. It uses
 26 |  * opencv for image reading.
 27 |  *
 28 |  * Created by liuxiao on 12/9/15.
 29 |  * Thanks to : pertusa, caprice-j, sofiawu, tqchen, piiswrong
 30 |  * Home Page: www.liuxiao.org
 31 |  * E-mail: liuxiao@foxmail.com
 32 | */
 33 | 
 34 | #include <cstdio>
 35 | #include <cstdlib>
 36 | #include <iostream>
 37 | #include <fstream>
 38 | #include <memory>
 39 | #include <iomanip>
 40 | #include <cassert>
 41 | 
 42 | #include "mxnet_model.h"
 43 | 
 44 | mxBufferFile::mxBufferFile(const std::string& file_path):file_path_(file_path) {
 45 | 
 46 |     std::ifstream ifs(file_path.c_str(), std::ios::in | std::ios::binary);
 47 |     if (!ifs) {
 48 |       std::cerr << "Can't open the file. Please check " << file_path << ". \n";
 49 |       return;
 50 |     }
 51 | 
 52 |     ifs.seekg(0, std::ios::end);
 53 |     length_ = static_cast<std::size_t>(ifs.tellg());
 54 |     ifs.seekg(0, std::ios::beg);
 55 |     std::cout << file_path.c_str() << " ... " << length_ << " bytes\n";
 56 | 
 57 |     buffer_.reset(new char[length_]);
 58 |     ifs.read(buffer_.get(), length_);
 59 |     ifs.close();
 60 | }
 61 | 
 62 | mxInputShape::mxInputShape(int width, int height, int channels) {
 63 |     input_shape_data[0] = 1;
 64 |     input_shape_data[1] = static_cast<mx_uint>(channels); 
 65 |     input_shape_data[2] = static_cast<mx_uint>(height);
 66 |     input_shape_data[3] = static_cast<mx_uint>(width);
 67 | }
 68 | 
 69 | 
 70 | void mxGetImageFile(const std::string& image_file, std::vector<mx_float> & image_data ) {
 71 |   // Read all kinds of file into a BGR color 3 channels image
 72 |   // the shape(height, width, channels)
 73 |   cv::Mat im = cv::imread(image_file, cv::IMREAD_COLOR);
 74 | 
 75 |   if (im.empty()) {
 76 |     std::cerr << "Can't open the image. Please check " << image_file << ". \n";
 77 |     assert(false);
 78 |   }
 79 | 
 80 |   int size = im.rows * im.cols * 3;
 81 |   image_data.resize(size);
 82 | 
 83 |   mx_float* ptr_image_r = image_data.data();
 84 |   mx_float* ptr_image_g = image_data.data() + size / 3;
 85 |   mx_float* ptr_image_b = image_data.data() + size / 3 * 2;
 86 | 
 87 |   for (int i = 0; i < im.rows; i++) {
 88 |     auto data = im.ptr<uchar>(i);
 89 | 
 90 |     for (int j = 0; j < im.cols; j++) {
 91 |         *ptr_image_b++ = static_cast<mx_float>(*data++);
 92 |         *ptr_image_g++ = static_cast<mx_float>(*data++);
 93 |         *ptr_image_r++ = static_cast<mx_float>(*data++);
 94 |     }
 95 |   }
 96 | }
 97 | 
 98 | void mxInfer ( PredictorHandle pred_hnd,         /* mxnet model */
 99 |            std::vector<mx_float> &image_data   /* input data */ )
100 | {         
101 |   // Set Input Image
102 |   int res = MXPredSetInput(pred_hnd, "data", image_data.data(), static_cast<mx_uint>(image_data.size()));
103 |   assert(res == 0);
104 |   // Do Predict Forward
105 |   res = MXPredForward(pred_hnd);
106 |   assert(res == 0);
107 | }
108 | 
109 | void mxOutputOfIndex ( PredictorHandle pred_hnd, /* mxnet model */
110 |            std::vector<float> &data,           /* output vector */
111 |            std::vector<int> &out_shape,        /* output tensor shape */
112 |            mx_uint output_index ) { 
113 | 
114 |   mx_uint* shape = nullptr;
115 |   mx_uint shape_len;
116 | 
117 |   // Get Output Result
118 |   int res = MXPredGetOutputShape(pred_hnd, output_index, &shape, &shape_len);
119 |   assert(res == 0);
120 | 
121 |   // std::cout << "output shape_len: " << shape_len << std::endl;
122 |   // for(mx_uint index=0; index<shape_len; index++) std::cout << shape[index] << std::endl;
123 | 
124 |   assert(shape_len==4);
125 | 
126 |   std::size_t size = 1;
127 |   out_shape.clear();
128 |   for (mx_uint i = 0; i < shape_len; ++i) { 
129 |       size *= shape[i]; 
130 |       out_shape.push_back(shape[i]);
131 |   }
132 | 
133 |   data.resize(size);
134 | 
135 |   res = MXPredGetOutput(pred_hnd, output_index, &(data[0]), static_cast<mx_uint>(size));
136 |   assert(res == 0);
137 | 
138 | }
139 | 
140 | void mxPrintOutputResult(const std::vector<float>& output) {
141 |     std::cout<< "embedding size: " << output.size() <<"\n";
142 |     for(int i=0; i < output.size(); ++i) {
143 |     std::cout << output[i];
144 |     if((i+1) % 16 == 0) {
145 |       std::cout << std::endl;
146 |     } else {
147 |       std::cout << " ";
148 |     }
149 |   }
150 | }
151 | 
152 | /*
153 |  * Load mxnet model
154 |  *
155 |  * Inputs:
156 |  * - json_file:  path to model-symbol.json
157 |  * - param_file: path to model-0000.params
158 |  * - shape: input shape to mxnet model (1, channels, height, width)
159 |  * - dev_type: 1: cpu, 2:gpu
160 |  * - dev_id: 0: arbitary
161 |  *
162 |  * Output:
163 |  * - PredictorHandle
164 |  */
165 | void mxLoadMXNetModel ( PredictorHandle* pred_hnd, /* Output */
166 |                       std::string json_file,     /* path to model-symbol.json */
167 |                       std::string param_file,    /* path to model-0000.params */
168 |                       mxInputShape shape,          /* input shape to mxnet model (1, channels, height, width) */
169 |                       int dev_type,              /* 1: cpu, 2:gpu */
170 |                       int dev_id ) {             /* 0: arbitary */
171 | 
172 |   mxBufferFile json_data(json_file);
173 |   mxBufferFile param_data(param_file);
174 | 
175 |   if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
176 |     std::cerr << "Cannot load mxnet model" << std::endl;
177 |     std::cerr << "\tjson file: " << json_file << std::endl;
178 |     std::cerr << "\tparams file: " << param_file << std::endl;
179 |     exit(EXIT_FAILURE);
180 |   }
181 | 
182 |   // Parameters
183 |   // int dev_type = 1;  // 1: cpu, 2: gpu
184 |   // int dev_id = 0;  // arbitrary.
185 |   mx_uint num_input_nodes = 1;  // 1 for feedforward
186 |   const char* input_key[1] = { "data" };
187 |   const char** input_keys = input_key;
188 | 
189 |   // Create Predictor
190 |   int res = MXPredCreate(static_cast<const char*>(json_data.GetBuffer()),
191 |                static_cast<const char*>(param_data.GetBuffer()),
192 |                static_cast<int>(param_data.GetLength()),
193 |                dev_type,
194 |                dev_id,
195 |                num_input_nodes,
196 |                input_keys,
197 |                shape.input_shape_indptr,
198 |                shape.input_shape_data,
199 |                pred_hnd);
200 |   assert(res==0);
201 | }
202 | 
203 | void mxHandleReshape(PredictorHandle  handle,   /* mxnet model handle */
204 |                      mxInputShape     shape,    /* new shape */
205 |                      PredictorHandle* out) {    /* new hanlde */
206 |   
207 |   mx_uint num_input_nodes = 1;  // 1 for feedforward
208 |   const char* input_key[1] = { "data" };
209 |   const char** input_keys = input_key;
210 |   // mxnet 1.2.1 has bug in c api MXPredReshape
211 |   // must merge bug fix commit 31244922d7861fef484fc315e633b4a0cc7e6ff6
212 |   int res = MXPredReshape(num_input_nodes,
213 |                 input_keys,
214 |                 shape.input_shape_indptr,
215 |                 shape.input_shape_data,
216 |                 handle,
217 |                 out );
218 |   assert(res == 0);
219 | }
220 | 


--------------------------------------------------------------------------------
/mxnet/ssh_detector/mxnet_model.h:
--------------------------------------------------------------------------------
 1 | #ifndef _MXNET_MODEL_H_
 2 | #define _MXNET_MODEL_H_
 3 | 
 4 | #include <vector>
 5 | #include <string>
 6 | 
 7 | #include <opencv2/opencv.hpp>
 8 | // Path for c_predict_api
 9 | #include "mxnet/c_predict_api.h"
10 | 
11 | // Read file to buffer
12 | class mxBufferFile {
13 |  public :
14 |   std::string file_path_;
15 |   std::size_t length_ = 0;
16 |   std::unique_ptr<char[]> buffer_;
17 | 
18 |   mxBufferFile(const std::string& file_path);
19 | 
20 |   inline std::size_t GetLength() {
21 |     return length_;
22 |   }
23 | 
24 |   inline char* GetBuffer() {
25 |     return buffer_.get();
26 |   }
27 | };
28 | 
29 | class mxInputShape {
30 | public:
31 |   mx_uint input_shape_indptr[2] = {0,4};
32 |   mx_uint input_shape_data[4];
33 | 
34 |   mxInputShape(int width, int height, int channels);
35 | };
36 | 
37 | void mxGetImageFile(const std::string& image_file, std::vector<mx_float>& image_data );
38 | 
39 | /*
40 |  * Load mxnet model
41 |  *
42 |  * Inputs:
43 |  * - json_file:  path to model-symbol.json
44 |  * - param_file: path to model-0000.params
45 |  * - shape: input shape to mxnet model (1, channels, height, width)
46 |  * - dev_type: 1: cpu, 2:gpu
47 |  * - dev_id: 0: arbitary
48 |  *
49 |  * Output:
50 |  * - PredictorHandle
51 |  */
52 | void mxLoadMXNetModel ( PredictorHandle* pred_hnd, /* Output */
53 |                       std::string json_file,     /* path to model-symbol.json */
54 |                       std::string param_file,    /* path to model-0000.params */
55 |                       mxInputShape shape,          /* input shape to mxnet model (1, channels, height, width) */
56 |                       int dev_type = 1,          /* 1: cpu, 2:gpu */
57 |                       int dev_id = 0             /* 0: arbitary */
58 |                     );
59 | 
60 | void mxHandleReshape(PredictorHandle  handle,   /* mxnet model handle */
61 |                      mxInputShape     shape,    /* new shape */
62 |                      PredictorHandle* out);     /* new hanlde */
63 | 
64 | void mxInfer ( PredictorHandle pred_hnd,         /* mxnet model */
65 |            std::vector<mx_float> &image_data);   /* input data */ 
66 | 
67 | void mxOutputOfIndex ( PredictorHandle pred_hnd, /* mxnet model */
68 |            std::vector<float> &data,           /* output vector */
69 |            std::vector<int> &out_shape,        /* output tensor shape */
70 |            mx_uint output_index );
71 | 
72 | void mxPrintOutputResult(const std::vector<float>& output);
73 | 
74 | #endif // _MXNET_MODEL_H_


--------------------------------------------------------------------------------
/mxnet/ssh_detector/ssh_detector.cpp:
--------------------------------------------------------------------------------
  1 | #include "ssh_detector_mxnet.h"
  2 | #include <iostream>
  3 | #include <iomanip>
  4 | #include <sys/time.h>
  5 | 
  6 | #include "anchors.h"
  7 | 
  8 | static float getElapse(struct timeval *tv1,struct timeval *tv2)
  9 | {
 10 |     float t = 0.0f;
 11 |     if (tv1->tv_sec == tv2->tv_sec)
 12 |         t = (tv2->tv_usec - tv1->tv_usec)/1000.0f;
 13 |     else
 14 |         t = ((tv2->tv_sec - tv1->tv_sec) * 1000 * 1000 + tv2->tv_usec - tv1->tv_usec)/1000.0f;
 15 |     return t;
 16 | }
 17 | 
 18 | SSH::SSH(const std::string& model_path, float threshold, float nms_threshold):
 19 |           json_data(model_path + "/mneti-symbol.json"),
 20 |           param_data(model_path + "/mneti-symbol.json")
 21 | {
 22 |     if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
 23 |         std::cerr << "Cannot load mxnet model" << std::endl;
 24 |         std::cerr << "\tjson file: " << json_data.file_path_ << std::endl;
 25 |         std::cerr << "\tparams file: " << param_data.file_path_ << std::endl;
 26 |         exit(EXIT_FAILURE);
 27 |     }
 28 | 
 29 |     generate_anchors_fpn(anchors_fpn, num_anchors);
 30 |     /*
 31 |     std::string json_file = model_path + "/mneti-symbol.json";
 32 |     std::string param_file = model_path + "/mneti-0000.params";
 33 | 
 34 |     int channels = 3;
 35 |     mxInputShape input_shape (w, h, channels);
 36 |     
 37 |     // Load model
 38 |     PredictorHandle pred_hnd = nullptr;
 39 |     mxLoadMXNetModel(&pred_hnd, json_file, param_file, input_shape);
 40 | 
 41 |     handle = (void *) pred_hnd;
 42 | 
 43 |     this->w = w;
 44 |     this->h = h;
 45 |     */
 46 |     this->threshold     = threshold;
 47 |     this->nms_threshold = nms_threshold;
 48 | 
 49 | }
 50 | 
 51 | SSH::SSH(const std::string& model_path, const std::string& model_name,
 52 |          float threshold, float nms_threshold,
 53 |          bool infer_blur_score):
 54 |           json_data(model_path + "/" + model_name + "-symbol.json"),
 55 |           param_data(model_path + "/" + model_name + "-0000.params")
 56 | {
 57 |     if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
 58 |         std::cerr << "Cannot load mxnet model" << std::endl;
 59 |         std::cerr << "\tjson file: " << json_data.file_path_ << std::endl;
 60 |         std::cerr << "\tparams file: " << param_data.file_path_ << std::endl;
 61 |         exit(EXIT_FAILURE);
 62 |     }
 63 | 
 64 |     generate_anchors_fpn(anchors_fpn, num_anchors);
 65 |     /*
 66 |     std::string json_file  = model_path + "/" + model_name + "-symbol.json";
 67 |     std::string param_file = model_path + "/" + model_name + "-0000.params";
 68 | 
 69 |     int channels = 3;
 70 |     mxInputShape input_shape (w, h, channels);
 71 |     
 72 |     // Load model
 73 |     PredictorHandle pred_hnd = nullptr;
 74 |     mxLoadMXNetModel(&pred_hnd, json_file, param_file, input_shape);
 75 | 
 76 |     handle = (void *) pred_hnd;
 77 | 
 78 |     this->w = w;
 79 |     this->h = h;
 80 |     */
 81 |     this->threshold     = threshold;
 82 |     this->nms_threshold = nms_threshold;
 83 | 
 84 |     this->infer_blur_score = infer_blur_score;
 85 |     if(infer_blur_score){
 86 |         pixel_means[0] = 0;
 87 |         pixel_means[1] = 0;
 88 |         pixel_means[2] = 0;
 89 |         pixel_stds[0] = 1;
 90 |         pixel_stds[1] = 1;
 91 |         pixel_stds[2] = 1;
 92 |         pixel_scale = 1;              
 93 |     }
 94 | 
 95 | }
 96 | 
 97 | 
 98 | SSH::SSH(const std::string& model_path, const std::string& model_name,
 99 |          std::vector<float> means, std::vector<float> stds, float scale,
100 |          float threshold, float nms_threshold,
101 |          bool infer_blur_score):
102 |           json_data(model_path + "/" + model_name + "-symbol.json"),
103 |           param_data(model_path + "/" + model_name + "-0000.params")
104 | {
105 |     if (json_data.GetLength() == 0 || param_data.GetLength() == 0) {
106 |         std::cerr << "Cannot load mxnet model" << std::endl;
107 |         std::cerr << "\tjson file: " << json_data.file_path_ << std::endl;
108 |         std::cerr << "\tparams file: " << param_data.file_path_ << std::endl;
109 |         exit(EXIT_FAILURE);
110 |     }
111 | 
112 |     generate_anchors_fpn(anchors_fpn, num_anchors);
113 |     /*
114 |     std::string json_file  = model_path + "/" + model_name + "-symbol.json";
115 |     std::string param_file = model_path + "/" + model_name + "-0000.params";
116 | 
117 |     int channels = 3;
118 |     mxInputShape input_shape (w, h, channels);
119 |     
120 |     // Load model
121 |     PredictorHandle pred_hnd = nullptr;
122 |     mxLoadMXNetModel(&pred_hnd, json_file, param_file, input_shape);
123 | 
124 |     handle = (void *) pred_hnd;
125 | 
126 |     this->w = w;
127 |     this->h = h;
128 |     */
129 |     this->threshold     = threshold;
130 |     this->nms_threshold = nms_threshold;
131 | 
132 |     assert(means.size()==3);
133 |     assert(stds.size()==3);
134 | 
135 |     for(int i=0;i<3;i++){
136 |         pixel_means[i] = means[i];
137 |         pixel_stds[i]  = stds[i];
138 |     }
139 |     pixel_scale = scale;
140 | 
141 |     this->infer_blur_score = infer_blur_score;
142 | }
143 | 
144 | /*
145 | SSH::~SSH(){
146 |     PredictorHandle pred_hnd = (PredictorHandle) handle;
147 |     MXPredFree(pred_hnd);
148 | }
149 | */
150 | 
151 | void SSH::detect(cv::Mat& im, std::vector<cv::Rect2f>  & target_boxes, 
152 |                               std::vector<cv::Point2f> & target_landmarks,
153 |                               std::vector<float>       & target_scores,
154 |                               std::vector<float>       & target_blur_scores) {
155 | 
156 |     assert(im.channels()==3);
157 |     int size = im.rows * im.cols * 3;
158 | 
159 |     PredictorHandle pred_hnd = nullptr;
160 |     mxInputShape input_shape(im.cols, im.rows, 3);
161 | 
162 |     #ifdef BENCH_SSH
163 |     struct timeval  tv1,tv2;
164 |     float sum_time = 0;
165 |     gettimeofday(&tv1,NULL);
166 |     #endif   
167 |     // mxHandleReshape((PredictorHandle) handle, input_shape, &pred_hnd);
168 |     // create a new mxnet infer handle
169 |     mx_uint num_input_nodes = 1;  // 1 for feedforward
170 |     const char* input_key[1] = { "data" };
171 |     const char** input_keys = input_key;
172 |     int res = MXPredCreate(static_cast<const char*>(json_data.GetBuffer()),
173 |                 static_cast<const char*>(param_data.GetBuffer()),
174 |                 static_cast<int>(param_data.GetLength()),
175 |                 1,
176 |                 0,
177 |                 num_input_nodes,
178 |                 input_keys,
179 |                 input_shape.input_shape_indptr,
180 |                 input_shape.input_shape_data,
181 |                 &pred_hnd);
182 |     assert(res==0);
183 | 
184 |     #ifdef BENCH_SSH
185 |     gettimeofday(&tv2,NULL);
186 |     sum_time += getElapse(&tv1, &tv2);
187 |     #endif
188 | 
189 |     std::vector<mx_float> image_data(size);
190 | 
191 |     mx_float* ptr_image_r = image_data.data();
192 |     mx_float* ptr_image_g = image_data.data() + size / 3;
193 |     mx_float* ptr_image_b = image_data.data() + size / 3 * 2;
194 | 
195 |     for (int i = 0; i < im.rows; i++) {
196 |         auto data = im.ptr<uchar>(i);
197 | 
198 |         for (int j = 0; j < im.cols; j++) {
199 |             *ptr_image_b = static_cast<mx_float>(((*data)/pixel_scale - pixel_means[0]) / pixel_stds[0]);
200 |             ptr_image_b++;
201 |             data++;
202 |             *ptr_image_g = static_cast<mx_float>(((*data)/pixel_scale - pixel_means[1]) / pixel_stds[1]);                
203 |             ptr_image_g++;
204 |             data++;
205 |             *ptr_image_r = static_cast<mx_float>(((*data)/pixel_scale - pixel_means[2]) / pixel_stds[2]);
206 |             ptr_image_r++;
207 |             data++;
208 |         }
209 |     }
210 |     
211 |     // for(size_t i = 0+size/3; i<10+size/3; i++) std::cout <<  std::setprecision(7) <<"image_data: " << image_data[i] << "\n";
212 | 
213 |     #ifdef BENCH_SSH
214 |     gettimeofday(&tv1,NULL);
215 |     #endif
216 |     mxInfer(pred_hnd, image_data);
217 |     #ifdef BENCH_SSH
218 |     gettimeofday(&tv2,NULL);
219 |     sum_time += getElapse(&tv1, &tv2);
220 |     #endif
221 |     // Inference
222 |     std::vector<float> scores;
223 |     std::vector<cv::Rect2f> boxes;
224 |     std::vector<float> blur_scores;
225 |     std::vector<cv::Point2f> landmarks;
226 | 
227 |     for(int i=0; i< 3; i++) {
228 |         std::vector<int> shape;
229 |         std::vector<float> scores1;
230 |         int index;
231 |         index = i*3;
232 |         #ifdef BENCH_SSH
233 |         gettimeofday(&tv1,NULL);
234 |         #endif
235 |         mxOutputOfIndex(pred_hnd, scores1, shape, index);
236 |         #ifdef BENCH_SSH
237 |         gettimeofday(&tv2,NULL);
238 |         sum_time += getElapse(&tv1, &tv2);
239 |         #endif
240 | 
241 |         int hscore = shape[2];
242 |         int wscore = shape[3];
243 |         std::vector<float> scores2;
244 |         int count = scores1.size()/2;
245 |         scores2.resize(count);
246 |         for(size_t i=0;i<scores2.size();i++){
247 |             scores2[i] = scores1[i+count];
248 |         }
249 |         std::vector<float> scores3;
250 |         tensor_reshape(scores2, scores3, hscore, wscore );
251 | 
252 |         index++;
253 |         std::vector<float> bbox_deltas;
254 | 
255 |         #ifdef BENCH_SSH
256 |         gettimeofday(&tv1,NULL);
257 |         #endif
258 |         mxOutputOfIndex(pred_hnd, bbox_deltas, shape, index);
259 |         #ifdef BENCH_SSH
260 |         gettimeofday(&tv2,NULL);
261 |         sum_time += getElapse(&tv1, &tv2);
262 |         #endif
263 | 
264 |         int h = shape[2];
265 |         int w = shape[3];
266 | 
267 |         int stride = stride_fpn[i];
268 |         std::vector<cv::Rect2f> anchors;
269 |         anchor_plane(h,w, stride, anchors_fpn[stride], anchors);
270 | 
271 |         std::vector<cv::Rect2f> boxes1;
272 |         std::vector<float> blur_scores1;
273 |         if(infer_blur_score){
274 |             int pred_len=0;
275 |             pred_len = shape[1] / num_anchors[stride];
276 |             bbox_pred_blur(anchors, boxes1, blur_scores1, bbox_deltas, pred_len, h, w);
277 |         } else
278 |             bbox_pred(anchors, boxes1, bbox_deltas, h, w);
279 |         clip_boxes(boxes1, im.rows, im.cols);
280 |         // for(size_t i=0; i<5; i++) std::cout <<  "boxes1: " << boxes1[i] << "\n";
281 |         // for(size_t i=0; i<5; i++) std::cout <<  "blur_scores1: " << blur_scores1[i] << "\n";
282 |         index++;
283 |         std::vector<float> landmark_deltas;
284 | 
285 |         #ifdef BENCH_SSH
286 |         gettimeofday(&tv1,NULL);
287 |         #endif
288 |         mxOutputOfIndex(pred_hnd, landmark_deltas, shape, index);
289 |         #ifdef BENCH_SSH
290 |         gettimeofday(&tv2,NULL);
291 |         sum_time += getElapse(&tv1, &tv2);
292 |         #endif
293 | 
294 |         std::vector<cv::Point2f> landmarks1;
295 |         landmark_pred(anchors, landmarks1, landmark_deltas, h, w);
296 |         // for(size_t i=0; i<20; i++) std::cout <<  "landmarks: " << landmarks1[i] << "\n";
297 | 
298 |         std::vector<bool> idx;
299 |         filter_threshold(idx, scores3, threshold);
300 |         // for(size_t i=0; i<idx.size(); i++) if(idx[i]==true)std::cout <<  "idx: " << i << "\n";
301 |         std::vector<float> scores4;
302 |         tensor_slice(scores3, scores4, idx, 1);
303 |         scores.insert(scores.end(), scores4.begin(), scores4.end());
304 |         std::vector<cv::Rect2f> boxes2;
305 |         std::vector<float> blur_scores2;
306 |         tensor_slice(boxes1, boxes2, idx, 1);
307 |         if(infer_blur_score)  tensor_slice(blur_scores1, blur_scores2, idx, 1);
308 |         boxes.insert(boxes.end(), boxes2.begin(), boxes2.end());
309 |         if(infer_blur_score) blur_scores.insert(blur_scores.end(), blur_scores2.begin(), blur_scores2.end());
310 |         std::vector<cv::Point2f> landmarks2;
311 |         tensor_slice(landmarks1, landmarks2, idx, 5);
312 |         landmarks.insert(landmarks.end(), landmarks2.begin(), landmarks2.end());
313 |     }
314 |     
315 |     std::vector<int> order;
316 |     argsort(order, scores);
317 |     // for(size_t i=0;i<order.size();i++) std::cout << "order: " << order[i] << "\n";
318 | 
319 |     std::vector<float> order_scores;
320 |     std::vector<cv::Rect2f> order_boxes;
321 |     std::vector<float> order_blur_scores;
322 |     std::vector<cv::Point2f> order_landmarks;
323 | 
324 |     sort_with_idx(scores, order_scores, order, 1);
325 |     sort_with_idx(boxes,  order_boxes, order, 1);
326 |     if(infer_blur_score) sort_with_idx(blur_scores,  order_blur_scores, order, 1);
327 |     sort_with_idx(landmarks, order_landmarks, order, 5);
328 | 
329 |     // for(auto & s: order_scores) std::cout << "scores: " << s << "\n";
330 |     // for(auto & b: order_boxes) std::cout  << "boxes: "  << b << "\n";
331 |     // for(auto & l: order_landmarks) std::cout << "landmarks: " << l << "\n";
332 | 
333 |     std::vector<bool> keep(order_scores.size(),false);
334 |     nms(order_scores, order_boxes, keep, nms_threshold);
335 |     // for(size_t i=0;i<keep.size();i++) if(keep[i]) std::cout << "keep: " << i << "\n";
336 | 
337 |     tensor_slice(order_boxes,     target_boxes,     keep, 1);
338 |     if(infer_blur_score) tensor_slice(order_blur_scores,     target_blur_scores,     keep, 1);
339 |     tensor_slice(order_landmarks, target_landmarks, keep, 5);
340 |     tensor_slice(order_scores,    target_scores,    keep, 1);
341 | 
342 |     #ifdef BENCH_SSH
343 |     gettimeofday(&tv1,NULL);
344 |     #endif
345 |     MXPredFree(pred_hnd);
346 |     #ifdef BENCH_SSH
347 |     gettimeofday(&tv2,NULL);
348 |     sum_time += getElapse(&tv1, &tv2);    
349 |     std::cout << "mxnet infer, time eclipsed: " << sum_time  << " ms\n";
350 |     #endif
351 | 
352 | }
353 | 
354 | void SSH::detect(cv::Mat& im, std::vector<cv::Rect2f>  & target_boxes, 
355 |                               std::vector<cv::Point2f> & target_landmarks,
356 |                               std::vector<float>       & target_scores){
357 |     std::vector<float>  target_blur_scores;
358 |     detect(im, target_boxes, target_landmarks, target_scores, target_blur_scores);
359 | }


--------------------------------------------------------------------------------
/mxnet/ssh_test.cpp:
--------------------------------------------------------------------------------
 1 | #include "ssh_detector_mxnet.h"
 2 | 
 3 | int main(int argc, char* argv[]) {
 4 | 
 5 |   const std::string keys =
 6 |       "{help h usage ? |                | print this message   }"
 7 |       "{model_path     |../../model     | path to ssh model  }"
 8 |       "{model_name     |mneti           | model name }"
 9 |       "{blur           |false           | if use blur scores }" 
10 |       "{@image         |../../1.png     | input image          }"
11 |   ;
12 | 
13 |   cv::CommandLineParser parser(argc, argv, keys);
14 |   parser.about("ssh detector");
15 |   if (parser.has("help")) {
16 |       parser.printMessage();
17 |       return 0;
18 |   }
19 | 
20 |   if (!parser.check()) {
21 |       parser.printErrors();
22 |       return EXIT_FAILURE;
23 |   }
24 | 
25 |   std::string model_path = parser.get<std::string>("model_path");
26 |   std::string model_name = parser.get<std::string>("model_name");
27 |   bool blur = parser.get<bool>("blur");
28 |   std::string image_path = parser.get<std::string>(0);
29 | 
30 |   cv::Mat im = cv::imread(image_path, cv::IMREAD_COLOR);
31 |   std::cout << "img resolution: " << im.cols << "*" << im.rows << "\n";
32 |   // SSH det(model_path, im.cols, im.rows);
33 |   std::vector<float> means;
34 |   std::vector<float> stds;
35 |   float scale;
36 | 
37 |   SSH det(model_path, model_name, 0.95, 0.3, blur);
38 | 
39 |   std::vector<cv::Rect2f> bboxes;
40 |   std::vector<cv::Point2f> landmarks;
41 |   std::vector<float> scores;
42 |   std::vector<float> blur_scores;
43 | 
44 |   if(blur)
45 |     det.detect(im,bboxes,landmarks,scores,blur_scores);
46 |   else
47 |     det.detect(im,bboxes,landmarks,scores);
48 | 
49 |   assert(bboxes.size()*5 == landmarks.size());
50 | 
51 |   for(auto & b: bboxes) std::cout << "b:" << b << "\n";
52 |   for(auto & l: landmarks) std::cout << "l:" << l << "\n";
53 |   for(auto & s: scores) std::cout << "s:" << s << "\n";
54 |   for(auto & b: blur_scores) std::cout << "b:" << b << "\n";
55 | 
56 |   if(bboxes.size()==0) std::cout << "detect no face!\n";
57 |   
58 | 
59 | }


--------------------------------------------------------------------------------
/tvm/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8.10)
 2 | 
 3 | project(ssh_detector_tvm)
 4 | 
 5 | add_definitions("-Wall -g -O2")
 6 | SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 7 | 
 8 | if(BENCH_SSH)
 9 |         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DBENCH_SSH")
10 | endif()
11 | 
12 | find_package( OpenCV REQUIRED )
13 | 
14 | message(STATUS "CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
15 | message(STATUS "CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
16 | include_directories(${OpenCV_INCLUDE_DIRS})
17 | 
18 | include_directories(${PROJECT_SOURCE_DIR}/../include)
19 | 
20 | message(STATUS "tvm root path: ${TVM_ROOT}")
21 | include_directories(${TVM_ROOT}/include ${TVM_ROOT}/include/tvm ${TVM_ROOT}/3rdparty/dmlc-core/include ${TVM_ROOT}/3rdparty/dlpack/include)
22 | 
23 | if (APPLE)
24 |     include_directories(/opt/local/include)
25 |     link_directories(/opt/local/lib)
26 | endif()
27 | 
28 | link_directories(
29 |   ${OpenCV_Install}/lib
30 |   /usr/local/lib
31 | )
32 | 
33 | add_subdirectory(${PROJECT_SOURCE_DIR}/ssh_detector)
34 | 
35 | add_executable(ssh_bench ssh_bench.cpp)
36 | target_link_libraries(ssh_bench ssh_detector  ${OpenCV_LIBS} pthread dl)


--------------------------------------------------------------------------------
/tvm/python/batch_converter.py:
--------------------------------------------------------------------------------
 1 | import nnvm
 2 | import tvm
 3 | import mxnet
 4 | 
 5 | import os
 6 | import argparse
 7 | 
 8 | parser = argparse.ArgumentParser(description='mxnet to tvm model convertor')
 9 | 
10 | args = parser.parse_args()
11 | 
12 | #cpu_target_list = ['broadwell','haswell','ivybridge']
13 | cpu_target_list = ['skylake']
14 | model_list = ['mneti']
15 | input_shape_list = [(2688,1520),(2592,1944),(960,960),(640,640),(480,480),(120,120)]
16 | 
17 | for cpu_target in cpu_target_list:
18 | 	for model in model_list:
19 | 		for input_shape in input_shape_list:
20 | 			batch_size = 1
21 | 			image_shape = (3, input_shape[1], input_shape[0])
22 | 			data_shape = (batch_size,) + image_shape
23 | 			prefix = './mxnet-model/' + model + '/' + model
24 | 			epoch = 0
25 | 			sym, arg_params, aux_params = mxnet.model.load_checkpoint(prefix, epoch)
26 | 			nnvm_sym, nnvm_params = nnvm.frontend.from_mxnet(sym, arg_params, aux_params)
27 | 			opt_level = 3
28 | 			target = 'llvm -mcpu=' + cpu_target
29 | 			target = tvm.target.create(target)
30 | 			prefix = 'tvm-model/' + cpu_target
31 | 			prefix +=  '/' + model + '/' + str(input_shape[0]) + '_' + str(input_shape[1]) + '/' 
32 | 			os.system('mkdir -p ' + prefix)
33 | 			with nnvm.compiler.build_config(opt_level=opt_level):
34 | 					graph, lib, params = nnvm.compiler.build(
35 | 							nnvm_sym, target=target, shape={"data": data_shape}, params=nnvm_params )
36 | 			lib.export_library(prefix + "deploy_lib.so")
37 | 			with open(prefix + "deploy_graph.json", "w") as fo:
38 | 					fo.write(graph.json())
39 | 			with open(prefix + "deploy_param.params", "wb") as fo:
40 | 					fo.write(nnvm.compiler.save_param_dict(params))
41 | 
42 | 


--------------------------------------------------------------------------------
/tvm/python/mxnet2tvm.py:
--------------------------------------------------------------------------------
 1 | import nnvm
 2 | import tvm
 3 | import mxnet
 4 | 
 5 | import os
 6 | import argparse
 7 | 
 8 | parser = argparse.ArgumentParser(description='mxnet to tvm model convertor')
 9 | parser.add_argument('--target', default='broadwell', help='target cpu microarchitect')
10 | parser.add_argument('--model', default='mneti', help='mxnet model prefix')
11 | parser.add_argument('--shape', default='2688,1520', help='input model shape')
12 | args = parser.parse_args()
13 | 
14 | batch_size = 1
15 | input_shape = args.shape.split(',')
16 | image_shape = (3, int(input_shape[1]), int(input_shape[0]))
17 | data_shape = (batch_size,) + image_shape
18 | 
19 | prefix = './mxnet-model/' + args.model + '/' + args.model
20 | epoch = 0
21 | 
22 | sym, arg_params, aux_params = mxnet.model.load_checkpoint(prefix, epoch)
23 | nnvm_sym, nnvm_params = nnvm.frontend.from_mxnet(sym, arg_params, aux_params)
24 | 
25 | opt_level = 3
26 | target = 'llvm -mcpu=' + args.target
27 | target = tvm.target.create(target)
28 | prefix = 'tvm-model/' + args.target
29 | prefix +=  '/' + args.model + '/' + input_shape[0] + '_' + input_shape[1] + '/' 
30 | os.system('mkdir -p ' + prefix)
31 | #target_host = 'llvm -target=aarch64-linux-gnu'
32 | #target_host = ''
33 | 
34 | with nnvm.compiler.build_config(opt_level=opt_level):
35 |     graph, lib, params = nnvm.compiler.build(
36 |         nnvm_sym, target=target, shape={"data": data_shape}, params=nnvm_params )
37 | 
38 | lib.export_library(prefix + "deploy_lib.so")
39 | with open(prefix + "deploy_graph.json", "w") as fo:
40 |     fo.write(graph.json())
41 | with open(prefix + "deploy_param.params", "wb") as fo:
42 |     fo.write(nnvm.compiler.save_param_dict(params))
43 | 
44 | 


--------------------------------------------------------------------------------
/tvm/ssh_bench.cpp:
--------------------------------------------------------------------------------
  1 | #include "ssh_detector.h"
  2 | 
  3 | #include <stdlib.h>
  4 | #include <sys/time.h>
  5 | 
  6 | #include <stdio.h>
  7 | #include <math.h>
  8 | 
  9 | static float getElapse(struct timeval *tv1,struct timeval *tv2)
 10 | {
 11 |     float t = 0.0f;
 12 |     if (tv1->tv_sec == tv2->tv_sec)
 13 |         t = (tv2->tv_usec - tv1->tv_usec)/1000.0f;
 14 |     else
 15 |         t = ((tv2->tv_sec - tv1->tv_sec) * 1000 * 1000 + tv2->tv_usec - tv1->tv_usec)/1000.0f;
 16 |     return t;
 17 | }
 18 | 
 19 | double tenengrad(const cv::Mat& src, int ksize)
 20 | {
 21 |     cv::Mat Gx, Gy;
 22 |     cv::Sobel(src, Gx, CV_64F, 1, 0, ksize);
 23 |     cv::Sobel(src, Gy, CV_64F, 0, 1, ksize);
 24 | 
 25 |     cv::Mat FM = Gx.mul(Gx) + Gy.mul(Gy);
 26 | 
 27 |     double focusMeasure = cv::mean(FM).val[0];
 28 |     return focusMeasure;
 29 | }
 30 | 
 31 | bool getEyeRoi(cv::Mat img, cv::Point2f eye_point, cv::Mat & res) {
 32 |     cv::Point2f p1(eye_point.x-5,eye_point.y-5);
 33 |     cv::Rect2f r1(p1.x,p1.y,10,10);
 34 |     cv::Rect2f rect_all(0,0,img.cols,img.rows);
 35 |     cv::Rect2f r2 = r1 & rect_all;
 36 |     if(r2.area()<=0) return false;
 37 |     res = img(r2);
 38 |     return true;
 39 | }
 40 | 
 41 | int main(int argc, char* argv[]) {
 42 | 
 43 |   const std::string keys =
 44 |       "{help h usage ? |                | print this message   }"
 45 |       "{model_path     |/Users/load/code/python/infinivision/tvm-convertor/tvm-model/broadwell/mneti/2688_1520     | path to ssh model  }"
 46 |       "{blur           |false           | if use blur scores }" 
 47 |       "{peroid         |15              | detect peroid }"
 48 |       "{threshold      |0.95            | threshold for detect score }"
 49 |       "{nms_threshold  |0.3             | nms_threshold for ssh detector }"      
 50 |       "{output         |./output        | path to save detect output  }"
 51 |       "{input          |../../../video  | path to input video file  }"      
 52 |       "{@video         |camera-244-crop-8p.mov     | input video file }"
 53 |   ;
 54 | 
 55 |   cv::CommandLineParser parser(argc, argv, keys);
 56 |   parser.about("ssh detector benchmark");
 57 |   if (parser.has("help")) {
 58 |       parser.printMessage();
 59 |       return 0;
 60 |   }
 61 | 
 62 |   if (!parser.check()) {
 63 |       parser.printErrors();
 64 |       return EXIT_FAILURE;
 65 |   }
 66 | 
 67 |   std::string model_path = parser.get<std::string>("model_path");
 68 |   bool blur = parser.get<bool>("blur");
 69 |   int peroid              = parser.get<int>("peroid");
 70 |   float threshold         = parser.get<float>("threshold");
 71 |   float nms_threshold     = parser.get<float>("nms_threshold");
 72 |   std::string output_path = parser.get<std::string>("output");
 73 |   std::string input_path  = parser.get<std::string>("input");
 74 |   std::string video_file  = parser.get<std::string>(0);
 75 |   std::string video_path  = input_path + "/" + video_file;
 76 | 
 77 |   std::string output_folder = output_path + "/" + video_file;
 78 |   std::string cmd = "rm -rf " + output_folder;
 79 |   system(cmd.c_str());
 80 |   cmd = "mkdir -p " + output_folder;
 81 |   system(cmd.c_str());
 82 | 
 83 |   cv::VideoCapture capture(video_path);
 84 |   cv::Mat frame,frame_r;
 85 |   int frame_count = 1;
 86 |   capture >> frame;
 87 |   resize(frame,frame_r,frame.size()/2);
 88 |   if(!frame.data) {
 89 |       std::cout<< "read first frame failed!";
 90 |       exit(1);
 91 |   }
 92 |   SSH * det = new SSH(model_path, threshold, nms_threshold, blur);
 93 | 
 94 |   std::cout << "frame resolution: " << frame.cols << "*" << frame.rows << "\n";
 95 | 
 96 |   std::vector<cv::Rect2f> boxes;
 97 |   std::vector<cv::Point2f> landmarks;
 98 |   std::vector<float> scores;
 99 |   std::vector<float> blur_scores;  
100 | 
101 |   struct timeval  tv1,tv2;
102 |   char keyboard = 0;
103 |   bool stop=false;
104 |   while(keyboard != 'q' && keyboard != 27){
105 | 
106 |       keyboard = (char)cv::waitKey( 30 );
107 |       
108 |       if(stop){
109 |           cv::imshow("Frame", frame_r);
110 |           if(keyboard==32) stop = false;
111 |           continue;
112 |       } else if(keyboard==32){
113 |           stop = true;
114 |           continue;
115 |       }
116 |       resize(frame,frame_r,frame.size()/2);
117 |       cv::imshow("Frame",frame_r);
118 |       capture >> frame;
119 |       if(!frame.data)
120 |         break;
121 | 
122 |       frame_count++;
123 |       if(frame_count%peroid!=0) continue;
124 |       
125 |       gettimeofday(&tv1,NULL);
126 |       if(blur)
127 |         det->detect(frame,boxes,landmarks,scores,blur_scores);
128 |       else
129 |         det->detect(frame,boxes,landmarks,scores);
130 |       gettimeofday(&tv2,NULL);
131 |       std::cout << "detected one frame " << scores.size() << " persons, time eclipsed: " <<  getElapse(&tv1, &tv2) << " ms\n";
132 | 
133 |       for(auto & p: landmarks)
134 |         cv::drawMarker(frame, p,  cv::Scalar(0, 255, 0), cv::MARKER_CROSS, 10, 1);
135 | 
136 |       for(int i=0;i<boxes.size();i++){
137 |         cv::rectangle( frame, boxes[i], cv::Scalar( 255, 0, 0 ), 2, 1 );
138 | 
139 |         // cv::circle(frame,landmarks[i*5], 5, cv::Scalar( 255, 0, 0 ), 2, 1);
140 |         // cv::circle(frame,landmarks[i*5+1], 5, cv::Scalar( 255, 0, 0 ), 2, 1);
141 |         /*
142 |         cv::Mat e1,e2;
143 |         double s1,s2;
144 |         bool res = getEyeRoi(frame, landmarks[i*5], e1);
145 |         if(res) 
146 |             s1 = tenengrad(e1,3);
147 |         else 
148 |             s1 = 0;
149 |         res = getEyeRoi(frame, landmarks[i*5+1],e2);
150 |         if(res) 
151 |             s2 = tenengrad(e1,3);
152 |         else 
153 |             s2 =0;
154 |         
155 |         double s = 0;
156 |         cv::Rect2f rect_ = boxes[i] & cv::Rect2f(0,0,frame.cols,frame.rows);
157 |         if(rect_.area()>0)  {
158 |             cv::Mat box = frame(boxes[i]);
159 |             s = tenengrad(box,7);
160 |         }
161 |         */
162 |         // s = s / 10000;
163 |         cv::Point middleHighPoint = cv::Point(boxes[i].x+boxes[i].width/2, boxes[i].y);
164 |         char buf_text[32];
165 |         if(blur)
166 |             snprintf(buf_text, 64, "s:%.3f, b: %.3f", scores[i], blur_scores[i]);
167 |         else
168 |             snprintf(buf_text, 64, "s:%.3e", scores[i]);
169 |         std::string text(buf_text);
170 |         // snprintf(buf_text, 64, "e:%.3f,m:%.3f", landmarks[i*5+1].x-landmarks[i*5+0].x, landmarks[i*5+4].x-landmarks[i*5+3].x);
171 |         // text = buf_text;
172 |         cv::putText(frame, text, middleHighPoint, cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 255, 255), 2);
173 | 
174 |       }
175 | 
176 |       //for(auto & p: landmarks)
177 |       //  cv::drawMarker(frame, p,  cv::Scalar(0, 255, 0), cv::MARKER_CROSS, 10, 1);
178 |       if(boxes.size()>0){
179 |           // cv::imwrite( output_folder + "/" + std::to_string(frame_count) + ".jpg", frame);
180 |           
181 |           for(auto b: boxes)
182 |             std::cout << "bouding box area: " << b.area() << "  len: " << sqrt(b.area()) << "\n";
183 |       }
184 | 
185 |   }
186 | 
187 |   delete det;
188 | 
189 | }


--------------------------------------------------------------------------------
/tvm/ssh_detector/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | file(GLOB ssh_detector_source "*.cpp" "*.cc")
2 | add_library( ssh_detector ${ssh_detector_source} )
3 | 


--------------------------------------------------------------------------------
/tvm/ssh_detector/ssh_detector.cpp:
--------------------------------------------------------------------------------
  1 | #include "ssh_detector.h"
  2 | #include "anchors.h"
  3 | 
  4 | #include "dlpack/dlpack.h"
  5 | #include <tvm/runtime/module.h>
  6 | #include <tvm/runtime/registry.h>
  7 | #include <tvm/runtime/packed_func.h>
  8 | 
  9 | #include <fstream>
 10 | #include <iomanip>
 11 | #include <sys/time.h>
 12 | 
 13 | 
 14 | static float getElapse(struct timeval *tv1,struct timeval *tv2)
 15 | {
 16 |     float t = 0.0f;
 17 |     if (tv1->tv_sec == tv2->tv_sec)
 18 |         t = (tv2->tv_usec - tv1->tv_usec)/1000.0f;
 19 |     else
 20 |         t = ((tv2->tv_sec - tv1->tv_sec) * 1000 * 1000 + tv2->tv_usec - tv1->tv_usec)/1000.0f;
 21 |     return t;
 22 | }
 23 | 
 24 | static void tvmOutputOfIndex(tvm::runtime::PackedFunc handler, /* mxnet model */
 25 |                              std::vector<float> &out_data,           /* output vector */
 26 |                              std::vector<int> &out_shape,        /* output tensor shape */
 27 |                              int output_index)
 28 | {
 29 |     // Get Output Result
 30 |     tvm::runtime::NDArray res = handler(output_index);
 31 | 
 32 |     out_shape.assign(res->shape, res->shape + res->ndim);
 33 | 
 34 |     int size = 1;
 35 |     for (int i = 0; i < res->ndim; ++i) {
 36 |         size *= res->shape[i];
 37 |     }
 38 | 
 39 |     float* data = (float*) res->data;
 40 |     out_data.assign(data, data + size);
 41 | }
 42 | 
 43 | SSH::SSH(const std::string& model_path, float threshold, float nms_threshold, bool infer_blur_score)
 44 | {
 45 |     generate_anchors_fpn(anchors_fpn, num_anchors);
 46 | 
 47 |     tvm::runtime::Module mod_syslib = tvm::runtime::Module::LoadFromFile(model_path + "/deploy_lib.so");
 48 |     std::ifstream json_in(model_path + "/deploy_graph.json");
 49 |     std::string json_data((std::istreambuf_iterator<char>(json_in)), std::istreambuf_iterator<char>());
 50 |     json_in.close();
 51 |     int device_type = kDLCPU;
 52 |     int device_id = 0;
 53 |     // get global function module for graph runtime
 54 |     tvm::runtime::Module mod = (*tvm::runtime::Registry::Get("tvm.graph_runtime.create"))(json_data, mod_syslib, device_type, device_id);
 55 |     this->handle = new tvm::runtime::Module(mod);
 56 |     std::ifstream params_in(model_path + "/deploy_param.params", std::ios::binary);
 57 |     std::string params_data((std::istreambuf_iterator<char>(params_in)), std::istreambuf_iterator<char>());
 58 |     params_in.close();
 59 |     TVMByteArray params_arr;
 60 |     params_arr.data = params_data.c_str();
 61 |     params_arr.size = params_data.length();
 62 |     tvm::runtime::PackedFunc load_params = mod.GetFunction("load_params");
 63 |     load_params(params_arr);
 64 | 
 65 |     this->nms_threshold = nms_threshold;
 66 |     this->threshold     = threshold;
 67 |     this->infer_blur_score = infer_blur_score;
 68 |     if(infer_blur_score){
 69 |         pixel_means[0] = 0;
 70 |         pixel_means[1] = 0;
 71 |         pixel_means[2] = 0;
 72 |         pixel_stds[0] = 1;
 73 |         pixel_stds[1] = 1;
 74 |         pixel_stds[2] = 1;
 75 |         pixel_scale = 1;
 76 |     }
 77 | }
 78 | 
 79 | SSH::SSH(const std::string& model_path, std::vector<float> means, std::vector<float> stds, float scale, 
 80 |                                         float threshold, float nms_threshold, bool infer_blur_score)
 81 | {
 82 |     generate_anchors_fpn(anchors_fpn, num_anchors);
 83 | 
 84 |     tvm::runtime::Module mod_syslib = tvm::runtime::Module::LoadFromFile(model_path + "/deploy_lib.so");
 85 |     std::ifstream json_in(model_path + "/deploy_graph.json");
 86 |     std::string json_data((std::istreambuf_iterator<char>(json_in)), std::istreambuf_iterator<char>());
 87 |     json_in.close();
 88 |     int device_type = kDLCPU;
 89 |     int device_id = 0;
 90 |     // get global function module for graph runtime
 91 |     tvm::runtime::Module mod = (*tvm::runtime::Registry::Get("tvm.graph_runtime.create"))(json_data, mod_syslib, device_type, device_id);
 92 |     this->handle = new tvm::runtime::Module(mod);
 93 |     std::ifstream params_in(model_path + "/deploy_param.params", std::ios::binary);
 94 |     std::string params_data((std::istreambuf_iterator<char>(params_in)), std::istreambuf_iterator<char>());
 95 |     params_in.close();
 96 |     TVMByteArray params_arr;
 97 |     params_arr.data = params_data.c_str();
 98 |     params_arr.size = params_data.length();
 99 |     tvm::runtime::PackedFunc load_params = mod.GetFunction("load_params");
100 |     load_params(params_arr);
101 | 
102 |     assert(means.size()==3);
103 |     assert(stds.size()==3);
104 |     for(int i=0;i<3;i++){
105 |         pixel_means[i] = means[i];
106 |         pixel_stds[i]  = stds[i];
107 |     }
108 |     pixel_scale = scale;
109 | 
110 |     this->nms_threshold = nms_threshold;
111 |     this->threshold     = threshold;
112 | 
113 |     this->infer_blur_score = infer_blur_score;
114 | }
115 | 
116 | SSH::~SSH()
117 | {
118 |     tvm::runtime::Module* mod = (tvm::runtime::Module*) handle;
119 |     delete mod;
120 |     if(infer_buff!=nullptr)
121 |         TVMArrayFree((DLTensor* )infer_buff);
122 | }
123 | 
124 | void SSH::detect(cv::Mat& im, std::vector<cv::Rect2f>  & target_boxes,
125 |                  std::vector<cv::Point2f> & target_landmarks,
126 |                  std::vector<float>       & target_scores,
127 |                  std::vector<float>       & target_blur_scores) {
128 | 
129 |     assert(im.channels() == 3);
130 | 
131 |     size_t size = im.channels() * im.rows * im.cols;
132 |     std::vector<float> image_data(size);
133 | 
134 |     float* ptr_image_r = image_data.data();
135 |     float* ptr_image_g = image_data.data() + size / 3;
136 |     float* ptr_image_b = image_data.data() + size / 3 * 2;
137 | 
138 |     for (int i = 0; i < im.rows; i++) {
139 |         auto data = im.ptr<uchar>(i);
140 | 
141 |         for (int j = 0; j < im.cols; j++) {
142 |             *ptr_image_b = static_cast<float>(((*data)/pixel_scale - pixel_means[0]) / pixel_stds[0]);
143 |             ptr_image_b++;
144 |             data++;
145 |             *ptr_image_g = static_cast<float>(((*data)/pixel_scale - pixel_means[1]) / pixel_stds[1]);
146 |             ptr_image_g++;
147 |             data++;
148 |             *ptr_image_r = static_cast<float>(((*data)/pixel_scale - pixel_means[2]) / pixel_stds[2]);
149 |             ptr_image_r++;
150 |             data++;
151 |         }
152 |     }
153 | 
154 |     constexpr int dtype_code = kDLFloat;
155 |     constexpr int dtype_bits = 32;
156 |     constexpr int dtype_lanes = 1;
157 |     constexpr int device_type = kDLCPU;
158 |     constexpr int device_id = 0;
159 | 
160 |     constexpr int in_ndim = 4;
161 |     const int64_t in_shape[in_ndim] = {1, 3, im.rows, im.cols};
162 | 
163 |     DLTensor* x;
164 | 
165 |     if(infer_buff==nullptr){
166 |         TVMArrayAlloc(in_shape, in_ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &x);
167 |         infer_buff = (void *) x;
168 |     } else {
169 |         x = (DLTensor*) infer_buff;
170 |     }
171 |         
172 | 
173 |     memcpy(x->data, &image_data[0], sizeof(image_data[0]) * image_data.size());
174 | 
175 |     tvm::runtime::Module* mod = (tvm::runtime::Module*) handle;
176 | 
177 |     // parameters in binary
178 | 
179 | 
180 | #ifdef BENCH_SSH
181 |     struct timeval  tv1,tv2;
182 |     float sum_time = 0;
183 |     gettimeofday(&tv1,NULL);
184 | #endif
185 | 
186 |     tvm::runtime::PackedFunc set_input = mod->GetFunction("set_input");
187 |     set_input("data", x);
188 | 
189 |     tvm::runtime::PackedFunc run = mod->GetFunction("run");
190 |     run();
191 | 
192 |     tvm::runtime::PackedFunc get_output = mod->GetFunction("get_output");
193 | 
194 | #ifdef BENCH_SSH
195 |     gettimeofday(&tv2,NULL);
196 |     sum_time += getElapse(&tv1, &tv2);
197 | #endif
198 | 
199 |     std::vector<float> scores;
200 |     std::vector<cv::Rect2f> boxes;
201 |     std::vector<float> blur_scores;
202 |     std::vector<cv::Point2f> landmarks;
203 | 
204 |     for (int index = 0; index < 3; ++index)
205 |     {
206 |         std::vector<int> shape;
207 |         std::vector<float> scores1;
208 | 
209 | #ifdef BENCH_SSH
210 |         gettimeofday(&tv1,NULL);
211 | #endif
212 |         tvmOutputOfIndex(get_output, scores1, shape, index * 3);
213 | #ifdef BENCH_SSH
214 |         gettimeofday(&tv2,NULL);
215 |         sum_time += getElapse(&tv1, &tv2);
216 | #endif
217 |         /*
218 |         std::cout << "output shape len: " << shape.size() << "\n";
219 |         for(auto s: shape)
220 |             std::cout << "output shape1: " << s << "\n";
221 |         */
222 |         int hscore = shape[2];
223 |         int wscore = shape[3];
224 |         std::vector<float> scores2;
225 |         int count = scores1.size()/2;
226 |         scores2.resize(count);
227 |         for(size_t i = 0; i < scores2.size(); i++)
228 |         {
229 |             scores2[i] = scores1[i + count];
230 |         }
231 |         std::vector<float> scores3;
232 |         tensor_reshape(scores2, scores3, hscore, wscore, 1);
233 | 
234 |         std::vector<float> bbox_deltas;
235 | 
236 | #ifdef BENCH_SSH
237 |         gettimeofday(&tv1,NULL);
238 | #endif
239 |         tvmOutputOfIndex(get_output, bbox_deltas, shape, index * 3 + 1);
240 | #ifdef BENCH_SSH
241 |         gettimeofday(&tv2,NULL);
242 |         sum_time += getElapse(&tv1, &tv2);
243 | #endif
244 | 
245 |         int h = shape[2];
246 |         int w = shape[3];
247 |         int c = 1;
248 |         if (shape.size() >= 5)
249 |         {
250 |             c = shape[4];
251 |         }
252 |         int stride = stride_fpn[index];
253 |         std::vector<cv::Rect2f> anchors;
254 |         anchor_plane(h,w, stride, anchors_fpn[stride], anchors);
255 |         /*
256 |         std::cout << "output shape len: " << shape.size() << "\n";
257 |         for(auto s: shape)
258 |             std::cout << "output shape2: " << s << "\n";
259 |         */
260 |         std::vector<cv::Rect2f> boxes1;
261 |         std::vector<float> blur_scores1;
262 |         if(infer_blur_score){
263 |             int pred_len=0;
264 |             if(shape.size() >= 5)
265 |                 pred_len = shape[4];
266 |             else
267 |                 pred_len = shape[1] / num_anchors[stride];
268 |             bbox_pred_blur(anchors, boxes1, blur_scores1, bbox_deltas, pred_len, h, w, c);
269 |         } else
270 |             bbox_pred(anchors, boxes1, bbox_deltas, h, w, c);
271 |         clip_boxes(boxes1, im.rows, im.cols);
272 | 
273 |         std::vector<float> landmark_deltas;
274 | 
275 | #ifdef BENCH_SSH
276 |         gettimeofday(&tv1,NULL);
277 | #endif
278 |         tvmOutputOfIndex(get_output, landmark_deltas, shape, index * 3 + 2);
279 | #ifdef BENCH_SSH
280 |         gettimeofday(&tv2,NULL);
281 |         sum_time += getElapse(&tv1, &tv2);
282 | #endif
283 |         c = 1;
284 |         if (shape.size() >= 5)
285 |         {
286 |             c = shape[4];
287 |         }
288 |         /*
289 |         std::cout << "output shape len: " << shape.size() << "\n";
290 |         for(auto s: shape)
291 |             std::cout << "output shape3: " << s << "\n";
292 |         */
293 | 
294 |         std::vector<cv::Point2f> landmarks1;
295 |         landmark_pred(anchors, landmarks1, landmark_deltas, h, w, c);
296 | 
297 |         std::vector<bool> idx;
298 |         filter_threshold(idx, scores3, threshold);
299 | 
300 |         std::vector<float> scores4;
301 |         tensor_slice(scores3, scores4, idx, 1);
302 |         scores.insert(scores.end(), scores4.begin(), scores4.end());
303 |         std::vector<cv::Rect2f> boxes2;
304 |         std::vector<float> blur_scores2;
305 |         tensor_slice(boxes1, boxes2, idx, 1);
306 |         if(infer_blur_score)  tensor_slice(blur_scores1, blur_scores2, idx, 1);
307 |         boxes.insert(boxes.end(), boxes2.begin(), boxes2.end());
308 |         if(infer_blur_score) blur_scores.insert(blur_scores.end(), blur_scores2.begin(), blur_scores2.end()); 
309 |         std::vector<cv::Point2f> landmarks2;
310 |         tensor_slice(landmarks1, landmarks2, idx, 5);
311 |         landmarks.insert(landmarks.end(), landmarks2.begin(), landmarks2.end());
312 |     }
313 | 
314 |     std::vector<int> order;
315 |     argsort(order, scores);
316 | 
317 |     std::vector<float> order_scores;
318 |     std::vector<cv::Rect2f> order_boxes;
319 |     std::vector<float> order_blur_scores;
320 |     std::vector<cv::Point2f> order_landmarks;
321 | 
322 |     sort_with_idx(scores, order_scores, order, 1);
323 |     sort_with_idx(boxes,  order_boxes, order, 1);
324 |     if(infer_blur_score) sort_with_idx(blur_scores,  order_blur_scores, order, 1);
325 |     sort_with_idx(landmarks, order_landmarks, order, 5);
326 | 
327 |     std::vector<bool> keep(order_scores.size(),false);
328 |     nms(order_scores, order_boxes, keep, nms_threshold);
329 | 
330 |     tensor_slice(order_boxes,     target_boxes,     keep, 1);
331 |     if(infer_blur_score) tensor_slice(order_blur_scores,     target_blur_scores,     keep, 1);
332 |     tensor_slice(order_landmarks, target_landmarks, keep, 5);
333 |     tensor_slice(order_scores,    target_scores,    keep, 1);
334 | 
335 | #ifdef BENCH_SSH
336 |     gettimeofday(&tv1,NULL);
337 | #endif
338 | #ifdef BENCH_SSH
339 |     gettimeofday(&tv2,NULL);
340 |     sum_time += getElapse(&tv1, &tv2);
341 |     std::cout << "tvm infer, time eclipsed: " << sum_time  << " ms\n";
342 | #endif
343 |     /*
344 |     for (auto& b: target_boxes)
345 |     {
346 |         cv::rectangle(im, b, cv::Scalar(255, 0, 0), 2, 1);
347 |     }
348 | 
349 |     for(auto & p: target_landmarks)
350 |     {
351 |         cv::drawMarker(im, p,  cv::Scalar(0, 255, 0), cv::MARKER_CROSS, 10, 1);
352 |     }
353 | 
354 |     if (boxes.size() > 0)
355 |         cv::imwrite("test_out.jpg", im);
356 |     */
357 | }
358 | 
359 | void SSH::detect(cv::Mat& im, std::vector<cv::Rect2f>  & target_boxes,
360 |                               std::vector<cv::Point2f> & target_landmarks,
361 |                               std::vector<float>       & target_scores){
362 |     std::vector<float>  target_blur_scores;
363 |     detect(im, target_boxes, target_landmarks, target_scores, target_blur_scores);
364 | }


--------------------------------------------------------------------------------
/tvm/ssh_detector/tvm_runtime_pack.cc:
--------------------------------------------------------------------------------
 1 | /*!
 2 |  * \brief This is an all in one TVM runtime file.
 3 |  *
 4 |  *   You only have to use this file to compile libtvm_runtime to
 5 |  *   include in your project.
 6 |  *
 7 |  *  - Copy this file into your project which depends on tvm runtime.
 8 |  *  - Compile with -std=c++11
 9 |  *  - Add the following include path
10 |  *     - /path/to/tvm/include/
11 |  *     - /path/to/tvm/3rdparty/dmlc-core/include/
12 |  *     - /path/to/tvm/3rdparty/dlpack/include/
13 |  *   - Add -lpthread -ldl to the linked library.
14 |  *   - You are good to go.
15 |  *   - See the Makefile in the same folder for example.
16 |  *
17 |  *  The include files here are presented with relative path
18 |  *  You need to remember to change it to point to the right file.
19 |  *
20 |  */
21 | #include "../src/runtime/cpu_device_api.cc"
22 | #include "../src/runtime/c_runtime_api.cc"
23 | #include "../src/runtime/workspace_pool.cc"
24 | #include "../src/runtime/module_util.cc"
25 | #include "../src/runtime/module.cc"
26 | #include "../src/runtime/registry.cc"
27 | #include "../src/runtime/file_util.cc"
28 | #include "../src/runtime/threading_backend.cc"
29 | #include "../src/runtime/thread_pool.cc"
30 | #include "../src/runtime/ndarray.cc"
31 | 
32 | // NOTE: all the files after this are optional modules
33 | // that you can include remove, depending on how much feature you use.
34 | 
35 | // Likely we only need to enable one of the following
36 | // If you use Module::Load, use dso_module
37 | // For system packed library, use system_lib_module
38 | #include "../src/runtime/dso_module.cc"
39 | #include "../src/runtime/system_lib_module.cc"
40 | 
41 | // Graph runtime
42 | #include "../src/runtime/graph/graph_runtime.cc"
43 | 
44 | // Uncomment the following lines to enable RPC
45 | // #include "../../src/runtime/rpc/rpc_session.cc"
46 | // #include "../../src/runtime/rpc/rpc_event_impl.cc"
47 | // #include "../../src/runtime/rpc/rpc_server_env.cc"
48 | 
49 | // These macros enables the device API when uncommented.
50 | #define TVM_CUDA_RUNTIME 1
51 | #define TVM_METAL_RUNTIME 1
52 | #define TVM_OPENCL_RUNTIME 1
53 | 
54 | // Uncomment the following lines to enable Metal
55 | // #include "../../src/runtime/metal/metal_device_api.mm"
56 | // #include "../../src/runtime/metal/metal_module.mm"
57 | 
58 | // Uncomment the following lines to enable CUDA
59 | // #include "../../src/runtime/cuda/cuda_device_api.cc"
60 | // #include "../../src/runtime/cuda/cuda_module.cc"
61 | 
62 | // Uncomment the following lines to enable OpenCL
63 | // #include "../../src/runtime/opencl/opencl_device_api.cc"
64 | // #include "../../src/runtime/opencl/opencl_module.cc"
65 | 


--------------------------------------------------------------------------------