├── CMakeLists.txt
├── README.md
├── main.cpp
├── main.qml
├── qml.qrc
└── siamfcpp
    ├── siamfcpp.cpp
    └── siamfcpp.h


/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 3.14)
 2 | 
 3 | project(siamFCppwarp LANGUAGES CXX)
 4 | 
 5 | set(CMAKE_INCLUDE_CURRENT_DIR ON)
 6 | 
 7 | set(CMAKE_AUTOUIC ON)
 8 | set(CMAKE_AUTOMOC ON)
 9 | set(CMAKE_AUTORCC ON)
10 | 
11 | set(CMAKE_CXX_STANDARD 11)
12 | set(CMAKE_CXX_STANDARD_REQUIRED ON)
13 | 
14 | 
15 | if(CMAKE_BUILD_TYPE MATCHES "release")
16 |    set(Torch_DIR   D:/environment/libtorch-win-shared-with-deps-1.9.0+cpu/libtorch/)
17 | 
18 |  ELSEif(CMAKE_BUILD_TYPE MATCHES "Debug")
19 |       set(Torch_DIR  D:/environment/libtorch-win-shared-with-deps-debug-1.9.0+cpu/libtorch/)
20 |         endif()
21 | 
22 | message(STATUS "Torch_Dir : ${Torch_Dir}")
23 | message(STATUS "CMAKE_BUILD_TYPE : ${CMAKE_BUILD_TYPE}")
24 | 
25 | find_package(Torch REQUIRED)
26 | find_package(Opencv REQUIRED)
27 | 
28 | include_directories(
29 | 
30 |     siamfcpp
31 |     )
32 | 
33 | #  Test
34 | # tool
35 | #"Test/*cpp"
36 | #"tool/*cpp"
37 | file(GLOB_RECURSE sourcefile
38 | 
39 | "siamfcpp/*cpp"
40 |     )
41 | 
42 | set(PROJECT_SOURCES
43 | 
44 |         main.cpp
45 | 
46 |         ${sourcefile}
47 | )
48 | add_executable(siamFCppwarp
49 |     ${PROJECT_SOURCES})
50 | 
51 | 
52 | 
53 | target_link_libraries(siamFCppwarp
54 | 
55 |   PRIVATE ${OpenCV_LIBS}
56 |   ${TORCH_LIBRARIES}
57 |   )
58 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 |  # siamFCppwarp
 2 | C++ Implementation siamfcpp  from [SiamTrackers:siamFcpp](https://github.com/HonglinChu/SiamTrackers)
 3 | test on GTX1660 CUDA 11.1
 4 | 
 5 | ## video sample
 6 | 1.CPU(FPS 13)
 7 | 
 8 | 
 9 | [cPU_implement](https://user-images.githubusercontent.com/51977158/130018392-93a7cd5c-f893-4e40-8b36-3206312a248b.mp4)
10 | 
11 | 2.Gpu(FPS 96)
12 | 
13 | 
14 | [GPU_implement](
15 | https://user-images.githubusercontent.com/51977158/130159650-60574617-cc02-44d2-89eb-438c85dffc4b.mp4
16 | )
17 | 
18 | # pre_model trans python-libtorch
19 | [baiduwangpan code:9m6d](https://pan.baidu.com/s/1Th7M6hQgGyKpe9u4ZpWxFA)
20 | # environment
21 | 1.opencv-4.5.1-vc14_vc15
22 | 
23 | 2.libtorch-win-shared-with-deps-1.9.0+cu111
24 | 
25 | #References 
26 | 
27 | 
28 | 
29 |       @inproceedings{xu2020siamfc++,
30 |       title={SiamFC++: Towards Robust and Accurate Visual Tracking with Target Estimation Guidelines.},
31 |       author={Xu, Yinda and Wang, Zeyu and Li, Zuoxin and Yuan, Ye and Yu, Gang},
32 |       booktitle={AAAI},
33 |       pages={12549--12556},
34 |       year={2020}
35 |     }
36 | 
37 | 
38 | 
39 | 
40 | 
41 | 


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
  1 | //#include <QGuiApplication>
  2 | //#include <QQmlApplicationEngine>
  3 | //#include<testpt.h>
  4 | #include<siamfcpp.h>
  5 | int main(int argc, char *argv[])
  6 | {
  7 |     /*
  8 | #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
  9 |     QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling);
 10 | #endif
 11 | 
 12 |     QGuiApplication app(argc, argv);
 13 | 
 14 |     QQmlApplicationEngine engine;
 15 |     const QUrl url(QStringLiteral("qrc:/main.qml"));
 16 |     QObject::connect(&engine, &QQmlApplicationEngine::objectCreated,
 17 |                      &app, [url](QObject *obj, const QUrl &objUrl) {
 18 |         if (!obj && url == objUrl)
 19 |             QCoreApplication::exit(-1);
 20 |     }, Qt::QueuedConnection);
 21 | 
 22 | */
 23 |   //  testpt testp(nullptr,torch::DeviceType::CUDA);
 24 | siamfcpp  siamfcpp(torch::DeviceType::CUDA);
 25 | 
 26 | 
 27 |  cv::VideoCapture   vs = cv::VideoCapture("../video/car.mp4");
 28 | 
 29 |   bool  frst = true;
 30 |   cv::Mat frame ;
 31 |         // ;
 32 |    float scores=0;
 33 |   float fps_c=0;
 34 |     int c=0;
 35 |     while ( vs.read(frame)){
 36 | 
 37 | 
 38 | c++;
 39 | 
 40 | 
 41 |             if (frst) {
 42 |                 cv::Rect box = cv::selectROI("window_name",
 43 |                                     frame
 44 | 
 45 |                                     );
 46 |              //  cv::Rect box(313,135,109,123);
 47 | 
 48 | 
 49 | 
 50 | 
 51 | 
 52 | 
 53 | 
 54 |                 siamfcpp.ini(frame, box);
 55 | 
 56 |                 frst= false;
 57 | 
 58 |             }
 59 |             else{
 60 | 
 61 |                  int start =cv::getTickCount();
 62 | 
 63 |                 cv::Rect roi=  siamfcpp.update(frame,scores);
 64 |                 int  end =cv::getTickCount();
 65 |                cv::Mat show_frame = frame.clone();
 66 |                 float cout=1/((end-start)/cv::getTickFrequency());
 67 |                 fps_c+=cout;
 68 | 
 69 | 
 70 |                 cv::putText(show_frame,
 71 |                            "FPS "+ std::to_string( cout), cv::Point(128, 20),
 72 |                             cv::FONT_HERSHEY_COMPLEX, 0.5, cv::Scalar(0, 0, 255),
 73 |                            1);
 74 | 
 75 | if(scores<=0.3)
 76 | {
 77 |     cv::putText(show_frame,
 78 |                "LOSS ", cv::Point(roi.x, roi.y-8),
 79 |                 cv::FONT_HERSHEY_COMPLEX, 1, cv::Scalar(255, 255, 0),
 80 |                1);
 81 |     cv::rectangle(show_frame, roi ,
 82 |                   cv::Scalar(255, 0,0),5);
 83 | }
 84 | else
 85 | {
 86 |                 cv::rectangle(show_frame, roi ,
 87 |                               cv::Scalar(0, 255,0),3);
 88 | }
 89 | 
 90 | 
 91 |                 cv::imshow("window_name2", show_frame);
 92 |                 cv::waitKey(1);
 93 | 
 94 |               //  break ;
 95 |             }
 96 | 
 97 | 
 98 |     }
 99 | 
100 | 
101 | 
102 | 
103 |   std::cout<<"Pfps "<<fps_c/c<<std::endl;
104 |  //engine.load(url);
105 |   //  return app.exec();
106 | }
107 | 


--------------------------------------------------------------------------------
/main.qml:
--------------------------------------------------------------------------------
 1 | import QtQuick 2.12
 2 | import QtQuick.Window 2.12
 3 | 
 4 | Window {
 5 |     width: 640
 6 |     height: 480
 7 |     visible: true
 8 |     title: qsTr("Hello World")
 9 | }
10 | 


--------------------------------------------------------------------------------
/qml.qrc:
--------------------------------------------------------------------------------
1 | <RCC>
2 |     <qresource prefix="/">
3 |         <file>main.qml</file>
4 |     </qresource>
5 | </RCC>
6 | 


--------------------------------------------------------------------------------
/siamfcpp/siamfcpp.cpp:
--------------------------------------------------------------------------------
  1 | #include "siamfcpp.h"
  2 | 
  3 | const std::string  siamfcpp::windowing("cosine");
  4 | /*
  5 | const float siamfcpp::context_amount=0.5;
  6 | const float siamfcpp::test_lr=0.52;
  7 | const float siamfcpp::penalty_k=0.08;
  8 | const float siamfcpp::window_influence=0.21;
  9 | const float siamfcpp::lossed_w=0.5;
 10 | */
 11 | 
 12 | const float siamfcpp::context_amount=0.5;
 13 | const float siamfcpp::test_lr=0.52;
 14 | const float siamfcpp::penalty_k=0.08;
 15 | const float siamfcpp::window_influence=0.21;
 16 | const float siamfcpp::AC_SCORE=0.3;
 17 | siamfcpp::siamfcpp( torch::DeviceType dvic)
 18 | {
 19 |     dvc=dvic;
 20 |     if(dvc==torch::DeviceType::CPU)
 21 |     {
 22 |       model= torch::jit::load( "../model/GOT/siamfcpp_fcpu.pt");
 23 |       trackmodel= torch::jit::load( "../model/GOT/siamfcpp_tcpu.pt");
 24 |     }
 25 |     else
 26 |     {
 27 |       model= torch::jit::load( "../model/VOT/siamfcpp_vot_fcuda.pt");
 28 |        trackmodel= torch::jit::load( "../model/VOT/siamfcpp_vot_tcuda.pt");
 29 |     }
 30 | 
 31 |       model.to(dvc);
 32 |       trackmodel.to(dvc);
 33 |       model.eval();
 34 |       trackmodel.eval();
 35 | }
 36 | siamfcpp::~siamfcpp(){
 37 | 
 38 | }
 39 | // TODO: What is this?
 40 | int siamfcpp::calculate_s_z() {
 41 |     float bb_half_perimeter = bounding_box.width + bounding_box.height;
 42 |     float w_z = bounding_box.width + context_amount * bb_half_perimeter;
 43 |     float h_z = bounding_box.height + context_amount * bb_half_perimeter;
 44 |     return round(sqrt(w_z * h_z));
 45 | }
 46 | 
 47 | 
 48 | 
 49 | torch::Tensor siamfcpp::convert_bbox(torch::Tensor loc) {
 50 |     // TODO: What are these?
 51 |     torch::Tensor delta = loc.permute({ 1, 2, 3, 0 }).contiguous().view({ 4, -1 });
 52 |     delta.narrow(0, 0, 1) = delta.narrow(0, 0, 1) * anchors.narrow(1, 2, 1).t() + anchors.narrow(1, 0, 1).t();
 53 |     delta.narrow(0, 1, 1) = delta.narrow(0, 1, 1) * anchors.narrow(1, 3, 1).t() + anchors.narrow(1, 1, 1).t();
 54 |     delta.narrow(0, 2, 1) = torch::exp(delta.narrow(0, 2, 1)) * anchors.narrow(1, 2, 1).t();
 55 |     delta.narrow(0, 3, 1) = torch::exp(delta.narrow(0, 3, 1)) * anchors.narrow(1, 3, 1).t();
 56 |     return delta;
 57 | }
 58 | /*
 59 | def cxywh2xywh(box):
 60 |     box = np.array(box, dtype=np.float32)
 61 |     return np.concatenate([
 62 |         box[..., [0]] - (box[..., [2]] - 1) / 2, box[..., [1]] -
 63 |         (box[..., [3]] - 1) / 2, box[..., [2]], box[..., [3]]
 64 |     ],
 65 |                           axis=-1)
 66 |   */
 67 | torch::Tensor siamfcpp::xyxy2cxywh(torch::Tensor box) {
 68 | /*
 69 |     torch::cat([
 70 |         box( 0) - (box[..., [2]] - 1) / 2, box[..., [1]] -
 71 |         (box[..., [3]] - 1) / 2, box[..., [2]], box[..., [3]]
 72 |     ],
 73 | 
 74 |  C= np.concatenate([(bbox[..., [0]] + bbox[..., [2]]) / 2,
 75 |                     (bbox[..., [1]] + bbox[..., [3]]) / 2,
 76 |                     bbox[..., [2]] - bbox[..., [0]] + 1,
 77 |                     bbox[..., [3]] - bbox[..., [1]] + 1],
 78 |                    axis=-1)
 79 |                           axis=-1)
 80 |             */
 81 | 
 82 | 
 83 | 
 84 | 
 85 | 
 86 | 
 87 |     torch::Tensor t0=  box.narrow(1,0,1);  //1维 第0 开始 取一排
 88 | 
 89 |     torch::Tensor t1=  box.narrow(1,1,1);
 90 | 
 91 |     torch::Tensor t2=  box.narrow(1,2,1);
 92 | 
 93 |     torch::Tensor t3= box.narrow(1,3,1);
 94 | 
 95 | 
 96 | 
 97 | 
 98 | torch::Tensor t=  torch::cat({ (t0 +t2) / 2,
 99 |                                (t1 + t3) / 2,
100 |                                t2 -t0 + 1,
101 |                               t3 - t1 + 1},-1);
102 | 
103 | return t;
104 | 
105 | 
106 | 
107 | }
108 | cv::Rect siamfcpp::cxywh2xywh(std::vector<float> roi) {
109 | 
110 | 
111 |     float x0=roi[0];
112 |     float x1=roi[1];
113 |     float x2=roi[2];
114 |     float x3=roi[3];
115 |  cv::Rect  list( x0 - (x2-1) / 2, x1 -
116 |                             (x3 - 1) / 2, x2, x3
117 | 
118 |                        );
119 | 
120 |     return list;
121 | 
122 | 
123 | }
124 | 
125 | std::vector<float>  siamfcpp::xywh2cxywh(cv::Rect roi){  //box_wh
126 |     /*
127 |     rect = np.array(rect, dtype=np.float32)
128 |         return np.concatenate([
129 |             rect[..., [0]] + (rect[..., [2]] - 1) / 2, rect[..., [1]] +
130 |             (rect[..., [3]] - 1) / 2, rect[..., [2]], rect[..., [3]]
131 |         ],
132 |                               axis=-1)
133 |             */
134 | 
135 |     float x0=roi.x;
136 |     float x1=roi.y;
137 |     float x2=roi.width;
138 |     float x3=roi.height;
139 |   std::vector<float> list={  x0 +( x2 - 1) / 2, x1 +
140 |                            (x3 - 1) / 2, x2, x3
141 |                        };
142 |     target_pos.push_back( list[0]);
143 |     target_pos.push_back( list[1]);
144 | 
145 |     target_sz.push_back( list[2]);
146 |     target_sz.push_back( list[3]);
147 |     //std::cout<<"xywh2cxywh"<<list<<std::endl;
148 |     return list;
149 | 
150 | 
151 | }
152 | void siamfcpp::ini(cv::Mat frame,cv::Rect roi){ // return  features im_z_crop avg_chans
153 | 
154 | 
155 | //
156 | im_w=frame.cols;
157 | im_h=frame.rows;
158 | 
159 |   std::cout<<"loadok"<<std::endl;
160 | 
161 |   xywh2cxywh(roi);//# bbox in xywh format is given for initialization in case of tracking
162 |  bounding_box = roi;
163 |  avg_chans = cv::mean(frame);
164 | 
165 |  /*  if self._hyper_params['windowing'] == 'cosine':
166 |             window = np.outer(np.hanning(score_size), np.hanning(score_size))
167 |             window = window.reshape(-1)
168 |             */
169 | 
170 | 
171 | 
172 | 
173 | 
174 | 
175 | torch::Tensor hannwindow = hann_window(score_size);
176 | 
177 | 
178 | window=torch::outer(hannwindow,hannwindow).to(dvc);
179 | window=window.reshape(-1);
180 | 
181 |  //calculate_s_z();
182 | 
183 | /*
184 | torch::Tensor z_crop= get_subwindow(image,z_size,Sz);
185 | 
186 | std::cout<<z_crop.sizes()<<std::endl;
187 |  output = model.forward({z_crop}).toTensorList();
188 | //auto outputs = model.forward({tensor_image}).toTuple();
189 | std::cout<<output.size()<<std::endl;
190 |  qDebug()<<"load_ok";
191 | */
192 |   float scale=0;
193 |  im_z_crop= get_crop(frame,target_pos,target_sz,z_size,0,scale,avg_chans,context_amount);
194 | 
195 |   torch::Tensor te=         Mat2tensor(im_z_crop).to(dvc);
196 | std::cout<<"input_inisize"<<te.sizes()<<std::endl;
197 | features = model.forward({te}).toTensorList();
198 | //std::cout<<"features"<<features.size()<<std::endl;
199 | 
200 | 
201 | 
202 | 
203 | }
204 | /*
205 | torch::Tensor siamfcpp::diagnoal(cv::Mat m)
206 | {
207 |     std::vector<float> diagv;
208 | 
209 | 
210 |         for(int i=0, j=0;i<m.rows;j++,i++)
211 |         {
212 |             float v=m.at<float>(i,j);
213 | diagv.push_back(v);
214 | if(i>=m.cols)
215 |     break;
216 | 
217 | 
218 | 
219 |     }
220 |         torch::Tensor t=torch::tensor(diagv).to(dvc);
221 |       //  std::cout<<"t"<<t<<std::endl;
222 |         return t;
223 | 
224 | }
225 | */
226 | cv::Rect siamfcpp::update( cv::Mat frame,float &scores){
227 | 
228 | 
229 | /*
230 |  rect = state  # bbox in xywh format is given for initialization in case of tracking
231 |             box = xywh2cxywh(rect).reshape(4)
232 |             target_pos_prior, target_sz_prior = box[:2], box[2:]
233 |         features = self._state['features']
234 |         */
235 | 
236 | 
237 |   std::vector<float> target_pos_prior, target_sz_prior;
238 |   target_pos_prior=target_pos;
239 |   target_sz_prior=target_sz;
240 | 
241 | return track(frame,target_pos_prior,target_sz_prior,features,scores);
242 | 
243 | 
244 | 
245 | }
246 | 
247 | cv::Rect siamfcpp::track( cv::Mat frame,std::vector<float> target_pos1,
248 |             std::vector<float> target_sz1,
249 |             torch::List<torch::Tensor>  features,float& scores,
250 |             bool   update_state ){
251 | float scale_x=0;
252 | cv::Mat im_x_crop;
253 | 
254 | 
255 | 
256 |  im_x_crop= get_crop(frame,target_pos1,target_sz1,z_size,x_size,scale_x,avg_chans,context_amount).clone();
257 | 
258 | 
259 |   torch::Tensor te=         Mat2tensor(im_x_crop);
260 | std::cout<<"im_x_crop"<<te.sizes()<<std::endl;
261 | 
262 |   torch::Tensor features1=features[0];
263 | 
264 |    torch::Tensor features2=features[1];
265 | 
266 |    std::vector<torch::IValue> output = trackmodel.forward({te.to(dvc),features1.to(dvc),features2.to(dvc)}).toTuple()->elements();
267 | 
268 | 
269 | 
270 | 
271 | 
272 | 
273 | 
274 | 
275 |             torch::Tensor score=  output[0].toTensor().select(0,0).squeeze().to(dvc);
276 | 
277 |             torch::Tensor box=    output[1].toTensor().select(0,0).to(dvc);
278 | 
279 |              torch::Tensor cls=   output[2].toTensor().select(0,0).to(dvc);
280 | 
281 |              torch::Tensor ctr=   output[3].toTensor().select(0,0).to(dvc);
282 | 
283 |              torch::Tensor box_wh=   xyxy2cxywh(box.to(dvc));
284 | 
285 | 
286 | 
287 |    /*
288 |     *   box = tensor_to_numpy(box[0])
289 |         score = tensor_to_numpy(score[0])[:, 0]
290 |         cls = tensor_to_numpy(cls[0])
291 |         ctr = tensor_to_numpy(ctr[0])
292 | # score post-processing
293 |        best_pscore_id, pscore, penalty = self._postprocess_score(
294 |            score, box_wh, target_sz, scale_x)
295 |        # box post-processing
296 |        new_target_pos, new_target_sz = self._postprocess_box(
297 |            best_pscore_id, score, box_wh, target_pos, target_sz, scale_x,
298 |            x_size, penalty)
299 |            */
300 |   torch::Tensor best_pscore_id, pscore, penalty ;
301 | std::vector< torch::Tensor> rv=postprocess_score(
302 |             score,box_wh,target_sz,scale_x
303 |             );
304 | best_pscore_id=rv[0].to(dvc);
305 | int best_id=best_pscore_id.item<int>();
306 | pscore=rv[1].to(dvc);
307 | penalty=rv[2].to(dvc);
308 | std::vector<float> rc= postprocess_box(best_id,score,box_wh,target_pos1,target_sz1,scale_x,x_size,penalty);
309 | //std::cout<<"bestid "<<best_id<<std::endl;
310 | //std::cout<<"rc "<<rc<<std::endl;
311 | rc=restrict_box(rc);
312 | scores=pscore[best_id].item<float>();
313 | pr_socres=scores;
314 | //update state
315 | if(pr_socres>AC_SCORE){
316 | target_pos[0]=rc[0];
317 | target_pos[1]=rc[1];
318 | target_sz[0]=rc[2];
319 | target_sz[1]=rc[3];
320 | }else
321 | {
322 | 
323 | 
324 | }
325 | 
326 | cv::Rect rr= cxywh2xywh(rc);
327 | //std::vector<float> rr1=xywh2xyxy
328 | 
329 | //std::cout<<"scores"<<scores<<std::endl;
330 | return rr;
331 | 
332 | }
333 | 
334 | 
335 | std::vector<torch::Tensor> siamfcpp::postprocess_score( torch::Tensor score,
336 |                         torch::Tensor box_wh,
337 |                         std::vector<float> target_sz1,
338 |                         float scale_x
339 |                             ){
340 |        /*
341 | # size penalty
342 |       penalty_k = self._hyper_params['penalty_k']
343 |       target_sz_in_crop = target_sz * scale_x
344 |       s_c = change(
345 |           sz(box_wh[:, 2], box_wh[:, 3]) /
346 |           (sz_wh(target_sz_in_crop)))  # scale penalty
347 |       r_c = change((target_sz_in_crop[0] / target_sz_in_crop[1]) /
348 |                    (box_wh[:, 2] / box_wh[:, 3]))  # ratio penalty
349 |       penalty = np.exp(-(r_c * s_c - 1) * penalty_k)
350 |       pscore = penalty * score
351 | 
352 |       # ipdb.set_trace()
353 |       # cos window (motion model)
354 |       window_influence = self._hyper_params['window_influence']
355 |       pscore = pscore * (
356 |           1 - window_influence) + self._state['window'] * window_influence
357 |       best_pscore_id = np.argmax(pscore)
358 | 
359 |       return best_pscore_id, pscore, penalty
360 |               */
361 | //(box_wh[:, 2], box_wh[:, 3]
362 |     torch::Tensor w=  box_wh.select(1,2).to(dvc);
363 |     torch::Tensor h=    box_wh.select(1,3).to(dvc);
364 |  std::vector<float> target_sz_in_crop1 ={ target_sz1[0] * scale_x,target_sz1[1] * scale_x};
365 |  torch::Tensor target_sz_in_crop= torch::tensor(target_sz_in_crop1).to(dvc);
366 |  torch::Tensor s_c=change(sz(w,h)/sz_wh(target_sz_in_crop)).to(dvc);
367 |  torch::Tensor r_c=change((target_sz_in_crop[0]/target_sz_in_crop[1])/
368 |          (w/h)
369 |          ).to(dvc);
370 |  torch::Tensor penalty =torch::exp(-(r_c*s_c-1)*penalty_k).to(dvc);
371 |   torch::Tensor pscore=penalty*score;
372 |   pscore=pscore*(
373 |               1-window_influence)+window*window_influence;
374 |   torch::Tensor best_pscore_id =torch::argmax(pscore).to(dvc);
375 |   std::vector<torch::Tensor> rv={best_pscore_id,pscore,penalty};
376 |   return rv;
377 | 
378 | 
379 | }
380 | std::vector<float> siamfcpp::postprocess_box(int best_pscore_id,
381 |                                  torch::Tensor score,
382 |                                  torch::Tensor box_wh,
383 |                                   std::vector<float> target_pos1,
384 |                                   std::vector<float> target_sz1,
385 |                                   float scale_x,
386 |                                  float x_size1,
387 |                                   torch::Tensor penalty
388 |                                  ){
389 | // std::cout<<"box_wh"<<box_wh.sizes()<<std::endl;
390 |    // std::cout<<"bestid"<<best_pscore_id<<std::endl;
391 |      torch::Tensor   pred_in_crop = box_wh.select(0,best_pscore_id) /scale_x;
392 | 
393 | 
394 |        torch::Tensor  lr = penalty[best_pscore_id] * score[best_pscore_id] * test_lr;
395 |        torch::Tensor  res_x = pred_in_crop[0] + target_pos1[0] - (int)(x_size1 / 2) / scale_x;
396 |        torch::Tensor       res_y = pred_in_crop[1] + target_pos1[1] - (int)(x_size1 / 2) / scale_x;
397 |        torch::Tensor    res_w = target_sz1[0] * (1 - lr) + pred_in_crop[2] * lr;
398 |        torch::Tensor    res_h = target_sz1[1] * (1 - lr) + pred_in_crop[3] * lr;
399 | 
400 |        // np.array([res_x, res_y])
401 |            //  np.array([res_w, res_h])
402 | 
403 |           std::vector<float> vec={res_x.item<float>(),res_y.item<float>(),res_w.item<float>(),res_h.item<float>()};
404 |            return vec;
405 | 
406 | }
407 | std::vector<float>  siamfcpp::restrict_box(std::vector<float> tragrect){
408 | 
409 |        tragrect[0] = fmax(0, fmin(im_w, tragrect[0]));
410 |           tragrect[1] = fmax(0, fmin(im_h, tragrect[1]));
411 |           tragrect[2] = fmax(min_w,
412 |                             fmin(im_w, tragrect[2] ));
413 |           tragrect[3]= fmax(min_h,
414 |                             fmin(im_h, tragrect[3]));
415 |           return tragrect;
416 | 
417 | }
418 | 
419 | 
420 | 
421 | cv::Mat siamfcpp::get_crop(cv::Mat im, std::vector<float> target_pos, std::vector<float> target_sz,const int z_size1,const int x_size1,float &scale ,cv::Scalar avg_chans,float context_amount ){
422 |    /*
423 |     Returns
424 |       -------
425 |           cropped & resized image, (output_size, output_size) if output_size provied,
426 |           otherwise, (x_size, x_size, 3) if x_size provided, (z_size, z_size, 3) otherwise
427 |       """
428 |       wc = target_sz[0] + context_amount * sum(target_sz)
429 |       hc = target_sz[1] + context_amount * sum(target_sz)
430 |       s_crop = np.sqrt(wc * hc)
431 |       scale = z_size / s_crop
432 | 
433 |       # im_pad = x_pad / scale
434 |       if x_size is None:
435 |           x_size = z_size
436 |       s_crop = x_size / scale
437 | 
438 |       if output_size is None:
439 |           output_size = x_size
440 |       if mask is not None:
441 |           im_crop, mask_crop = func_get_subwindow(im,
442 |                                                   target_pos,
443 |                                                   output_size,
444 |                                                   round(s_crop),
445 |                                                   avg_chans,
446 |                                                   mask=mask)
447 |           return im_crop, mask_crop, scale
448 |       else:
449 |           im_crop = func_get_subwindow(im, target_pos, output_size, round(s_crop),
450 |                                        avg_chans)
451 |           return im_crop, scale
452 |                           */
453 |     float wc= target_sz[0] + context_amount *(target_sz[0]+target_sz[1]);
454 | 
455 |     float hc= target_sz[1] + context_amount *(target_sz[0]+target_sz[1]);
456 | 
457 |     float   s_crop = sqrt(wc * hc);
458 |   int output_size=0;
459 |      scale=z_size1 / s_crop;  //127/
460 |     if (x_size1 == 0)
461 |     {
462 |       output_size = z_size1;//127
463 |     s_crop = z_size1 / scale;  //s_crop 目标大小 target size
464 |      }else
465 |     {
466 |      output_size=x_size1;//303
467 |        s_crop = x_size1 / scale; //303  *
468 |     }
469 | s_crop_rc=round(s_crop);
470 | cv::Mat im_crop=get_subwindow_tracking(im,target_pos,output_size,round(s_crop),avg_chans);
471 | 
472 | 
473 | 
474 | return im_crop ;
475 | 
476 | 
477 | 
478 | 
479 | }
480 | //   tar_pos,303, 303/
481 | cv::Mat siamfcpp::get_subwindow_tracking(cv::Mat im,
482 |                                std::vector<float> pos,
483 |                              float  model_sz,
484 |                              float  original_sz,
485 |                               cv::Scalar avg_chans
486 |                                ){
487 |     /*
488 |     Returns
489 |        -------
490 |        numpy.array
491 |            image patch within _original_sz_ in _im_ and  resized to _model_sz_, padded by _avg_chans_
492 |            (model_sz, model_sz, 3)
493 |        """
494 |        crop_cxywh = np.concatenate(
495 |            [np.array(pos), np.array((original_sz, original_sz))], axis=-1)
496 |        crop_xyxy = cxywh2xyxy(crop_cxywh)
497 |        # warpAffine transform matrix
498 |        M_13 = crop_xyxy[0]
499 |        M_23 = crop_xyxy[1]
500 |        M_11 = (crop_xyxy[2] - M_13) / (model_sz - 1)
501 |        M_22 = (crop_xyxy[3] - M_23) / (model_sz - 1)
502 |        mat2x3 = np.array([
503 |            M_11,
504 |            0,
505 |            M_13,
506 |            0,
507 |            M_22,
508 |            M_23,
509 |        ]).reshape(2, 3)
510 |        im_patch = cv2.warpAffine(im,
511 |                                  mat2x3, (model_sz, model_sz),
512 |                                  flags=(cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP),
513 |                                  borderMode=cv2.BORDER_CONSTANT,
514 |                                  borderValue=tuple(map(int, avg_chans)))
515 |        if mask is not None:
516 |            mask_patch = cv2.warpAffine(mask,
517 |                                        mat2x3, (model_sz, model_sz),
518 |                                        flags=(cv2.INTER_NEAREST
519 |                                               | cv2.WARP_INVERSE_MAP))
520 |            return im_patch, mask_patch
521 |        return im_patch
522 |                          */
523 | 
524 | 
525 |     std::vector<float> xywh={pos[0],pos[1],original_sz,original_sz};
526 |     torch::Tensor  crop_cxywh=torch::tensor(xywh);
527 |       torch::Tensor  crop_xyxy = cxywh2xyxy(crop_cxywh.to(dvc));
528 | 
529 | // warpAffine transform matrix
530 | float M_13 = crop_xyxy[0].item<float>();; //0维切片
531 | float M_23 = crop_xyxy[1].item<float>();
532 | float M_11 = ((crop_xyxy[2] - M_13) / (model_sz - 1)).item<float>(); //w/(303-1)   original_sz/303缩放系数 a
533 | float M_22 = ((crop_xyxy[3] - M_23) / (model_sz - 1)).item<float>();//h/(303-1)
534 | 
535 | 
536 | std::vector<float> ten={M_11,0,M_13,0,M_22,
537 |                                 M_23};
538 | std::cout<<"M_11"<< M_11<<std::endl;
539 | std::cout<<"M_13"<< M_13<<std::endl;
540 | std::cout<<"M_22"<< M_22<<std::endl;
541 | std::cout<<"M_23"<< M_23<<std::endl;
542 | 
543 | /*
544 |  a=scale*cos0  b=scale *sin0
545 |  a 0 (1-a)*center.x
546 |  0 a  (1-a)*center.y
547 |  */
548 | torch::Tensor mat2x3 =torch::tensor(ten).reshape({2,3}).to(dvc);//calculate
549 | cv::Mat im_patch ;
550 | 
551 | 
552 | cv::Mat im_patch_test ;
553 | //std::cout<<"mat2x3"<<mat2x3<<std::endl;
554 | 
555 | cv::warpAffine(im,im_patch,
556 |                         tensor2Mat(mat2x3) , cv::Size(model_sz, model_sz),
557 |                         cv::INTER_LINEAR | cv::WARP_INVERSE_MAP,
558 |                          cv::BORDER_CONSTANT,
559 |                         avg_chans);
560 | 
561 | cv::warpAffine(im,im_patch_test,
562 |                         tensor2Mat(mat2x3) , cv::Size(),
563 |                         cv::INTER_LINEAR | cv::WARP_INVERSE_MAP,
564 |                          cv::BORDER_CONSTANT,
565 |                         avg_chans);
566 | cv::imshow("warpAffinetest",im_patch_test);
567 | cv::imshow("warpAffine",im_patch);
568 | return im_patch;
569 | }
570 | torch::Tensor siamfcpp::cxywh2xyxy( torch::Tensor box ){
571 | /*
572 |     box = np.array(box, dtype=np.float32)
573 |         return np.concatenate([
574 |             box[..., [0]] - (box[..., [2]] - 1) / 2, box[..., [1]] -
575 |             (box[..., [3]] - 1) / 2, box[..., [0]] +
576 |             (box[..., [2]] - 1) / 2, box[..., [1]] + (box[..., [3]] - 1) / 2
577 |         ],
578 |                               axis=-1)
579 |             */
580 | 
581 |   //  std::cout<<"cxywh2xyxy box"<<box.sizes()<<std::endl;
582 | 
583 | /*
584 |       std::cout<<box.sizes()<<std::endl;
585 |     torch::Tensor t0=  box.select(0,0);  //1维 第0 开始 取一排
586 |   std::cout<<"t0 "<<(t0).item<float>()<<std::endl;
587 |     torch::Tensor t1=  box.select(0,1);
588 |   std::cout<<"t1 "<<(t1).item<float>()<<std::endl;
589 |     torch::Tensor t2=  box.select(0,2);
590 |   std::cout<<"t2 "<<(t2).item<float>()<<std::endl;
591 |     torch::Tensor t3= box.select(0,3);
592 |   std::cout<<"t3 "<<(t3).item<float>()<<std::endl;
593 | torch::Tensor x0=t2 - 1;
594 |   std::cout<<"x0 "<<(t2 - 1).item<float>()<<std::endl;
595 | 
596 |   */
597 | 
598 |    float t0=  box.select(0,0).item<float>();  //1维 第0 开始 取一排
599 | 
600 |    float t1=  box.select(0,1).item<float>();
601 | 
602 |    float t2=  box.select(0,2).item<float>();
603 | 
604 |    float t3= box.select(0,3).item<float>();
605 | 
606 | 
607 | 
608 |   std::vector<float> t={t0 - (t2 - 1) / 2,t1 -
609 |                         (t3 - 1) / 2, t0 +
610 |                         (t2 - 1) / 2, t1 + (t3 - 1) / 2
611 |                     };
612 | 
613 | 
614 | torch::Tensor tt=torch::tensor(t).to(dvc);
615 |   //  std::cout<<"tt "<<tt<<std::endl;
616 | 
617 |     return tt;
618 | 
619 | 
620 | }
621 | 
622 | 
623 | 
624 | std::vector<float> siamfcpp::xywh2xyxy(cv::Rect roi ){
625 | 
626 | 
627 | 
628 | 
629 |     float x0=roi.x;
630 |     float x1=roi.y;
631 |     float x2=roi.width;
632 |     float x3=roi.height;
633 |   std::vector<float> list={
634 |                             x0,x1,x2 + x0 - 1,
635 |                             x3+ x1 - 1
636 |                        };
637 | 
638 | 
639 |     return list;
640 | 
641 | 
642 | }
643 | 
644 | 
645 | /*
646 | 
647 | def tensor_to_imarray(t):
648 |     r"""
649 |     Perform naive detach / cpu / numpy process and then transpose
650 |     cast dtype to np.uint8
651 |     :param t: torch.Tensor, (1, C, H, W)
652 |     :return: numpy.array, (H, W, C)
653 |     """
654 |     arr = t.detach().cpu().numpy().astype(np.uint8)
655 |     if arr.ndim == 4:
656 |         arr = arr[0]
657 |     return arr.transpose(1, 2, 0)
658 | 
659 | */
660 | /*
661 | cv::Mat siamfcpp::tensor_to_imarray(torch::Tensor out_tensor,int img_h,int img_w){
662 | 
663 |     //s1:sequeeze去掉多余维度,(1,C,H,W)->(C,H,W)；s2:permute执行通道顺序调整,(C,H,W)->(H,W,C)
664 |     out_tensor = out_tensor.squeeze().detach().permute({ 1, 2, 0 });
665 |     out_tensor = out_tensor.mul(255).clamp(0, 255).to(torch::kU8); //s3:*255，转uint8
666 |     out_tensor = out_tensor.to(torch::kCPU); //迁移至CPU
667 |     cv::Mat resultImg(img_h, img_w, CV_8UC3, out_tensor.data_ptr()); // 将Tensor数据拷贝至Mat
668 |     return resultImg ;
669 | 
670 | }
671 | */
672 | 
673 | cv::Mat siamfcpp::tensor2Mat(torch::Tensor &i_tensor)
674 | {
675 |     int height = i_tensor.size(0), width = i_tensor.size(1);
676 |     //i_tensor = i_tensor.to(torch::kF32);
677 |     i_tensor = i_tensor.to(torch::kCPU);
678 |     cv::Mat o_Mat(cv::Size(width, height), CV_32F, i_tensor.data_ptr());
679 | 
680 | 
681 |   //  std::cout<<"o_Mat "<<o_Mat<<std::endl;
682 |     return o_Mat;
683 | }
684 | 
685 | torch::Tensor siamfcpp::Mat2tensor(cv::Mat im){
686 | 
687 |   torch::Tensor ts=  torch::from_blob(
688 |             im.data,
689 |             { 1, im.rows, im.cols, im.channels() },
690 |             torch::TensorOptions(torch::kByte)
691 |         ).permute({ 0, 3, 1, 2 }).toType(torch::kFloat);
692 | 
693 |   return ts;
694 | }
695 | 
696 | 
697 | torch::Tensor siamfcpp::change(torch::Tensor r){
698 | 
699 |     return torch::maximum(r,1.0/r);
700 | 
701 | }
702 | 
703 | torch::Tensor siamfcpp::sz(torch::Tensor w,torch::Tensor h){
704 |     torch::Tensor pad=(w+h)*0.5;
705 |     torch::Tensor sz2=(w+pad)*(h+pad);
706 | return torch::sqrt(sz2);
707 | }
708 | 
709 | torch::Tensor siamfcpp::sz_wh(torch::Tensor wh){
710 |     torch::Tensor pad=(wh[0]+wh[1])*0.5;
711 |     torch::Tensor sz2=(wh[0]+pad)*(wh[1]+pad);
712 | return torch::sqrt(sz2);
713 | }
714 | torch::Tensor siamfcpp::hann_window(int window_length ,torch::DeviceType dev) {
715 |     return torch::arange(window_length).mul(M_PI * 2. / static_cast<double>(window_length - 1)).cos().mul(-0.5).add(0.5).to(dev);
716 | }
717 | 


--------------------------------------------------------------------------------
/siamfcpp/siamfcpp.h:
--------------------------------------------------------------------------------
  1 | #ifndef SIAMFCPP_H
  2 | #define SIAMFCPP_H
  3 | 
  4 | #include<opencv2/opencv.hpp>
  5 | //#undef slots
  6 | #include <torch/torch.h>
  7 | #include<torch/script.h>
  8 | //#define slots Q_SLOTS
  9 | //#include <QObject>
 10 | //#include<QDebug>
 11 | #include <cmath>
 12 | 
 13 | class siamfcpp //:public QObject
 14 | {
 15 |    // Q_OBJECT
 16 | public:
 17 |     siamfcpp( torch::DeviceType dvic=torch::DeviceType::CUDA);
 18 |     ~siamfcpp();
 19 |     //tracking model
 20 |      torch::jit::script::Module trackmodel;
 21 |  torch::jit::script::Module model;
 22 |     //ini value
 23 |      std::vector<float> target_pos, target_sz;
 24 | 
 25 | torch::List<torch::Tensor>  features;
 26 | cv::Mat  im_z_crop;
 27 |    cv::Scalar avg_chans;
 28 |    //device
 29 |     torch::DeviceType dvc;
 30 | 
 31 |     float pr_socres=0;
 32 |    float   s_crop_rc;//record s_crop
 33 | 
 34 |    void ini( cv::Mat frame,cv::Rect roi);
 35 |    cv::Rect update( cv::Mat frame,float &scores);
 36 | 
 37 | cv::Rect track( cv::Mat frame,std::vector<float> target_pos1,
 38 |             std::vector<float> target_sz1,
 39 |            torch::List<torch::Tensor> features,float& scores,
 40 |             bool   update_state=true );
 41 |   std::vector<torch::Tensor> postprocess_score( torch::Tensor score,
 42 |                           torch::Tensor box_wh,
 43 |                           std::vector<float> target_sz1,
 44 |                           float scale_x
 45 |                           );
 46 | std::vector<float> restrict_box(std::vector<float> tragrect);
 47 | torch::Tensor hann_window(int window_length,torch::DeviceType dev=torch::DeviceType::CPU);
 48 | std::vector<float> postprocess_box(int best_pscore_id,
 49 |                                    torch::Tensor score,
 50 |                                    torch::Tensor box_wh,
 51 |                                     std::vector<float> target_pos1,
 52 |                                     std::vector<float> target_sz1,
 53 |                                     float scale_x,
 54 |                                    float x_size1,
 55 |                                     torch::Tensor penalty
 56 |                                    );
 57 |   torch::Tensor change(torch::Tensor r);
 58 | 
 59 | 
 60 |   torch::Tensor sz(torch::Tensor w,torch::Tensor h);
 61 | 
 62 |   torch::Tensor sz_wh(torch::Tensor wh);
 63 | 
 64 | 
 65 | int calculate_s_z();
 66 | torch::Tensor convert_bbox(torch::Tensor loc) ;
 67 | 
 68 | cv::Rect cxywh2xywh(std::vector<float> roi) ;
 69 | torch::Tensor xyxy2cxywh(torch::Tensor box) ;
 70 | //void  postprocess_score();
 71 | 
 72 | std::vector<float>  xywh2cxywh(cv::Rect roi); //box_wh
 73 | cv::Mat get_crop(cv::Mat im, std::vector<float> target_pos, std::vector<float> target_sz,
 74 |                  const int z_size1,const int x_size1,float &scale,
 75 |                  cv::Scalar avg_chans,float context_amount=0.5   );
 76 | cv::Mat get_subwindow_tracking(cv::Mat im,
 77 |                                std::vector<float> pos,
 78 |                              float  model_sz,
 79 |                              float  original_sz,
 80 |                               cv::Scalar avg_chans={0.0,0.0, 0.0, 0.0}
 81 |                               );
 82 | std::vector<float> xywh2xyxy(cv::Rect roi );
 83 |  torch::Tensor cxywh2xyxy( torch::Tensor crop_cxywh );
 84 | // cv::Mat tensor_to_imarray(torch::Tensor out_tensor,int img_h,int img_w);
 85 | torch::Tensor Mat2tensor(cv::Mat im);
 86 | //torch::Tensor Matarr2tensor(cv::Mat im);
 87 | //torch::Tensor diagnoal(cv::Mat m);
 88 | cv::Mat tensor2Mat(torch::Tensor &i_tensor);
 89 | protected:
 90 | 
 91 | 
 92 |          static const int     total_stride=8;
 93 |          static const int   score_size=17;
 94 |         static const int  score_offset=87;
 95 |            static const float context_amount;
 96 |            static const float test_lr;
 97 |            static const float penalty_k;
 98 |            static const float window_influence;
 99 |       static const std::string    windowing;
100 |        static const int   z_size=127;
101 | 
102 | 
103 |        static const int    x_size=303;
104 |         static const int  num_conv3x3=3;
105 |       static const int   min_w=10;
106 |        static const int   min_h=10;
107 |      //    phase_init="feature",
108 |       //   phase_track="track",
109 |       //   corr_fea_output=False,
110 | 
111 |        static const float AC_SCORE;//信分数
112 | 
113 | 
114 | 
115 | 
116 | 
117 | 
118 |    cv::Rect bounding_box;
119 | 
120 | 
121 |    // TODO: What are these?
122 | 
123 |    torch::List<torch::Tensor> zf;
124 |    torch::Tensor anchors;
125 |    torch::Tensor window;
126 |    int  im_w;//image  width
127 |   int im_h;  // image height
128 | };
129 | 
130 | #endif // SIAMFCPP_H
131 | 


--------------------------------------------------------------------------------