├── README.md
├── ECO
    ├── regularization_filter.hpp
    ├── metrics.hpp
    ├── optimize_scores.hpp
    ├── scale_filter.hpp
    ├── interpolator.hpp
    ├── metrics.cc
    ├── wrappers.hpp
    ├── gradient.hpp
    ├── feature_operator.hpp
    ├── feature_extractor.hpp
    ├── sse.hpp
    ├── sample_update.hpp
    ├── eco.hpp
    ├── training.hpp
    ├── interpolator.cc
    ├── regularization_filter.cc
    ├── ffttools.hpp
    ├── recttools.hpp
    ├── optimize_scores.cc
    ├── fhog.hpp
    ├── scale_filter.cc
    ├── parameters.hpp
    ├── feature_operator.cc
    ├── eco_unittest.cc
    ├── sample_update.cc
    ├── debug.hpp
    ├── gradient.cpp
    └── ffttools.cc
├── KCF
    ├── labdata.hpp
    ├── recttools.hpp
    ├── fhog.hpp
    ├── ffttools.hpp
    └── kcftracker.hpp
├── MTCNN
    ├── helpers.hpp
    ├── face_detector.hpp
    └── face_detector.cpp
└── DaSiamRPN
    └── dasiamrpntracker.h


/README.md:
--------------------------------------------------------------------------------
 1 | # Object-tracking-system-based-on-deep-learning
 2 | Object tracking system based on deep learning. 
 3 | 
 4 | Master-Jetson TX2, ROS-image + control, image (DL) - detection + tracking, control - TurtleBot
 5 | 
 6 | 
 7 | Video:https://www.bilibili.com/video/av49678702/
 8 | 
 9 | Blog:https://blog.csdn.net/qq_35999634/article/details/89375456
10 | 
11 | Wechat Official Account: ScienceRecord
12 | 


--------------------------------------------------------------------------------
/ECO/regularization_filter.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef REGULARIZATION_FILTER_HPP
 2 | #define REGULARIZATION_FILTER_HPP
 3 | 
 4 | #include <opencv2/opencv.hpp>
 5 | #include <cmath>
 6 | 
 7 | #include "parameters.hpp"
 8 | #include "ffttools.hpp"
 9 | #include "debug.hpp"
10 | 
11 | namespace eco
12 | {
13 | cv::Mat get_regularization_filter(cv::Size sz,
14 |                                   cv::Size2f target_sz,
15 |                                   const EcoParameters &params);
16 | }
17 | #endif


--------------------------------------------------------------------------------
/ECO/metrics.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef METRICS_HPP
 2 | #define METRICS_HPP
 3 | 
 4 | //#include <cmath>
 5 | #include <math.h>
 6 | #include <opencv2/core.hpp>
 7 | 
 8 | class Metrics
 9 | {
10 |   public:
11 |     float center_error(const cv::Rect2f bbox, const cv::Rect2f bboxGroundtruth);
12 |     float iou(const cv::Rect2f bbox, const cv::Rect2f bboxGroundtruth);
13 |     cv::Rect2f intersection(const cv::Rect2f bbox,
14 |                             const cv::Rect2f bboxGroundtruth);
15 |     float auc();
16 | };
17 | 
18 | #endif


--------------------------------------------------------------------------------
/KCF/labdata.hpp:
--------------------------------------------------------------------------------
 1 | namespace kcf
 2 | {
 3 | const int nClusters = 15;
 4 | float data[nClusters][3] = {
 5 | {161.317504, 127.223401, 128.609333},
 6 | {142.922425, 128.666965, 127.532319},
 7 | {67.879757, 127.721830, 135.903311},
 8 | {92.705062, 129.965717, 137.399500},
 9 | {120.172257, 128.279647, 127.036493},
10 | {195.470568, 127.857070, 129.345415},
11 | {41.257102, 130.059468, 132.675336},
12 | {12.014861, 129.480555, 127.064714},
13 | {226.567086, 127.567831, 136.345727},
14 | {154.664210, 131.676606, 156.481669},
15 | {121.180447, 137.020793, 153.433743},
16 | {87.042204, 137.211742, 98.614874},
17 | {113.809537, 106.577104, 157.818094},
18 | {81.083293, 170.051905, 148.904079},
19 | {45.015485, 138.543124, 102.402528}};
20 | }


--------------------------------------------------------------------------------
/ECO/optimize_scores.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef OPTIMIZE_SCORES_HPP
 2 | #define OPTIMIZE_SCORES_HPP
 3 | 
 4 | //#include <opencv2/features2d/features2d.hpp>
 5 | #include <opencv2/opencv.hpp>
 6 | 
 7 | #include "ffttools.hpp"
 8 | #include "debug.hpp"
 9 | 
10 | namespace eco
11 | {
12 | class OptimizeScores
13 | {
14 |   public:
15 | 	virtual ~OptimizeScores() {}
16 | 	OptimizeScores() {} 
17 | 	OptimizeScores(std::vector<cv::Mat> &scores_fs, int ite)
18 | 		: scores_fs_(scores_fs), iterations_(ite) {}
19 | 
20 | 	void compute_scores();
21 | 
22 | 	inline int get_scale_ind() const { return scale_ind_; }
23 | 	inline float get_disp_row() const { return disp_row_; }
24 | 	inline float get_disp_col() const { return disp_col_; }
25 | 	inline float get_max_score() const { return max_score_; }
26 | 
27 |   private:
28 | 	std::vector<cv::Mat> scores_fs_;
29 | 	int iterations_;
30 | 
31 | 	int scale_ind_;
32 | 	float disp_row_;
33 | 	float disp_col_;
34 | 	float max_score_;
35 | };
36 | } // namespace eco
37 | #endif
38 | 


--------------------------------------------------------------------------------
/ECO/scale_filter.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef SCALE_FILTER_HPP
 2 | #define SCALE_FILTER_HPP
 3 | 
 4 | #include "parameters.hpp"
 5 | #include "ffttools.hpp"
 6 | #include "recttools.hpp"
 7 | #include "feature_extractor.hpp"
 8 | #include "debug.hpp"
 9 | #include <opencv2/core.hpp>
10 | 
11 | namespace eco
12 | {
13 | 
14 | class ScaleFilter
15 | {
16 |   public:
17 |     ScaleFilter(){};
18 |     virtual ~ScaleFilter(){};
19 |     void init(int &nScales, float &scale_step, const EcoParameters &params);
20 |     float scale_filter_track(const cv::Mat &im, const cv::Point2f &pos, const cv::Size2f &base_target_sz, const float &currentScaleFactor, const EcoParameters &params);
21 |     cv::Mat extract_scale_sample(const cv::Mat &im, const cv::Point2f &posf, const cv::Size2f &base_target_sz, vector<float> &scaleFactors, const cv::Size &scale_model_sz);
22 |     
23 |   private:
24 |     vector<float> scaleSizeFactors_;
25 |     vector<float> interpScaleFactors_;
26 |     cv::Mat yf_;
27 |     vector<float> window_;
28 |     bool max_scale_dim_;
29 | };
30 | } // namespace eco
31 | #endif


--------------------------------------------------------------------------------
/ECO/interpolator.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef INTERPOLATOR_HPP
 2 | #define INTERPOLATOR_HPP
 3 | 
 4 | #include <opencv2/opencv.hpp>
 5 | #include <math.h>
 6 | #include "debug.hpp"
 7 | 
 8 | namespace eco{
 9 | class Interpolator
10 | {
11 |   public:
12 | 	Interpolator();
13 | 	virtual ~Interpolator();
14 | 	static inline float mat_cos1(float x)
15 | 	{
16 | 		return (cos(x * M_PI));
17 | 	}
18 | 	static inline float mat_sin1(float x)
19 | 	{
20 | 		return (sin(x * M_PI));
21 | 	}
22 | 	static inline float mat_cos2(float x)
23 | 	{
24 | 		return (cos(2 * x * M_PI));
25 | 	}
26 | 	static inline float mat_sin2(float x)
27 | 	{
28 | 		return (sin(2 * x * M_PI));
29 | 	}
30 | 	static inline float mat_cos4(float x)
31 | 	{
32 | 		return (cos(4 * x * M_PI));
33 | 	}
34 | 	static inline float mat_sin4(float x)
35 | 	{
36 | 		return (sin(4 * x * M_PI));
37 | 	}
38 | 
39 | 	static void get_interp_fourier(cv::Size filter_sz, 
40 | 								   cv::Mat &interp1_fs,
41 | 								   cv::Mat &interp2_fs, 
42 | 								   float a);
43 | 
44 | 	static cv::Mat cubic_spline_fourier(cv::Mat f, float a);
45 | };
46 | }
47 | 
48 | #endif


--------------------------------------------------------------------------------
/MTCNN/helpers.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _HELPERS_HPP_
 2 | #define _HELPERS_HPP_
 3 | 
 4 | #include <chrono>
 5 | #include <opencv2/opencv.hpp>
 6 | 
 7 | class Timer {
 8 | private:
 9 | 	std::chrono::time_point<std::chrono::system_clock> t1;
10 | public:
11 | 	inline void start() {
12 | 		t1 = std::chrono::system_clock::now();
13 | 	} 
14 | 	inline double stop() {
15 | 		auto t2 = std::chrono::system_clock::now();
16 | 		return std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1).count();
17 | 	}
18 | };
19 | 
20 | inline cv::Mat cropImage(cv::Mat img, cv::Rect r) {
21 | 	cv::Mat m = cv::Mat::zeros(r.height, r.width, img.type());
22 | 	int dx = std::abs(std::min(0, r.x));
23 | 	if (dx > 0) { r.x = 0; }
24 | 	r.width -= dx;
25 | 	int dy = std::abs(std::min(0, r.y));
26 | 	if (dy > 0) { r.y = 0; }	
27 | 	r.height -= dy;
28 | 	int dw = std::abs(std::min(0, img.cols - 1 - (r.x + r.width)));
29 | 	r.width -= dw;
30 | 	int dh = std::abs(std::min(0, img.rows - 1 - (r.y + r.height)));
31 | 	r.height -= dh;
32 | 	if (r.width > 0 && r.height > 0) {
33 | 		img(r).copyTo(m(cv::Range(dy, dy + r.height), cv::Range(dx, dx + r.width)));
34 | 	}
35 | 	return m;
36 | }
37 | 
38 | inline void drawAndShowFace(cv::Mat img, cv::Rect r, const std::vector<cv::Point>& pts) {
39 | 	cv::Mat outImg;
40 | 	img.convertTo(outImg, CV_8UC3);
41 | 	cv::rectangle(outImg, r, cv::Scalar(0, 0, 255));
42 | 	for (size_t i = 0; i < pts.size(); ++i) {
43 | 		cv::circle(outImg, pts[i], 3, cv::Scalar(0, 0, 255));
44 | 	}
45 | 	cv::imshow("test", outImg);
46 | 	cv::waitKey(0);
47 | }
48 | 
49 | #endif //_HELPERS_HPP_


--------------------------------------------------------------------------------
/ECO/metrics.cc:
--------------------------------------------------------------------------------
 1 | #include "metrics.hpp"
 2 | 
 3 | float Metrics::center_error(const cv::Rect2f bbox, const cv::Rect2f bboxGroundtruth)
 4 | {
 5 |     float cx = bbox.x + bbox.width / 2.0f;
 6 |     float cy = bbox.y + bbox.height / 2.0f;
 7 |     float cx_gt = bboxGroundtruth.x + bboxGroundtruth.width / 2.0f;
 8 |     float cy_gt = bboxGroundtruth.y + bboxGroundtruth.height / 2.0f;
 9 |     float result = std::sqrt(std::pow((cx - cx_gt), 2) +
10 |                              std::pow((cy - cy_gt), 2));
11 |     return result;
12 | }
13 | 
14 | float Metrics::iou(const cv::Rect2f bbox, const cv::Rect2f bboxGroundtruth)
15 | {
16 |     cv::Rect2f inter = Metrics::intersection(bbox, bboxGroundtruth);
17 |     float area_bbox = bbox.area();
18 |     float area_bbox_gt = bboxGroundtruth.area();
19 |     float area_intersection = inter.area();
20 |     float iou = area_bbox + area_bbox_gt - area_intersection;
21 |     iou = area_intersection / (iou + 1e-12);
22 |     return iou;
23 | }
24 | 
25 | cv::Rect2f Metrics::intersection(const cv::Rect2f bbox,
26 |                                  const cv::Rect2f bboxGroundtruth)
27 | {
28 |     float x1, y1, x2, y2, w, h;
29 |     x1 = std::max(bbox.x, bboxGroundtruth.x);
30 |     y1 = std::max(bbox.y, bboxGroundtruth.y);
31 |     x2 = std::min(bbox.x + bbox.width, bboxGroundtruth.x + bboxGroundtruth.width);
32 |     y2 = std::min(bbox.y + bbox.height, bboxGroundtruth.y + bboxGroundtruth.height);
33 |     w = std::max(0.0f, x2 - x1);
34 |     h = std::max(0.0f, y2 - y1);
35 | 
36 |     cv::Rect2f result(x1, y1, w, h);
37 |     return result;
38 | }
39 | 
40 | float Metrics::auc()
41 | {
42 | }


--------------------------------------------------------------------------------
/ECO/wrappers.hpp:
--------------------------------------------------------------------------------
 1 | /*******************************************************************************
 2 | * Piotr's Computer Vision Matlab Toolbox      Version 3.00
 3 | * Copyright 2014 Piotr Dollar.  [pdollar-at-gmail.com]
 4 | * Licensed under the Simplified BSD License [see external/bsd.txt]
 5 | *******************************************************************************/
 6 | #ifndef _WRAPPERS_HPP_
 7 | #define _WRAPPERS_HPP_
 8 | 
 9 | #include <stdlib.h>
10 | 
11 | // wrapper functions if compiling from C/C++
12 | inline void wrError(const char *errormsg) { throw errormsg; }
13 | inline void *wrCalloc(size_t num, size_t size) { return calloc(num, size); }
14 | inline void *wrMalloc(size_t size) { return malloc(size); }
15 | inline void wrFree(void *ptr) { free(ptr); }
16 | 
17 | // platform independent aligned memory allocation (see also alFree)
18 | // __m128 should be 128/8=16 byte aligned
19 | inline void *alMalloc(size_t size, int alignment)
20 | {
21 |   const size_t pSize = sizeof(void *), a = alignment - 1;
22 |   void *raw = wrMalloc(size + a + pSize);
23 |   // get the aligned address, alignment should be 2^N.
24 |   void *aligned = (void *)(((size_t)raw + pSize + a) & ~a); 
25 |   *(void **)((size_t)aligned - pSize) = raw; // save address of raw in -1
26 |   return aligned;
27 | }
28 | 
29 | // platform independent alignned memory de-allocation (see also alMalloc)
30 | inline void alFree(void *aligned)
31 | {
32 |   // raw: the address of (void *) pointer.
33 |   // aligned: the address of a (void *) pointer now point to (char *)
34 |   // - sizeof(void *): minus the address by sizeof(void *)
35 |   // (void **): the address of a pointer point to a (void *) pointer
36 |   // *: the pointer point to a (void *) pointer = the address of (void *)pointer
37 |   void *raw = *(void **)((char *)aligned - sizeof(void *));
38 |   wrFree(raw);
39 | }
40 | 
41 | #endif
42 | 


--------------------------------------------------------------------------------
/MTCNN/face_detector.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _FACE_DETECTOR_HPP_
 2 | #define _FACE_DETECTOR_HPP_
 3 | 
 4 | #include <string>
 5 | 
 6 | #include <caffe/caffe.hpp>
 7 | #include <opencv2/opencv.hpp>
 8 | 
 9 | namespace mtcnn {
10 | 
11 | const int NUM_REGRESSIONS = 4;
12 | const int NUM_PTS = 5;
13 | 
14 | struct BBox {
15 | 	float x1;
16 | 	float y1;
17 | 	float x2;
18 | 	float y2;
19 | 	cv::Rect getRect() const;
20 | 	BBox getSquare() const;
21 | };
22 | 
23 | struct Face {
24 | 	BBox bbox;
25 | 	float regression[NUM_REGRESSIONS];
26 | 	float score;
27 | 	float ptsCoords[2 * NUM_PTS];
28 | 	
29 | 	static void applyRegression(std::vector<Face>& faces, bool addOne = false);
30 | 	static void bboxes2Squares(std::vector<Face>& faces);
31 | };
32 | 
33 | class FaceDetector {
34 | private:
35 | 	boost::shared_ptr< caffe::Net<float> > pNet_;
36 | 	boost::shared_ptr< caffe::Net<float> > rNet_;
37 | 	boost::shared_ptr< caffe::Net<float> > oNet_;
38 | 	boost::shared_ptr< caffe::Net<float> > lNet_;
39 | 	float pThreshold_;
40 | 	float rThreshold_;
41 | 	float oThreshold_;
42 | 	bool useLNet_;
43 | 	void initNetInput(boost::shared_ptr< caffe::Net<float> > net, cv::Mat img);
44 | 	void initNetInput(boost::shared_ptr< caffe::Net<float> > net, std::vector<cv::Mat>& imgs);
45 | 	std::vector<Face> step1(cv::Mat img, float minFaceSize, float scaleFactor);
46 | 	std::vector<Face> step2(cv::Mat img, const std::vector<Face>& faces);
47 | 	std::vector<Face> step3(cv::Mat img, const std::vector<Face>& faces);
48 | 	std::vector<Face> step4(cv::Mat img, const std::vector<Face>& faces);
49 | 	std::vector<Face> composeFaces(const caffe::Blob<float>* regressionsBlob, 
50 | 										   const caffe::Blob<float>* scoresBlob,
51 | 										   float scale);
52 | 	static std::vector<Face> nonMaximumSuppression(std::vector<Face> faces, float threshold, bool useMin = false);
53 | public:
54 | 	FaceDetector(const std::string& modelDir, 
55 | 				 float pThreshold = 0.6f, 
56 | 				 float rThreshold = 0.7f, 
57 | 				 float oThreshold = 0.7f,
58 | 				 bool useLNet = false, 
59 | 				 bool useGPU = true, 
60 | 				 int deviceID = 0);
61 | 	std::vector<Face> detect(cv::Mat img, float minFaceSize, float scaleFactor);
62 | };
63 | 
64 | } // namespace mtcnn
65 | 
66 | #endif // _FACE_DETECTOR_HPP_
67 | 


--------------------------------------------------------------------------------
/ECO/gradient.hpp:
--------------------------------------------------------------------------------
 1 | /*******************************************************************************
 2 | * Piotr's Computer Vision Matlab Toolbox      Version 3.30
 3 | * Copyright 2014 Piotr Dollar & Ron Appel.  [pdollar-at-gmail.com]
 4 | * Licensed under the Simplified BSD License [see external/bsd.txt]
 5 | *******************************************************************************/
 6 | #ifndef GRADIENTMEX_HPP
 7 | #define GRADIENTMEX_HPP
 8 | 
 9 | #include "wrappers.hpp"
10 | #include <math.h>
11 | #include "string.h"
12 | #include "sse.hpp"
13 | #include <stdlib.h>
14 | 
15 | #include <assert.h>
16 | #include <stdio.h>
17 | 
18 | // compute x and y gradients for just one column (uses sse)
19 | void grad1(float *I, float *Gx, float *Gy, int h, int w, int x);
20 | 
21 | // compute x and y gradients at each location (uses sse)
22 | void grad2(float *I, float *Gx, float *Gy, int h, int w, int d);
23 | 
24 | // build lookup table a[] s.t. a[x*n]~=acos(x) for x in [-1,1]
25 | float *acosTable();
26 | 
27 | // compute gradient magnitude and orientation at each location (uses sse)
28 | void gradMag(float *I, float *M, float *O, int h, int w, int d, bool full);
29 | 
30 | // normalize gradient magnitude at each location (uses sse)
31 | void gradMagNorm(float *M, float *S, int h, int w, float norm);
32 | 
33 | // helper for gradHist, quantize O and M into O0, O1 and M0, M1 (uses sse)
34 | void gradQuantize(float *O, float *M, int *O0, int *O1, float *M0, float *M1,
35 |                   int nb, int n, float norm, int nOrients, bool full, bool interpolate);
36 | 
37 | // compute nOrients gradient histograms per bin x bin block of pixels
38 | void gradHist(float *M, float *O, float *H, int h, int w,
39 |               int bin, int nOrients, int softBin, bool full);
40 | 
41 | /******************************************************************************/
42 | 
43 | // HOG helper: compute 2x2 block normalization values (padded by 1 pixel)
44 | float *hogNormMatrix(float *H, int nOrients, int hb, int wb, int bin);
45 | 
46 | // HOG helper: compute HOG or FHOG channels
47 | void hogChannels(float *H, const float *R, const float *N,
48 |                  int hb, int wb, int nOrients, float clip, int type);
49 | 
50 | // compute HOG features
51 | void hog(float *M, float *O, float *H, int h, int w, int binSize,
52 |          int nOrients, int softBin, bool full, float clip);
53 | 
54 | // compute FHOG features
55 | void fhog(float *M, float *O, float *H, int h, int w, int binSize,
56 |           int nOrients, int softBin, float clip);
57 | 
58 | #endif


--------------------------------------------------------------------------------
/ECO/feature_operator.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef FEATURE_OPERATOR_HPP
 2 | #define FEATURE_OPERATOR_HPP
 3 | 
 4 | #include <iostream>
 5 | #include <algorithm>
 6 | #include <opencv2/core.hpp>
 7 | #include "parameters.hpp"
 8 | #include "ffttools.hpp"
 9 | #include "recttools.hpp"
10 | #include "debug.hpp"
11 | 
12 | namespace eco
13 | {
14 | template <typename T>
15 | extern std::vector<T> operator+(const std::vector<T> &a,
16 | 								const std::vector<T> &b)
17 | {
18 | 	assert(a.size() == b.size());
19 | 	std::vector<T> result;
20 | 	for (unsigned int i = 0; i < a.size(); ++i)
21 | 	{
22 | 		result.push_back(a[i] + b[i]);
23 | 	}
24 | 	return result;
25 | }
26 | 
27 | template <typename T>
28 | extern std::vector<T> operator-(const std::vector<T> &a,
29 | 								const std::vector<T> &b)
30 | {
31 | 	assert(a.size() == b.size());
32 | 	std::vector<T> result;
33 | 	for (unsigned int i = 0; i < a.size(); ++i)
34 | 	{
35 | 		result.push_back(a[i] - b[i]);
36 | 	}
37 | 	return result;
38 | }
39 | 
40 | template <typename T>
41 | extern std::vector<T> operator*(const std::vector<T> &a, const float scale)
42 | {
43 | 	std::vector<T> result;
44 | 	for (unsigned int i = 0; i < a.size(); ++i)
45 | 	{
46 | 		result.push_back(a[i] * scale);
47 | 	}
48 | 	return result;
49 | }
50 | 
51 | extern ECO_FEATS do_dft(const ECO_FEATS &xlw);
52 | extern ECO_FEATS do_windows(const ECO_FEATS &xl, vector<cv::Mat> &cos_win);
53 | 
54 | extern void FilterSymmetrize(ECO_FEATS &hf);
55 | extern vector<cv::Mat> init_projection_matrix(const ECO_FEATS &init_sample,
56 | 											  const vector<int> &compressed_dim,
57 | 											  const vector<int> &feature_dim);
58 | extern ECO_FEATS FeatureProjection(const ECO_FEATS &x,
59 | 								   const std::vector<cv::Mat> &projection_matrix);
60 | extern ECO_FEATS FeatureProjectionMultScale(const ECO_FEATS &x,
61 | 											const std::vector<cv::Mat> &projection_matrix);
62 | 
63 | extern float FeatureComputeInnerProduct(const ECO_FEATS &feat1,
64 | 										const ECO_FEATS &feat2);
65 | extern float FeatureComputeEnergy(const ECO_FEATS &feat);
66 | extern ECO_FEATS FeautreComputePower2(const ECO_FEATS &feats);
67 | extern std::vector<cv::Mat> FeatureComputeScores(const ECO_FEATS &x,
68 | 												 const ECO_FEATS &f);
69 | extern std::vector<cv::Mat> FeatureVectorization(const ECO_FEATS &x);
70 | 
71 | extern ECO_FEATS FeatureVectorMultiply(const ECO_FEATS &x,
72 | 									   const std::vector<cv::Mat> &y,
73 | 									   const bool _conj = 0); // feature * yf
74 | 
75 | extern ECO_FEATS FeatureDotMultiply(const ECO_FEATS &a, const ECO_FEATS &b);
76 | extern ECO_FEATS FeatureDotDivide(const ECO_FEATS &a, const ECO_FEATS &b);
77 | } // namespace eco
78 | 
79 | #endif


--------------------------------------------------------------------------------
/ECO/feature_extractor.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef FEATURE_EXTRACTOR_HPP
 2 | #define FEATURE_EXTRACTOR_HPP
 3 | 
 4 | #include <stdio.h>
 5 | #include <stdlib.h>
 6 | #include <iostream>
 7 | #include <string>
 8 | #include <math.h>
 9 | #include <vector>
10 | #include <numeric>
11 | #include <opencv2/core/core.hpp>
12 | 
13 | #include "parameters.hpp"
14 | #include "ffttools.hpp"
15 | #include "recttools.hpp"
16 | #include "fhog.hpp"
17 | #include "debug.hpp"
18 | 
19 | #ifdef USE_SIMD
20 | #include "gradient.hpp"
21 | #endif
22 | 
23 | #ifdef USE_CAFFE
24 | #include <caffe/caffe.hpp>
25 | #include <caffe/util/io.hpp>
26 | #include <caffe/caffe.hpp>
27 | #endif
28 | 
29 | namespace eco
30 | {
31 | class FeatureExtractor
32 | {
33 |   public:
34 | 	FeatureExtractor() {}
35 | 	virtual ~FeatureExtractor(){};
36 | 
37 | 	ECO_FEATS extractor(const cv::Mat image,
38 | 						const cv::Point2f pos,
39 | 						const vector<float> scales,
40 | 						const EcoParameters &params,
41 | 						const bool &is_color_image);
42 | 
43 | 	cv::Mat sample_patch(const cv::Mat im,
44 | 						 const cv::Point2f pos,
45 | 						 cv::Size2f sample_sz,
46 | 						 cv::Size2f input_sz);
47 | 
48 | #ifdef USE_SIMD
49 | 	vector<cv::Mat> get_hog_features_simd(const vector<cv::Mat> ims);
50 | #else
51 | 	vector<cv::Mat> get_hog_features(const vector<cv::Mat> ims);
52 | #endif
53 | 	vector<cv::Mat> hog_feature_normalization(vector<cv::Mat> &hog_feat_maps);
54 | 	inline vector<cv::Mat> get_hog_feats() const { return hog_feat_maps_; }
55 | 
56 | 	vector<cv::Mat> get_cn_features(const vector<cv::Mat> ims);
57 | 	vector<cv::Mat> cn_feature_normalization(vector<cv::Mat> &cn_feat_maps);
58 | 	inline vector<cv::Mat> get_cn_feats() const { return cn_feat_maps_; }
59 | 
60 | #ifdef USE_CAFFE
61 | 	ECO_FEATS get_cnn_layers(vector<cv::Mat> im, const cv::Mat &deep_mean_mat);
62 | 	cv::Mat sample_pool(const cv::Mat &im, int smaple_factor, int stride);
63 | 	void cnn_feature_normalization(ECO_FEATS &feature);
64 | 	inline ECO_FEATS get_cnn_feats() const { return cnn_feat_maps_; }
65 | #endif
66 | 
67 |   private:
68 | 	EcoParameters params_;
69 | 
70 | 	HogFeatures hog_features_;
71 | 	int hog_feat_ind_ = -1;
72 | 	vector<cv::Mat> hog_feat_maps_;
73 | 
74 | 	ColorspaceFeatures colorspace_features_;
75 | 	int colorspace_feat_ind_ = -1;
76 | 	vector<cv::Mat> colorspace_feat_maps_;
77 | 
78 | 	CnFeatures cn_features_;
79 | 	int cn_feat_ind_ = -1;
80 | 	vector<cv::Mat> cn_feat_maps_;
81 | 
82 | 	IcFeatures ic_features_;
83 | 	int ic_feat_ind_ = -1;
84 | 	vector<cv::Mat> ic_feat_maps_;
85 | 
86 | #ifdef USE_CAFFE
87 | 	boost::shared_ptr<caffe::Net<float>> net_;
88 | 	CnnFeatures cnn_features_;
89 | 	int cnn_feat_ind_ = -1;
90 | 	ECO_FEATS cnn_feat_maps_;
91 | #endif
92 | };
93 | } // namespace eco
94 | #endif
95 | 


--------------------------------------------------------------------------------
/ECO/sse.hpp:
--------------------------------------------------------------------------------
 1 | /*******************************************************************************
 2 | * Piotr's Computer Vision Matlab Toolbox      Version 3.23
 3 | * Copyright 2014 Piotr Dollar.  [pdollar-at-gmail.com]
 4 | * Licensed under the Simplified BSD License [see external/bsd.txt]
 5 | *******************************************************************************/
 6 | // Intel Intrinsics Guide: https://software.intel.com/sites/landingpage/IntrinsicsGuide/#
 7 | 
 8 | #ifndef _SSE_HPP_
 9 | #define _SSE_HPP_
10 | 
11 | #ifdef USE_NEON
12 | #include "sse2neon.h"
13 | #else
14 | #include <emmintrin.h>// SSE2:<e*.h>, SSE3:<p*.h>, SSE4:<s*.h>
15 | #endif
16 | 
17 | #define RETf inline __m128
18 | #define RETi inline __m128i
19 | 
20 | // set, load and store values
21 | RETf SET( const float &x ) { return _mm_set1_ps(x); }
22 | RETf SET( float x, float y, float z, float w ) { return _mm_set_ps(x,y,z,w); }
23 | RETi SET( const int &x ) { return _mm_set1_epi32(x); }
24 | RETf LD( const float &x ) { return _mm_load_ps(&x); }
25 | RETf LDu( const float &x ) { return _mm_loadu_ps(&x); }
26 | RETf STR( float &x, const __m128 y ) { _mm_store_ps(&x,y); return y; }
27 | RETf STR1( float &x, const __m128 y ) { _mm_store_ss(&x,y); return y; }
28 | RETf STRu( float &x, const __m128 y ) { _mm_storeu_ps(&x,y); return y; }
29 | RETf STR( float &x, const float y ) { return STR(x,SET(y)); }
30 | 
31 | // arithmetic operators
32 | RETi ADD( const __m128i x, const __m128i y ) { return _mm_add_epi32(x,y); }
33 | RETf ADD( const __m128 x, const __m128 y ) { return _mm_add_ps(x,y); }
34 | RETf ADD( const __m128 x, const __m128 y, const __m128 z ) {
35 |   return ADD(ADD(x,y),z); }
36 | RETf ADD( const __m128 a, const __m128 b, const __m128 c, const __m128 &d ) {
37 |   return ADD(ADD(ADD(a,b),c),d); }
38 | RETf SUB( const __m128 x, const __m128 y ) { return _mm_sub_ps(x,y); }
39 | RETf MUL( const __m128 x, const __m128 y ) { return _mm_mul_ps(x,y); }
40 | RETf MUL( const __m128 x, const float y ) { return MUL(x,SET(y)); }
41 | RETf MUL( const float x, const __m128 y ) { return MUL(SET(x),y); }
42 | RETf INC( __m128 &x, const __m128 y ) { return x = ADD(x,y); }
43 | RETf INC( float &x, const __m128 y ) { __m128 t=ADD(LD(x),y); return STR(x,t); }
44 | RETf DEC( __m128 &x, const __m128 y ) { return x = SUB(x,y); }
45 | RETf DEC( float &x, const __m128 y ) { __m128 t=SUB(LD(x),y); return STR(x,t); }
46 | RETf MINN( const __m128 x, const __m128 y ) { return _mm_min_ps(x,y); }
47 | RETf RCP( const __m128 x ) { return _mm_rcp_ps(x); }
48 | RETf RCPSQRT( const __m128 x ) { return _mm_rsqrt_ps(x); }
49 | 
50 | // logical operators
51 | RETf AND( const __m128 x, const __m128 y ) { return _mm_and_ps(x,y); }
52 | RETi AND( const __m128i x, const __m128i y ) { return _mm_and_si128(x,y); }
53 | RETf ANDNOT( const __m128 x, const __m128 y ) { return _mm_andnot_ps(x,y); }
54 | RETf OR( const __m128 x, const __m128 y ) { return _mm_or_ps(x,y); }
55 | RETf XOR( const __m128 x, const __m128 y ) { return _mm_xor_ps(x,y); }
56 | 
57 | // comparison operators
58 | RETf CMPGT( const __m128 x, const __m128 y ) { return _mm_cmpgt_ps(x,y); }
59 | RETf CMPLT( const __m128 x, const __m128 y ) { return _mm_cmplt_ps(x,y); }
60 | RETi CMPGT( const __m128i x, const __m128i y ) { return _mm_cmpgt_epi32(x,y); }
61 | RETi CMPLT( const __m128i x, const __m128i y ) { return _mm_cmplt_epi32(x,y); }
62 | 
63 | // conversion operators
64 | RETf CVT( const __m128i x ) { return _mm_cvtepi32_ps(x); }
65 | RETi CVT( const __m128 x ) { return _mm_cvttps_epi32(x); }
66 | 
67 | #undef RETf
68 | #undef RETi
69 | #endif
70 | 


--------------------------------------------------------------------------------
/ECO/sample_update.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef SAMPLE_UPDATE_HPP
  2 | #define SAMPLE_UPDATE_HPP
  3 | 
  4 | #include <numeric>
  5 | #include <vector>
  6 | 
  7 | #include <opencv2/opencv.hpp>
  8 | 
  9 | #include "parameters.hpp"
 10 | #include "ffttools.hpp"
 11 | #include "feature_operator.hpp"
 12 | #include "debug.hpp"
 13 | 
 14 | namespace eco
 15 | {
 16 | 
 17 | class SampleUpdate
 18 | {
 19 |   public:
 20 | 	SampleUpdate(){};
 21 | 	virtual ~SampleUpdate(){};
 22 | 
 23 | 	void init(const std::vector<cv::Size> &filter,
 24 | 			  const std::vector<int> &feature_dim,
 25 | 			  const size_t nSamples,
 26 | 			  const float learning_rate);
 27 | 
 28 | 	void update_sample_space_model(const ECO_FEATS &new_train_sample); 
 29 | 
 30 | 	void update_distance_matrix(cv::Mat &gram_vector, float new_sample_norm,
 31 | 								int id1, int id2, float w1, float w2);
 32 | 
 33 | 	inline cv::Mat find_gram_vector(const ECO_FEATS &new_train_sample) 
 34 | 	{
 35 | 		cv::Mat result(cv::Size(1, nSamples_), CV_32FC2);
 36 | 		for (size_t i = 0; i < (size_t)result.rows; i++) // init to INF;
 37 | 			result.at<cv::Vec<float, 2>>(i, 0) = cv::Vec<float, 2>(INF, 0);
 38 | 
 39 | 		std::vector<float> distance_vector;
 40 | 		for (size_t i = 0; i < num_training_samples_; i++) // calculate the distance;
 41 | 			distance_vector.push_back(2 * 
 42 | 			FeatureComputeInnerProduct(samples_f_[i], new_train_sample));
 43 | 
 44 | 		for (size_t i = 0; i < distance_vector.size(); i++)
 45 | 			result.at<cv::Vec<float, 2>>(i, 0) = 
 46 | 				cv::Vec<float, 2>(distance_vector[i], 0);
 47 | 
 48 | 		return result;
 49 | 	};
 50 | 	// find the minimum element in prior_weights_;
 51 | 	inline void findMin(float &min_w, size_t &index) const
 52 | 	{
 53 | 		std::vector<float>::const_iterator pos = std::min_element(prior_weights_.begin(), prior_weights_.end());
 54 | 		min_w = *pos;
 55 | 		index = pos - prior_weights_.begin();
 56 | 	};
 57 | 
 58 | 	inline ECO_FEATS merge_samples(const ECO_FEATS &sample1,
 59 | 								   const ECO_FEATS &sample2,
 60 | 								   const float w1, const float w2,
 61 | 								   const std::string sample_merge_type = "merge")
 62 | 	{
 63 | 		float alpha1 = w1 / (w1 + w2);
 64 | 		float alpha2 = 1 - alpha1;
 65 | 		ECO_FEATS merged_sample = sample1;
 66 | 
 67 | 		if (sample_merge_type == std::string("replace"))
 68 | 		{
 69 | 		}
 70 | 		else if (sample_merge_type == std::string("merge"))
 71 | 		{
 72 | 			for (size_t i = 0; i < sample1.size(); i++)
 73 | 				for (size_t j = 0; j < sample1[i].size(); j++)
 74 | 					merged_sample[i][j] = alpha1 * sample1[i][j] + alpha2 * sample2[i][j];
 75 | 		}
 76 | 		return merged_sample;
 77 | 	};
 78 | 
 79 | 	inline void replace_sample(const ECO_FEATS &new_sample, const size_t idx)
 80 | 	{
 81 | 		samples_f_[idx] = new_sample;
 82 | 	};
 83 | 
 84 | 	inline void set_gram_matrix(const int r, const int c, const float val)
 85 | 	{
 86 | 		gram_matrix_.at<float>(r, c) = val;
 87 | 	};
 88 | 
 89 | 	int get_merged_sample_id() const { return merged_sample_id_; }
 90 | 
 91 | 	int get_new_sample_id() const { return new_sample_id_; }
 92 | 
 93 | 	std::vector<float> get_prior_weights() const { return prior_weights_; }
 94 | 
 95 | 	std::vector<ECO_FEATS> get_samples() const { return samples_f_; }
 96 | 
 97 |   private:
 98 | 	cv::Mat distance_matrix_, gram_matrix_; // distance matrix and its kernel
 99 | 
100 | 	size_t nSamples_ = 50;
101 | 
102 | 	float learning_rate_ = 0.009;
103 | 
104 | 	const float minmum_sample_weight_ = 0.0036;
105 | 
106 | 	std::vector<float> sample_weight_;
107 | 
108 | 	std::vector<ECO_FEATS> samples_f_; // all samples frontier
109 | 
110 | 	size_t num_training_samples_ = 0;
111 | 
112 | 	std::vector<float> prior_weights_;
113 | 
114 | 	ECO_FEATS new_sample_, merged_sample_;
115 | 
116 | 	int new_sample_id_ = -1, merged_sample_id_ = -1;
117 | };
118 | 
119 | } // namespace eco
120 | 
121 | #endif


--------------------------------------------------------------------------------
/ECO/eco.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef ECO_HPP
  2 | #define ECO_HPP
  3 | 
  4 | #include <iostream>
  5 | #include <string>
  6 | #include <math.h>
  7 | #include <algorithm>
  8 | #include <opencv2/core.hpp>
  9 | #include <opencv2/core/ocl.hpp>
 10 | 
 11 | #ifdef USE_CAFFE
 12 | #include <caffe/caffe.hpp>
 13 | #include <caffe/util/io.hpp>
 14 | #endif
 15 | /*
 16 | #ifdef USE_CUDA
 17 | #include <opencv2/opencv.hpp>
 18 | #include <opencv2/core/cuda.hpp>
 19 | #endif 
 20 | */
 21 | #ifdef USE_MULTI_THREAD
 22 | #include <pthread.h>
 23 | #include <unistd.h>
 24 | #endif
 25 | 
 26 | #include "parameters.hpp"
 27 | #include "interpolator.hpp"
 28 | #include "regularization_filter.hpp"
 29 | #include "feature_extractor.hpp"
 30 | #include "feature_operator.hpp"
 31 | #include "sample_update.hpp"
 32 | #include "optimize_scores.hpp"
 33 | #include "training.hpp"
 34 | #include "ffttools.hpp"
 35 | #include "scale_filter.hpp"
 36 | #include "debug.hpp"
 37 | 
 38 | namespace eco
 39 | {
 40 | class ECO
 41 | {
 42 |   public:
 43 | 	ECO() {};
 44 | 	virtual ~ECO() {}
 45 | 
 46 | 	void init(cv::Mat &im, const cv::Rect2f &rect, const eco::EcoParameters &paramters); 
 47 | 
 48 | 	bool update(const cv::Mat &frame, cv::Rect2f &roi);
 49 | 	
 50 | 	void init_parameters(const eco::EcoParameters &parameters);
 51 | 
 52 | 	void init_features(); 
 53 | #ifdef USE_CAFFE
 54 | 	void read_deep_mean(const string &mean_file);
 55 | #endif
 56 | 	void yf_gaussian(); // the desired outputs of features, real part of (9) in paper C-COT
 57 | 
 58 | 	void cos_window(); 	// construct cosine window of features;
 59 | 
 60 | 	ECO_FEATS interpolate_dft(const ECO_FEATS &xlf, 
 61 | 							  vector<cv::Mat> &interp1_fs,
 62 | 							  vector<cv::Mat> &interp2_fs);
 63 | 
 64 | 	ECO_FEATS compact_fourier_coeff(const ECO_FEATS &xf);
 65 | 
 66 | 	ECO_FEATS full_fourier_coeff(const ECO_FEATS &xf);
 67 | 
 68 | 	vector<cv::Mat> project_mat_energy(vector<cv::Mat> proj, 
 69 | 									   vector<cv::Mat> yf);
 70 | 	
 71 | 	ECO_FEATS shift_sample(ECO_FEATS &xf, 
 72 | 						   cv::Point2f shift,
 73 | 						   std::vector<cv::Mat> kx, 
 74 | 						   std::vector<cv::Mat> ky);
 75 | #ifdef USE_MULTI_THREAD
 76 | 	static void *thread_train(void *params);
 77 | #endif
 78 | 
 79 |   private:
 80 | 	bool				is_color_image_;
 81 | 	EcoParameters 		params_;
 82 | 	cv::Point2f 		pos_; 							// final result
 83 | 	size_t 				frames_since_last_train_; 	 	// used for update;
 84 | 
 85 | 	// The max size of feature and its index, output_sz is T in (9) of C-COT paper
 86 | 	size_t 				output_size_, output_index_; 	
 87 | 
 88 | 	cv::Size2f 			base_target_size_; 	// target size without scale
 89 | 	cv::Size2i			img_sample_size_;  	// base_target_sz * sarch_area_scale
 90 | 	cv::Size2i			img_support_size_;	// the corresponding size in the image
 91 | 
 92 | 	vector<cv::Size> 	feature_size_, filter_size_;
 93 | 	vector<int> 		feature_dim_, compressed_dim_;
 94 | 
 95 | 	ScaleFilter 		scale_filter_;
 96 | 	int 				nScales_;				// number of scales;
 97 | 	float 				scale_step_;
 98 | 	vector<float>		scale_factors_;
 99 | 	float 				currentScaleFactor_; 	// current img scale 
100 | 
101 | 	// Compute the Fourier series indices 
102 | 	// kx_, ky_ is the k in (9) of C-COT paper, yf_ is the left part of (9);
103 | 	vector<cv::Mat> 	ky_, kx_, yf_; 
104 | 	vector<cv::Mat> 	interp1_fs_, interp2_fs_; 
105 | 	vector<cv::Mat> 	cos_window_;
106 | 	vector<cv::Mat> 	projection_matrix_;
107 | 
108 | 	vector<cv::Mat> 	reg_filter_;
109 | 	vector<float> 		reg_energy_;
110 | 
111 | 	FeatureExtractor 	feature_extractor_;
112 | 
113 | 	SampleUpdate 		sample_update_;
114 | 	ECO_FEATS 			sample_energy_;
115 | 
116 | 	EcoTrain 			eco_trainer_;
117 | 
118 | 	ECO_FEATS 			hf_full_;
119 | 
120 | #ifdef USE_MULTI_THREAD
121 | 	bool 				thread_flag_train_;
122 |   public:
123 | 	pthread_t			thread_train_;
124 | #endif
125 | 	
126 | };
127 | 
128 | } // namespace eco
129 | 
130 | #endif
131 | 


--------------------------------------------------------------------------------
/ECO/training.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef TRAINING_HPP
  2 | #define TRAINING_HPP
  3 | 
  4 | #include <iostream>
  5 | #include <string>
  6 | #include <math.h>
  7 | #include <stdio.h>
  8 | #include <algorithm>
  9 | 
 10 | #include <opencv2/opencv.hpp>
 11 | 
 12 | #include "ffttools.hpp"
 13 | #include "recttools.hpp"
 14 | #include "parameters.hpp"
 15 | #include "feature_operator.hpp"
 16 | #include "debug.hpp"
 17 | 
 18 | namespace eco
 19 | {
 20 | class EcoTrain
 21 | {
 22 |   public:
 23 | 	EcoTrain();
 24 | 	virtual ~EcoTrain();
 25 | 
 26 | 	struct STATE
 27 | 	{
 28 | 		ECO_FEATS p, r_prev;
 29 | 		float rho;
 30 | 	};
 31 | 	// the right and left side of the equation (18) of suppl. paper ECO
 32 | 	struct ECO_EQ
 33 | 	{
 34 | 		ECO_EQ() {}
 35 | 		ECO_EQ(ECO_FEATS up_part, std::vector<cv::Mat> low_part) : up_part_(up_part), low_part_(low_part) {}
 36 | 
 37 | 		ECO_FEATS up_part_;			    // this is f + delta(f)
 38 | 		std::vector<cv::Mat> low_part_; // this is delta(P)
 39 | 
 40 | 		ECO_EQ operator+(const ECO_EQ data);
 41 | 		ECO_EQ operator-(const ECO_EQ data);
 42 | 		ECO_EQ operator*(const float scale);
 43 | 	};
 44 | 
 45 | 	void train_init(const ECO_FEATS &hf,
 46 | 					const ECO_FEATS &hf_inc,
 47 | 					const vector<cv::Mat> &proj_matrix,
 48 | 					const ECO_FEATS &xlf,
 49 | 					const vector<cv::Mat> &yf,
 50 | 					const vector<cv::Mat> &reg_filter,
 51 | 					const ECO_FEATS &sample_energy,
 52 | 					const vector<float> &reg_energy,
 53 | 					const vector<cv::Mat> &proj_energy,
 54 | 					const EcoParameters &params);
 55 | 
 56 | 	// Filter training and Projection updating(for the 1st Frame)==============
 57 | 	void train_joint();
 58 | 
 59 | 	ECO_EQ pcg_eco_joint(const ECO_FEATS &init_samplef_proj,
 60 | 						   const vector<cv::Mat> &reg_filter,
 61 | 						   const ECO_FEATS &init_samplef,
 62 | 						   const vector<cv::Mat> &init_samplesf_H,
 63 | 						   const ECO_FEATS &init_hf,
 64 | 						   const ECO_EQ &rhs_samplef,
 65 | 						   const ECO_EQ &diag_M, // preconditionor
 66 | 						   const ECO_EQ &hf);
 67 | 
 68 | 	ECO_EQ lhs_operation_joint(const ECO_EQ &hf,
 69 | 								  const ECO_FEATS &samplesf,
 70 | 								  const vector<cv::Mat> &reg_filter,
 71 | 								  const ECO_FEATS &init_samplef,
 72 | 								  const vector<cv::Mat> &XH,
 73 | 								  const ECO_FEATS &init_hf);
 74 | 	// Only filter training(for tracker update)===============================
 75 | 	void train_filter(const vector<ECO_FEATS> &samplesf,
 76 | 					  const vector<float> &sample_weights,
 77 | 					  const ECO_FEATS &sample_energy);
 78 | 
 79 | 	ECO_FEATS pcg_eco_filter(const vector<ECO_FEATS> &samplesf,
 80 | 							 const vector<cv::Mat> &reg_filter,
 81 | 							 const vector<float> &sample_weights,
 82 | 							 const ECO_FEATS &rhs_samplef,
 83 | 							 const ECO_FEATS &diag_M,
 84 | 							 const ECO_FEATS &hf);
 85 | 
 86 | 	ECO_FEATS lhs_operation_filter(const ECO_FEATS &hf,
 87 | 								   const vector<ECO_FEATS> &samplesf,
 88 | 								   const vector<cv::Mat> &reg_filter,
 89 | 								   const vector<float> &sample_weights);
 90 | 	// joint structure basic operation================================
 91 | 	ECO_EQ jointDotDivision(const ECO_EQ &a, const ECO_EQ &b);
 92 | 	float inner_product_joint(const ECO_EQ &a, const ECO_EQ &b);
 93 | 	float inner_product_filter(const ECO_FEATS &a, const ECO_FEATS &b);
 94 | 	vector<cv::Mat> get_proj() const { return projection_matrix_; }
 95 | 	ECO_FEATS get_hf() const { return hf_; }
 96 | 
 97 |   private:
 98 | 	ECO_FEATS hf_, hf_inc_; // filter parameters and its increament
 99 | 
100 | 	ECO_FEATS xlf_, sample_energy_;
101 | 
102 | 	vector<cv::Mat> yf_; // the label of sample
103 | 
104 | 	vector<cv::Mat> reg_filter_;
105 | 	vector<float> reg_energy_;
106 | 
107 | 	vector<cv::Mat> projection_matrix_, proj_energy_;
108 | 
109 | 	EcoParameters params_;
110 | 	STATE state_;
111 | }; // end of class
112 | } // namespace eco
113 | #endif


--------------------------------------------------------------------------------
/ECO/interpolator.cc:
--------------------------------------------------------------------------------
 1 | #include "interpolator.hpp"
 2 | 
 3 | namespace eco
 4 | {
 5 | Interpolator::Interpolator() {}
 6 | 
 7 | Interpolator::~Interpolator() {}
 8 | 
 9 | void Interpolator::get_interp_fourier(cv::Size filter_sz,
10 | 									  cv::Mat &interp1_fs,
11 | 									  cv::Mat &interp2_fs,
12 | 									  float a)
13 | {
14 | 	cv::Mat temp1(filter_sz.height, 1, CV_32FC1);
15 | 	cv::Mat temp2(1, filter_sz.width, CV_32FC1);
16 | 	for (int j = 0; j < temp1.rows; j++)
17 | 	{
18 | 		temp1.at<float>(j, 0) = j - temp1.rows / 2;
19 | 	}
20 | 	for (int j = 0; j < temp2.cols; j++)
21 | 	{
22 | 		temp2.at<float>(0, j) = j - temp2.cols / 2;
23 | 	}
24 | 
25 | 	interp1_fs = cubic_spline_fourier(temp1 / filter_sz.height, a) / filter_sz.height;
26 | 	interp2_fs = cubic_spline_fourier(temp2 / filter_sz.width, a) / filter_sz.width;
27 | 
28 | 	// Multiply Fourier coeff with e ^ (-i*pi*k / N):[cos(pi*k/N), -sin(pi*k/N)]
29 | 	cv::Mat result1(temp1.size(), CV_32FC1), result2(temp1.size(), CV_32FC1);
30 | 	temp1 = temp1 / filter_sz.height;
31 | 	temp2 = temp2 / filter_sz.width;
32 | 	std::transform(temp1.begin<float>(), temp1.end<float>(), result1.begin<float>(), Interpolator::mat_cos1);
33 | 	std::transform(temp1.begin<float>(), temp1.end<float>(), result2.begin<float>(), Interpolator::mat_sin1);
34 | 
35 | 	//cv::Mat planes1[] = {interp1_fs.mul(result1), -interp1_fs.mul(result2)};
36 | 	//cv::merge(planes1, 2, interp1_fs);
37 | 	//interp2_fs = interp1_fs.t();
38 | 	cv::Mat temp = cv::Mat(interp1_fs.size(), CV_32FC2);
39 | 	cv::Mat tempT = cv::Mat(interp1_fs.cols, interp1_fs.rows, CV_32FC2);
40 | 	for(int r = 0; r < temp1.rows; r++)
41 | 	{
42 | 		for(int c = 0; c < temp1.cols; c++)
43 | 		{
44 | 
45 | 			temp.at<cv::Vec2f>(r, c)[0] = interp1_fs.at<float>(r, c) * result1.at<float>(r, c);
46 | 			temp.at<cv::Vec2f>(r, c)[1] = -interp1_fs.at<float>(r, c) * result2.at<float>(r, c);
47 | 			tempT.at<cv::Vec2f>(c, r)[0] = temp.at<cv::Vec2f>(r, c)[0];
48 | 			tempT.at<cv::Vec2f>(c, r)[1] = temp.at<cv::Vec2f>(r, c)[1];
49 | 		}
50 | 	}
51 | 	interp1_fs = temp;
52 | 	interp2_fs = tempT;
53 | }
54 | 
55 | cv::Mat Interpolator::cubic_spline_fourier(cv::Mat f, float a)
56 | {
57 | 	if (f.empty())
58 | 	{
59 | 		assert(0 && "error: input mat is empty!");
60 | 	}
61 | /*
62 | 	cv::Mat bf(f.size(), CV_32FC1),
63 | 		temp_cos2(f.size(), CV_32FC1),
64 | 		temp_cos4(f.size(), CV_32FC1),
65 | 		temp_sin2(f.size(), CV_32FC1),
66 | 		temp_sin4(f.size(), CV_32FC1);
67 | 	std::transform(f.begin<float>(), f.end<float>(), temp_cos2.begin<float>(), Interpolator::mat_cos2);
68 | 	std::transform(f.begin<float>(), f.end<float>(), temp_cos4.begin<float>(), Interpolator::mat_cos4);
69 | 	std::transform(f.begin<float>(), f.end<float>(), temp_sin2.begin<float>(), Interpolator::mat_sin2);
70 | 	std::transform(f.begin<float>(), f.end<float>(), temp_sin4.begin<float>(), Interpolator::mat_sin4);
71 | 
72 | 	bf = 6 * (cv::Mat::ones(f.size(), CV_32FC1) - temp_cos2) + 3 * a * (cv::Mat::ones(f.size(), CV_32FC1) - temp_cos4) - (6 + a * 8) * CV_PI * f.mul(temp_sin2) - 2 * a * CV_PI * f.mul(temp_sin4);
73 | 
74 | 	cv::Mat L(f.size(), CV_32FC1);
75 | 	cv::pow(f, 4, L);
76 | 	cv::divide(bf, 4 * L * cv::pow(CV_PI, 4), bf);
77 | 	bf.at<float>(bf.rows / 2, bf.cols / 2) = 1;
78 | */
79 | 	cv::Mat bf(f.size(), CV_32FC1);
80 | 	for(int r = 0; r < bf.rows; r++)
81 | 	{
82 | 		for(int c = 0; c < bf.cols; c++)
83 | 		{
84 | 			bf.at<float>(r, c) = 6.0f * (1 - cos(2.0f * f.at<float>(r, c) * M_PI)) 
85 | 			+ 3.0f * a * (1.0f - cos(4 * f.at<float>(r, c) * M_PI))
86 | 			- (6.0f + a * 8.0f) * M_PI * f.at<float>(r, c) * sin(2.0f * f.at<float>(r, c) * M_PI)
87 | 			- 2.0f * a * M_PI * f.at<float>(r, c) * sin(4.0f * f.at<float>(r, c) * M_PI);
88 | 			float L = 4.0f * pow(f.at<float>(r, c) * M_PI, 4);
89 | 			bf.at<float>(r, c) /= L;
90 | 		}
91 | 	}
92 | 	bf.at<float>(bf.rows / 2, bf.cols / 2) = 1;
93 | 	//printMat(bf);
94 | 	//showmat1channels(bf, 2);
95 | 
96 | 	return bf;
97 | }
98 | } // namespace eco


--------------------------------------------------------------------------------
/ECO/regularization_filter.cc:
--------------------------------------------------------------------------------
  1 | #include "regularization_filter.hpp"
  2 | 
  3 | namespace eco
  4 | {
  5 | cv::Mat get_regularization_filter(cv::Size sz,
  6 | 								  cv::Size2f target_sz,
  7 | 								  const EcoParameters &params)
  8 | {
  9 | 	cv::Mat result;
 10 | 
 11 | 	if (params.use_reg_window)
 12 | 	{
 13 | 		cv::Size2d reg_scale = cv::Size2d(target_sz.width * 0.5,
 14 | 										  target_sz.height * 0.5);
 15 | 
 16 | 		// construct the regularization window
 17 | 		cv::Mat reg_window(sz, CV_64FC1);
 18 | 		for (double x = -0.5 * (sz.height - 1), counter1 = 0;
 19 | 			 counter1 < sz.height; x += 1, ++counter1)
 20 | 			for (double y = -0.5 * (sz.width - 1), counter2 = 0;
 21 | 				 counter2 < sz.width; y += 1, ++counter2)
 22 | 			{ // use abs() directly will cause error because it returns int!!!
 23 | 				reg_window.at<double>(counter1, counter2) =
 24 | 					(params.reg_window_edge - params.reg_window_min) *
 25 | 						(std::pow(std::abs(x / reg_scale.height), params.reg_window_power) +
 26 | 						 std::pow(std::abs(y / reg_scale.width), params.reg_window_power)) +
 27 | 					params.reg_window_min;
 28 | 			}
 29 | 		/* debug
 30 | 		debug("%f %f", reg_scale.height, reg_scale.width);
 31 | 		debug("%d %d", sz.height, sz.width);
 32 | 		debug("Channels: %d",reg_window.channels());
 33 | 		printMat(reg_window);
 34 | 		//showmat1channels(reg_window, 3);
 35 | 		debug("%lf, %lf", reg_window.at<double>(46, 23), reg_window.at<double>(143,89));
 36 | 		*/
 37 | 
 38 | 		// compute the DFT and enforce sparsity
 39 | 		cv::Mat reg_window_dft = dft(reg_window) / sz.area();
 40 | 		cv::Mat reg_win_abs(sz, CV_64FC1);
 41 | 		reg_win_abs = magnitude(reg_window_dft);
 42 | 		double minv = 0.0, maxv = 0.0;
 43 | 		cv::minMaxLoc(reg_win_abs, &minv, &maxv);
 44 | 		// set to zero while the element smaller than threshold
 45 | 		for (size_t i = 0; i < (size_t)reg_window_dft.rows; i++)
 46 | 			for (size_t j = 0; j < (size_t)reg_window_dft.cols; j++)
 47 | 			{
 48 | 				if (reg_win_abs.at<double>(i, j) < (params.reg_sparsity_threshold * maxv))
 49 | 					reg_window_dft.at<cv::Vec<double, 2>>(i, j) = cv::Vec<double, 2>(0.0, 0.0);
 50 | 			}
 51 | 		/* debug
 52 | 		showmat2channels(reg_window_dft, 3);
 53 | 		debug("%lf, %lf", reg_window_dft.at<cv::Vec<double, 2>>(0, 0)[0], reg_window_dft.at<cv::Vec<double, 2>>(0,1)[0]);
 54 | 		*/
 55 | 
 56 | 		// do the inverse transform, correct window minimum
 57 | 		cv::Mat reg_window_sparse = real(dft(reg_window_dft, true));
 58 | 		//showmat1channels(reg_window_sparse, 3);
 59 | 		cv::minMaxLoc(reg_window_sparse, &minv, &maxv);
 60 | 		//debug("%lf, %lf, %lf, %d", minv, maxv, params.reg_window_min, sz.area());
 61 | 		reg_window_dft.at<double>(0, 0) = reg_window_dft.at<double>(0, 0) - sz.area() * minv + params.reg_window_min;
 62 | 		reg_window_dft = fftshift(reg_window_dft);
 63 | 
 64 | 		// find the regularization filter by removing the zeros
 65 | 		cv::Mat tmp;
 66 | 		for (size_t i = 0; i < (size_t)reg_window_dft.rows; i++)
 67 | 		{
 68 | 			for (size_t j = 0; j < (size_t)reg_window_dft.cols; j++)
 69 | 			{
 70 | 				if (((reg_window_dft.at<cv::Vec<double, 2>>(i, j) !=
 71 | 					  cv::Vec<double, 2>(0, 0)) &&
 72 | 					 (reg_window_dft.at<cv::Vec<double, 2>>(i, j) !=
 73 | 					  cv::Vec<double, 2>(2, 0))))
 74 | 				{
 75 | 					tmp.push_back(reg_window_dft.row(i));
 76 | 					break;
 77 | 				}
 78 | 			} //end for
 79 | 		}	 //end for
 80 | 
 81 | 		tmp = tmp.t();
 82 | 		for (size_t i = 0; i < (size_t)tmp.rows; i++)
 83 | 		{
 84 | 			for (size_t j = 0; j < (size_t)tmp.cols; j++)
 85 | 			{
 86 | 				if (((tmp.at<cv::Vec<double, 2>>(i, j) !=
 87 | 					  cv::Vec<double, 2>(0, 0)) &&
 88 | 					 (tmp.at<cv::Vec<double, 2>>(i, j) !=
 89 | 					  cv::Vec<double, 2>(1, 0))))
 90 | 				{
 91 | 					result.push_back(real(tmp.row(i)));
 92 | 					break;
 93 | 				}
 94 | 			} //end for
 95 | 		}	 //end for
 96 | 		result = result.t();
 97 | 	} // if params.use_reg_window
 98 | 	else
 99 | 	{
100 | 		result.push_back(params.reg_window_min);
101 | 	}
102 | 
103 | 	return result;
104 | }
105 | } // namespace eco


--------------------------------------------------------------------------------
/ECO/ffttools.hpp:
--------------------------------------------------------------------------------
  1 | /* 
  2 | Author: Christian Bailer
  3 | Contact address: Christian.Bailer@dfki.de 
  4 | Department Augmented Vision DFKI 
  5 | 
  6 |                           License Agreement
  7 |                For Open Source Computer Vision Library
  8 |                        (3-clause BSD License)
  9 | 
 10 | Redistribution and use in source and binary forms, with or without modification,
 11 | are permitted provided that the following conditions are met:
 12 | 
 13 |   * Redistributions of source code must retain the above copyright notice,
 14 |     this list of conditions and the following disclaimer.
 15 | 
 16 |   * Redistributions in binary form must reproduce the above copyright notice,
 17 |     this list of conditions and the following disclaimer in the documentation
 18 |     and/or other materials provided with the distribution.
 19 | 
 20 |   * Neither the names of the copyright holders nor the names of the contributors
 21 |     may be used to endorse or promote products derived from this software
 22 |     without specific prior written permission.
 23 | 
 24 | This software is provided by the copyright holders and contributors "as is" and
 25 | any express or implied warranties, including, but not limited to, the implied
 26 | warranties of merchantability and fitness for a particular purpose are disclaimed.
 27 | In no event shall copyright holders or contributors be liable for any direct,
 28 | indirect, incidental, special, exemplary, or consequential damages
 29 | (including, but not limited to, procurement of substitute goods or services;
 30 | loss of use, data, or profits; or business interruption) however caused
 31 | and on any theory of liability, whether in contract, strict liability,
 32 | or tort (including negligence or otherwise) arising in any way out of
 33 | the use of this software, even if advised of the possibility of such damage.
 34 | */
 35 | 
 36 | #ifndef FFTTOOLS_HPP
 37 | #define FFTTOOLS_HPP
 38 | 
 39 | #include <opencv2/imgproc/imgproc.hpp>
 40 | #include "debug.hpp"
 41 | /*
 42 | #ifdef USE_CUDA
 43 | #include <opencv2/opencv.hpp>
 44 | #include <opencv2/core/cuda.hpp>
 45 | #endif 
 46 | */
 47 | #ifdef USE_SIMD
 48 | #include "sse.hpp"
 49 | #include "wrappers.hpp"
 50 | #endif
 51 | 
 52 | #ifdef USE_FFTW
 53 | #include <fftw3.h>
 54 | #endif
 55 | 
 56 | namespace eco
 57 | {
 58 | cv::Mat dft(const cv::Mat img_org, const bool backwards = false);
 59 | cv::Mat fftshift(const cv::Mat img_org,
 60 | 				   const bool rowshift = true,
 61 | 				   const bool colshift = true,
 62 | 				   const bool reverse = 0);
 63 | 
 64 | cv::Mat real(const cv::Mat img);
 65 | cv::Mat imag(const cv::Mat img);
 66 | cv::Mat magnitude(const cv::Mat img);
 67 | cv::Mat complexDotMultiplication(const cv::Mat &a, const cv::Mat &b);
 68 | cv::Mat complexDotMultiplicationCPU(const cv::Mat &a, const cv::Mat &b);
 69 | #ifdef USE_SIMD
 70 | cv::Mat complexDotMultiplicationSIMD(const cv::Mat &a, const cv::Mat &b);
 71 | #endif
 72 | /*
 73 | #ifdef USE_CUDA
 74 | cv::Mat complexDotMultiplicationGPU(const cv::Mat &a, const cv::Mat &b);
 75 | #endif
 76 | */
 77 | cv::Mat complexDotDivision(const cv::Mat a, const cv::Mat b);
 78 | cv::Mat complexMatrixMultiplication(const cv::Mat &a, const cv::Mat &b);
 79 | cv::Mat complexConvolution(const cv::Mat a_input,
 80 | 						   const cv::Mat b_input,
 81 | 						   const bool valid = 0);
 82 | 
 83 | cv::Mat real2complex(const cv::Mat &x);
 84 | cv::Mat mat_conj(const cv::Mat &org);
 85 | float mat_sum_f(const cv::Mat &org);
 86 | double mat_sum_d(const cv::Mat &org);
 87 | 
 88 | inline bool SizeCompare(cv::Size &a, cv::Size &b)
 89 | {
 90 | 	return a.height < b.height;
 91 | }
 92 | 
 93 | inline void rot90(cv::Mat &matImage, int rotflag)
 94 | {
 95 | 	if (rotflag == 1)
 96 | 	{
 97 | 		cv::transpose(matImage, matImage);
 98 | 		cv::flip(matImage, matImage, 1); // flip around y-axis
 99 | 	}
100 | 	else if (rotflag == 2)
101 | 	{
102 | 		cv::transpose(matImage, matImage);
103 | 		cv::flip(matImage, matImage, 0); // flip around x-axis
104 | 	}
105 | 	else if (rotflag == 3)
106 | 	{
107 | 		cv::flip(matImage, matImage, -1); // flip around both axis
108 | 	}
109 | 	else if (rotflag != 0) // 0: keep the same
110 | 	{
111 | 		assert(0 && "error: unknown rotation flag!");
112 | 	}
113 | }
114 | 
115 | } // namespace eco
116 | 
117 | #endif


--------------------------------------------------------------------------------
/ECO/recttools.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | Author: Christian Bailer
  3 | Contact address: Christian.Bailer@dfki.de
  4 | Department Augmented Vision DFKI
  5 | 
  6 | License Agreement
  7 | For Open Source Computer Vision Library
  8 | (3-clause BSD License)
  9 | 
 10 | Redistribution and use in source and binary forms, with or without modification,
 11 | are permitted provided that the following conditions are met:
 12 | 
 13 | * Redistributions of source code must retain the above copyright notice,
 14 | this list of conditions and the following disclaimer.
 15 | 
 16 | * Redistributions in binary form must reproduce the above copyright notice,
 17 | this list of conditions and the following disclaimer in the documentation
 18 | and/or other materials provided with the distribution.
 19 | 
 20 | * Neither the names of the copyright holders nor the names of the contributors
 21 | may be used to endorse or promote products derived from this software
 22 | without specific prior written permission.
 23 | 
 24 | This software is provided by the copyright holders and contributors "as is" and
 25 | any express or implied warranties, including, but not limited to, the implied
 26 | warranties of merchantability and fitness for a particular purpose are disclaimed.
 27 | In no event shall copyright holders or contributors be liable for any direct,
 28 | indirect, incidental, special, exemplary, or consequential damages
 29 | (including, but not limited to, procurement of substitute goods or services;
 30 | loss of use, data, or profits; or business interruption) however caused
 31 | and on any theory of liability, whether in contract, strict liability,
 32 | or tort (including negligence or otherwise) arising in any way out of
 33 | the use of this software, even if advised of the possibility of such damage.
 34 | */
 35 | 
 36 | #ifndef RECTTOOLS_HPP
 37 | #define RECTTOOLS_HPP
 38 | 
 39 | #include <math.h>
 40 | #include <opencv2/opencv.hpp>
 41 | #include "debug.hpp"
 42 | 
 43 | namespace eco
 44 | {
 45 | template <typename t>
 46 | inline cv::Vec<t, 2> center(const cv::Rect_<t> &rect)
 47 | {
 48 | 	return cv::Vec<t, 2>(rect.x + rect.width / (t)2.0f, rect.y + rect.height / (t)2.0f);
 49 | }
 50 | 
 51 | template <typename t>
 52 | inline t x2(const cv::Rect_<t> &rect)
 53 | {
 54 | 	return rect.x + rect.width;
 55 | }
 56 | 
 57 | template <typename t>
 58 | inline t y2(const cv::Rect_<t> &rect)
 59 | {
 60 | 	return rect.y + rect.height;
 61 | }
 62 | 
 63 | template <typename t>
 64 | inline void resize(cv::Rect_<t> &rect, float scalex, float scaley = 0)
 65 | {
 66 | 	if (!scaley)
 67 | 	{
 68 | 		scaley = scalex;
 69 | 	}
 70 | 	rect.x -= rect.width * (scalex - 1.0f) / 2.0f;
 71 | 	rect.width *= scalex;
 72 | 
 73 | 	rect.y -= rect.height * (scaley - 1.0f) / 2.0f;
 74 | 	rect.height *= scaley;
 75 | }
 76 | 
 77 | template <typename t>
 78 | inline void limit(cv::Rect_<t> &rect, cv::Rect_<t> limit)
 79 | {
 80 | 	if (rect.x + rect.width > limit.x + limit.width)
 81 | 	{
 82 | 		rect.width = limit.x + limit.width - rect.x;
 83 | 	}
 84 | 	if (rect.y + rect.height > limit.y + limit.height)
 85 | 	{
 86 | 		rect.height = limit.y + limit.height - rect.y;
 87 | 	}
 88 | 	if (rect.x < limit.x)
 89 | 	{
 90 | 		rect.width -= (limit.x - rect.x);
 91 | 		rect.x = limit.x;
 92 | 	}
 93 | 	if (rect.y < limit.y)
 94 | 	{
 95 | 		rect.height -= (limit.y - rect.y);
 96 | 		rect.y = limit.y;
 97 | 	}
 98 | 	if (rect.width < 0)
 99 | 	{
100 | 		rect.width = 0;
101 | 	}
102 | 	if (rect.height < 0)
103 | 	{
104 | 		rect.height = 0;
105 | 	}
106 | }
107 | 
108 | template <typename t>
109 | inline void limit(cv::Rect_<t> &rect, t width, t height, t x = 0, t y = 0)
110 | {
111 | 	limit(rect, cv::Rect_<t>(x, y, width, height));
112 | }
113 | 
114 | template <typename t>
115 | inline cv::Rect getBorder(const cv::Rect_<t> &original, cv::Rect_<t> &limited)
116 | {
117 | 	cv::Rect_<t> res;
118 | 	res.x = limited.x - original.x;
119 | 	res.y = limited.y - original.y;
120 | 	res.width = x2(original) - x2(limited);
121 | 	res.height = y2(original) - y2(limited);
122 | 	assert(res.x >= 0 && res.y >= 0 && res.width >= 0 && res.height >= 0);
123 | 	return res;
124 | }
125 | // cut "window" out from "input".
126 | inline cv::Mat subwindow(const cv::Mat &input, const cv::Rect &window, int borderType = cv::BORDER_CONSTANT)
127 | {
128 | 	cv::Mat res;
129 | 	cv::Rect cutWindow = window;
130 | 	limit(cutWindow, input.cols, input.rows);
131 | 	//debug("cutWindow: %d x %d", cutWindow.height, cutWindow.width);
132 | 	if (cutWindow.height <= 0 || cutWindow.width <= 0)
133 | 	{
134 | 		//assert(0 && "error: cutWindow size error!\n");
135 | 		return res;//cv::Mat(window.height,window.width,input.type(),0) ;
136 | 	}
137 | 	cv::Rect border = getBorder(window, cutWindow);
138 | 	res = input(cutWindow);
139 | 	if (border != cv::Rect(0, 0, 0, 0))
140 | 	{
141 | 		cv::copyMakeBorder(res, res, border.y, border.height, border.x, border.width, borderType);
142 | 	}
143 | 	return res;
144 | }
145 | 
146 | inline cv::Mat getGrayImage(cv::Mat img)
147 | {
148 | 	cv::cvtColor(img, img, CV_BGR2GRAY);
149 | 	img.convertTo(img, CV_32F, 1 / 255.f);
150 | 	return img;
151 | }
152 | 
153 | } // namespace eco
154 | 
155 | #endif


--------------------------------------------------------------------------------
/DaSiamRPN/dasiamrpntracker.h:
--------------------------------------------------------------------------------
  1 | /* DaSiamRPNCaffe2
  2 | # Licensed under The MIT License
  3 | # Written by wzq*/
  4 | #ifndef DASIAMRPNTRACKER_H
  5 | #define DASIAMRPNTRACKER_H
  6 | #include <array>
  7 | #include <vector>
  8 | #include <math.h>
  9 | #include <iostream>
 10 | #include <caffe2/core/init.h>
 11 | #include <caffe2/core/context.h>
 12 | #include <caffe2/core/context_gpu.h>
 13 | #include <caffe2/utils/proto_utils.h>
 14 | #include <caffe2/core/predictor.h>
 15 | #include <opencv2/opencv.hpp>
 16 | 
 17 | #include <eigen3/Eigen/Core>
 18 | #include <eigen3/Eigen/Dense>
 19 | #define DEBUG_ 0
 20 | static std::string global_init_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/global_init_net.pb";
 21 | static std::string temple_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/temple_pred_net.pb";
 22 | static std::string track_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/track_pred_net.pb";
 23 | 
 24 | static std::string adjust_init_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/adjust_init_net.pb";
 25 | static std::string adjust_pred_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/adjust_pred_net.pb";
 26 | 
 27 | static std::string Correlation_init_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/Correlation_init_net.pb";
 28 | static std::string Correlation_pred_net_file = "/home/nvidia/Develop/Project/ROS/Tracking/src/tracking/DaSiamRPN/Model/Correlation_pred_net.pb";
 29 | 
 30 | struct TrackerConfig{
 31 | // These are the default hyper-params for DaSiamRPN 0.3827
 32 |    std::string windowing = "cosine";// # to penalize large displacements [cosine/uniform]
 33 |    Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic,Eigen::RowMajor> window;
 34 |    // Params from the network architecture, have to be consistent with the training
 35 |    int exemplar_size = 127;  // input z size
 36 |    int instance_size = 271;  // input x size (search region)
 37 |    int total_stride = 8;
 38 |    int score_size = (instance_size-exemplar_size)/total_stride+1;
 39 |    float context_amount = 0.5;  // context amount for the exemplar
 40 |    Eigen::Array<float, 5, 1> ratios;
 41 |    Eigen::Array<float, 1, 1> scales;
 42 |    int anchor_num;
 43 |    Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic,Eigen::RowMajor> anchor;
 44 |    float penalty_k = 0.055;
 45 |    float window_influence = 0.42;
 46 |    float lr = 0.295;
 47 |    TrackerConfig(){
 48 |         ratios <<0.33,0.5,1,2,3;
 49 |         scales << 8;
 50 |         anchor_num = ratios.size()*scales.size();
 51 |    }
 52 | };
 53 | struct BoxInfo{
 54 |     float xc;
 55 |     float yc;
 56 |     float w;
 57 |     float h;
 58 |     float best_score;
 59 | };
 60 | 
 61 | struct TrackInfo{
 62 |     int im_h;
 63 |     int im_w;
 64 |     BoxInfo binfo;
 65 |     std::string window;
 66 |     TrackerConfig cfg;
 67 |     cv::Scalar avg_chans;
 68 | };
 69 | 
 70 | 
 71 | using namespace caffe2;
 72 | class DaSiamRPNTracker
 73 | {
 74 | public:
 75 |     DaSiamRPNTracker();
 76 |     void SiamRPN_init(const cv::Mat& mat,BoxInfo&info,TrackInfo& output,const std::string&global_init_net_file,
 77 |                       const std::string temple_net_file,const std::string& track_net_file,caffe2::DeviceType d,int gpuid);
 78 |     void tracker_eval(cv::Mat x_crop,
 79 |                       BoxInfo &binfo,
 80 |                       float& scale_z,
 81 |                       TrackerConfig &p);
 82 |     void SiamRPN_track(cv::Mat mat,TrackInfo&info);
 83 | 
 84 | private:
 85 |     std::unique_ptr<caffe2::NetDef> global_init_net;
 86 |     std::unique_ptr<caffe2::NetDef> temple_net;
 87 |     std::unique_ptr<caffe2::NetDef> track_net;
 88 |     std::unique_ptr<caffe2::NetDef> adjust_net;
 89 |     std::unique_ptr<caffe2::NetDef> adjsut_init_net;
 90 |     std::shared_ptr<caffe2::Workspace> trackerEngine;
 91 |     caffe2::DeviceType mode;
 92 |     int gpuid;
 93 |     caffe2::DeviceOption devOption;
 94 |     TrackInfo trackInfo;
 95 |     //for gpu inference
 96 |     std::vector<caffe2::TensorCUDA*>cuda_r1_kernels;
 97 |     std::vector<caffe2::TensorCUDA*>cuda_cls_kernels;
 98 |    //for cpu inference
 99 |     std::vector<caffe2::TensorCPU*>cpu_r1_kernels;
100 |     std::vector<caffe2::TensorCPU*>cpu_cls_kernels;
101 | protected:
102 |     //these functions are only called internal, set them protected.
103 |     void GetTensorToHost(const Tensor<CUDAContext>* tensor,std::vector<float>& data ) {
104 |         data.resize(tensor->size());
105 |         CUDAContext context_;
106 |         context_.template Copy<float,CUDAContext,CPUContext>(
107 |                     data.size(),tensor->template data<float>(),data.data());
108 |     }
109 | 
110 |     void GetTensorToHost( Tensor<CUDAContext>* tensor, Tensor<CPUContext>* ctensor ) {
111 |         ctensor->Resize(tensor->dims());
112 |         CUDAContext context_;
113 |         context_.template Copy<float,CUDAContext,CPUContext>(
114 |                     ctensor->size(),tensor->template data<float>(),ctensor->template mutable_data<float>());
115 |     }
116 | 
117 |     void FeedInputToNet(cv::Mat& input,const std::string& blob_name);
118 |     void regress_adjust();
119 | };
120 | 
121 | #endif // DASIAMRPNTRACKER_H
122 | 


--------------------------------------------------------------------------------
/ECO/optimize_scores.cc:
--------------------------------------------------------------------------------
  1 | #include"optimize_scores.hpp"
  2 | 
  3 | namespace eco{
  4 | void OptimizeScores::compute_scores()
  5 | {
  6 | 	std::vector<cv::Mat> sampled_scores;
  7 | 	// Do inverse fft to the scores in the Fourier domain back to spacial domain
  8 | 	for (size_t i = 0; i < scores_fs_.size(); ++i) // for each scale
  9 | 	{
 10 | 		int area = scores_fs_[i].size().area();
 11 | 		cv::Mat tmp = dft(fftshift(scores_fs_[i], 1, 1, 1), 1);// inverse dft
 12 | 		sampled_scores.push_back(real(tmp * area)); // spacial domain only contains real part
 13 | 	}
 14 | 
 15 | 	// to store the position of maximum value of response
 16 | 	std::vector<int> row, col; 
 17 | 	std::vector<float> 	init_max_score; // inialized max score
 18 | 	for (size_t i = 0; i < scores_fs_.size(); ++i) // for each scale
 19 | 	{
 20 | 		cv::Point pos;
 21 | 		double maxValue = 0, minValue = 0;
 22 | 		cv::minMaxLoc(sampled_scores[i], &minValue, &maxValue, NULL, &pos);
 23 | 		row.push_back(pos.y);
 24 | 		col.push_back(pos.x);
 25 | 		init_max_score.push_back(sampled_scores[i].at<float>(pos.y, pos.x));
 26 | 		//debug("init_max_score %lu: value: %lf %lf y:%d x:%d", i, minValue, maxValue, pos.y, pos.x);
 27 | 	}
 28 | 	
 29 | 	// Shift and rescale the coordinate system to [-pi, pi]
 30 | 	int h = scores_fs_[0].rows, w = scores_fs_[0].cols;
 31 | 	std::vector<float> max_pos_y, max_pos_x, init_pos_y, init_pos_x;
 32 | 	for (size_t i = 0; i < row.size(); ++i)
 33 | 	{
 34 | 		max_pos_y.push_back( (row[i] + (h - 1) / 2) %  h - (h - 1) / 2);
 35 | 		max_pos_y[i] *= 2 * CV_PI / h;
 36 | 		max_pos_x.push_back( (col[i] + (w - 1) / 2) %  w - (w - 1) / 2); 
 37 | 		max_pos_x[i] *= 2 * CV_PI / w;
 38 | 	}
 39 | 	init_pos_y = max_pos_y; init_pos_x = max_pos_x;
 40 | 	// Construct grid
 41 | 	std::vector<float> ky, kx, ky2, kx2;
 42 | 	for (int i = 0; i < h; ++i)
 43 | 	{
 44 | 		ky.push_back(i - (h - 1) / 2);
 45 | 		ky2.push_back(ky[i] * ky[i]);
 46 | 	}
 47 | 	for (int i = 0; i < w; ++i)
 48 | 	{
 49 | 		kx.push_back(i - (w - 1) / 2);
 50 | 		kx2.push_back(kx[i] * kx[i]);
 51 | 	}
 52 | 	// Pre-compute complex exponential 
 53 | 	std::vector<cv::Mat> exp_iky, exp_ikx;
 54 | 	for (unsigned int i = 0; i < scores_fs_.size(); ++i)
 55 | 	{
 56 | 		cv::Mat tempy(1, h, CV_32FC2);
 57 | 		cv::Mat tempx(w, 1, CV_32FC2);
 58 | 		for (int y = 0; y < h; ++y)
 59 | 			tempy.at<cv::Vec<float, 2>>(0, y) = cv::Vec<float, 2>(cos(ky[y] * max_pos_y[i]), sin(ky[y] * max_pos_y[i]));
 60 | 		for (int x = 0; x < w; ++x)
 61 | 			tempx.at<cv::Vec<float, 2>>(x, 0) = cv::Vec<float, 2>(cos(kx[x] * max_pos_x[i]), sin(kx[x] * max_pos_x[i]));
 62 | 		exp_iky.push_back(tempy);
 63 | 		exp_ikx.push_back(tempx);
 64 | 	}
 65 | 
 66 | 	cv::Mat kyMat(1, h, CV_32FC1, &ky[0]);
 67 | 	cv::Mat ky2Mat(1, h, CV_32FC1, &ky2[0]);
 68 | 	cv::Mat kxMat(w, 1, CV_32FC1, &kx[0]);
 69 | 	cv::Mat kx2Mat(w, 1, CV_32FC1, &kx2[0]);
 70 | 
 71 | 	for (int ite = 0; ite < iterations_; ++ite)
 72 | 	{
 73 | 		// Compute gradient
 74 | 		std::vector<cv::Mat> ky_exp_ky, kx_exp_kx, y_resp, resp_x, grad_y, grad_x;
 75 | 		std::vector<cv::Mat> ival, H_yy, H_xx, H_xy, det_H;
 76 | 		for (unsigned int i = 0; i < scores_fs_.size(); i++)
 77 | 		{
 78 | 			ky_exp_ky.push_back(complexDotMultiplication(kyMat, exp_iky[i]));
 79 | 			kx_exp_kx.push_back(complexDotMultiplication(kxMat, exp_ikx[i]));
 80 | 		
 81 | 			y_resp.push_back(exp_iky[i] * scores_fs_[i]);
 82 | 			resp_x.push_back(scores_fs_[i] * exp_ikx[i]);
 83 | 
 84 | 			grad_y.push_back(-1 * imag(ky_exp_ky[i] * resp_x[i]));
 85 | 			grad_x.push_back(-1 * imag(y_resp[i] * kx_exp_kx[i]));
 86 | 
 87 | 			// Compute Hessian
 88 | 			ival.push_back(exp_iky[i] * resp_x[i]);
 89 | 			std::vector<cv::Mat> tmp;
 90 | 			cv::split(ival[i], tmp);
 91 | 			cv::merge(std::vector<cv::Mat>({-1 * tmp[1], tmp[0]}), ival[i]);
 92 | 
 93 | 			H_yy.push_back(real(-1 * complexDotMultiplication(ky2Mat, exp_iky[i]) * resp_x[i] + ival[i]));
 94 | 			H_xx.push_back(real(-1 * y_resp[i] * complexDotMultiplication(kx2Mat, exp_ikx[i]) + ival[i]));
 95 | 			H_xy.push_back(real(-1 * ky_exp_ky[i] * (scores_fs_[i] * kx_exp_kx[i])));
 96 | 
 97 | 			det_H.push_back(H_yy[i].mul(H_xx[i]) - H_xy[i].mul(H_xy[i]));
 98 | 		
 99 | 			// Compute new position using newtons method
100 | 			cv::Mat tmp1, tmp2;
101 | 			cv::divide(H_xx[i].mul(grad_y[i]) - H_xy[i].mul(grad_x[i]), det_H[i], tmp1);
102 | 			max_pos_y[i] -= tmp1.at<float>(0, 0);
103 | 			cv::divide(H_yy[i].mul(grad_x[i]) - H_xy[i].mul(grad_y[i]), det_H[i], tmp2);
104 | 			max_pos_x[i] -= tmp2.at<float>(0, 0);
105 | 
106 | 			// Evaluate maximum
107 | 			cv::Mat tempy(1, h, CV_32FC2), tempx(w, 1, CV_32FC2);
108 | 			for (int y = 0; y < h; ++y)
109 | 				tempy.at<cv::Vec<float, 2>>(0, y) = cv::Vec<float, 2>(cos(ky[y] * max_pos_y[i]), sin(ky[y] * max_pos_y[i]));
110 | 			for (int x = 0; x < w; ++x)
111 | 				tempx.at<cv::Vec<float, 2>>(x, 0) = cv::Vec<float, 2>(cos(kx[x] * max_pos_x[i]), sin(kx[x] * max_pos_x[i]));
112 | 			exp_iky[i] = tempy;
113 | 			exp_ikx[i] = tempx;
114 | 		}
115 | 	}
116 | 	// Evaluate the Fourier series at the estimated locations to find the corresponding scores.
117 | 	std::vector<float> max_score;
118 | 	for (size_t i = 0; i < sampled_scores.size(); ++i)
119 | 	{
120 | 		float new_scores = real(exp_iky[i] * scores_fs_[i] * exp_ikx[i]).at<float>(0, 0);
121 | 		// check for scales that have not increased in score
122 | 		if (new_scores > init_max_score[i])
123 | 		{
124 | 			max_score.push_back(new_scores);
125 | 		}
126 | 		else
127 | 		{
128 | 			max_score.push_back(init_max_score[i]);
129 | 			max_pos_y[i] = init_pos_y[i];
130 | 			max_pos_x[i] = init_pos_x[i];
131 | 		}
132 | 			
133 | 	}
134 | 	 
135 | 	// Find the scale with the maximum response 
136 | 	std::vector<float>::iterator pos = max_element(max_score.begin(), max_score.end());
137 | 	scale_ind_ = pos - max_score.begin();
138 | 	max_score_ = *pos;
139 | 
140 | 	// Scale the coordinate system to output_sz
141 | 	disp_row_ = (fmod(max_pos_y[scale_ind_] + CV_PI, CV_PI * 2.0) - CV_PI) / (CV_PI * 2.0) * h;
142 | 	disp_col_ = (fmod(max_pos_x[scale_ind_] + CV_PI, CV_PI * 2.0) - CV_PI) / (CV_PI * 2.0) * w;
143 | 	//return sampled_scores;
144 | }
145 | }


--------------------------------------------------------------------------------
/KCF/recttools.hpp:
--------------------------------------------------------------------------------
  1 | /* 
  2 | Author: Christian Bailer
  3 | Contact address: Christian.Bailer@dfki.de 
  4 | Department Augmented Vision DFKI 
  5 | 
  6 |                           License Agreement
  7 |                For Open Source Computer Vision Library
  8 |                        (3-clause BSD License)
  9 | 
 10 | Redistribution and use in source and binary forms, with or without modification,
 11 | are permitted provided that the following conditions are met:
 12 | 
 13 |   * Redistributions of source code must retain the above copyright notice,
 14 |     this list of conditions and the following disclaimer.
 15 | 
 16 |   * Redistributions in binary form must reproduce the above copyright notice,
 17 |     this list of conditions and the following disclaimer in the documentation
 18 |     and/or other materials provided with the distribution.
 19 | 
 20 |   * Neither the names of the copyright holders nor the names of the contributors
 21 |     may be used to endorse or promote products derived from this software
 22 |     without specific prior written permission.
 23 | 
 24 | This software is provided by the copyright holders and contributors "as is" and
 25 | any express or implied warranties, including, but not limited to, the implied
 26 | warranties of merchantability and fitness for a particular purpose are disclaimed.
 27 | In no event shall copyright holders or contributors be liable for any direct,
 28 | indirect, incidental, special, exemplary, or consequential damages
 29 | (including, but not limited to, procurement of substitute goods or services;
 30 | loss of use, data, or profits; or business interruption) however caused
 31 | and on any theory of liability, whether in contract, strict liability,
 32 | or tort (including negligence or otherwise) arising in any way out of
 33 | the use of this software, even if advised of the possibility of such damage.
 34 | */
 35 | 
 36 | #pragma once
 37 | 
 38 | //#include <cv.h>
 39 | #include <math.h>
 40 | 
 41 | #ifndef _OPENCV_RECTTOOLS_HPP_
 42 | #define _OPENCV_RECTTOOLS_HPP_
 43 | #endif
 44 | 
 45 | namespace kcf
 46 | {
 47 | 
 48 | template <typename t>
 49 | inline cv::Vec<t, 2 > center(const cv::Rect_<t> &rect)
 50 | {
 51 |     return cv::Vec<t, 2 > (rect.x + rect.width / (t) 2, rect.y + rect.height / (t) 2);
 52 | }
 53 | 
 54 | template <typename t>
 55 | inline t x2(const cv::Rect_<t> &rect)
 56 | {
 57 |     return rect.x + rect.width;
 58 | }
 59 | 
 60 | template <typename t>
 61 | inline t y2(const cv::Rect_<t> &rect)
 62 | {
 63 |     return rect.y + rect.height;
 64 | }
 65 | 
 66 | template <typename t>
 67 | inline void resize(cv::Rect_<t> &rect, float scalex, float scaley = 0)
 68 | {
 69 |     if (!scaley)scaley = scalex;
 70 |     rect.x -= rect.width * (scalex - 1.f) / 2.f;
 71 |     rect.width *= scalex;
 72 | 
 73 |     rect.y -= rect.height * (scaley - 1.f) / 2.f;
 74 |     rect.height *= scaley;
 75 | 
 76 | }
 77 | 
 78 | template <typename t>
 79 | inline void limit(cv::Rect_<t> &rect, cv::Rect_<t> limit)
 80 | {
 81 |     if (rect.x + rect.width > limit.x + limit.width)
 82 |         rect.width = (limit.x + limit.width - rect.x);
 83 |     if (rect.y + rect.height > limit.y + limit.height)
 84 |         rect.height = (limit.y + limit.height - rect.y);
 85 |     if (rect.x < limit.x)
 86 |     {
 87 |         rect.width -= (limit.x - rect.x);
 88 |         rect.x = limit.x;
 89 |     }
 90 |     if (rect.y < limit.y)
 91 |     {
 92 |         rect.height -= (limit.y - rect.y);
 93 |         rect.y = limit.y;
 94 |     }
 95 | 
 96 |     if(rect.width<0)rect.width=0;
 97 |     if(rect.height<0)rect.height=0;
 98 | }
 99 | 
100 | template <typename t>
101 | inline void limit(cv::Rect_<t> &rect, t width, t height, t x = 0, t y = 0)
102 | {
103 |     limit(rect, cv::Rect_<t > (x, y, width, height));
104 | 
105 | }
106 | 
107 | template <typename t>
108 | inline cv::Rect getBorder(const cv::Rect_<t > &original, cv::Rect_<t > & limited)
109 | {
110 |     cv::Rect_<t > res;
111 |     res.x = limited.x - original.x;
112 |     res.y = limited.y - original.y;
113 |     res.width = x2(original) - x2(limited);
114 |     res.height = y2(original) - y2(limited);
115 |     assert(res.x >= 0 && res.y >= 0 && res.width >= 0 && res.height >= 0);
116 |     return res;
117 | }
118 | 
119 | inline cv::Mat subwindow(const cv::Mat &in, const cv::Rect & window, int borderType = cv::BORDER_CONSTANT)
120 | {
121 |     cv::Rect cutWindow = window;
122 |     limit(cutWindow, in.cols, in.rows); //limit cutwindow inside Mat in;
123 | 
124 |     if (cutWindow.height <= 0 || cutWindow.width <= 0) assert(0); 
125 |     //    return cv::Mat(window.height,window.width,in.type(),0) ;
126 | 
127 |     cv::Rect border = getBorder(window, cutWindow); //get the border of cutWindow in window;
128 |     cv::Mat res = in(cutWindow);
129 | 
130 |     if (border != cv::Rect(0, 0, 0, 0))
131 |     {// back to the same size of window with border filled with constant
132 |         cv::copyMakeBorder(res, res, border.y, border.height, border.x, border.width, borderType);
133 |     }
134 |     return res;
135 | }
136 | 
137 | inline cv::Mat getGrayImage(cv::Mat img)
138 | {
139 |     cv::cvtColor(img, img, CV_BGR2GRAY);
140 |     img.convertTo(img, CV_32F, 1 / 255.f);
141 |     return img;
142 | }
143 | 
144 | inline void cutOutsize(float &num, int limit)
145 | {
146 |   if(num < 0)
147 |     num = 0;
148 |   else if(num > limit - 1)
149 |     num = limit - 1;
150 | }
151 | 
152 | inline cv::Mat extractImage(const cv::Mat &in, float cx, float cy, float patch_width, float patch_height)
153 | {
154 | 
155 |     float xs_s = floor(cx) - floor(patch_width / 2);
156 |     cutOutsize(xs_s, in.cols);
157 | 
158 |     float xs_e = floor(cx + patch_width - 1) - floor(patch_width / 2);
159 |     cutOutsize(xs_e, in.cols);
160 | 
161 |     float ys_s = floor(cy) - floor(patch_height / 2);
162 |     cutOutsize(ys_s, in.rows);
163 | 
164 |     float ys_e = floor(cy + patch_height - 1) - floor(patch_height / 2);
165 |     cutOutsize(ys_e, in.rows);
166 | 
167 | 
168 |     return in(cv::Rect(xs_s, ys_s, xs_e - xs_s, ys_e - ys_s));
169 | }
170 | 
171 | }
172 | 
173 | 
174 | 
175 | 


--------------------------------------------------------------------------------
/ECO/fhog.hpp:
--------------------------------------------------------------------------------
  1 | /*M///////////////////////////////////////////////////////////////////////////////////////
  2 | //
  3 | //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4 | //
  5 | //  By downloading, copying, installing or using the software you agree to this license.
  6 | //  If you do not agree to this license, do not download, install,
  7 | //  copy or use the software.
  8 | //
  9 | //
 10 | //                           License Agreement
 11 | //                For Open Source Computer Vision Library
 12 | //
 13 | // Copyright (C) 2010-2013, University of Nizhny Novgorod, all rights reserved.
 14 | // Third party copyrights are property of their respective owners.
 15 | //
 16 | // Redistribution and use in source and binary forms, with or without modification,
 17 | // are permitted provided that the following conditions are met:
 18 | //
 19 | //   * Redistribution's of source code must retain the above copyright notice,
 20 | //     this list of conditions and the following disclaimer.
 21 | //
 22 | //   * Redistribution's in binary form must reproduce the above copyright notice,
 23 | //     this list of conditions and the following disclaimer in the documentation
 24 | //     and/or other materials provided with the distribution.
 25 | //
 26 | //   * The name of the copyright holders may not be used to endorse or promote products
 27 | //     derived from this software without specific prior written permission.
 28 | //
 29 | // This software is provided by the copyright holders and contributors "as is" and
 30 | // any express or implied warranties, including, but not limited to, the implied
 31 | // warranties of merchantability and fitness for a particular purpose are disclaimed.
 32 | // In no event shall the Intel Corporation or contributors be liable for any direct,
 33 | // indirect, incidental, special, exemplary, or consequential damages
 34 | // (including, but not limited to, procurement of substitute goods or services;
 35 | // loss of use, data, or profits; or business interruption) however caused
 36 | // and on any theory of liability, whether in contract, strict liability,
 37 | // or tort (including negligence or otherwise) arising in any way out of
 38 | // the use of this software, even if advised of the possibility of such damage.
 39 | //
 40 | //M*/
 41 | 
 42 | //Modified from latentsvm module's "_lsvmc_latentsvm.h".
 43 | 
 44 | /*****************************************************************************/
 45 | /*                      Latent SVM prediction API                            */
 46 | /*****************************************************************************/
 47 | 
 48 | #ifndef _FHOG_H_
 49 | #define _FHOG_H_
 50 | 
 51 | #include <stdio.h>
 52 | 
 53 | #include <opencv2/imgproc.hpp>
 54 | 
 55 | namespace eco
 56 | {
 57 | // DataType: STRUCT featureMap
 58 | // FEATURE MAP DESCRIPTION
 59 | //   Rectangular map (sizeX x sizeY),
 60 | //   every cell stores feature vector (dimension = numFeatures)
 61 | // map             - matrix of feature vectors
 62 | //                   to set and get feature vectors (i,j)
 63 | //                   used formula map[(j * sizeX + i) * p + k], where
 64 | //                   k - component of feature vector in cell (i, j)
 65 | typedef struct
 66 | {
 67 |     int sizeX;
 68 |     int sizeY;
 69 |     int numFeatures;
 70 |     float *map;
 71 | } CvLSVMFeatureMapCaskade;
 72 | 
 73 | #include "float.h"
 74 | 
 75 | #define PI CV_PI
 76 | #define EPS 0.000001
 77 | #define F_MAX FLT_MAX
 78 | #define F_MIN -FLT_MAX
 79 | 
 80 | // The number of elements in bin
 81 | // The number of sectors in gradient histogram building
 82 | #define NUM_SECTOR 9
 83 | 
 84 | // The number of levels in image resize procedure
 85 | // We need Lambda levels to resize image twice
 86 | #define LAMBDA 10
 87 | 
 88 | // Block size. Used in feature pyramid building procedure
 89 | #define SIDE_LENGTH 8
 90 | 
 91 | #define VAL_OF_TRUNCATE 0.2f
 92 | 
 93 | //modified from "_lsvm_error.h"
 94 | #define LATENT_SVM_OK 0
 95 | #define LATENT_SVM_MEM_NULL 2
 96 | #define DISTANCE_TRANSFORM_OK 1
 97 | #define DISTANCE_TRANSFORM_GET_INTERSECTION_ERROR -1
 98 | #define DISTANCE_TRANSFORM_ERROR -2
 99 | #define DISTANCE_TRANSFORM_EQUAL_POINTS -3
100 | #define LATENT_SVM_GET_FEATURE_PYRAMID_FAILED -4
101 | #define LATENT_SVM_SEARCH_OBJECT_FAILED -5
102 | #define LATENT_SVM_FAILED_SUPERPOSITION -6
103 | #define FILTER_OUT_OF_BOUNDARIES -7
104 | #define LATENT_SVM_TBB_SCHEDULE_CREATION_FAILED -8
105 | #define LATENT_SVM_TBB_NUMTHREADS_NOT_CORRECT -9
106 | #define FFT_OK 2
107 | #define FFT_ERROR -10
108 | #define LSVM_PARSER_FILE_NOT_FOUND -11
109 | 
110 | /*
111 | // Getting feature map for the selected subimage  
112 | //
113 | // API
114 | // int getFeatureMaps(const IplImage * image, const int k, featureMap **map);
115 | // INPUT
116 | // image             - selected subimage
117 | // k                 - size of cells
118 | // OUTPUT
119 | // map               - feature map
120 | // RESULT
121 | // Error status
122 | */
123 | int getFeatureMaps(const IplImage *image, const int k,
124 |                    CvLSVMFeatureMapCaskade **map);
125 | 
126 | /*
127 | // Feature map Normalization and Truncation 
128 | //
129 | // API
130 | // int normalizationAndTruncationFeatureMaps(featureMap *map, const float alfa);
131 | // INPUT
132 | // map               - feature map
133 | // alfa              - truncation threshold
134 | // OUTPUT
135 | // map               - truncated and normalized feature map
136 | // RESULT
137 | // Error status
138 | */
139 | int normalizeAndTruncate(CvLSVMFeatureMapCaskade *map, const float alfa);
140 | 
141 | /*
142 | // Feature map reduction
143 | // In each cell we reduce dimension of the feature vector
144 | // according to original paper special procedure
145 | //
146 | // API
147 | // int PCAFeatureMaps(featureMap *map)
148 | // INPUT
149 | // map               - feature map
150 | // OUTPUT
151 | // map               - feature map
152 | // RESULT
153 | // Error status
154 | */
155 | int PCAFeatureMaps(CvLSVMFeatureMapCaskade *map);
156 | 
157 | int allocFeatureMapObject(CvLSVMFeatureMapCaskade **obj,
158 |                           const int sizeX, const int sizeY, const int p);
159 | 
160 | int freeFeatureMapObject(CvLSVMFeatureMapCaskade **obj);
161 | } // namespace eco
162 | 
163 | #endif
164 | 


--------------------------------------------------------------------------------
/KCF/fhog.hpp:
--------------------------------------------------------------------------------
  1 | /*M///////////////////////////////////////////////////////////////////////////////////////
  2 | //
  3 | //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4 | //
  5 | //  By downloading, copying, installing or using the software you agree to this license.
  6 | //  If you do not agree to this license, do not download, install,
  7 | //  copy or use the software.
  8 | //
  9 | //
 10 | //                           License Agreement
 11 | //                For Open Source Computer Vision Library
 12 | //
 13 | // Copyright (C) 2010-2013, University of Nizhny Novgorod, all rights reserved.
 14 | // Third party copyrights are property of their respective owners.
 15 | //
 16 | // Redistribution and use in source and binary forms, with or without modification,
 17 | // are permitted provided that the following conditions are met:
 18 | //
 19 | //   * Redistribution's of source code must retain the above copyright notice,
 20 | //     this list of conditions and the following disclaimer.
 21 | //
 22 | //   * Redistribution's in binary form must reproduce the above copyright notice,
 23 | //     this list of conditions and the following disclaimer in the documentation
 24 | //     and/or other materials provided with the distribution.
 25 | //
 26 | //   * The name of the copyright holders may not be used to endorse or promote products
 27 | //     derived from this software without specific prior written permission.
 28 | //
 29 | // This software is provided by the copyright holders and contributors "as is" and
 30 | // any express or implied warranties, including, but not limited to, the implied
 31 | // warranties of merchantability and fitness for a particular purpose are disclaimed.
 32 | // In no event shall the Intel Corporation or contributors be liable for any direct,
 33 | // indirect, incidental, special, exemplary, or consequential damages
 34 | // (including, but not limited to, procurement of substitute goods or services;
 35 | // loss of use, data, or profits; or business interruption) however caused
 36 | // and on any theory of liability, whether in contract, strict liability,
 37 | // or tort (including negligence or otherwise) arising in any way out of
 38 | // the use of this software, even if advised of the possibility of such damage.
 39 | //
 40 | //M*/
 41 | 
 42 | 
 43 | //Modified from latentsvm module's "_lsvmc_latentsvm.h".
 44 | 
 45 | 
 46 | /*****************************************************************************/
 47 | /*                      Latent SVM prediction API                            */
 48 | /*****************************************************************************/
 49 | 
 50 | #ifndef _FHOG_H_
 51 | #define _FHOG_H_
 52 | 
 53 | #include <stdio.h>
 54 | //#include "_lsvmc_types.h"
 55 | //#include "_lsvmc_error.h"
 56 | //#include "_lsvmc_routine.h"
 57 | 
 58 | //#include "opencv2/imgproc.hpp"
 59 | #include "opencv2/imgproc/imgproc_c.h"
 60 | 
 61 | namespace kcf
 62 | {
 63 | //modified from "_lsvmc_types.h"
 64 | 
 65 | // DataType: STRUCT featureMap
 66 | // FEATURE MAP DESCRIPTION
 67 | //   Rectangular map (sizeX x sizeY), 
 68 | //   every cell stores feature vector (dimension = numFeatures)
 69 | // map             - matrix of feature vectors
 70 | //                   to set and get feature vectors (i,j) 
 71 | //                   used formula map[(j * sizeX + i) * p + k], where
 72 | //                   k - component of feature vector in cell (i, j)
 73 | typedef struct{
 74 |     int sizeX;
 75 |     int sizeY;
 76 |     int numFeatures;
 77 |     float *map;
 78 | } CvLSVMFeatureMapCaskade;
 79 | 
 80 | 
 81 | #include "float.h"
 82 | 
 83 | #define PI    CV_PI
 84 | 
 85 | #define EPS 0.000001
 86 | 
 87 | #define F_MAX FLT_MAX
 88 | #define F_MIN -FLT_MAX
 89 | 
 90 | // The number of elements in bin
 91 | // The number of sectors in gradient histogram building
 92 | #define NUM_SECTOR 9
 93 | 
 94 | // The number of levels in image resize procedure
 95 | // We need Lambda levels to resize image twice
 96 | #define LAMBDA 10
 97 | 
 98 | // Block size. Used in feature pyramid building procedure
 99 | #define SIDE_LENGTH 8
100 | 
101 | #define VAL_OF_TRUNCATE 0.2f 
102 | 
103 | 
104 | //modified from "_lsvm_error.h"
105 | #define LATENT_SVM_OK 0
106 | #define LATENT_SVM_MEM_NULL 2
107 | #define DISTANCE_TRANSFORM_OK 1
108 | #define DISTANCE_TRANSFORM_GET_INTERSECTION_ERROR -1
109 | #define DISTANCE_TRANSFORM_ERROR -2
110 | #define DISTANCE_TRANSFORM_EQUAL_POINTS -3
111 | #define LATENT_SVM_GET_FEATURE_PYRAMID_FAILED -4
112 | #define LATENT_SVM_SEARCH_OBJECT_FAILED -5
113 | #define LATENT_SVM_FAILED_SUPERPOSITION -6
114 | #define FILTER_OUT_OF_BOUNDARIES -7
115 | #define LATENT_SVM_TBB_SCHEDULE_CREATION_FAILED -8
116 | #define LATENT_SVM_TBB_NUMTHREADS_NOT_CORRECT -9
117 | #define FFT_OK 2
118 | #define FFT_ERROR -10
119 | #define LSVM_PARSER_FILE_NOT_FOUND -11
120 | 
121 | 
122 | 
123 | /*
124 | // Getting feature map for the selected subimage  
125 | //
126 | // API
127 | // int getFeatureMaps(const IplImage * image, const int k, featureMap **map);
128 | // INPUT
129 | // image             - selected subimage
130 | // k                 - size of cells
131 | // OUTPUT
132 | // map               - feature map
133 | // RESULT
134 | // Error status
135 | */
136 | int getFeatureMaps(const IplImage * image, const int k, CvLSVMFeatureMapCaskade **map);
137 | 
138 | 
139 | /*
140 | // Feature map Normalization and Truncation 
141 | //
142 | // API
143 | // int normalizationAndTruncationFeatureMaps(featureMap *map, const float alfa);
144 | // INPUT
145 | // map               - feature map
146 | // alfa              - truncation threshold
147 | // OUTPUT
148 | // map               - truncated and normalized feature map
149 | // RESULT
150 | // Error status
151 | */
152 | int normalizeAndTruncate(CvLSVMFeatureMapCaskade *map, const float alfa);
153 | 
154 | /*
155 | // Feature map reduction
156 | // In each cell we reduce dimension of the feature vector
157 | // according to original paper special procedure
158 | //
159 | // API
160 | // int PCAFeatureMaps(featureMap *map)
161 | // INPUT
162 | // map               - feature map
163 | // OUTPUT
164 | // map               - feature map
165 | // RESULT
166 | // Error status
167 | */
168 | int PCAFeatureMaps(CvLSVMFeatureMapCaskade *map);
169 | 
170 | 
171 | //modified from "lsvmc_routine.h"
172 | 
173 | int allocFeatureMapObject(CvLSVMFeatureMapCaskade **obj, const int sizeX, const int sizeY,
174 |                           const int p);
175 | 
176 | int freeFeatureMapObject (CvLSVMFeatureMapCaskade **obj);
177 | 
178 | }
179 | 
180 | #endif
181 | 


--------------------------------------------------------------------------------
/KCF/ffttools.hpp:
--------------------------------------------------------------------------------
  1 | /* 
  2 | Author: Christian Bailer
  3 | Contact address: Christian.Bailer@dfki.de 
  4 | Department Augmented Vision DFKI 
  5 | 
  6 |                           License Agreement
  7 |                For Open Source Computer Vision Library
  8 |                        (3-clause BSD License)
  9 | 
 10 | Redistribution and use in source and binary forms, with or without modification,
 11 | are permitted provided that the following conditions are met:
 12 | 
 13 |   * Redistributions of source code must retain the above copyright notice,
 14 |     this list of conditions and the following disclaimer.
 15 | 
 16 |   * Redistributions in binary form must reproduce the above copyright notice,
 17 |     this list of conditions and the following disclaimer in the documentation
 18 |     and/or other materials provided with the distribution.
 19 | 
 20 |   * Neither the names of the copyright holders nor the names of the contributors
 21 |     may be used to endorse or promote products derived from this software
 22 |     without specific prior written permission.
 23 | 
 24 | This software is provided by the copyright holders and contributors "as is" and
 25 | any express or implied warranties, including, but not limited to, the implied
 26 | warranties of merchantability and fitness for a particular purpose are disclaimed.
 27 | In no event shall copyright holders or contributors be liable for any direct,
 28 | indirect, incidental, special, exemplary, or consequential damages
 29 | (including, but not limited to, procurement of substitute goods or services;
 30 | loss of use, data, or profits; or business interruption) however caused
 31 | and on any theory of liability, whether in contract, strict liability,
 32 | or tort (including negligence or otherwise) arising in any way out of
 33 | the use of this software, even if advised of the possibility of such damage.
 34 | */
 35 | 
 36 | #pragma once
 37 | 
 38 | //#include <cv.h>
 39 | 
 40 | #ifndef _OPENCV_FFTTOOLS_HPP_
 41 | #define _OPENCV_FFTTOOLS_HPP_
 42 | #endif
 43 | 
 44 | namespace kcf
 45 | {
 46 | cv::Mat dft_d(cv::Mat img, bool backwards = false, bool byRow = false); //byRow=true: 1d trasform, else 2d;
 47 | cv::Mat real(cv::Mat img);
 48 | cv::Mat imag(cv::Mat img);
 49 | cv::Mat magnitude(cv::Mat img);
 50 | cv::Mat complexDotMultiplication(cv::Mat a, cv::Mat b);
 51 | cv::Mat complexDotDivision(cv::Mat a, cv::Mat b);
 52 | void rearrange(cv::Mat &img);
 53 | void normalizedLogTransform(cv::Mat &img);
 54 | 
 55 | cv::Mat dft_d(cv::Mat img, bool backwards, bool byRow)
 56 | {
 57 |     if (img.channels() == 1)
 58 |     {
 59 |         cv::Mat planes[] = {cv::Mat_<float>(img),
 60 |                             cv::Mat_<float>::zeros(img.size())};
 61 |         cv::merge(planes, 2, img);
 62 |     }
 63 |     if (byRow)
 64 |         cv::dft(img, img, (cv::DFT_ROWS | cv::DFT_COMPLEX_OUTPUT));
 65 |     else
 66 |         cv::dft(img, img, backwards ? (cv::DFT_INVERSE | cv::DFT_SCALE) : 0); //do scale when ifft;
 67 | 
 68 |     return img;
 69 | }
 70 | 
 71 | cv::Mat real(cv::Mat img)
 72 | {
 73 |     std::vector<cv::Mat> planes;
 74 |     cv::split(img, planes);
 75 |     return planes[0];
 76 | }
 77 | 
 78 | cv::Mat imag(cv::Mat img)
 79 | {
 80 |     std::vector<cv::Mat> planes;
 81 |     cv::split(img, planes);
 82 |     return planes[1];
 83 | }
 84 | 
 85 | cv::Mat magnitude(cv::Mat img)
 86 | {
 87 |     cv::Mat res;
 88 |     std::vector<cv::Mat> planes;
 89 |     cv::split(img, planes); // planes[0] = Re(DFT(I), planes[1] = Im(DFT(I))
 90 |     if (planes.size() == 1)
 91 |         res = cv::abs(img);
 92 |     else if (planes.size() == 2)
 93 |         cv::magnitude(planes[0], planes[1], res); // planes[0] = magnitude
 94 |     else
 95 |         assert(0);
 96 |     return res;
 97 | }
 98 | 
 99 | cv::Mat complexDotMultiplication(cv::Mat a, cv::Mat b)
100 | {
101 |     std::vector<cv::Mat> pa;
102 |     std::vector<cv::Mat> pb;
103 |     cv::split(a, pa);
104 |     cv::split(b, pb);
105 | 
106 |     std::vector<cv::Mat> pres;
107 |     pres.push_back(pa[0].mul(pb[0]) - pa[1].mul(pb[1]));
108 |     pres.push_back(pa[0].mul(pb[1]) + pa[1].mul(pb[0]));
109 | 
110 |     cv::Mat res;
111 |     cv::merge(pres, res);
112 | 
113 |     return res;
114 | }
115 | 
116 | cv::Mat complexDotDivisionReal(cv::Mat a, cv::Mat b)
117 | {
118 |     std::vector<cv::Mat> pa;
119 |     cv::split(a, pa);
120 | 
121 |     std::vector<cv::Mat> pres;
122 | 
123 |     cv::Mat divisor = 1. / b;
124 | 
125 |     pres.push_back(pa[0].mul(divisor));
126 |     pres.push_back(pa[1].mul(divisor));
127 | 
128 |     cv::Mat res;
129 |     cv::merge(pres, res);
130 |     return res;
131 | }
132 | 
133 | cv::Mat complexDotDivision(cv::Mat a, cv::Mat b)
134 | {
135 |     std::vector<cv::Mat> pa;
136 |     std::vector<cv::Mat> pb;
137 |     cv::split(a, pa);
138 |     cv::split(b, pb);
139 | 
140 |     cv::Mat divisor = 1. / (pb[0].mul(pb[0]) + pb[1].mul(pb[1]));
141 | 
142 |     std::vector<cv::Mat> pres;
143 | 
144 |     pres.push_back((pa[0].mul(pb[0]) + pa[1].mul(pb[1])).mul(divisor));
145 |     pres.push_back((pa[1].mul(pb[0]) + pa[0].mul(pb[1])).mul(divisor));
146 | 
147 |     cv::Mat res;
148 |     cv::merge(pres, res);
149 |     return res;
150 | }
151 | 
152 | void rearrange(cv::Mat &img) //KCF page 11 Figure 6;
153 | {
154 |     // img = img(cv::Rect(0, 0, img.cols & -2, img.rows & -2));
155 |     int cx = img.cols / 2;
156 |     int cy = img.rows / 2;
157 | 
158 |     cv::Mat q0(img, cv::Rect(0, 0, cx, cy));   // Top-Left - Create a ROI per quadrant
159 |     cv::Mat q1(img, cv::Rect(cx, 0, cx, cy));  // Top-Right
160 |     cv::Mat q2(img, cv::Rect(0, cy, cx, cy));  // Bottom-Left
161 |     cv::Mat q3(img, cv::Rect(cx, cy, cx, cy)); // Bottom-Right
162 | 
163 |     cv::Mat tmp; // swap quadrants (Top-Left with Bottom-Right)
164 |     q0.copyTo(tmp);
165 |     q3.copyTo(q0);
166 |     tmp.copyTo(q3);
167 |     q1.copyTo(tmp); // swap quadrant (Top-Right with Bottom-Left)
168 |     q2.copyTo(q1);
169 |     tmp.copyTo(q2);
170 | }
171 | /*
172 | template < typename type>
173 | cv::Mat fouriertransFull(const cv::Mat & in)
174 | {
175 |     return dft_d(in);
176 | 
177 |     cv::Mat planes[] = {cv::Mat_<type > (in), cv::Mat_<type>::zeros(in.size())};
178 |     cv::Mat t;
179 |     assert(planes[0].depth() == planes[1].depth());
180 |     assert(planes[0].size == planes[1].size);
181 |     cv::merge(planes, 2, t);
182 |     cv::dft(t, t);
183 | 
184 |     //cv::normalize(a, a, 0, 1, CV_MINMAX);
185 |     //cv::normalize(t, t, 0, 1, CV_MINMAX);
186 | 
187 |     // cv::imshow("a",real(a));
188 |     //  cv::imshow("b",real(t));
189 |     // cv::waitKey(0);
190 | 
191 |     return t;
192 | }*/
193 | 
194 | void normalizedLogTransform(cv::Mat &img)
195 | {
196 |     img = cv::abs(img);
197 |     img += cv::Scalar::all(1);
198 |     cv::log(img, img);
199 |     // cv::normalize(img, img, 0, 1, CV_MINMAX);
200 | }
201 | } // namespace kcf
202 | 


--------------------------------------------------------------------------------
/ECO/scale_filter.cc:
--------------------------------------------------------------------------------
  1 | #include "scale_filter.hpp"
  2 | 
  3 | namespace eco
  4 | {
  5 | void ScaleFilter::init(int &nScales, float &scale_step, const EcoParameters &params)
  6 | {
  7 |     nScales = params.number_of_scales_filter;
  8 |     scale_step = params.scale_step_filter;
  9 |     float scale_sigma = params.number_of_interp_scales * params.scale_sigma_factor;
 10 |     vector<float> scale_exp, scale_exp_shift;
 11 |     int scalemin = floor((1.0 - (float)nScales) / 2.0);
 12 |     int scalemax = floor(((float)nScales - 1.0) / 2.0);
 13 |     for (int i = scalemin; i <= scalemax; i++)
 14 |     {
 15 |         scale_exp.push_back(i * params.number_of_interp_scales / (float)nScales);
 16 |     }
 17 |     for (int i = 0; i < nScales; i++)
 18 |     {
 19 |         scale_exp_shift.push_back(scale_exp[(i + nScales / 2) % nScales]);
 20 |     }
 21 |     /*    debug("scale: min:%d, max:%d", scalemin, scalemax);
 22 |     debug("scale_exp_shift:");
 23 |     for (int i = 0; i < nScales; i++)
 24 |     {
 25 |         printf("%d:%f; ", i, scale_exp_shift[i]);
 26 |     }
 27 |     printf("\n");
 28 | */
 29 |     vector<float> interp_scale_exp, interp_scale_exp_shift;
 30 |     scalemin = floor((1.0 - (float)params.number_of_interp_scales) / 2.0);
 31 |     scalemax = floor(((float)params.number_of_interp_scales - 1.0) / 2.0);
 32 |     for (int i = scalemin; i <= scalemax; i++)
 33 |     {
 34 |         interp_scale_exp.push_back(i);
 35 |     }
 36 |     for (int i = 0; i < params.number_of_interp_scales; i++)
 37 |     {
 38 |         interp_scale_exp_shift.push_back(interp_scale_exp[(i + params.number_of_interp_scales / 2) % params.number_of_interp_scales]);
 39 |     }
 40 |     /*    debug("scale: min:%d, max:%d", scalemin, scalemax);
 41 |     debug("interp_scale_exp_shift:");
 42 |     for (int i = 0; i < params.number_of_interp_scales; i++)
 43 |     {
 44 |         printf("%d:%f; ", i, interp_scale_exp_shift[i]);
 45 |     }
 46 |     printf("\n");
 47 | */
 48 |     for (int i = 0; i < nScales; i++)
 49 |     {
 50 |         scaleSizeFactors_.push_back(std::pow(scale_step, scale_exp[i]));
 51 |     }
 52 |     /*    debug("scaleSizeFactors_:");
 53 |     for (int i = 0; i < nScales; i++)
 54 |     {
 55 |         printf("%d:%f; ", i, scaleSizeFactors_[i]);
 56 |     }
 57 |     printf("\n");
 58 | */
 59 |     for (int i = 0; i < params.number_of_interp_scales; i++)
 60 |     {
 61 |         interpScaleFactors_.push_back(std::pow(scale_step, interp_scale_exp_shift[i]));
 62 |     }
 63 |     /*    debug("interpScaleFactors_:");
 64 |     for (int i = 0; i < params.number_of_interp_scales; i++)
 65 |     {
 66 |         printf("%d:%f; ", i, interpScaleFactors_[i]);
 67 |     }
 68 |     printf("\n");
 69 | */
 70 | 
 71 |     cv::Mat ys_mat = cv::Mat(cv::Size(nScales, 1), CV_32FC1);
 72 |     for (int i = 0; i < nScales; i++)
 73 |     {
 74 |         ys_mat.at<float>(0, i) = std::exp(-0.5f * scale_exp_shift[i] * scale_exp_shift[i] / scale_sigma / scale_sigma);
 75 |     }
 76 |     /*
 77 |     debug("ys:");
 78 |     printMat(ys_mat);
 79 |     showmat1channels(ys_mat,2);
 80 | */
 81 |     yf_ = real(dft(ys_mat, false));
 82 |     /*
 83 |     debug("yf:");
 84 |     printMat(yf_);
 85 |     showmat1channels(yf_,2);
 86 | */
 87 | 
 88 |     for (int i = 0; i < nScales; i++)
 89 |     {
 90 |         window_.push_back(0.5f * (1.0f - std::cos(2 * M_PI * i / (nScales - 1.0f))));
 91 |     }
 92 |     /*
 93 |     debug("window_:");
 94 |     for (int i = 0; i < nScales; i++)
 95 |     {
 96 |         printf("%d:%f; ", i, window_[i]);
 97 |     }
 98 | */
 99 |     //max_scale_dim_ = !params.s_num_compressed_dim.compare("MAX");
100 |     //debug("max_scale_dim_: %d", max_scale_dim_);
101 | }
102 | 
103 | float ScaleFilter::scale_filter_track(const cv::Mat &im, const cv::Point2f &pos, const cv::Size2f &base_target_sz, const float &currentScaleFactor, const EcoParameters &params)
104 | {
105 |     debug("%f", currentScaleFactor);
106 |     vector<float> scales;
107 |     for (unsigned int i = 0; i < scaleSizeFactors_.size(); i++)
108 |     {
109 |         scales.push_back(scaleSizeFactors_[i] * currentScaleFactor);
110 |         //printf("%f ", scaleSizeFactors_[i]);
111 |     }
112 |     cv::Mat xs = extract_scale_sample(im, pos, base_target_sz, scales, params.scale_model_sz);
113 | 
114 |     debug("Not finished!-------------------");
115 |     assert(0);
116 | 
117 |     float scale_change_factor;
118 |     return scale_change_factor;
119 | }
120 | 
121 | cv::Mat ScaleFilter::extract_scale_sample(const cv::Mat &im, const cv::Point2f &posf, const cv::Size2f &base_target_sz, vector<float> &scaleFactors, const cv::Size &scale_model_sz)
122 | {
123 |     //printMat(new_im);
124 |     //showmat3channels(new_im, 0);
125 |     //debug("pos: %f %f", posf.x, posf.y);
126 |     cv::Point2i pos(posf);
127 |     int nScales = scaleFactors.size();
128 |     int df = std::floor(*std::min_element(std::begin(scaleFactors), std::end(scaleFactors)));
129 |     // debug("df:%d", df);
130 | 
131 |     cv::Mat new_im;
132 |     im.copyTo(new_im);
133 |     if (df > 1)
134 |     {
135 |         // compute offset and new center position
136 |         cv::Point os((pos.x - 1) % df, ((pos.y - 1) % df));
137 |         pos.x = (pos.x - os.x - 1) / df + 1;
138 |         pos.y = (pos.y - os.y - 1) / df + 1;
139 | 
140 |         for (unsigned int i = 0; i < scaleFactors.size(); i++)
141 |         {
142 |             scaleFactors[i] /= df;
143 |         }
144 |         // down sample image
145 |         int r = (im.rows - os.y) / df + 1;
146 |         int c = (im.cols - os.x) / df;
147 |         cv::Mat new_im2(r, c, im.type());
148 |         new_im = new_im2;
149 |         for (size_t i = 0 + os.y, m = 0;
150 |              i < (size_t)im.rows && m < (size_t)new_im.rows;
151 |              i += df, ++m)
152 |         {
153 |             for (size_t j = 0 + os.x, n = 0;
154 |                  j < (size_t)im.cols && n < (size_t)new_im.cols;
155 |                  j += df, ++n)
156 |             {
157 | 
158 |                 if (im.channels() == 1)
159 |                 {
160 |                     new_im.at<uchar>(m, n) = im.at<uchar>(i, j);
161 |                 }
162 |                 else
163 |                 {
164 |                     new_im.at<cv::Vec3b>(m, n) = im.at<cv::Vec3b>(i, j);
165 |                 }
166 |             }
167 |         }
168 |     }
169 | 
170 |     for (int s = 0; s < nScales; s++)
171 |     {
172 |         cv::Size patch_sz;
173 |         patch_sz.width = std::max(std::floor(base_target_sz.width * scaleFactors[s]), 2.0f);
174 |         patch_sz.height = std::max(std::floor(base_target_sz.height * scaleFactors[s]), 2.0f);
175 |         //debug("patch_sz:%d %d", patch_sz.height, patch_sz.width);
176 | 
177 |         cv::Point pos2(pos.x - floor((patch_sz.width + 1) / 2),
178 |                        pos.y - floor((patch_sz.height + 1) / 2));
179 | 
180 |         cv::Mat im_patch = subwindow(new_im, cv::Rect(pos2, patch_sz), IPL_BORDER_REPLICATE);
181 | 
182 |         cv::Mat im_patch_resized;
183 |         if (im_patch.cols == 0 || im_patch.rows == 0)
184 |         {
185 |             return im_patch_resized;
186 |         }
187 |         cv::resize(im_patch, im_patch_resized, scale_model_sz);
188 |         //printMat(im_patch);
189 |         //showmat3channels(im_patch, 0);
190 |         //printMat(im_patch_resized);
191 |         //showmat3channels(im_patch_resized, 0);
192 | 
193 |         vector<cv::Mat> im_vector, temp_hog;
194 |         im_vector.push_back(im_patch);
195 |         FeatureExtractor feature_extractor;
196 | #ifdef USE_SIMD
197 | 		temp_hog = feature_extractor.get_hog_features_simd(im_vector);
198 | #else
199 | 		temp_hog = feature_extractor.get_hog_features(im_vector);
200 | #endif
201 | 		temp_hog = feature_extractor.hog_feature_normalization(temp_hog);
202 | 
203 |         debug("Not finished!-------------------");
204 |         assert(0);
205 |     }
206 | 
207 |     cv::Mat scale_sample;
208 |     return scale_sample;
209 | }
210 | 
211 | } // namespace eco


--------------------------------------------------------------------------------
/KCF/kcftracker.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 | 
  3 | Tracker based on Kernelized Correlation Filter (KCF) [1] and Circulant Structure with Kernels (CSK) [2].
  4 | CSK is implemented by using raw gray level features, since it is a single-channel filter.
  5 | KCF is implemented by using HOG features (the default), since it extends CSK to multiple channels.
  6 | 
  7 | [1] J. F. Henriques, R. Caseiro, P. Martins, J. Batista,
  8 | "High-Speed Tracking with Kernelized Correlation Filters", TPAMI 2015.
  9 | 
 10 | [2] J. F. Henriques, R. Caseiro, P. Martins, J. Batista,
 11 | "Exploiting the Circulant Structure of Tracking-by-detection with Kernels", ECCV 2012.
 12 | 
 13 | Authors: Joao Faro, Christian Bailer, Joao F. Henriques
 14 | Contacts: joaopfaro@gmail.com, Christian.Bailer@dfki.de, henriques@isr.uc.pt
 15 | Institute of Systems and Robotics - University of Coimbra / Department Augmented Vision DFKI
 16 | 
 17 | 
 18 | Constructor parameters, all boolean:
 19 |     hog: use HOG features (default), otherwise use raw pixels
 20 |     fixed_window: fix window size (default), otherwise use ROI size (slower but more accurate)
 21 |     multiscale: use multi-scale tracking (default; cannot be used with fixed_window = true)
 22 | 
 23 | Default values are set for all properties of the tracker depending on the above choices.
 24 | Their values can be customized further before calling init():
 25 |     interp_factor: linear interpolation factor for adaptation
 26 |     sigma: gaussian kernel bandwidth
 27 |     lambda: regularization
 28 |     cell_size: HOG cell size
 29 |     padding: horizontal area surrounding the target, relative to its size
 30 |     output_sigma_factor: bandwidth of gaussian target
 31 |     template_size: template size in pixels, 0 to use ROI size
 32 |     scale_step: scale step for multi-scale estimation, 1 to disable it
 33 |     scale_weight: to downweight detection scores of other scales for added stability
 34 | 
 35 | For speed, the value (template_size/cell_size) should be a power of 2 or a product of small prime numbers.
 36 | 
 37 | Inputs to init():
 38 |    image is the initial frame.
 39 |    roi is a cv::Rect with the target positions in the initial frame
 40 | 
 41 | Inputs to update():
 42 |    image is the current frame.
 43 | 
 44 | Outputs of update():
 45 |    cv::Rect with target positions for the current frame
 46 | 
 47 | 
 48 | By downloading, copying, installing or using the software you agree to this license.
 49 | If you do not agree to this license, do not download, install,
 50 | copy or use the software.
 51 | 
 52 | 
 53 |                           License Agreement
 54 |                For Open Source Computer Vision Library
 55 |                        (3-clause BSD License)
 56 | 
 57 | Redistribution and use in source and binary forms, with or without modification,
 58 | are permitted provided that the following conditions are met:
 59 | 
 60 |   * Redistributions of source code must retain the above copyright notice,
 61 |     this list of conditions and the following disclaimer.
 62 | 
 63 |   * Redistributions in binary form must reproduce the above copyright notice,
 64 |     this list of conditions and the following disclaimer in the documentation
 65 |     and/or other materials provided with the distribution.
 66 | 
 67 |   * Neither the names of the copyright holders nor the names of the contributors
 68 |     may be used to endorse or promote products derived from this software
 69 |     without specific prior written permission.
 70 | 
 71 | This software is provided by the copyright holders and contributors "as is" and
 72 | any express or implied warranties, including, but not limited to, the implied
 73 | warranties of merchantability and fitness for a particular purpose are disclaimed.
 74 | In no event shall copyright holders or contributors be liable for any direct,
 75 | indirect, incidental, special, exemplary, or consequential damages
 76 | (including, but not limited to, procurement of substitute goods or services;
 77 | loss of use, data, or profits; or business interruption) however caused
 78 | and on any theory of liability, whether in contract, strict liability,
 79 | or tort (including negligence or otherwise) arising in any way out of
 80 | the use of this software, even if advised of the possibility of such damage.
 81 |  */
 82 | 
 83 | #pragma once
 84 | #include <opencv2/opencv.hpp>
 85 | #include <string>
 86 | #ifndef _OPENCV_KCFTRACKER_HPP_
 87 | #define _OPENCV_KCFTRACKER_HPP_
 88 | #endif
 89 | 
 90 | namespace kcf
 91 | {
 92 | class KCFTracker
 93 | {
 94 | public:
 95 |     // Constructor
 96 |     KCFTracker(bool hog = true, bool fixed_window = true, bool multiscale = true, bool lab = true, bool dsst = false);
 97 | 
 98 |     // Initialize tracker 
 99 |     void init( const cv::Mat image, const cv::Rect2d& roi);
100 |     
101 |     // Update position based on the new frame
102 |     //cv::Rect update(cv::Mat image);
103 |     bool update( const cv::Mat image, cv::Rect2d& roi);
104 | 
105 |     float detect_thresh_kcf; // thresh hold for tracking error or not
106 |     float sigma; // gaussian kernel bandwidth
107 |     float lambda; // regularization
108 |     float interp_factor; // linear interpolation factor for adaptation
109 |     int cell_size; // HOG cell size
110 |     int cell_sizeQ; // cell size^2, to avoid repeated operations
111 |     float padding; // extra area surrounding the target
112 |     float output_sigma_factor; // bandwidth of gaussian target
113 |     int template_size; // template size
114 | 
115 |     float scale_step; // scale step for multi-scale estimation
116 |     float scale_weight;  // to downweight detection scores of other scales for added stability
117 | 
118 | //=====dsst====
119 |     float detect_thresh_dsst; // thresh hold for tracking error or not
120 |     int base_width_dsst; // initial ROI widt
121 |     int base_height_dsst; // initial ROI height
122 |     int scale_max_area; // max ROI size before compressing
123 |     float scale_padding; // extra area surrounding the target for scaling
124 | //    float scale_step; // scale step for multi-scale estimation
125 |     float scale_sigma_factor; // bandwidth of gaussian target
126 |     int n_scales; // # of scaling windows
127 |     float scale_lr; // scale learning rate
128 |     float *scaleFactors; // all scale changing rate, from larger to smaller with 1 to be the middle
129 |     int scale_model_width; // the model width for scaling
130 |     int scale_model_height; // the model height for scaling
131 |     float min_scale_factor; // min scaling rate
132 |     float max_scale_factor; // max scaling rate
133 |     float scale_lambda; // regularization
134 | //===========
135 | 
136 | protected:
137 |     bool update_kcf( const cv::Mat image, cv::Rect2d& roi);
138 |     // Detect object in the current frame.
139 |     cv::Point2f detect(cv::Mat z, cv::Mat x, float &peak_value); // paper Algorithm 1 , _alpha updated in train();
140 | 
141 |     // train tracker with a single image, to update _alphaf;
142 |     void train(cv::Mat x, float train_interp_factor);
143 | 
144 |     // Evaluates a Gaussian kernel with bandwidth SIGMA for all relative shifts between input images X and Y, which must both be MxN. They must    also be periodic (ie., pre-processed with a cosine window).
145 |     cv::Mat gaussianCorrelation(cv::Mat x1, cv::Mat x2); // paper (30)
146 | 
147 |     // Create Gaussian Peak. Function called only in the first frame.
148 |     cv::Mat createGaussianPeak(int sizey, int sizex);
149 | 
150 |     // Obtain sub-window from image, with replication-padding and extract features
151 |     cv::Mat getFeatures(const cv::Mat & image, bool inithann, float scale_adjust = 1.0f);
152 | 
153 |     // Initialize Hanning window. Function called only in the first frame.
154 |     void createHanningMats();
155 | 
156 |     // Calculate sub-pixel peak for one dimension
157 |     float subPixelPeak(float left, float center, float right);
158 | 
159 | //=====dsst====
160 |     // Initialization for scales
161 |     void init_dsst(const cv::Mat image, const cv::Rect2d& roi);
162 | 
163 |     bool update_dsst( const cv::Mat image, cv::Rect2d& roi);
164 |     // Detect the new scaling rate
165 |     cv::Point2i detect_dsst(cv::Mat image);
166 | 
167 |     // Train method for scaling
168 |     void train_dsst(cv::Mat image, bool ini = false);
169 | 
170 |     // Compute the F^l in the paper
171 |     cv::Mat get_sample_dsst(const cv::Mat & image);
172 | 
173 |     // Compute the FFT Guassian Peak for scaling
174 |     cv::Mat createGaussianPeak_dsst();
175 | 
176 |     // Compute the hanning window for scaling
177 |     cv::Mat createHanningMats_dsst();
178 | //===========
179 | 
180 |     cv::Mat _alphaf;//alpha in paper, use this to calculate the detect result, changed in train();
181 |     cv::Mat _prob;  //Gaussian Peak(training outputs);
182 |     cv::Mat _tmpl;  //features of image (or the normalized gray image itself when raw), changed in train();
183 |     cv::Mat _num;   //numerator: use to update as MOSSE
184 |     cv::Mat _den;   //denumerator: use to update as MOSSE
185 |     cv::Mat _labCentroids;
186 | 
187 |     cv::Rect_<float> _roi;
188 | 
189 | private:
190 |     int _size_patch[3];//0:rows;1:cols;2:numFeatures; init in getFeatures();
191 |     cv::Mat _hann;
192 |     cv::Size _tmpl_sz;
193 |     float _scale;
194 |     int _gaussian_size;
195 |     bool _hogfeatures;
196 |     bool _labfeatures;
197 |     float _peak_value;
198 | 
199 | //=====dsst====
200 |     bool _dsst;
201 |     float _scale_dsst;
202 |     cv::Mat _den_dsst;
203 |     cv::Mat _num_dsst;
204 |     cv::Mat _hann_dsst;
205 |     cv::Mat _prob_dsst;
206 | //============    
207 | };
208 | }
209 | 


--------------------------------------------------------------------------------
/ECO/parameters.hpp:
--------------------------------------------------------------------------------
  1 | // Set the value the same as testing_ECO_gpu.m
  2 | #ifndef PARAMETERS_HPP
  3 | #define PARAMETERS_HPP
  4 | 
  5 | #ifdef USE_CAFFE
  6 | #include <caffe/caffe.hpp>
  7 | #include <caffe/util/io.hpp>
  8 | #endif
  9 | 
 10 | #include <vector>
 11 | #include <string>
 12 | #include <opencv2/core.hpp>
 13 | 
 14 | #define INF 0x7f800000 //0x7fffffff
 15 | 
 16 | using std::string;
 17 | using std::vector;
 18 | 
 19 | namespace eco
 20 | {
 21 | // ECO feature[Num_features][Dimension_of_the_feature];
 22 | typedef std::vector<std::vector<cv::Mat>> ECO_FEATS;
 23 | typedef cv::Vec<float, 2> COMPLEX; // represent a complex number;
 24 | 
 25 | // cnn   feature   configuration =========================================
 26 | #ifdef USE_CAFFE
 27 | struct CnnParameters
 28 | {
 29 | 	string proto = "/home/nvidia/Develop/Project/Tracker/OpenTracker/eco/model/imagenet-vgg-m-2048.prototxt";
 30 | 	string model = "/home/nvidia/Develop/Project/Tracker/OpenTracker/eco/model/VGG_CNN_M_2048.caffemodel";
 31 | 	string mean_file = "/home/nvidia/Develop/Project/Tracker/OpenTracker/eco/model/VGG_mean.binaryproto";
 32 | 
 33 | 	boost::shared_ptr<caffe::Net<float>> net;
 34 | 	cv::Mat deep_mean_mat, deep_mean_mean_mat;
 35 | 
 36 | 	string nn_name = "imagenet-vgg-m-2048.mat";
 37 | 	vector<int> stride = {2, 16};			// stride in total
 38 | 	vector<int> cell_size = {4, 16};		// downsample_factor
 39 | 	vector<int> output_layer = {3, 14};		// Which layers to use
 40 | 	vector<int> downsample_factor = {2, 1}; // How much to downsample each output layer
 41 | 	int input_size_scale = 1;				// Extra scale factor of the input samples to the network (1 is no scaling)
 42 | 	vector<int> nDim = {96, 512};			// Original dimension of features (ECO Paper Table 1)
 43 | 	vector<int> compressed_dim = {16, 64};  // Compressed dimensionality of each output layer (ECO Paper Table 1)
 44 | 	vector<float> penalty = {0, 0};
 45 | 
 46 | 	vector<int> start_ind = {3, 3, 1, 1};	 // sample feature start index
 47 | 	vector<int> end_ind = {106, 106, 13, 13}; // sample feature end index
 48 | };
 49 | struct CnnFeatures
 50 | {
 51 | 	CnnParameters fparams;
 52 | 	cv::Size img_input_sz = cv::Size(224, 224); // VGG default input sample size
 53 | 	cv::Size img_sample_sz;						// the size of sample
 54 | 	cv::Size data_sz_block0, data_sz_block1;
 55 | 	cv::Mat mean;
 56 | };
 57 | #endif
 58 | 
 59 | // hog parameters cofiguration =========================================
 60 | struct HogParameters
 61 | {
 62 | 	int cell_size = 6;
 63 | 	int compressed_dim = 10; // Compressed dimensionality of each output layer (ECO Paper Table 1)
 64 | 	int nOrients = 9;
 65 | 	size_t nDim = 31; // Original dimension of feature
 66 | 	float penalty = 0;
 67 | };
 68 | struct HogFeatures
 69 | {
 70 | 	HogParameters fparams;
 71 | 	cv::Size img_input_sz;  // input sample size
 72 | 	cv::Size img_sample_sz; // the size of sample
 73 | 	cv::Size data_sz_block0;
 74 | };
 75 | 
 76 | // CN parameters configuration =========================================
 77 | struct ColorspaceParameters
 78 | {
 79 | 	string colorspace = "gray";
 80 | 	int cell_size = 1;
 81 | };
 82 | struct ColorspaceFeatures
 83 | {
 84 | 	ColorspaceParameters fparams;
 85 | 	cv::Size img_input_sz;  
 86 | 	cv::Size img_sample_sz;
 87 | 	cv::Size data_sz_block0;
 88 | };
 89 | //---------------------------
 90 | struct CnParameters // only used for Color image
 91 | {
 92 | 	string tablename = "look_tables/CNnorm.txt";
 93 | 	float table[32768][10];
 94 | 	int cell_size = 4;
 95 | 	int compressed_dim = 3;
 96 | 	size_t nDim = 10; 
 97 | 	float penalty = 0;
 98 | };
 99 | struct CnFeatures
100 | {
101 | 	CnParameters fparams;
102 | 	cv::Size img_input_sz; 
103 | 	cv::Size img_sample_sz; 
104 | 	cv::Size data_sz_block0;
105 | };
106 | //---------------------------
107 | struct IcParameters // only used for gray image
108 | {
109 | 	string tablename = "look_tables/intensityChannelNorm6";
110 | 	float table[256][5];
111 | 	int cell_size = 4;
112 | 	int compressed_dim = 3;
113 | 	size_t nDim = 5; 
114 | 	float penalty = 0;
115 | };
116 | struct IcFeatures
117 | {
118 | 	IcParameters fparams;
119 | 	cv::Size img_input_sz;
120 | 	cv::Size img_sample_sz;
121 | 	cv::Size data_sz_block0;
122 | };
123 | 
124 | // Cojugate Gradient Options Structure =====================================
125 | struct CgOpts 
126 | {
127 | 	bool debug;
128 | 	bool CG_use_FR;
129 | 	float tol;
130 | 	bool CG_standard_alpha;
131 | 	float init_forget_factor;
132 | 	int maxit;
133 | };
134 | 
135 | // Parameters set exactly the same as 'testing_ECO_HC.m'====================
136 | struct EcoParameters
137 | {
138 | 	// Features
139 | 	bool useDeepFeature = false;
140 | 	bool useHogFeature = true;
141 | 	bool useColorspaceFeature = false;// not implemented yet
142 | 	bool useCnFeature = false;
143 | 	bool useIcFeature = true;
144 | 
145 | #ifdef USE_CAFFE
146 | 	CnnFeatures cnn_features;
147 | #endif
148 | 	HogFeatures hog_features;
149 | 	ColorspaceFeatures colorspace_feature;
150 | 	CnFeatures cn_features;
151 | 	IcFeatures ic_features;
152 | 	
153 | 	// extra parameters
154 | 	CgOpts CG_opts;
155 | 	float max_score_threshhold = 0.1;
156 | 
157 | 	// Global feature parameters1s
158 | 	int normalize_power = 2;
159 | 	bool normalize_size = true;
160 | 	bool normalize_dim = true;
161 | 
162 | 	// img sample parameters
163 | 	string search_area_shape = "square"; // The shape of the samples
164 | 	float search_area_scale = 4.0;		 // The scaling of the target size to get the search area
165 | 	int min_image_sample_size = 22500;   // Minimum area of image samples, 200x200
166 | 	int max_image_sample_size = 40000;   // Maximum area of image samples, 250x250
167 | 
168 | 	// Detection parameters
169 | 	int refinement_iterations = 1; // Number of iterations used to refine the resulting position in a frame
170 | 	int newton_iterations = 5;	 // The number of Newton iterations used for optimizing the detection score
171 | 	bool clamp_position = false;   // Clamp the target position to be inside the image
172 | 
173 | 	// Learning parameters
174 | 	float output_sigma_factor = 1.0f / 16.0f; // Label function sigma
175 | 	float learning_rate = 0.009; // Learning rate
176 | 	size_t nSamples = 30; // Maximum number of stored training samples
177 | 	string sample_replace_strategy = "lowest_prior"; // Which sample to replace when the memory is full
178 | 	bool lt_size = 0; // The size of the long - term memory(where all samples have equal weight)
179 | 	int train_gap = 5; // The number of intermediate frames with no training(0 corresponds to training every frame)
180 | 	int skip_after_frame = 10; // After which frame number the sparse update scheme should start(1 is directly)
181 | 	bool use_detection_sample = true; // Use the sample that was extracted at the detection stage also for learning
182 | 
183 | 	// Factorized convolution parameters
184 | 	bool use_projection_matrix = true;	// Use projection matrix, i.e. use the factorized convolution formulation
185 | 	bool update_projection_matrix = true; // Whether the projection matrix should be optimized or not
186 | 	string proj_init_method = "pca"; // Method for initializing the projection matrix
187 | 	float projection_reg = 1e-7; // Regularization paremeter of the projection matrix (lambda)
188 | 
189 | 	// Generative sample space model parameters
190 | 	bool use_sample_merge = true; // Use the generative sample space model to merge samples
191 | 	string sample_merge_type = "Merge"; // Strategy for updating the samples
192 | 	string distance_matrix_update_type = "exact"; // Strategy for updating the distance matrix
193 | 
194 | 	// Conjugate Gradient parameters
195 | 	int CG_iter = 5; // The number of Conjugate Gradient iterations in each update after the first frame
196 | 	int init_CG_iter = 10 * 15; // The total number of Conjugate Gradient iterations used in the first frame
197 | 	int init_GN_iter = 10; // The number of Gauss-Newton iterations used in the first frame(only if the projection matrix is updated)
198 | 	bool CG_use_FR = false; // Use the Fletcher-Reeves(true) or Polak-Ribiere(false) formula in the Conjugate Gradient
199 | 	bool CG_standard_alpha = true;  // Use the standard formula for computing the step length in Conjugate Gradient
200 | 	int CG_forgetting_rate = 50; // Forgetting rate of the last conjugate direction
201 | 	float precond_data_param = 0.75; // Weight of the data term in the preconditioner
202 | 	float precond_reg_param = 0.25; // Weight of the regularization term in the preconditioner
203 | 	int precond_proj_param = 40; // Weight of the projection matrix part in the preconditioner
204 | 
205 | 	// Regularization window parameters
206 | 	bool use_reg_window = true; // Use spatial regularization or not
207 | 	double reg_window_min = 1e-4; // The minimum value of the regularization window
208 | 	double reg_window_edge = 10e-3; // The impact of the spatial regularization
209 | 	size_t reg_window_power = 2; // The degree of the polynomial to use(e.g. 2 is a quadratic window)
210 | 	float reg_sparsity_threshold = 0.05; // A relative threshold of which DFT coefficients that should be set to zero
211 | 
212 | 	// Interpolation parameters
213 | 	string interpolation_method = "bicubic"; // The kind of interpolation kernel
214 | 	float interpolation_bicubic_a = -0.75;   // The parameter for the bicubic interpolation kernel
215 | 	bool interpolation_centering = true; // Center the kernel at the feature sample
216 | 	bool interpolation_windowing = false; // Do additional windowing on the Fourier coefficients of the kernel
217 | 
218 | 	// Scale parameters for the translation model
219 | 	// Only used if: use_scale_filter = false
220 | 	size_t number_of_scales = 7; // Number of scales to run the detector
221 | 	float scale_step = 1.01f; // The scale factor
222 | 	float min_scale_factor;
223 | 	float max_scale_factor;
224 | 
225 | 	// Scale filter parameters
226 | 	// Only used if: use_scale_filter = true
227 | 	bool use_scale_filter = false; // Use the fDSST scale filter or not (for speed)
228 | 	float scale_sigma_factor = 1.0f / 16.0f; // Scale label function sigma
229 | 	float scale_learning_rate = 0.025;		 // Scale filter learning rate
230 | 	int number_of_scales_filter = 17;		 // Number of scales
231 | 	int number_of_interp_scales = 33;		 // Number of interpolated scales
232 | 	float scale_model_factor = 1.0;			 // Scaling of the scale model
233 | 	float scale_step_filter = 1.02;			 // The scale factor for the scale filter
234 | 	float scale_model_max_area = 32 * 16;	 // Maximume area for the scale sample patch
235 | 	string scale_feature = "HOG4";			 // Features for the scale filter (only HOG4 supported)
236 | 	int s_num_compressed_dim = 17;	 // Number of compressed feature dimensions in the scale filter
237 | 	float lambda = 1e-2;					 // Scale filter regularization
238 | 	float do_poly_interp = true;			 // Do 2nd order polynomial interpolation to obtain more accurate scale
239 | 	cv::Size scale_model_sz;
240 | 
241 | 
242 | 	bool debug = 0; // to show heatmap or not
243 | 
244 | 	// GPU
245 | 	bool use_gpu = true; // whether Caffe use gpu or not
246 | 	int gpu_id = 0;
247 | };
248 | } // namespace eco
249 | #endif
250 | 


--------------------------------------------------------------------------------
/ECO/feature_operator.cc:
--------------------------------------------------------------------------------
  1 | #include "feature_operator.hpp"
  2 | 
  3 | namespace eco
  4 | {
  5 | ECO_FEATS do_dft(const ECO_FEATS &xlw)
  6 | {
  7 | 	ECO_FEATS xlf;
  8 | 	for (size_t i = 0; i < xlw.size(); i++)
  9 | 	{
 10 | 		std::vector<cv::Mat> temp;
 11 | 		for (size_t j = 0; j < xlw[i].size(); j++)
 12 | 		{
 13 | 			int size = xlw[i][j].rows;
 14 | 			if (size % 2 == 1)
 15 | 				temp.push_back(fftshift(dft(xlw[i][j])));
 16 | 			else
 17 | 			{
 18 | 				cv::Mat xf = fftshift(dft(xlw[i][j]));
 19 | 				cv::Mat xf_pad = subwindow(xf, cv::Rect(cv::Point(0, 0), cv::Size(size + 1, size + 1)));
 20 | 				for (size_t k = 0; k < (size_t)xf_pad.rows; k++)
 21 | 				{
 22 | 					xf_pad.at<cv::Vec<float, 2>>(size, k) = xf_pad.at<cv::Vec<float, 2>>(size - 1, k).conj();
 23 | 					xf_pad.at<cv::Vec<float, 2>>(k, size) = xf_pad.at<cv::Vec<float, 2>>(k, size - 1).conj();
 24 | 				}
 25 | 				temp.push_back(xf_pad);
 26 | 			}
 27 | 		}
 28 | 
 29 | 		xlf.push_back(temp);
 30 | 	}
 31 | 	return xlf;
 32 | }
 33 | // Do the element-wise multiplication for the two matrix.
 34 | ECO_FEATS do_windows(const ECO_FEATS &xl, vector<cv::Mat> &cos_win)
 35 | {
 36 | 	ECO_FEATS xlw;
 37 | 	for (size_t i = 0; i < xl.size(); i++) // for each feature
 38 | 	{
 39 | 		vector<cv::Mat> temp;
 40 | 		//debug("xl[%lu]: %lu", i, xl[i].size()); //96, 512, 31
 41 | 		for (size_t j = 0; j < xl[i].size(); j++) // for the dimensions fo the feature
 42 | 			temp.push_back(cos_win[i].mul(xl[i][j]));
 43 | 		xlw.push_back(temp);
 44 | 	}
 45 | 	return xlw;
 46 | }
 47 | 
 48 | void FilterSymmetrize(ECO_FEATS &hf)
 49 | {
 50 | 
 51 | 	for (size_t i = 0; i < hf.size(); i++)
 52 | 	{
 53 | 		int dc_ind = (hf[i][0].rows + 1) / 2;
 54 | 		for (size_t j = 0; j < (size_t)hf[i].size(); j++)
 55 | 		{
 56 | 			int c = hf[i][j].cols - 1;
 57 | 			for (size_t r = dc_ind; r < (size_t)hf[i][j].rows; r++)
 58 | 			{
 59 | 				//cout << hf[i][j].at<COMPLEX>(r, c);
 60 | 				hf[i][j].at<COMPLEX>(r, c) = hf[i][j].at<COMPLEX>(2 * dc_ind - r - 2, c).conj();
 61 | 			}
 62 | 		}
 63 | 	}
 64 | }
 65 | 
 66 | vector<cv::Mat> init_projection_matrix(const ECO_FEATS &init_sample,
 67 | 									   const vector<int> &compressed_dim,
 68 | 									   const vector<int> &feature_dim)
 69 | {
 70 | 	vector<cv::Mat> result;
 71 | 	for (size_t i = 0; i < init_sample.size(); i++) // for each feature
 72 | 	{
 73 | 		// vectorize mat init_sample
 74 | 		cv::Mat feat_vec(init_sample[i][0].size().area(), feature_dim[i], CV_32FC1);
 75 | 		//cv::Mat mean(init_sample[i][0].size().area(), feature_dim[i], CV_32FC1);
 76 | 		for (unsigned int j = 0; j < init_sample[i].size(); j++) // for each dimension of the feature
 77 | 		{
 78 | 			float mean = cv::mean(init_sample[i][j])[0]; // get the mean value of the mat;
 79 | 			for (size_t r = 0; r < (size_t)init_sample[i][j].rows; r++)
 80 | 				for (size_t c = 0; c < (size_t)init_sample[i][j].cols; c++)
 81 | 					feat_vec.at<float>(c * init_sample[i][j].rows + r, j) = init_sample[i][j].at<float>(r, c) - mean;
 82 | 		}
 83 | 		result.push_back(feat_vec);
 84 | 	}
 85 | 	//printMat(result[0]); // 3844 x 31
 86 | 
 87 | 	vector<cv::Mat> proj_mat;
 88 | 	// do SVD and dimension reduction for each feature
 89 | 	for (size_t i = 0; i < result.size(); i++)
 90 | 	{
 91 | 		cv::Mat S, V, D;
 92 | 		cv::SVD::compute(result[i].t() * result[i], S, V, D);
 93 | 		vector<cv::Mat> V_;
 94 | 		V_.push_back(V);									  // real part
 95 | 		V_.push_back(cv::Mat::zeros(V.size(), CV_32FC1));	 // image part
 96 | 		cv::merge(V_, V);									  // merge to 2 channel mat, represent a complex
 97 | 		proj_mat.push_back(V.colRange(0, compressed_dim[i])); // get the previous compressed number components
 98 | 	}
 99 | 
100 | 	return proj_mat;
101 | }
102 | // Do projection row vector x_mat[i] * projection_matrix[i]
103 | ECO_FEATS FeatureProjection(const ECO_FEATS &x, const std::vector<cv::Mat> &projection_matrix)
104 | {
105 | 	ECO_FEATS result;
106 | 	for (size_t i = 0; i < x.size(); i++) // for each feature
107 | 	{
108 | 		// vectorize the mat
109 | 		cv::Mat x_mat;
110 | 		for (size_t j = 0; j < x[i].size(); j++) // for each channel of original feature
111 | 		{
112 | 			// transform x[i][j]:
113 | 			// x1 x4
114 | 			// x2 x5
115 | 			// x3 x6
116 | 			// to [x1 x2 x3 x4 x5 x6]
117 | 			cv::Mat t = x[i][j].t();
118 | 			x_mat.push_back(cv::Mat(1, x[i][j].size().area(), CV_32FC2, t.data)); // vectorize each channel map to row vector
119 | 		}
120 | 		//debug("x_mat %lu: %d x %d", i, x_mat.rows, x_mat.cols);
121 | 		x_mat = x_mat.t();
122 | 		// do projection by matrix production
123 | 		cv::Mat res_temp = x_mat * projection_matrix[i]; // each col is a reduced feature
124 | 
125 | 		// transform back to standard formation
126 | 		std::vector<cv::Mat> temp;
127 | 		for (size_t j = 0; j < (size_t)res_temp.cols; j++) // for each channel of reduced feature
128 | 		{
129 | 			cv::Mat temp2 = res_temp.col(j); // just a header of the col, no data copied
130 | 			cv::Mat tt;
131 | 			temp2.copyTo(tt); // the memory should be continous, prepare for transform back
132 | 			cv::Mat temp3(x[i][0].cols, x[i][0].rows, CV_32FC2, tt.data);
133 | 			temp.push_back(temp3.t());
134 | 		}
135 | 		result.push_back(temp);
136 | 	}
137 | 	return result;
138 | }
139 | 
140 | ECO_FEATS FeatureProjectionMultScale(const ECO_FEATS &x, const std::vector<cv::Mat> &projection_matrix)
141 | {
142 | 	ECO_FEATS result;
143 | 	//vector<cv::Mat> featsVec = FeatureVectorization(x);
144 | 	for (size_t i = 0; i < x.size(); i++)
145 | 	{
146 | 		int org_dim = projection_matrix[i].rows;
147 | 		int numScales = x[i].size() / org_dim;
148 | 		std::vector<cv::Mat> temp;
149 | 
150 | 		for (size_t s = 0; s < (size_t)numScales; s++) // for every scale
151 | 		{
152 | 			cv::Mat x_mat;
153 | 			for (size_t j = s * org_dim; j < (s + 1) * org_dim; j++)
154 | 			{
155 | 				cv::Mat t = x[i][j].t();
156 | 				x_mat.push_back(cv::Mat(1, x[i][j].size().area(), CV_32FC1, t.data));
157 | 			}
158 | 
159 | 			x_mat = x_mat.t();
160 | 
161 | 			cv::Mat res_temp = x_mat * real(projection_matrix[i]);
162 | 
163 | 			//**** reconver to standard formation ****
164 | 			for (size_t j = 0; j < (size_t)res_temp.cols; j++)
165 | 			{
166 | 				cv::Mat temp2 = res_temp.col(j);
167 | 				cv::Mat tt;
168 | 				temp2.copyTo(tt);											  // the memory should be continous!!!!!!!!!!
169 | 				cv::Mat temp3(x[i][0].cols, x[i][0].rows, CV_32FC1, tt.data); //(x[i][0].cols, x[i][0].rows, CV_32FC2, temp2.data) int size[2] = { x[i][0].cols, x[i][0].rows };cv::Mat temp3 = temp2.reshape(2, 2, size)
170 | 				temp.push_back(temp3.t());
171 | 			}
172 | 		}
173 | 		result.push_back(temp);
174 | 	}
175 | 	return result;
176 | }
177 | // inputs: a+bi, c+di; output: ac+bd; // gpu_implemented
178 | float FeatureComputeInnerProduct(const ECO_FEATS &feat1, const ECO_FEATS &feat2)
179 | {
180 | 	if (feat1.size() != feat2.size())
181 | 		return 0;
182 | 
183 | 	float dist = 0;
184 | 	for (size_t i = 0; i < feat1.size(); i++)
185 | 	{
186 | 		for (size_t j = 0; j < feat1[i].size(); j++)
187 | 		{
188 | 			cv::Mat feat2_conj = mat_conj(feat2[i][j]);
189 | 			dist += mat_sum_f(real(complexDotMultiplication(feat1[i][j], 
190 | 															feat2_conj)));
191 | 		}
192 | 	}
193 | 	return dist;
194 | }
195 | // compute the energy of the feature
196 | // input: a+bi; output: a^2+b^2; // gpu_implemented
197 | float FeatureComputeEnergy(const ECO_FEATS &feat)
198 | {
199 | 	float res = 0;
200 | 	if (feat.empty())
201 | 		return res;
202 | 
203 | 	res = FeatureComputeInnerProduct(feat, feat);
204 | 	return res;
205 | }
206 | // compute the conv(feats) .* feats
207 | ECO_FEATS FeautreComputePower2(const ECO_FEATS &feats)
208 | {
209 | 	ECO_FEATS result;
210 | 	if (feats.empty())
211 | 	{
212 | 		return feats;
213 | 	}
214 | 
215 | 	for (size_t i = 0; i < feats.size(); i++) // for each feature
216 | 	{
217 | 		std::vector<cv::Mat> feat_vec;
218 | 		for (size_t j = 0; j < (size_t)feats[i].size(); j++) // for each dimension
219 | 		{
220 | 			cv::Mat temp(feats[i][0].size(), CV_32FC2);
221 | 			feats[i][j].copyTo(temp);
222 | 			for (size_t r = 0; r < (size_t)feats[i][j].rows; r++)
223 | 			{
224 | 				for (size_t c = 0; c < (size_t)feats[i][j].cols; c++)
225 | 				{
226 | 					temp.at<COMPLEX>(r, c)[0] = 
227 | 							std::pow(temp.at<COMPLEX>(r, c)[0], 2) + 
228 | 							std::pow(temp.at<COMPLEX>(r, c)[1], 2);
229 | 					temp.at<COMPLEX>(r, c)[1] = 0;
230 | 				}
231 | 			}
232 | 			feat_vec.push_back(temp);
233 | 		}
234 | 		result.push_back(feat_vec);
235 | 	}
236 | 	return result;
237 | }
238 | // sum up for each feature // gpu_implemented
239 | // DotMultiply for each dimension of each feature and
240 | // sum up all the dimensions for each feature
241 | std::vector<cv::Mat> FeatureComputeScores(const ECO_FEATS &x, 
242 | 										  const ECO_FEATS &f)
243 | {
244 | 	std::vector<cv::Mat> res;
245 | 	ECO_FEATS res_temp = FeatureDotMultiply(x, f);
246 | 	for (size_t i = 0; i < res_temp.size(); i++) // for each feature
247 | 	{
248 | 		cv::Mat temp(cv::Mat::zeros(res_temp[i][0].size(), 
249 | 									res_temp[i][0].type()));
250 | 		for (size_t j = 0; j < res_temp[i].size(); j++) // for each dimension
251 | 		{
252 | 			temp = temp + res_temp[i][j];
253 | 		}
254 | 		res.push_back(temp);
255 | 	}
256 | 	return res;
257 | }
258 | // vectorize features
259 | std::vector<cv::Mat> FeatureVectorization(const ECO_FEATS &x)
260 | {
261 | 	if (x.empty())
262 | 		return std::vector<cv::Mat>();
263 | 
264 | 	std::vector<cv::Mat> res;
265 | 	for (size_t i = 0; i < x.size(); i++)
266 | 	{
267 | 		cv::Mat temp;
268 | 		for (size_t j = 0; j < x[i].size(); j++)
269 | 		{
270 | 			cv::Mat temp2 = x[i][j].t();
271 | 			temp.push_back(cv::Mat(1, x[i][j].size().area(), 
272 | 								   CV_32FC2, temp2.data));
273 | 		}
274 | 		res.push_back(temp);
275 | 	}
276 | 	return res;
277 | }
278 | 
279 | ECO_FEATS FeatureVectorMultiply(const ECO_FEATS &x,
280 | 								const std::vector<cv::Mat> &y,
281 | 								const bool _conj)
282 | {
283 | 	if (x.size() != y.size())
284 | 		assert(0 && "Feature and Vector Size Unmatched!");
285 | 
286 | 	ECO_FEATS res;
287 | 	for (size_t i = 0; i < x.size(); i++)
288 | 	{
289 | 		vector<cv::Mat> temp;
290 | 		for (size_t j = 0; j < x[i].size(); j++)
291 | 		{
292 | 			if (_conj)
293 | 				temp.push_back(complexDotMultiplication(mat_conj(x[i][j]), y[i]));
294 | 			else
295 | 				temp.push_back(complexDotMultiplication(x[i][j], y[i]));
296 | 		}
297 | 		res.push_back(temp);
298 | 	}
299 | 	return res;
300 | }
301 | 
302 | // features operation 
303 | ECO_FEATS FeatureDotMultiply(const ECO_FEATS &a, const ECO_FEATS &b)
304 | {
305 | 	ECO_FEATS res;
306 | 	if (a.size() != b.size())
307 | 		assert(0 && "Unmatched feature size!");
308 | 
309 | 	for (size_t i = 0; i < a.size(); i++)
310 | 	{
311 | 		std::vector<cv::Mat> temp;
312 | 		for (size_t j = 0; j < a[i].size(); j++)
313 | 		{
314 | 			temp.push_back(complexDotMultiplication(a[i][j], b[i][j]));
315 | 		}
316 | 		res.push_back(temp);
317 | 	}
318 | 	return res;
319 | }
320 | ECO_FEATS FeatureDotDivide(const ECO_FEATS &a, const ECO_FEATS &b)
321 | {
322 | 	ECO_FEATS res;
323 | 	if (a.size() != b.size())
324 | 		assert(0 && "Unmatched feature size!");
325 | 
326 | 	for (size_t i = 0; i < a.size(); i++)
327 | 	{
328 | 		std::vector<cv::Mat> temp;
329 | 		for (size_t j = 0; j < a[i].size(); j++)
330 | 		{
331 | 			temp.push_back(complexDotDivision(a[i][j], b[i][j]));
332 | 		}
333 | 		res.push_back(temp);
334 | 	}
335 | 	return res;
336 | }
337 | } // namespace eco


--------------------------------------------------------------------------------
/ECO/eco_unittest.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2005, Google Inc.
  2 | // All rights reserved.
  3 | //
  4 | // Redistribution and use in source and binary forms, with or without
  5 | // modification, are permitted provided that the following conditions are
  6 | // met:
  7 | //
  8 | //     * Redistributions of source code must retain the above copyright
  9 | // notice, this list of conditions and the following disclaimer.
 10 | //     * Redistributions in binary form must reproduce the above
 11 | // copyright notice, this list of conditions and the following disclaimer
 12 | // in the documentation and/or other materials provided with the
 13 | // distribution.
 14 | //     * Neither the name of Google Inc. nor the names of its
 15 | // contributors may be used to endorse or promote products derived from
 16 | // this software without specific prior written permission.
 17 | //
 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 | // A sample program demonstrating using Google C++ testing framework.
 31 | //
 32 | // Author: wan@google.com (Zhanyong Wan)
 33 | 
 34 | #include "gtest/gtest.h"
 35 | #include "ffttools.hpp"
 36 | #include "debug.hpp"
 37 | 
 38 | #ifdef USE_FFTW
 39 | #include <fftw3.h>
 40 | #endif
 41 | 
 42 | namespace
 43 | {
 44 | TEST(ffttoolsTest, dft_float)
 45 | {
 46 |   int N = 5;
 47 |   cv::Mat_<float> mat_float(N, 2*N, CV_32FC1);
 48 |   for (int j = 0; j < mat_float.rows; j++)
 49 |     for (int i = 0; i < mat_float.cols; i++)
 50 |       mat_float.at<float>(j, i) = i + j * mat_float.cols;
 51 |   //showmat1channels(mat_float, 2);
 52 | 
 53 |   double timer = (double)cv::getTickCount();
 54 |   float timedft = 0;
 55 | 
 56 |   cv::Mat res = eco::dft(mat_float)/(mat_float.rows * mat_float.cols);
 57 |   //printMat(res);
 58 |   //showmat2channels(res, 2);
 59 | 
 60 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
 61 |   debug("dft time: %f", timedft);
 62 | 	timer = (double)cv::getTickCount();
 63 | 
 64 |   res = eco::dft(res, 1);
 65 |   //showmat2channels(res, 2);
 66 | 
 67 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
 68 |   debug("dft time: %f", timedft);
 69 | }
 70 | 
 71 | TEST(ffttoolsTest, dft_double)
 72 | {
 73 |   int N = 5;
 74 |   cv::Mat_<double> mat_double(N, 2*N, CV_64FC1);
 75 |   for (int j = 0; j < mat_double.rows; j++)
 76 |     for (int i = 0; i < mat_double.cols; i++)
 77 |       mat_double.at<double>(j, i) = i + j * mat_double.cols;
 78 |   //showmat1channels(mat_double, 3);
 79 | 
 80 |   double timer = (double)cv::getTickCount();
 81 |   float timedft = 0;
 82 | 
 83 |   cv::Mat res = eco::dft(mat_double)/(mat_double.rows * mat_double.cols);
 84 |   //showmat2channels(res, 3);
 85 | 
 86 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
 87 |   debug("dft_d time: %f", timedft);
 88 | 	timer = (double)cv::getTickCount();
 89 | 
 90 |   res = eco::dft(res, 1);
 91 |   //showmat2channels(res, 3);
 92 | 
 93 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
 94 |   debug("dft_d time: %f", timedft);
 95 | }
 96 | /*
 97 | TEST(ffttoolsTest, fftshift)
 98 | {
 99 |   cv::Mat_<float> mat_float(10, 10, CV_32FC1);
100 |   for (int j = 0; j < mat_float.rows; j++)
101 |     for (int i = 0; i < mat_float.cols; i++)
102 |       mat_float.at<float>(j, i) = i + j * mat_float.cols;
103 |   debug("channels: %d", mat_float.channels());
104 | 
105 |   showmat1channels(mat_float, 2);
106 |   cv::Mat res;
107 |   res = eco::fftshift(mat_float);
108 |   showmat1channels(res, 2);
109 | 
110 |   res = eco::dft(mat_float, 1);
111 |   showmat2channels(res, 2);
112 | 
113 |   res = eco::fftshift(res);
114 |   showmat2channels(res, 2);
115 | }
116 | 
117 | TEST(ffttoolsTest, fftshift)
118 | {
119 |   cv::Mat_<double> mat_double(10, 10, CV_32FC1);
120 |   for (int j = 0; j < mat_double.rows; j++)
121 |     for (int i = 0; i < mat_double.cols; i++)
122 |       mat_double.at<double>(j, i) = i + j * mat_double.cols;
123 |   debug("channels: %d", mat_double.channels());
124 | 
125 |   showmat1channels(mat_double, 3);
126 |   cv::Mat res;
127 |   res = eco::fftshift(mat_double);
128 |   showmat1channels(res, 3);
129 | 
130 |   res = eco::dft_d(mat_double, 1);
131 |   showmat2channels(res, 3);
132 | 
133 |   res = eco::fftshift(res);
134 |   showmat2channels(res, 3);
135 | }
136 | 
137 | TEST(ffttoolsTest, complexDotDivision)
138 | {
139 |   cv::Mat mat_float(10, 10, CV_32FC2);
140 |   for (int j = 0; j < mat_float.rows; j++)
141 |     for (int i = 0; i < mat_float.cols; i++)
142 |     {
143 |       mat_float.at<cv::Vec2f>(j, i)[0] = i + j * mat_float.cols;
144 |       mat_float.at<cv::Vec2f>(j, i)[1] = i + j * mat_float.cols;
145 |     }
146 |   debug("channels: %d", mat_float.channels());
147 |   showmat2channels(mat_float, 2);
148 | 
149 |   cv::Mat mat_float1(10, 10, CV_32FC2);
150 |   for (int j = 0; j < mat_float1.rows; j++)
151 |     for (int i = 0; i < mat_float1.cols; i++)
152 |     {
153 |       mat_float1.at<cv::Vec2f>(j, i)[0] = i + j * mat_float1.cols;
154 |       mat_float1.at<cv::Vec2f>(j, i)[1] = -i;
155 |     }
156 |   debug("channels: %d", mat_float1.channels());
157 |   showmat2channels(mat_float1, 2);
158 | 
159 |   cv::Mat res;
160 |   res = eco::complexDotDivision(mat_float, mat_float1);
161 |   showmat2channels(res, 2);
162 | }
163 | 
164 | TEST(ffttoolsTest, complexMatrixMultiplication)
165 | {
166 |   cv::Mat mat_float(10, 10, CV_32FC2);
167 |   for (int j = 0; j < mat_float.rows; j++)
168 |     for (int i = 0; i < mat_float.cols; i++)
169 |     {
170 |       mat_float.at<cv::Vec2f>(j, i)[0] = i + j * mat_float.cols;
171 |       mat_float.at<cv::Vec2f>(j, i)[1] = i + j * mat_float.cols;
172 |     }
173 |   debug("channels: %d", mat_float.channels());
174 |   showmat2channels(mat_float, 2);
175 | 
176 |   cv::Mat mat_float1(10, 10, CV_32FC2);
177 |   for (int j = 0; j < mat_float1.rows; j++)
178 |     for (int i = 0; i < mat_float1.cols; i++)
179 |     {
180 |       mat_float1.at<cv::Vec2f>(j, i)[0] = i + j * mat_float1.cols;
181 |       mat_float1.at<cv::Vec2f>(j, i)[1] = -i;
182 |     }
183 |   debug("channels: %d", mat_float1.channels());
184 |   showmat2channels(mat_float1, 2);
185 | 
186 |   cv::Mat res;
187 |   res = eco::complexMatrixMultiplication(mat_float, mat_float1);
188 |   showmat2channels(res, 2);
189 | }
190 | 
191 | TEST(ffttoolsTest, mat_sum_f)
192 | {
193 |   cv::Mat_<float> mat_float(10, 10, CV_32FC1);
194 |   for (int j = 0; j < mat_float.rows; j++)
195 |     for (int i = 0; i < mat_float.cols; i++)
196 |       mat_float.at<float>(j, i) = i + j * mat_float.cols;
197 | 
198 |   EXPECT_EQ(4950, eco::mat_sum_f(mat_float));
199 | }
200 | 
201 | TEST(ffttoolsTest, mat_sum_d)
202 | {
203 |   cv::Mat_<double> mat_double(10, 10, CV_64FC1);
204 |   for (int j = 0; j < mat_double.rows; j++)
205 |     for (int i = 0; i < mat_double.cols; i++)
206 |       mat_double.at<double>(j, i) = i + j * mat_double.cols;
207 | 
208 |   EXPECT_EQ(4950, eco::mat_sum_d(mat_double));
209 | }
210 | 
211 | TEST(ffttoolsTest, rot90)
212 | {
213 |   cv::Mat_<float> mat_float(10, 10, CV_32FC1);
214 |   for (int j = 0; j < mat_float.rows; j++)
215 |     for (int i = 0; i < mat_float.cols; i++)
216 |       mat_float.at<float>(j, i) = i + j * mat_float.cols;
217 | 
218 |   showmat1channels(mat_float, 2);
219 |   debug("==============");
220 |   eco::rot90(mat_float, 1);
221 |   showmat1channels(mat_float, 2);
222 |   debug("==============");
223 |   eco::rot90(mat_float, 2);
224 |   showmat1channels(mat_float, 2);
225 |   debug("==============");
226 |   eco::rot90(mat_float, 3);
227 |   showmat1channels(mat_float, 2);
228 | }
229 | 
230 | TEST(ffttoolsTest, complexConvolution)
231 | {
232 |   cv::Mat mat_float(14, 14, CV_32FC2);
233 |   for (int j = 0; j < mat_float.rows; j++)
234 |     for (int i = 0; i < mat_float.cols; i++)
235 |     {
236 |       mat_float.at<cv::Vec2f>(j, i)[0] = i + j * mat_float.cols;
237 |       mat_float.at<cv::Vec2f>(j, i)[1] = i + j * mat_float.cols;
238 |     }
239 |   debug("channels: %d", mat_float.channels());
240 |   showmat2channels(mat_float, 2);
241 | 
242 |   cv::Mat mat_float1(9, 9, CV_32FC2);
243 |   for (int j = 0; j < mat_float1.rows; j++)
244 |     for (int i = 0; i < mat_float1.cols; i++)
245 |     {
246 |       mat_float1.at<cv::Vec2f>(j, i)[0] = i + j * mat_float1.cols;
247 |       mat_float1.at<cv::Vec2f>(j, i)[1] = -i;
248 |     }
249 |   debug("channels: %d", mat_float1.channels());
250 |   showmat2channels(mat_float1, 2);
251 | 
252 |   cv::Mat res;
253 |   res = eco::complexConvolution(mat_float, mat_float1, 0);
254 |   showmat2channels(res, 2);
255 |   res = eco::complexConvolution(mat_float, mat_float1, 1);
256 |   showmat2channels(res, 2);
257 | }
258 | 
259 | TEST(debug, copyTo_clone_Difference)
260 | {
261 |   copyTo_clone_Difference();
262 | }
263 | */
264 | 
265 | TEST(ffttoolsTest, complexDotMultiplication)
266 | {
267 |   int N = 5;
268 |   cv::Mat mat_float(N, N*2, CV_32FC2);
269 |   for (int j = 0; j < mat_float.rows; j++)
270 |     for (int i = 0; i < mat_float.cols; i++)
271 |     {
272 |       mat_float.at<cv::Vec2f>(j, i)[0] = i + j * mat_float.cols;
273 |       mat_float.at<cv::Vec2f>(j, i)[1] = i + j * mat_float.cols;
274 |     }
275 |   //showmat2channels(mat_float, 2);
276 | 
277 |   cv::Mat mat_float1(N, N*2, CV_32FC2);
278 |   for (int j = 0; j < mat_float1.rows; j++)
279 |     for (int i = 0; i < mat_float1.cols; i++)
280 |     {
281 |       mat_float1.at<cv::Vec2f>(j, i)[0] = i + j * mat_float1.cols;
282 |       mat_float1.at<cv::Vec2f>(j, i)[1] = -i;
283 |     }
284 |   //showmat2channels(mat_float1, 2);
285 | 
286 |   // complexDotMultiplicationCPU
287 |   cv::Mat res;
288 |   res = eco::complexDotMultiplicationCPU(mat_float, mat_float1);  
289 |   int iter = 70;
290 |   double timer = (double)cv::getTickCount();
291 |   float timedft = 0;
292 |   while (iter > 0)
293 |   {
294 |     res = eco::complexDotMultiplicationCPU(mat_float, mat_float1);  
295 |     iter--;
296 |   }
297 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
298 |   debug("complexDotMultiplicationCPU time: %f", timedft);
299 |   //showmat2channels(res, 2);
300 |   
301 |   // complexDotMultiplicationSIMD
302 | #ifdef USE_SIMD
303 |   res = eco::complexDotMultiplicationSIMD(mat_float, mat_float1);
304 |   iter = 70;
305 |   timer = (double)cv::getTickCount();
306 |   while (iter > 0)
307 |   {
308 |     res = eco::complexDotMultiplicationSIMD(mat_float, mat_float1);  
309 |     iter--;
310 |   }
311 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
312 |   debug("complexDotMultiplicationSIMD time: %f", timedft);
313 |   //showmat2channels(res, 2);
314 | #endif
315 | 
316 | /*
317 | #ifdef USE_CUDA
318 |   cv::cuda::setDevice(0);
319 |   debug("%d", cv::cuda::getDevice());
320 |   //res = eco::complexDotMultiplicationGPU(mat_float, mat_float1);
321 |   iter = 1;
322 |   timer = (double)cv::getTickCount();
323 |   while (iter > 0)
324 |   {
325 |     res = eco::complexDotMultiplicationGPU(mat_float, mat_float1);
326 |     iter--;
327 |   }
328 |   timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
329 |   debug("complexDotMultiplicationGPU time: %f", timedft);
330 |   //showmat2channels(res, 2);
331 | #endif
332 | */
333 | }//TEST
334 | TEST(matReferenceTest, matReferenceTest)
335 | {
336 |   eco::matReferenceTest();
337 | }
338 | } //namespace


--------------------------------------------------------------------------------
/ECO/sample_update.cc:
--------------------------------------------------------------------------------
  1 | #include "sample_update.hpp"
  2 | 
  3 | namespace eco
  4 | {
  5 | void SampleUpdate::init(const std::vector<cv::Size> &filter,
  6 | 						const std::vector<int> &feature_dim,
  7 | 						const size_t nSamples,
  8 | 						const float learning_rate)
  9 | {
 10 | 	distance_matrix_.release();
 11 | 	gram_matrix_.release();
 12 | 	nSamples_ = nSamples;
 13 | 	learning_rate_ = learning_rate;
 14 | 	sample_weight_.clear();
 15 | 	samples_f_.clear();
 16 | 	num_training_samples_ = 0;
 17 | 	prior_weights_.clear();
 18 | 	new_sample_.clear();
 19 | 	merged_sample_.clear();
 20 | 	new_sample_id_ = -1;
 21 | 	merged_sample_id_ = -1;
 22 | 
 23 | 	// distance matrix initialization
 24 | 	distance_matrix_.create(cv::Size(nSamples_, nSamples_), CV_32FC2);
 25 | 	gram_matrix_.create(cv::Size(nSamples_, nSamples_), CV_32FC2);
 26 | 
 27 | 	// initialization to INF
 28 | 	for (size_t i = 0; i < (size_t)distance_matrix_.rows; i++)
 29 | 	{
 30 | 		for (size_t j = 0; j < (size_t)distance_matrix_.cols; j++)
 31 | 		{
 32 | 			distance_matrix_.at<cv::Vec<float, 2>>(i, j) = cv::Vec<float, 2>(INF, 0);
 33 | 			gram_matrix_.at<cv::Vec<float, 2>>(i, j) = cv::Vec<float, 2>(INF, 0);
 34 | 		}
 35 | 	}
 36 | 	// Samples memory initialization
 37 | 	samples_f_.clear();
 38 | 
 39 | 	for (size_t n = 0; n < nSamples_; n++)
 40 | 	{
 41 | 		ECO_FEATS temp;
 42 | 		for (size_t j = 0; j < (size_t)feature_dim.size(); j++) // for each feature
 43 | 		{
 44 | 			std::vector<cv::Mat> temp_single_feat;
 45 | 			for (size_t i = 0; i < (size_t)feature_dim[j]; i++) // for each dimension of the feature
 46 | 				temp_single_feat.push_back(cv::Mat::zeros(
 47 | 					cv::Size((filter[j].height + 1) / 2, filter[j].width),
 48 | 					CV_32FC2));
 49 | 			temp.push_back(temp_single_feat);
 50 | 		}
 51 | 		samples_f_.push_back(temp);
 52 | 	}
 53 | 
 54 | 	// resize prior weights to the same as nSamples_
 55 | 
 56 | 	prior_weights_.resize(nSamples_);
 57 | 
 58 | 	// Show debug.
 59 | 	for (size_t j = 0; j < (size_t)feature_dim.size(); j++)
 60 | 	{
 61 | 		debug("samples: %lu, feature %lu, size: %lu, mat: %d x %d",
 62 | 			  nSamples_, j, samples_f_[nSamples_ - 1][j].size(),
 63 | 			  samples_f_[nSamples_ - 1][j][feature_dim[j] - 1].rows,
 64 | 			  samples_f_[nSamples_ - 1][j][feature_dim[j] - 1].cols);
 65 | 	}
 66 | 	/*
 67 | 	debug("prior_weights_ size: %lu ", prior_weights_.size());
 68 | 	for (size_t j = 0; j < (size_t)prior_weights_.size(); j++)
 69 | 	{
 70 | 		printf("%f ", prior_weights_[j]);
 71 | 	}
 72 | 	printf("\n");
 73 | 	*/
 74 | }
 75 | 
 76 | void SampleUpdate::update_sample_space_model(const ECO_FEATS &new_train_sample)
 77 | {
 78 | 	// Calculate the distance
 79 | 	cv::Mat gram_vector = find_gram_vector(new_train_sample);				  // 32FC2 50 x 1, 2(ac+bd)
 80 | 	float new_train_sample_norm = 2 * FeatureComputeEnergy(new_train_sample); //2(c^2+d^2)
 81 | 	cv::Mat distance(nSamples_, 1, CV_32FC2);
 82 | 	for (size_t i = 0; i < nSamples_; i++)
 83 | 	{
 84 | 		// a^2 + b^2 + c^2 + d^2 - 2(ac+bd)
 85 | 		float temp = new_train_sample_norm + gram_matrix_.at<cv::Vec<float, 2>>(i, i)[0] - 2 * gram_vector.at<cv::Vec<float, 2>>(i, 0)[0];
 86 | 		if (i < (size_t)num_training_samples_)
 87 | 			distance.at<cv::Vec<float, 2>>(i, 0) =
 88 | 				cv::Vec<float, 2>(std::max(temp, 0.0f), 0);
 89 | 		else
 90 | 			distance.at<cv::Vec<float, 2>>(i, 0) = cv::Vec<float, 2>(INF, 0);
 91 | 	}
 92 | 	// End of calcualte the distance
 93 | 
 94 | 	if (num_training_samples_ == nSamples_) // if memory is full
 95 | 	{
 96 | 		float min_sample_weight_ = INF;
 97 | 		size_t min_sample_id = 0;
 98 | 		findMin(min_sample_weight_, min_sample_id);
 99 | 		//	debug("min_sample: %d %f", min_sample_id, min_sample_weight_);
100 | 
101 | 		if (min_sample_weight_ < minmum_sample_weight_)
102 | 		// If any prior weight is less than the minimum allowed weight,
103 | 		// replace that sample with the new sample
104 | 		{
105 | 			update_distance_matrix(gram_vector, new_train_sample_norm, min_sample_id, -1, 0, 1);
106 | 			prior_weights_[min_sample_id] = 0;
107 | 			// normalize the prior_weights_
108 | 			float sum = std::accumulate(prior_weights_.begin(), prior_weights_.end(), 0.0f);
109 | 			for (size_t i = 0; i < (size_t)nSamples_; i++)
110 | 			{
111 | 				prior_weights_[i] = prior_weights_[i] *
112 | 									(1 - learning_rate_) / sum;
113 | 			}
114 | 			// set the new sample's weight as learning_rate_
115 | 			prior_weights_[min_sample_id] = learning_rate_;
116 | 
117 | 			// update sampel space.
118 | 			merged_sample_id_ = -1;
119 | 			new_sample_id_ = min_sample_id;
120 | 			new_sample_ = new_train_sample;
121 | 			replace_sample(new_sample_, new_sample_id_);
122 | 		}
123 | 		else // If no sample has low enough prior weight, then we either merge
124 | 			 // the new sample with an existing sample, or merge two of the
125 | 			 // existing samples and insert the new sample in the vacated position
126 | 		{
127 | 			// Find the minimum distance between new sample and exsiting samples.
128 | 			double new_sample_min_dist;
129 | 			cv::Point min_sample_id;
130 | 			cv::minMaxLoc(real(distance), &new_sample_min_dist, 0, &min_sample_id);
131 | 
132 | 			// Find the closest pair amongst existing samples.
133 | 			double existing_samples_min_dist;
134 | 			cv::Point closest_exist_sample_pair;
135 | 			cv::Mat duplicate = distance_matrix_.clone();
136 | 			cv::minMaxLoc(real(duplicate), &existing_samples_min_dist, 0, &closest_exist_sample_pair);
137 | 
138 | 			if (closest_exist_sample_pair.x == closest_exist_sample_pair.y)
139 | 				assert(0 && "error: distance matrix diagonal filled wrongly.");
140 | 
141 | 			if (new_sample_min_dist < existing_samples_min_dist)
142 | 			{
143 | 				// If the min distance of the new sample to the existing samples is less than the min distance
144 | 				// amongst any of the existing samples, we merge the new sample with the nearest existing
145 | 
146 | 				// renormalize prior weights
147 | 				for (size_t i = 0; i < prior_weights_[i]; i++)
148 | 					prior_weights_[i] *= (1 - learning_rate_);
149 | 
150 | 				// Set the position of the merged sample
151 | 				merged_sample_id_ = min_sample_id.y;
152 | 
153 | 				// Extract the existing sample to merge
154 | 				ECO_FEATS existing_sample_to_merge = samples_f_[merged_sample_id_];
155 | 
156 | 				// Merge the new_train_sample with existing sample
157 | 				merged_sample_ =
158 | 					merge_samples(existing_sample_to_merge,
159 | 								  new_train_sample,
160 | 								  prior_weights_[merged_sample_id_], learning_rate_,
161 | 								  std::string("merge"));
162 | 
163 | 				// Update distance matrix and the gram matrix
164 | 				update_distance_matrix(gram_vector, new_train_sample_norm, merged_sample_id_, -1, prior_weights_[merged_sample_id_], learning_rate_);
165 | 
166 | 				// Update the prior weight of the merged sample
167 | 				prior_weights_[min_sample_id.y] += learning_rate_;
168 | 
169 | 				// update the merged sample and discard new sample
170 | 				replace_sample(merged_sample_, merged_sample_id_);
171 | 			}
172 | 			else
173 | 			{
174 | 				// we merge the nearest existing samples and insert the new sample in the vacated position
175 | 
176 | 				// renormalize prior weights
177 | 				for (size_t i = 0; i < prior_weights_[i]; i++)
178 | 					prior_weights_[i] *= (1 - learning_rate_);
179 | 
180 | 				// Ensure that the sample with higher prior weight is assigned id1.
181 | 				if (prior_weights_[closest_exist_sample_pair.x] >
182 | 					prior_weights_[closest_exist_sample_pair.y])
183 | 					std::swap(closest_exist_sample_pair.x,
184 | 							  closest_exist_sample_pair.y);
185 | 
186 | 				// Merge the existing closest samples
187 | 				merged_sample_ =
188 | 					merge_samples(samples_f_[closest_exist_sample_pair.x],
189 | 								  samples_f_[closest_exist_sample_pair.y],
190 | 								  prior_weights_[closest_exist_sample_pair.x], prior_weights_[closest_exist_sample_pair.y],
191 | 								  std::string("merge"));
192 | 
193 | 				// Update distance matrix and the gram matrix
194 | 				update_distance_matrix(gram_vector, new_train_sample_norm, closest_exist_sample_pair.x, closest_exist_sample_pair.y, prior_weights_[closest_exist_sample_pair.x], prior_weights_[closest_exist_sample_pair.y]);
195 | 
196 | 				// Update prior weights for the merged sample and the new sample
197 | 				prior_weights_[closest_exist_sample_pair.x] +=
198 | 					prior_weights_[closest_exist_sample_pair.y];
199 | 				prior_weights_[closest_exist_sample_pair.y] = learning_rate_;
200 | 
201 | 				// Update the merged sample and insert new sample
202 | 				merged_sample_id_ = closest_exist_sample_pair.x;
203 | 				new_sample_id_ = closest_exist_sample_pair.y;
204 | 				new_sample_ = new_train_sample;
205 | 				replace_sample(merged_sample_, merged_sample_id_);
206 | 				replace_sample(new_sample_, new_sample_id_);
207 | 			}
208 | 		}
209 | 	}	// end if memory is full
210 | 	else // if memory is not full
211 | 	{
212 | 		size_t sample_position = num_training_samples_;
213 | 		update_distance_matrix(gram_vector, new_train_sample_norm, sample_position, -1, 0, 1);
214 | 
215 | 		if (sample_position == 0)
216 | 		{
217 | 			prior_weights_[sample_position] = 1;
218 | 		}
219 | 		else
220 | 		{
221 | 			for (size_t i = 0; i < sample_position; i++)
222 | 				prior_weights_[i] *= (1 - learning_rate_);
223 | 			prior_weights_[sample_position] = learning_rate_;
224 | 		}
225 | 		// update sample space
226 | 		new_sample_id_ = sample_position;
227 | 		new_sample_ = new_train_sample;
228 | 		replace_sample(new_sample_, new_sample_id_);
229 | 
230 | 		num_training_samples_++;
231 | 	}
232 | 	//debug("num_training_samples_: %lu", num_training_samples_);
233 | }
234 | 
235 | void SampleUpdate::update_distance_matrix(cv::Mat &gram_vector, float new_sample_norm, int id1, int id2, float w1, float w2)
236 | {
237 | 	float alpha1 = w1 / (w1 + w2);
238 | 	float alpha2 = 1 - alpha1;
239 | 	//debug("alpha1: %f, alpha2: %f", alpha1, alpha2);
240 | 	if (id2 < 0) //
241 | 	{
242 | 		COMPLEX norm_id1 = gram_matrix_.at<COMPLEX>(id1, id1);
243 | 
244 | 		// update the matrix
245 | 		if (alpha1 == 0)
246 | 		{
247 | 			gram_vector.col(0).copyTo(gram_matrix_.col(id1));
248 | 			cv::Mat tt = gram_vector.t();
249 | 			tt.row(0).copyTo(gram_matrix_.row(id1));
250 | 			gram_matrix_.at<COMPLEX>(id1, id1) = COMPLEX(new_sample_norm, 0);
251 | 		}
252 | 		else if (alpha2 == 0)
253 | 		{
254 | 			//  do nothing discard new sample
255 | 		}
256 | 		else
257 | 		{ // The new sample is merge with an existing sample
258 | 			cv::Mat t = alpha1 * gram_matrix_.col(id1) + alpha2 * gram_vector.col(0), t_t;
259 | 			t.col(0).copyTo(gram_matrix_.col(id1));
260 | 			t_t = t.t();
261 | 			t_t.row(0).copyTo(gram_matrix_.row(id1));
262 | 			gram_matrix_.at<COMPLEX>(id1, id1) =
263 | 				COMPLEX(std::pow(alpha1, 2) * norm_id1[0] + std::pow(alpha2, 2) * new_sample_norm + 2 * alpha1 * alpha2 * gram_vector.at<COMPLEX>(id1)[0], 0);
264 | 		}
265 | 
266 | 		// Update distance matrix
267 | 		cv::Mat distance(nSamples_, 1, CV_32FC2);
268 | 		for (size_t i = 0; i < nSamples_; i++)
269 | 		{
270 | 			float temp = gram_matrix_.at<COMPLEX>(id1, id1)[0] +
271 | 						 gram_matrix_.at<COMPLEX>(i, i)[0] -
272 | 						 2 * gram_matrix_.at<COMPLEX>(i, id1)[0];
273 | 			distance.at<COMPLEX>(i, 0) = COMPLEX(std::max(temp, 0.0f), 0);
274 | 		}
275 | 		distance.col(0).copyTo(distance_matrix_.col(id1));
276 | 		cv::Mat tt = distance.t();
277 | 		tt.row(0).copyTo(distance_matrix_.row(id1));
278 | 		distance_matrix_.at<COMPLEX>(id1, id1) = COMPLEX(INF, 0);
279 | 	}
280 | 	else
281 | 	{
282 | 		if (alpha1 == 0 || alpha2 == 0)
283 | 		{
284 | 			assert(0 && "error: alpha1 or alpha2 equals 0");
285 | 		}
286 | 		// Two existing samples are merged and the new sample fills the empty
287 | 		COMPLEX norm_id1 = gram_matrix_.at<COMPLEX>(id1, id1);
288 | 		COMPLEX norm_id2 = gram_matrix_.at<COMPLEX>(id2, id2);
289 | 		COMPLEX ip_id1_id2 = gram_matrix_.at<COMPLEX>(id1, id2);
290 | 		//debug("%d %d, %f %f, %f %f %f", id1, id2, w1, w2, norm_id1[0], norm_id2[0], ip_id1_id2[0]);
291 | 		// Handle the merge of existing samples
292 | 		cv::Mat t = alpha1 * gram_matrix_.col(id1) +
293 | 					alpha2 * gram_matrix_.col(id2),
294 | 				t_t;
295 | 
296 | 		t.col(0).copyTo(gram_matrix_.col(id1));
297 | 
298 | 		cv::Mat tt = t.t();
299 | 		tt.row(0).copyTo(gram_matrix_.row(id1));
300 | 
301 | 		gram_matrix_.at<COMPLEX>(id1, id1) =
302 | 			COMPLEX(std::pow(alpha1, 2) * norm_id1[0] +
303 | 						std::pow(alpha2, 2) * norm_id2[0] +
304 | 						2 * alpha1 * alpha2 * ip_id1_id2[0],
305 | 					0);
306 | 		gram_vector.at<COMPLEX>(id1) =
307 | 			COMPLEX(alpha1 * gram_vector.at<COMPLEX>(id1, 0)[0] +
308 | 						alpha2 * gram_vector.at<COMPLEX>(id2, 0)[0],
309 | 					0);
310 | 
311 | 		// Handle the new sample
312 | 		gram_vector.col(0).copyTo(gram_matrix_.col(id2));
313 | 		tt = gram_vector.t();
314 | 		tt.row(0).copyTo(gram_matrix_.row(id2));
315 | 		gram_matrix_.at<COMPLEX>(id2, id2) = new_sample_norm;
316 | 
317 | 		// Update the distance matrix
318 | 		cv::Mat distance(nSamples_, 1, CV_32FC2);
319 | 		std::vector<int> id({id1, id2});
320 | 		for (size_t i = 0; i < 2; i++)
321 | 		{
322 | 			for (size_t j = 0; j < nSamples_; j++)
323 | 			{
324 | 				float temp = gram_matrix_.at<COMPLEX>(id[i], id[i])[0] +
325 | 							 gram_matrix_.at<COMPLEX>(j, j)[0] -
326 | 							 2 * gram_matrix_.at<COMPLEX>(j, id[i])[0];
327 | 				distance.at<COMPLEX>(j, 0) = COMPLEX(std::max(temp, 0.0f), 0);
328 | 			}
329 | 			distance.col(0).copyTo(distance_matrix_.col(id[i]));
330 | 			cv::Mat tt = distance.t();
331 | 			tt.row(0).copyTo(distance_matrix_.row(id[i]));
332 | 			distance_matrix_.at<COMPLEX>(id[i], id[i]) = COMPLEX(INF, 0);
333 | 		}
334 | 	} //if end
335 | } //function end
336 | } // namespace eco
337 | 


--------------------------------------------------------------------------------
/MTCNN/face_detector.cpp:
--------------------------------------------------------------------------------
  1 | #include "face_detector.hpp"
  2 | #include "helpers.hpp"
  3 | 
  4 | namespace mtcnn {
  5 | 
  6 | const std::string P_NET_PROTO = "/det1.prototxt";
  7 | const std::string P_NET_WEIGHTS = "/det1.caffemodel";
  8 | const std::string P_NET_REGRESSION_BLOB_NAME = "conv4-2";
  9 | const std::string P_NET_SCORE_BLOB_NAME = "prob1";
 10 | const float P_NET_WINDOW_SIDE = 12.f;
 11 | const int P_NET_STRIDE = 2;
 12 | 
 13 | const std::string R_NET_PROTO = "/det2.prototxt";
 14 | const std::string R_NET_WEIGHTS = "/det2.caffemodel";
 15 | const std::string R_NET_REGRESSION_BLOB_NAME = "conv5-2";
 16 | const std::string R_NET_SCORE_BLOB_NAME = "prob1";
 17 | 
 18 | const std::string O_NET_PROTO = "/det3.prototxt";
 19 | const std::string O_NET_WEIGHTS = "/det3.caffemodel";
 20 | const std::string O_NET_REGRESSION_BLOB_NAME = "conv6-2";
 21 | const std::string O_NET_SCORE_BLOB_NAME = "prob1";
 22 | const std::string O_NET_PTS_BLOB_NAME = "conv6-3";
 23 | 
 24 | const std::string L_NET_PROTO = "/det4.prototxt";
 25 | const std::string L_NET_WEIGHTS = "/det4.caffemodel";
 26 | const float L_THRESHOLD = 0.35f;
 27 | 
 28 | const float IMG_MEAN = 127.5f;
 29 | const float IMG_INV_STDDEV = 1.f / 128.f;
 30 | 
 31 | FaceDetector::FaceDetector(const std::string& modelDir, 
 32 | 							float pThreshold, 
 33 | 							float rThreshold, 
 34 | 							float oThreshold, 
 35 | 							bool useLNet,
 36 | 							bool useGPU, 
 37 | 							int deviceID) : pThreshold_(pThreshold), 
 38 | 											rThreshold_(rThreshold), 
 39 | 											oThreshold_(oThreshold),
 40 | 											useLNet_(useLNet) {
 41 | 	if (useGPU) {
 42 | 		caffe::Caffe::set_mode(caffe::Caffe::GPU);
 43 | 		caffe::Caffe::SetDevice(deviceID);
 44 |                 printf("!!!GPU!!!\r\n");
 45 | 	} else {
 46 | 		caffe::Caffe::set_mode(caffe::Caffe::CPU);
 47 |                 printf("CPU\r\n");
 48 | 	}
 49 | 	pNet_.reset( new caffe::Net<float> (modelDir + P_NET_PROTO, caffe::TEST));
 50 | 	pNet_->CopyTrainedLayersFrom(modelDir + P_NET_WEIGHTS);
 51 | 	rNet_.reset( new caffe::Net<float> (modelDir + R_NET_PROTO, caffe::TEST));
 52 | 	rNet_->CopyTrainedLayersFrom(modelDir + R_NET_WEIGHTS);
 53 | 	oNet_.reset( new caffe::Net<float> (modelDir + O_NET_PROTO, caffe::TEST));
 54 | 	oNet_->CopyTrainedLayersFrom(modelDir + O_NET_WEIGHTS);
 55 | 	if (useLNet_) {
 56 | 		lNet_.reset( new caffe::Net<float> (modelDir + L_NET_PROTO, caffe::TEST));
 57 | 		lNet_->CopyTrainedLayersFrom(modelDir + L_NET_WEIGHTS);
 58 | 	}
 59 | }
 60 | 
 61 | std::vector<Face> FaceDetector::detect(cv::Mat img, float minFaceSize, float scaleFactor) {
 62 | 	cv::Mat rgbImg;
 63 | 	if (img.channels() == 3) {
 64 | 		cv::cvtColor(img, rgbImg, CV_BGR2RGB);
 65 | 	} else if (img.channels() == 4) {
 66 | 		cv::cvtColor(img, rgbImg, CV_BGRA2RGB);
 67 | 	}
 68 | 	if (rgbImg.empty()) {
 69 | 		return std::vector<Face>();
 70 | 	}
 71 | 	rgbImg.convertTo(rgbImg, CV_32FC3);
 72 | 	rgbImg = rgbImg.t();
 73 | 	std::vector<Face> faces = step1(rgbImg, minFaceSize, scaleFactor);
 74 | 	faces = step2(rgbImg, faces);
 75 | 	faces = step3(rgbImg, faces);
 76 | 	if (useLNet_) {
 77 | 		faces = step4(rgbImg, faces);
 78 | 	}
 79 | 	for (size_t i = 0; i < faces.size(); ++i) {
 80 | 		std::swap(faces[i].bbox.x1, faces[i].bbox.y1);
 81 | 		std::swap(faces[i].bbox.x2, faces[i].bbox.y2);
 82 | 		for (int p = 0; p < NUM_PTS; ++p) {
 83 | 			std::swap(faces[i].ptsCoords[2 * p], faces[i].ptsCoords[2 * p + 1]);
 84 | 		}
 85 | 	}
 86 | 	return faces;
 87 | }
 88 | 
 89 | void FaceDetector::initNetInput(boost::shared_ptr< caffe::Net<float> > net, cv::Mat img) {
 90 | 	std::vector<cv::Mat> channels;
 91 | 	cv::split(img, channels);	
 92 | 	caffe::Blob<float>* inputLayer = net->input_blobs()[0];
 93 | 	assert(inputLayer->channels() == static_cast<int>(channels.size()));
 94 | 	if (img.rows != inputLayer->height() || img.cols != inputLayer->width()) {
 95 | 		inputLayer->Reshape(1, channels.size(), img.rows, img.cols);
 96 | 		net->Reshape();
 97 | 	}
 98 | 	float* inputData = inputLayer->mutable_cpu_data();
 99 | 	for (size_t i = 0; i < channels.size(); ++i) {
100 | 		channels[i] -= IMG_MEAN;
101 | 		channels[i] *= IMG_INV_STDDEV;
102 | 		memcpy(inputData, channels[i].data, sizeof(float) * img.cols * img.rows);
103 | 		inputData += img.cols * img.rows;
104 | 	}
105 | }
106 | 
107 | void FaceDetector::initNetInput(boost::shared_ptr< caffe::Net<float> > net, std::vector<cv::Mat>& imgs) {
108 | 	caffe::Blob<float>* inputLayer = net->input_blobs()[0];
109 | 	float* inputData = inputLayer->mutable_cpu_data();
110 | 	for (size_t i = 0; i < imgs.size(); ++i) {
111 | 		std::vector<cv::Mat> channels;
112 | 		cv::split(imgs[i], channels);	
113 | 		for (size_t c = 0; c < channels.size(); ++c) {
114 | 			channels[c] -= IMG_MEAN;
115 | 			channels[c] *= IMG_INV_STDDEV;
116 | 			memcpy(inputData, channels[c].data, sizeof(float) * imgs[i].cols * imgs[i].rows);
117 | 			inputData += imgs[i].cols * imgs[i].rows;
118 | 		}
119 | 	}
120 | }
121 | 
122 | std::vector<Face> FaceDetector::step1(cv::Mat img, float minFaceSize, float scaleFactor) {
123 | 	std::vector<Face> finalFaces;
124 | 	float maxFaceSize = static_cast<float>(std::min(img.rows, img.cols));
125 | 	float faceSize = minFaceSize;
126 | 	while (faceSize <= maxFaceSize) {
127 | 		float currentScale = (P_NET_WINDOW_SIDE) / faceSize;
128 | 		int imgHeight = std::ceil(img.rows * currentScale);
129 | 		int imgWidth = std::ceil(img.cols * currentScale);
130 | 		cv::Mat resizedImg;
131 | 		cv::resize(img, resizedImg, cv::Size(imgWidth, imgHeight), 0, 0, cv::INTER_AREA);
132 | 		initNetInput(pNet_, resizedImg);
133 | 
134 | 		pNet_->Forward();
135 | 		const caffe::Blob<float>* regressionsBlob = pNet_->blob_by_name(P_NET_REGRESSION_BLOB_NAME).get();
136 | 		const caffe::Blob<float>* scoresBlob = pNet_->blob_by_name(P_NET_SCORE_BLOB_NAME).get();
137 | 		std::vector<Face> faces = composeFaces(regressionsBlob, scoresBlob, currentScale);
138 | 		std::vector<Face> facesNMS = FaceDetector::nonMaximumSuppression(faces, 0.8f);
139 | 		
140 | 		if (!facesNMS.empty()) {
141 | 			finalFaces.insert(finalFaces.end(), facesNMS.begin(), facesNMS.end()); 
142 | 		}
143 | 		faceSize /= scaleFactor;
144 | 	}
145 | 	finalFaces = FaceDetector::nonMaximumSuppression(finalFaces, 0.8f);
146 | 	Face::applyRegression(finalFaces, false);
147 | 	Face::bboxes2Squares(finalFaces);
148 | 	return finalFaces;
149 | }
150 | 
151 | std::vector<Face> FaceDetector::step2(cv::Mat img, const std::vector<Face>& faces) {
152 | 	std::vector<Face> finalFaces;
153 | 	cv::Size windowSize = cv::Size(rNet_->input_blobs()[0]->width(), rNet_->input_blobs()[0]->height());
154 | 	for (size_t i = 0; i < faces.size(); ++i) {
155 | 		cv::Mat sample = cropImage(img, faces[i].bbox.getRect());
156 | 		cv::resize(sample, sample, windowSize, 0, 0, cv::INTER_AREA);
157 | 		initNetInput(rNet_, sample);
158 | 		rNet_->Forward();
159 | 		const caffe::Blob<float>* regressionBlob = rNet_->blob_by_name(R_NET_REGRESSION_BLOB_NAME).get();
160 | 		const caffe::Blob<float>* scoreBlob = rNet_->blob_by_name(R_NET_SCORE_BLOB_NAME).get();
161 | 		float score = scoreBlob->cpu_data()[1];
162 | 		if (score < rThreshold_) {
163 | 			continue;
164 | 		}
165 | 		const float* regressionData = regressionBlob->cpu_data();
166 | 		Face face = faces[i];
167 | 		face.regression[0] = regressionData[0];
168 | 		face.regression[1] = regressionData[1];
169 | 		face.regression[2] = regressionData[2];
170 | 		face.regression[3] = regressionData[3];
171 | 		face.score = score;
172 | 		finalFaces.push_back(face);
173 | 	}
174 | 	finalFaces = FaceDetector::nonMaximumSuppression(finalFaces, 0.8f);
175 | 	Face::applyRegression(finalFaces, true);
176 | 	Face::bboxes2Squares(finalFaces);
177 | 	return finalFaces;
178 | }
179 | 
180 | std::vector<Face> FaceDetector::step3(cv::Mat img, const std::vector<Face>& faces) {
181 | 	std::vector<Face> finalFaces;
182 | 	cv::Size windowSize = cv::Size(oNet_->input_blobs()[0]->width(), oNet_->input_blobs()[0]->height());
183 | 	for (size_t i = 0; i < faces.size(); ++i) {
184 | 		cv::Mat sample = cropImage(img, faces[i].bbox.getRect());
185 | 		cv::resize(sample, sample, windowSize, 0, 0, cv::INTER_AREA);
186 | 		initNetInput(oNet_, sample);
187 | 		oNet_->Forward();
188 | 		const caffe::Blob<float>* regressionBlob = oNet_->blob_by_name(O_NET_REGRESSION_BLOB_NAME).get();
189 | 		const caffe::Blob<float>* scoreBlob = oNet_->blob_by_name(O_NET_SCORE_BLOB_NAME).get();
190 | 		const caffe::Blob<float>* ptsBlob = oNet_->blob_by_name(O_NET_PTS_BLOB_NAME).get();
191 | 		float score = scoreBlob->cpu_data()[1];
192 | 		if (score < oThreshold_) {
193 | 			continue;
194 | 		}
195 | 		const float* regressionData = regressionBlob->cpu_data();
196 | 		Face face = faces[i];
197 | 		face.regression[0] = regressionData[0];
198 | 		face.regression[1] = regressionData[1];
199 | 		face.regression[2] = regressionData[2];
200 | 		face.regression[3] = regressionData[3];
201 | 		face.score = score;
202 | 		const float* ptsData = ptsBlob->cpu_data();
203 | 		for (int p = 0; p < NUM_PTS; ++p) {
204 | 			face.ptsCoords[2 * p] = face.bbox.x1 + ptsData[p + NUM_PTS] * (face.bbox.x2 - face.bbox.x1 + 1) - 1;
205 | 			face.ptsCoords[2 * p + 1] = face.bbox.y1 + ptsData[p] * (face.bbox.y2 - face.bbox.y1 + 1) - 1;
206 | 		}
207 | 		finalFaces.push_back(face);
208 | 	}
209 | 	Face::applyRegression(finalFaces, true);
210 | 	finalFaces = FaceDetector::nonMaximumSuppression(finalFaces, 0.8f, true);
211 | 	return finalFaces;
212 | }
213 | 
214 | std::vector<Face> FaceDetector::step4(cv::Mat img, const std::vector<Face>& faces) {
215 | 	std::vector<Face> finalFaces;
216 | 	cv::Size windowSize = cv::Size(lNet_->input_blobs()[0]->width(), lNet_->input_blobs()[0]->height());
217 | 	for (size_t i = 0; i < faces.size(); ++i) {
218 | 		std::vector<cv::Mat> samples;
219 | 		std::vector<cv::Rect> patches;
220 | 		for (int p = 0; p < NUM_PTS; ++p) {
221 | 			float maxSide = std::max(faces[i].bbox.x2 - faces[i].bbox.x1, faces[i].bbox.y2 - faces[i].bbox.y1);
222 | 			int patchSide = std::floor(0.25f *(maxSide + 1));
223 | 			if (patchSide % 2 == 1) { 
224 | 				++patchSide;
225 | 			}
226 | 			int patchX = std::floor(faces[i].ptsCoords[2 * p] - 0.5f * patchSide);
227 | 			int patchY = std::floor(faces[i].ptsCoords[2 * p + 1] - 0.5f * patchSide);
228 | 			cv::Rect patch(patchX, patchY, patchSide, patchSide);
229 | 			cv::Mat sample = cropImage(img, patch);
230 | 			cv::resize(sample, sample, windowSize, 0, 0, cv::INTER_AREA);
231 | 			samples.push_back(sample);
232 | 			patches.push_back(patch);
233 | 		}
234 | 		initNetInput(lNet_, samples);
235 | 		lNet_->Forward();
236 | 		Face face = faces[i];
237 | 		for (int p = 0; p < NUM_PTS; ++p) {
238 | 			const caffe::Blob<float>* regressionBlob = lNet_->output_blobs()[p];
239 | 			const float* regressionData = regressionBlob->cpu_data();
240 | 			float dx = regressionData[1];
241 | 			float dy = regressionData[0];
242 | 			if (std::abs(dx - 0.5f) < L_THRESHOLD && std::abs(dy - 0.5f) < L_THRESHOLD) { 
243 | 				face.ptsCoords[2 * p] += dx * patches[p].width - 0.5f * patches[p].width;
244 | 				face.ptsCoords[2 * p + 1] += dy * patches[p].height - 0.5f * patches[p].height;
245 | 			}
246 | 		}
247 | 		finalFaces.push_back(face);
248 | 	}
249 | 	return finalFaces;
250 | }
251 | 
252 | std::vector<Face> FaceDetector::nonMaximumSuppression(std::vector<Face> faces, float threshold, bool useMin) {
253 | 	std::vector<Face> facesNMS;
254 | 	if (faces.empty()) {
255 | 		return facesNMS;
256 | 	}
257 | 	std::sort(faces.begin(), faces.end(), [](const Face& f1, const Face& f2) {
258 | 		return f1.score > f2.score;
259 | 	});
260 | 	std::vector<int> indices(faces.size());
261 | 	for (size_t i = 0; i < indices.size(); ++i) {
262 | 		indices[i] = i;
263 | 	}
264 | 	while (indices.size() > 0) {
265 | 		int idx = indices[0];
266 | 		facesNMS.push_back(faces[idx]);
267 | 		std::vector<int> tmpIndices = indices;
268 | 		indices.clear();
269 | 		for(size_t i = 1; i < tmpIndices.size(); ++i) {
270 | 			int tmpIdx = tmpIndices[i];
271 | 			float interX1 = std::max(faces[idx].bbox.x1, faces[tmpIdx].bbox.x1);
272 | 			float interY1 = std::max(faces[idx].bbox.y1, faces[tmpIdx].bbox.y1);
273 | 			float interX2 = std::min(faces[idx].bbox.x2, faces[tmpIdx].bbox.x2);
274 | 			float interY2 = std::min(faces[idx].bbox.y2, faces[tmpIdx].bbox.y2);
275 | 			 
276 | 			float bboxWidth = std::max(0.f, (interX2 - interX1 + 1));
277 | 			float bboxHeight = std::max(0.f, (interY2 - interY1 + 1));
278 | 
279 | 			float interArea = bboxWidth * bboxHeight;
280 | 			// TODO: compute outside the loop
281 | 			float area1 = (faces[idx].bbox.x2 - faces[idx].bbox.x1 + 1) * 
282 | 							(faces[idx].bbox.y2 - faces[idx].bbox.y1 + 1);
283 | 			float area2 = (faces[tmpIdx].bbox.x2 - faces[tmpIdx].bbox.x1 + 1) * 
284 | 							(faces[tmpIdx].bbox.y2 - faces[tmpIdx].bbox.y1 + 1);
285 | 			float o = 0.f;
286 | 			if (useMin) {
287 | 				o = interArea / std::min(area1, area2);           
288 | 			} else {
289 | 				o = interArea / (area1 + area2 - interArea);
290 | 			}
291 | 			if(o <= threshold) {
292 | 			    indices.push_back(tmpIdx);
293 | 			}
294 | 		}
295 | 	}
296 | 	return facesNMS;
297 | }
298 | 
299 | std::vector<Face> FaceDetector::composeFaces(const caffe::Blob<float>* regressionsBlob, 
300 | 									  const caffe::Blob<float>* scoresBlob, 
301 | 									  float scale) {
302 | 	assert(regressionsBlob->num() == 1 && scoresBlob->num() == 1);
303 | 	assert(regressionsBlob->channels() == 4 && scoresBlob->channels() == 2);
304 | 	std::vector<Face> faces;
305 | 	const int windowSide = static_cast<int>(P_NET_WINDOW_SIDE);
306 | 
307 | 	const int height = regressionsBlob->height();
308 | 	const int width = regressionsBlob->width();
309 | 
310 | 	const float* regressionsData = regressionsBlob->cpu_data();
311 | 	const float* scoresData = scoresBlob->cpu_data();
312 | 	for (int y = 0; y < height; ++y) {
313 | 		for (int x = 0; x < width; ++x) {
314 | 			float score = scoresData[1 * width * height + y * width + x];
315 | 			if (score < pThreshold_) {
316 | 				continue;
317 | 			}
318 | 			Face face;
319 | 			face.bbox.x1 = std::floor((P_NET_STRIDE * x + 1) / scale);
320 | 			face.bbox.y1 = std::floor((P_NET_STRIDE * y + 1) / scale);
321 | 			face.bbox.x2 = std::floor((P_NET_STRIDE * x + windowSide) / scale);
322 | 			face.bbox.y2 = std::floor((P_NET_STRIDE * y + windowSide) / scale);
323 | 			face.regression[0] = regressionsData[0 * width * height + y * width + x];
324 | 			face.regression[1] = regressionsData[1 * width * height + y * width + x];
325 | 			face.regression[2] = regressionsData[2 * width * height + y * width + x];
326 | 			face.regression[3] = regressionsData[3 * width * height + y * width + x];
327 | 			face.score = score;
328 | 			faces.push_back(face);
329 | 		}
330 | 	}
331 | 	return faces;
332 | }
333 | 
334 | cv::Rect BBox::getRect() const {
335 | 	return cv::Rect(x1, y1, x2 - x1, y2 - y1);
336 | }
337 | 
338 | BBox BBox::getSquare() const {
339 | 	BBox bbox;
340 | 	float bboxWidth = x2 - x1;
341 | 	float bboxHeight = y2 - y1;
342 | 	float side = std::max(bboxWidth, bboxHeight);
343 | 	bbox.x1 = static_cast<int>(x1 + (bboxWidth - side) * 0.5f);
344 | 	bbox.y1 = static_cast<int>(y1 + (bboxHeight - side) * 0.5f);
345 | 	bbox.x2 = static_cast<int>(bbox.x1 + side);
346 | 	bbox.y2 = static_cast<int>(bbox.y1 + side);
347 | 	return bbox;
348 | }
349 | 
350 | void Face::applyRegression(std::vector<Face>& faces, bool addOne) {
351 | 	for (size_t i = 0; i < faces.size(); ++i) {
352 | 		float bboxWidth = faces[i].bbox.x2 - faces[i].bbox.x1 + static_cast<float>(addOne);
353 | 		float bboxHeight = faces[i].bbox.y2 - faces[i].bbox.y1 + static_cast<float>(addOne);
354 | 		faces[i].bbox.x1 = faces[i].bbox.x1 + faces[i].regression[1] * bboxWidth;
355 | 		faces[i].bbox.y1 = faces[i].bbox.y1 + faces[i].regression[0] * bboxHeight;
356 | 		faces[i].bbox.x2 = faces[i].bbox.x2 + faces[i].regression[3] * bboxWidth;
357 | 		faces[i].bbox.y2 = faces[i].bbox.y2 + faces[i].regression[2] * bboxHeight;
358 | 	}
359 | }
360 | 
361 | void Face::bboxes2Squares(std::vector<Face>& faces) {
362 | 	for (size_t i = 0; i < faces.size(); ++i) {
363 | 		faces[i].bbox = faces[i].bbox.getSquare();
364 | 	}
365 | }
366 | 
367 | }
368 | 


--------------------------------------------------------------------------------
/ECO/debug.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef DEBUG_HPP
  2 | #define DEBUG_HPP
  3 | 
  4 | #include <stdio.h>
  5 | #include <string>
  6 | #include <vector>
  7 | #include <numeric>
  8 | #include <iostream>
  9 | 
 10 | #include <iostream>
 11 | #include <cstdio>
 12 | #include <ctime>
 13 | 
 14 | #include <opencv2/opencv.hpp>
 15 | #include "parameters.hpp"
 16 | 
 17 | namespace eco
 18 | {
 19 | #define debug(a, args...) //printf("%s(%s:%d) " a "\n", __func__, __FILE__, __LINE__, ##args)
 20 | #define ddebug(a, args...) //printf("%s(%s:%d) " a "\n", __func__, __FILE__, __LINE__, ##args)
 21 | 
 22 | using namespace std;
 23 | 
 24 | void timerExample();
 25 | inline void timerExample()
 26 | {
 27 | 	std::clock_t start;
 28 | 	double duration;
 29 | 
 30 | 	start = std::clock();
 31 | 
 32 | 	/* Your algorithm here */
 33 | 
 34 | 	duration = (std::clock() - start) / (double)CLOCKS_PER_SEC;
 35 | 	std::cout << "time: " << duration << '\n';
 36 | }
 37 | 
 38 | void timerExampleCV();
 39 | inline void timerExampleCV()
 40 | {
 41 | 	double timer = (double)cv::getTickCount();
 42 | 	float timedft = 0;
 43 | 
 44 | 	// your test code here
 45 | 
 46 | 	timedft = ((double)cv::getTickCount() - timer) / cv::getTickFrequency();
 47 | 	debug("time: %f", timedft);
 48 | }
 49 | 
 50 | // Show the type of a Mat
 51 | // Using like this:
 52 | //	string ty =  type2str( im.type() );
 53 | //	printf("im: %s %d x %d \n", ty.c_str(), im.cols, im.rows );
 54 | void printMat(cv::Mat mat);
 55 | inline void printMat(cv::Mat mat)
 56 | {
 57 | 	int type = mat.type();
 58 | 	string r;
 59 | 
 60 | 	uchar depth = type & CV_MAT_DEPTH_MASK;
 61 | 	uchar chans = 1 + (type >> CV_CN_SHIFT);
 62 | 
 63 | 	switch (depth)
 64 | 	{
 65 | 	case CV_8U:
 66 | 		r = "8U";
 67 | 		break;
 68 | 	case CV_8S:
 69 | 		r = "8S";
 70 | 		break;
 71 | 	case CV_16U:
 72 | 		r = "16U";
 73 | 		break;
 74 | 	case CV_16S:
 75 | 		r = "16S";
 76 | 		break;
 77 | 	case CV_32S:
 78 | 		r = "32S";
 79 | 		break;
 80 | 	case CV_32F:
 81 | 		r = "32F";
 82 | 		break;
 83 | 	case CV_64F:
 84 | 		r = "64F";
 85 | 		break;
 86 | 	default:
 87 | 		r = "User";
 88 | 		break;
 89 | 	}
 90 | 
 91 | 	r += "C";
 92 | 	r += (chans + '0');
 93 | 
 94 | 	debug("%s %d x %d", r.c_str(), mat.rows, mat.cols);
 95 | 	//return r;
 96 | }
 97 | 
 98 | void printECO_FEATS(ECO_FEATS a);
 99 | inline void printECO_FEATS(ECO_FEATS a)
100 | {
101 | 	printMat(a[0][0]);
102 | 	for (size_t i = 0; i < a.size(); i++)
103 | 	{
104 | 		debug("%lu, %lu, %d x %d", i, a[i].size(), a[i][0].rows, a[i][0].cols);
105 | 	}
106 | }
107 | 
108 | void printVector_Mat(std::vector<cv::Mat> a);
109 | inline void printVector_Mat(std::vector<cv::Mat> a)
110 | {
111 | 	printMat(a[0]);
112 | 	for (size_t i = 0; i < a.size(); i++)
113 | 	{
114 | 		debug("%lu, %lu, %d x %d", i, a.size(), a[i].rows, a[i].cols);
115 | 	}
116 | }
117 | 
118 | void printMaxmin();
119 | inline void printMaxmin(cv::Mat mat)
120 | {
121 | 	cv::Point p;
122 | 	double maxValue = -1, minValue = 256;
123 | 	cv::minMaxLoc(mat, &minValue, &maxValue, NULL, &p);
124 | 	printf("mat: min: %lf max: %lf loc: %d %d \n", minValue, maxValue, p.x, p.y);
125 | }
126 | 
127 | void showmat1channels(cv::Mat mat, int type);
128 | inline void showmat1channels(cv::Mat mat, int type)
129 | {
130 | 	for (int i = 0; i < mat.rows; i++)
131 | 	{
132 | 		for (int j = 0; j < mat.cols; j++)
133 | 		{
134 | 			if (type == 0)
135 | 			{ // char
136 | 				printf("%d ", mat.at<unsigned char>(i, j));
137 | 			}
138 | 			else if (type == 1)
139 | 			{ // int
140 | 				printf("%d ", mat.at<int>(i, j));
141 | 			}
142 | 			else if (type == 2)
143 | 			{ //float
144 | 				printf("%f ", mat.at<float>(i, j));
145 | 			}
146 | 			else if (type == 3)
147 | 			{ //double
148 | 				printf("%lf ", mat.at<double>(i, j));
149 | 			}
150 | 		}
151 | 		printf("\n");
152 | 	}
153 | 	printf("-------------------------End of 1 channel mat\n");
154 | }
155 | 
156 | void showmat2channels(cv::Mat mat, int type);
157 | 
158 | inline void showmat2channels(cv::Mat mat, int type)
159 | {
160 | 	for (int k = 0; k < mat.channels(); k++)
161 | 	{	
162 | 		printf("\n");
163 | 		for (int i = 0; i < mat.rows; i++)
164 | 		{
165 | 			for (int j = 0; j < mat.cols; j++)
166 | 			{
167 | 				if (type == 0)
168 | 				{ // char
169 | 					printf("%d ", mat.at<cv::Vec2b>(i, j)[k]);
170 | 				}
171 | 				else if (type == 1)
172 | 				{ // int
173 | 					printf("%d ", mat.at<cv::Vec2i>(i, j)[k]);
174 | 				}
175 | 				else if (type == 2)
176 | 				{ //float
177 | 					printf("%f ", mat.at<cv::Vec2f>(i, j)[k]);
178 | 				}
179 | 				else if (type == 3)
180 | 				{ //float
181 | 					printf("%lf ", mat.at<cv::Vec2d>(i, j)[k]);
182 | 				}
183 | 			}
184 | 			printf("\n");
185 | 		}
186 | 		printf("-------------------------");
187 | 	}
188 | 	printf("End of 2 channels mat\n");
189 | }
190 | 
191 | void showmat3channels(cv::Mat mat, int type);
192 | 
193 | inline void showmat3channels(cv::Mat mat, int type)
194 | {
195 | 	for (int k = 0; k < mat.channels(); k++)
196 | 	{
197 | 		for (int i = 0; i < mat.rows; i++)
198 | 		{
199 | 			for (int j = 0; j < mat.cols; j++)
200 | 			{
201 | 				if (type == 0)
202 | 				{ // char
203 | 					printf("%d ", mat.at<cv::Vec3b>(i, j)[k]);
204 | 				}
205 | 				else if (type == 1)
206 | 				{ // int
207 | 					printf("%d ", mat.at<cv::Vec3i>(i, j)[k]);
208 | 				}
209 | 				else if (type == 2)
210 | 				{ //float
211 | 					printf("%f ", mat.at<cv::Vec3f>(i, j)[k]);
212 | 				}
213 | 				else if (type == 3)
214 | 				{ //float
215 | 					printf("%lf ", mat.at<cv::Vec3d>(i, j)[k]);
216 | 				}
217 | 			}
218 | 			printf("\n");
219 | 		}
220 | 		printf("\n\n");
221 | 		break;
222 | 	}
223 | 	printf("End of 3 channel mat\n");
224 | }
225 | 
226 | void showmat1ch(cv::Mat mat, int type);
227 | inline void showmat1ch(cv::Mat mat, int type)
228 | {
229 | 	printf("\nFirst row: \n");
230 | 	for (int j = 0; j < mat.cols; j++)
231 | 	{
232 | 		if (type == 1)
233 | 		{ // int
234 | 			printf("%d ", mat.at<int>(0, j));
235 | 		}
236 | 		else if (type == 2)
237 | 		{ //float
238 | 			printf("%f ", mat.at<float>(0, j));
239 | 		}
240 | 		else if (type == 3)
241 | 		{ // first row
242 | 			printf("%lf ", mat.at<double>(0, j));
243 | 		}
244 | 	}
245 | 	printf("\nrow %d: \n", mat.cols - 1);
246 | 	for (int j = 0; j < mat.cols; j++)
247 | 	{
248 | 		if (type == 1)
249 | 		{ // last row
250 | 			printf("%d ", mat.at<int>(mat.cols - 1, j));
251 | 		}
252 | 		else if (type == 2)
253 | 		{ // last row
254 | 			printf("%f ", mat.at<float>(mat.cols - 1, j));
255 | 		}
256 | 		else if (type == 3)
257 | 		{ // last row
258 | 			printf("%lf ", mat.at<double>(mat.cols - 1, j));
259 | 		}
260 | 	}
261 | 	printf("\nFirst col: \n");
262 | 	for (int i = 0; i < mat.rows; i++)
263 | 	{
264 | 		if (type == 1)
265 | 		{ // first col
266 | 			printf("%d ", mat.at<int>(i, 0));
267 | 		}
268 | 		else if (type == 2)
269 | 		{ // first col
270 | 			printf("%f ", mat.at<float>(i, 0));
271 | 		}
272 | 		else if (type == 3)
273 | 		{ // first col
274 | 			printf("%lf ", mat.at<double>(i, 0));
275 | 		}
276 | 	}
277 | 	printf("\ncol %d: \n", mat.rows - 1);
278 | 	for (int i = 0; i < mat.rows; i++)
279 | 	{
280 | 		if (type == 1)
281 | 		{ // last col
282 | 			printf("%d ", mat.at<int>(i, mat.rows - 1));
283 | 		}
284 | 		else if (type == 2)
285 | 		{ // last col
286 | 			printf("%f ", mat.at<float>(i, mat.rows - 1));
287 | 		}
288 | 		else if (type == 3)
289 | 		{ // last col
290 | 			printf("%lf ", mat.at<double>(i, mat.rows - 1));
291 | 		}
292 | 	}
293 | 	printf("\nEnd of 2 channel mat\n");
294 | }
295 | 
296 | void showmat2ch(cv::Mat mat, int type);
297 | inline void showmat2ch(cv::Mat mat, int type)
298 | {
299 | 	printf("First row: \n");
300 | 	for (int k = 0; k < mat.channels(); k++)
301 | 	{
302 | 		for (int j = 0; j < mat.cols; j++)
303 | 		{
304 | 			if (type == 3)
305 | 			{ // first row
306 | 				printf("%lf ", mat.at<cv::Vec2d>(0, j)[k]);
307 | 			}
308 | 		}
309 | 		printf("\n\n");
310 | 	}
311 | 	printf("\n");
312 | 	printf("row %d: \n", mat.cols - 1);
313 | 	for (int k = 0; k < mat.channels(); k++)
314 | 	{
315 | 		for (int j = 0; j < mat.cols; j++)
316 | 		{
317 | 			if (type == 3)
318 | 			{ // last row
319 | 				printf("%lf ", mat.at<cv::Vec2d>(mat.cols - 1, j)[k]);
320 | 			}
321 | 		}
322 | 		printf("\n\n");
323 | 	}
324 | 	printf("\n");
325 | 	printf("First col: \n");
326 | 	for (int k = 0; k < mat.channels(); k++)
327 | 	{
328 | 		for (int i = 0; i < mat.rows; i++)
329 | 		{
330 | 			if (type == 3)
331 | 			{ // first col
332 | 				printf("%lf ", mat.at<cv::Vec2d>(i, 0)[k]);
333 | 			}
334 | 		}
335 | 		printf("\n\n");
336 | 	}
337 | 	printf("\n");
338 | 	printf("col %d: \n", mat.rows - 1);
339 | 	for (int k = 0; k < mat.channels(); k++)
340 | 	{
341 | 		for (int i = 0; i < mat.rows; i++)
342 | 		{
343 | 			if (type == 3)
344 | 			{ // last col
345 | 				printf("%lf ", mat.at<cv::Vec2d>(i, mat.rows - 1)[k]);
346 | 			}
347 | 		}
348 | 		printf("\n\n");
349 | 	}
350 | 	printf("End of 2 channel mat\n");
351 | }
352 | 
353 | // Attention!!!! opencv/caffe: BGR, matlab: RGB, different order!!!
354 | void showmat3ch(cv::Mat mat, int type);
355 | inline void showmat3ch(cv::Mat mat, int type)
356 | {
357 | 	std::vector<cv::Mat> splitmat;
358 | 	cv::split(mat, splitmat);
359 | 
360 | 	debug("channels: %lu", splitmat.size());
361 | 
362 | 	printf("First row of channel 0: \n");
363 | 	for (int j = 0; j < mat.cols; j += 1)
364 | 	{
365 | 		if (type == 0)
366 | 		{ // 2: means Red, 0 means first row
367 | 			printf("%d ", splitmat[2].at<uchar>(0, j));
368 | 		}
369 | 		else if (type == 1)
370 | 		{
371 | 			printf("%d ", splitmat[2].at<int>(0, j));
372 | 		}
373 | 		else if (type == 2)
374 | 		{
375 | 			printf("%f ", splitmat[2].at<float>(0, j));
376 | 		}
377 | 		else if (type == 3)
378 | 		{
379 | 			printf("%lf ", splitmat[2].at<double>(0, j));
380 | 		}
381 | 	}
382 | 	printf("\n\n");
383 | 
384 | 	printf("\n");
385 | 	printf("First col of  channel 0: \n");
386 | 	for (int i = 0; i < mat.rows; i += 1)
387 | 	{
388 | 		if (type == 0)
389 | 		{
390 | 			printf("%d ", splitmat[2].at<uchar>(i, 0));
391 | 		}
392 | 		else if (type == 1)
393 | 		{
394 | 			printf("%d ", splitmat[2].at<int>(i, 0));
395 | 		}
396 | 		else if (type == 2)
397 | 		{
398 | 			printf("%f ", splitmat[2].at<float>(i, 0));
399 | 		}
400 | 		else if (type == 3)
401 | 		{
402 | 			printf("%lf ", splitmat[2].at<double>(i, 0));
403 | 		}
404 | 	}
405 | 	printf("\n\n");
406 | 
407 | 	printf("End of feature mat\n");
408 | }
409 | 
410 | void showmatNch(cv::Mat mat, int type);
411 | inline void showmatNch(cv::Mat mat, int type)
412 | {
413 | 	std::vector<cv::Mat> splitmat;
414 | 	cv::split(mat, splitmat);
415 | 
416 | 	debug("channels: %lu", splitmat.size());
417 | 	printf("1th row of channel 0: \n");
418 | 	for (int j = 0; j < mat.cols; j += 1)
419 | 	{
420 | 		if (type == 0)
421 | 		{
422 | 			printf("%d ", splitmat[0].at<uchar>(1, j));
423 | 		}
424 | 		else if (type == 1)
425 | 		{ // first row
426 | 			printf("%d ", splitmat[0].at<int>(1, j));
427 | 		}
428 | 		else if (type == 2)
429 | 		{ // first row
430 | 			printf("%f ", splitmat[0].at<float>(1, j));
431 | 		}
432 | 		else if (type == 3)
433 | 		{ // first row
434 | 			printf("%lf ", splitmat[0].at<double>(1, j));
435 | 		}
436 | 	}	
437 | 	printf("\n");
438 | 	printf("1th col of  channel 0: \n");
439 | 	for (int i = 0; i < mat.rows; i += 1)
440 | 	{
441 | 		if (type == 0)
442 | 		{ // first row
443 | 			printf("%d ", splitmat[0].at<uchar>(i, 1));
444 | 		}
445 | 		else if (type == 1)
446 | 		{ // first row
447 | 			printf("%d ", splitmat[0].at<int>(i, 1));
448 | 		}
449 | 		else if (type == 2)
450 | 		{ // first row
451 | 			printf("%f ", splitmat[0].at<float>(i, 1));
452 | 		}
453 | 		else if (type == 3)
454 | 		{ // first col
455 | 			printf("%lf ", splitmat[0].at<double>(i, 1));
456 | 		}
457 | 	}
458 | 	printf("\n");
459 | 	printf("10th row of channel 0: \n");
460 | 	for (int j = 0; j < mat.cols; j += 1)
461 | 	{
462 | 		if (type == 0)
463 | 		{
464 | 			printf("%d ", splitmat[0].at<uchar>(10, j));
465 | 		}
466 | 		else if (type == 1)
467 | 		{ // first row
468 | 			printf("%d ", splitmat[0].at<int>(10, j));
469 | 		}
470 | 		else if (type == 2)
471 | 		{ // first row
472 | 			printf("%f ", splitmat[0].at<float>(10, j));
473 | 		}
474 | 		else if (type == 3)
475 | 		{ // first row
476 | 			printf("%lf ", splitmat[0].at<double>(10, j));
477 | 		}
478 | 	}
479 | 	printf("\n");
480 | 	printf("20th col of  channel 0: \n");
481 | 	for (int i = 0; i < mat.rows; i += 1)
482 | 	{
483 | 		if (type == 0)
484 | 		{ // first row
485 | 			printf("%d ", splitmat[0].at<uchar>(i, 20));
486 | 		}
487 | 		else if (type == 1)
488 | 		{ // first row
489 | 			printf("%d ", splitmat[0].at<int>(i, 20));
490 | 		}
491 | 		else if (type == 2)
492 | 		{ // first row
493 | 			printf("%f ", splitmat[0].at<float>(i, 20));
494 | 		}
495 | 		else if (type == 3)
496 | 		{ // first col
497 | 			printf("%lf ", splitmat[0].at<double>(i, 20));
498 | 		}
499 | 	}
500 | 	printf("\n");
501 | 	printf("24th row of channel -1: \n");
502 | 	for (int j = 0; j < mat.cols; j += 1)
503 | 	{
504 | 		if (type == 0)
505 | 		{
506 | 			printf("%d ", splitmat[splitmat.size()-1].at<uchar>(24, j));
507 | 		}
508 | 		else if (type == 1)
509 | 		{ // first row
510 | 			printf("%d ", splitmat[splitmat.size()-1].at<int>(24, j));
511 | 		}
512 | 		else if (type == 2)
513 | 		{ // first row
514 | 			printf("%f ", splitmat[splitmat.size()-1].at<float>(24, j));
515 | 		}
516 | 		else if (type == 3)
517 | 		{ // first row
518 | 			printf("%lf ", splitmat[splitmat.size()-1].at<double>(24, j));
519 | 		}
520 | 	}
521 | 	printf("\n\n");
522 | 	printf("End of feature mat\n");
523 | }
524 | 
525 | //TEST==========================================================================
526 | // Simple test of the structure of mat in opencv; channel->x->y;
527 | void opencvTest();
528 | inline void opencvTest()
529 | {
530 | 	printf("opencvTest begin=======================================\n");
531 | 	float *newdata = (float *)malloc(sizeof(float) * (2 * 3 * 4));
532 | 
533 | 	for (int i = 0; i < 2 * 3 * 4; i++)
534 | 	{
535 | 		newdata[i] = i;
536 | 	}
537 | 
538 | 	cv::Mat mat = cv::Mat(cv::Size(3, 4), CV_32FC(2), newdata);
539 | 
540 | 	printf("\nInfo of original mat:");
541 | 	printMat(mat);
542 | 	for (int i = 0; i < 2 * 3 * 4; i++)
543 | 	{
544 | 		printf("%f ", mat.at<float>(0, i));
545 | 	}
546 | 	printf("\n");
547 | 
548 | 	std::vector<cv::Mat> splitmat;
549 | 	cv::split(mat, splitmat);
550 | 
551 | 	printf("\nInfo of splited mat:");
552 | 	printMat(splitmat[0]);
553 | 
554 | 	printf("channel 0:\n");
555 | 	for (int j = 0; j < mat.rows; j++)
556 | 	{
557 | 		for (int i = 0; i < mat.cols; i++)
558 | 		{
559 | 			printf("%f ", splitmat[0].at<float>(j, i));
560 | 		}
561 | 		printf("\n");
562 | 	}
563 | 	printf("\n");
564 | 	printf("channel 1:\n");
565 | 	for (int j = 0; j < mat.rows; j++)
566 | 	{
567 | 		for (int i = 0; i < mat.cols; i++)
568 | 		{
569 | 			printf("%f ", splitmat[1].at<float>(j, i));
570 | 		}
571 | 		printf("\n");
572 | 	}
573 | 	printf("\n");
574 | 	printf("%p, %p, %p, %p, %p\n", newdata, 
575 | 			&mat.at<cv::Vec2f>(0, 0)[0], 
576 | 			&mat.at<cv::Vec2f>(0, 0)[1],
577 | 			&mat.at<cv::Vec2f>(0, 1)[0],
578 | 			&mat.at<cv::Vec2f>(1, 0)[0]
579 | 	);
580 | 
581 | 	free(newdata);
582 | 	printf("opencvTest end=======================================\n");
583 | }
584 | 
585 | void absTest();
586 | inline void absTest()
587 | {
588 | 	printf("absTest begin=======================================\n");
589 | 	std::vector<float> v{0.1, 0.2};
590 | 	//sometimes this works:
591 | 	//```
592 | 	//float abs = abs(1.23f);
593 | 	//```
594 | 	//but it use a different liberay from eigen, cause error
595 | 	//so remember to add `std::` before!
596 | 	float abs = std::abs(1.23f);
597 | 	debug("False abs:%f", abs);
598 | 
599 | 	abs = std::abs(1.23f);
600 | 	debug("True abs:%f", abs);
601 | 	printf("absTest end=======================================\n");
602 | }
603 | 
604 | void accumulateTest();
605 | inline void accumulateTest()
606 | {
607 | 	printf("accumulateTest begin=======================================\n");
608 | 	std::vector<float> v{0.1, 0.2};
609 | 	float sum = std::accumulate(v.begin(), v.end(), 0);
610 | 	debug("False sum:%f", sum);
611 | 
612 | 	sum = std::accumulate(v.begin(), v.end(), 0.0f);
613 | 	debug("True sum:%f", sum);
614 | 	printf("accumulateTest end=======================================\n");
615 | }
616 | /* Compare the differences of function copyTo() and clone():
617 | [0, 0, 0, 0, 0]
618 | [0, 0, 0, 0, 0]
619 | [0, 0, 0, 0, 0]
620 | [1, 1, 1, 1, 1]
621 | */
622 | void copyTo_clone_Difference();
623 | inline void copyTo_clone_Difference()
624 | {
625 | 	// copyTo will not change the address of the destination matrix.
626 | 	cv::Mat mat1 = cv::Mat::ones(1, 5, CV_32F);
627 | 	cv::Mat mat2 = mat1;
628 | 	cv::Mat mat3 = cv::Mat::zeros(1, 5, CV_32F);
629 | 	mat3.copyTo(mat1);
630 | 	std::cout << mat1 << std::endl; // it has a old address with new value
631 | 	std::cout << mat2 << std::endl; // it has a old address with new value
632 | 	// clone will always allocate a new address for the destination matrix.
633 | 	mat1 = cv::Mat::ones(1, 5, CV_32F);
634 | 	mat2 = mat1;
635 | 	mat3 = cv::Mat::zeros(1, 5, CV_32F);
636 | 	mat1 = mat3.clone();
637 | 	std::cout << mat1 << std::endl; // it has a new address with new value
638 | 	std::cout << mat2 << std::endl; // it has a old address with old value
639 | }
640 | 
641 | void matReferenceTest();
642 | inline void matReferenceTest()
643 | {
644 | 	cv::Mat mat;
645 | 	mat.release();
646 | 	debug("%p, %d", mat.data, mat.data==NULL);
647 | 	mat.create(cv::Size(10, 10), CV_32FC2);
648 | 	debug("%p, %d", mat.data, mat.data==NULL);
649 | 	mat.release();
650 | 	debug("%p, %d", mat.data, mat.data==NULL);
651 | }
652 | 
653 | }
654 | #endif


--------------------------------------------------------------------------------
/ECO/gradient.cpp:
--------------------------------------------------------------------------------
  1 | /*******************************************************************************
  2 | * Piotr's Computer Vision Matlab Toolbox      Version 3.30
  3 | * Copyright 2014 Piotr Dollar & Ron Appel.  [pdollar-at-gmail.com]
  4 | * Licensed under the Simplified BSD License [see external/bsd.txt]
  5 | *******************************************************************************/
  6 | 
  7 | #include "gradient.hpp"
  8 | 
  9 | #define PI 3.14159265f
 10 | 
 11 | // compute x and y gradients for just one column (uses sse)
 12 | void grad1( float *I, float *Gx, float *Gy, int h, int w, int x ) {
 13 |   int y, y1; float *Ip, *In, r; __m128 *_Ip, *_In, *_G, _r;
 14 |   // compute column of Gx
 15 |   Ip=I-h; In=I+h; r=.5f;
 16 |   if(x==0) { r=1; Ip+=h; } else if(x==w-1) { r=1; In-=h; }
 17 |   // if h not 4x or address not 16-aligned, calculate normally,
 18 |   if( h<4 || h%4>0 || (size_t(I)&15) || (size_t(Gx)&15) ) {
 19 |     for( y=0; y<h; y++ ) *Gx++=(*In++-*Ip++)*r;
 20 |   } else { // use sse
 21 |     _G=(__m128*) Gx; _Ip=(__m128*) Ip; _In=(__m128*) In; _r = SET(r);
 22 |     for(y=0; y<h; y+=4) *_G++=MUL(SUB(*_In++,*_Ip++),_r);
 23 |   }
 24 |   // compute column of Gy
 25 |   #define GRADY(r) *Gy++=(*In++-*Ip++)*r;
 26 |   Ip=I; In=Ip+1;
 27 |   // all calculate normally:
 28 |   // GRADY(1); Ip--; for(y=1; y<h-1; y++) GRADY(.5f); In--; GRADY(1);
 29 |   // sse speed-up:
 30 |   // calculate normally for the leading elements(not 16x)
 31 |   y1=((~((size_t) Gy) + 1) & 15)/4; if(y1==0) y1=4; if(y1>h-1) y1=h-1;
 32 |   GRADY(1); Ip--; for(y=1; y<y1; y++) GRADY(.5f); 
 33 |   // use sse for the main elements
 34 |   _r = SET(.5f); _G=(__m128*) Gy;
 35 |   for(; y+4<h-1; y+=4, Ip+=4, In+=4, Gy+=4)
 36 |     *_G++=MUL(SUB(LDu(*In),LDu(*Ip)),_r);
 37 |   // calculate normally for the final elements(not 16x)
 38 |   for(; y<h-1; y++) GRADY(.5f); In--; GRADY(1); 
 39 |   #undef GRADY
 40 | }
 41 | 
 42 | // compute x and y gradients at each location (uses sse)
 43 | void grad2( float *I, float *Gx, float *Gy, int h, int w, int d ) {
 44 |   int o, x, c, a=w*h; for(c=0; c<d; c++) for(x=0; x<w; x++) {
 45 |     o=c*a+x*h; grad1( I+o, Gx+o, Gy+o, h, w, x );
 46 |   }
 47 | }
 48 | 
 49 | // build lookup table a[] s.t. a[x*n]~=acos(x) for x in [-1,1]
 50 | float* acosTable() {
 51 |   const int n=10000, b=10; int i;
 52 |   static float a[n*2+b*2]; static bool init=false;
 53 |   float *a1=a+n+b; if( init ) return a1;
 54 |   for( i=-n-b; i<-n; i++ )   a1[i]=PI;
 55 |   for( i=-n; i<n; i++ )      a1[i]=float(acos(i/float(n)));
 56 |   for( i=n; i<n+b; i++ )     a1[i]=0;
 57 |   for( i=-n-b; i<n/10; i++ ) if( a1[i] > PI-1e-6f ) a1[i]=PI-1e-6f;
 58 |   init=true; return a1;
 59 | }
 60 | 
 61 | // compute gradient magnitude and orientation at each location (uses sse)
 62 | void gradMag( float *I, float *M, float *O, int h, int w, int d, bool full ) {
 63 |   int x, y, y1, c, h4, s; float *Gx, *Gy, *M2; __m128 *_Gx, *_Gy, *_M2, _m;
 64 |   float *acost = acosTable(), acMult=10000.0f;
 65 |   // allocate memory for storing one column of output (padded so h4%4==0)
 66 |   h4=(h%4==0) ? h : h-(h%4)+4; s=d*h4*sizeof(float);
 67 |   M2=(float*) alMalloc(s,16); _M2=(__m128*) M2;
 68 |   Gx=(float*) alMalloc(s,16); _Gx=(__m128*) Gx;
 69 |   Gy=(float*) alMalloc(s,16); _Gy=(__m128*) Gy;
 70 |   // compute gradient magnitude and orientation for each column
 71 |   for( x=0; x<w; x++ ) {
 72 |     // compute gradients (Gx, Gy) with maximum squared magnitude (M2)
 73 |     for(c=0; c<d; c++) {
 74 |       grad1( I+x*h+c*w*h, Gx+c*h4, Gy+c*h4, h, w, x );
 75 |       for( y=0; y<h4/4; y++ ) {
 76 |         y1=h4/4*c+y;
 77 |         _M2[y1]=ADD(MUL(_Gx[y1],_Gx[y1]),MUL(_Gy[y1],_Gy[y1]));
 78 |         if( c==0 ) continue; _m = CMPGT( _M2[y1], _M2[y] );
 79 |         _M2[y] = OR( AND(_m,_M2[y1]), ANDNOT(_m,_M2[y]) );
 80 |         _Gx[y] = OR( AND(_m,_Gx[y1]), ANDNOT(_m,_Gx[y]) );
 81 |         _Gy[y] = OR( AND(_m,_Gy[y1]), ANDNOT(_m,_Gy[y]) );
 82 |       }
 83 |     }
 84 |     // compute gradient mangitude (M) and normalize Gx
 85 |     for( y=0; y<h4/4; y++ ) {
 86 |       _m = MINN( RCPSQRT(_M2[y]), SET(1e10f) );
 87 |       _M2[y] = RCP(_m);
 88 |       if(O) _Gx[y] = MUL( MUL(_Gx[y],_m), SET(acMult) );
 89 |       if(O) _Gx[y] = XOR( _Gx[y], AND(_Gy[y], SET(-0.f)) );
 90 |     };
 91 |     memcpy( M+x*h, M2, h*sizeof(float) );
 92 |     // compute and store gradient orientation (O) via table lookup
 93 |     if( O!=0 ) for( y=0; y<h; y++ ) O[x*h+y] = acost[(int)Gx[y]];
 94 |     if( O!=0 && full ) {
 95 |       y1=((~size_t(O+x*h)+1)&15)/4; y=0;
 96 |       for( ; y<y1; y++ ) O[y+x*h]+=(Gy[y]<0)*PI;
 97 |       for( ; y<h-4; y+=4 ) STRu( O[y+x*h],
 98 |         ADD( LDu(O[y+x*h]), AND(CMPLT(LDu(Gy[y]),SET(0.f)),SET(PI)) ) );
 99 |       for( ; y<h; y++ ) O[y+x*h]+=(Gy[y]<0)*PI;
100 |     }
101 |   }
102 |   alFree(Gx); alFree(Gy); alFree(M2);
103 | }
104 | 
105 | // normalize gradient magnitude at each location (uses sse)
106 | void gradMagNorm( float *M, float *S, int h, int w, float norm ) {
107 |   __m128 *_M, *_S, _norm; int i=0, n=h*w, n4=n/4;
108 |   _S = (__m128*) S; _M = (__m128*) M; _norm = SET(norm);
109 |   bool sse = !(size_t(M)&15) && !(size_t(S)&15);// address 16x or not
110 |   if(sse) for(; i<n4; i++) { *_M=MUL(*_M,RCP(ADD(*_S++,_norm))); _M++; }
111 |   if(sse) i*=4; for(; i<n; i++) M[i] /= (S[i] + norm);
112 | }
113 | 
114 | // helper for gradHist, quantize O and M into O0, O1 and M0, M1 (uses sse)
115 | void gradQuantize( float *O, float *M, int *O0, int *O1, float *M0, float *M1,
116 |   int nb, int n, float norm, int nOrients, bool full, bool interpolate )
117 | {
118 |   // assumes all *OUTPUT* matrices are 4-byte aligned
119 |   int i, o0, o1; float o, od, m;
120 |   __m128i _o0, _o1, *_O0, *_O1; __m128 _o, _od, _m, *_M0, *_M1;
121 |   // define useful constants
122 |   const float oMult=(float)nOrients/(full?2*PI:PI); const int oMax=nOrients*nb;
123 |   const __m128 _norm=SET(norm), _oMult=SET(oMult), _nbf=SET((float)nb);
124 |   const __m128i _oMax=SET(oMax), _nb=SET(nb);
125 |   /* original version
126 |   // O0=O*oMult*wb*hb; M0=M/bin/bin; M1,O1=0; 
127 |   for(int i = 0; i<n; i++ ) {
128 |     o=O[i]*oMult; o0=(int) (o+.5f);
129 |     o0*=nb; if(o0>=oMax) o0=0; O0[i]=o0;
130 |     M0[i]=M[i]*norm; M1[i]=0; O1[i]=0;
131 |   }
132 |   */
133 |   // speed-up version
134 |   // perform the majority of the work with sse
135 |   _O0=(__m128i*) O0; _O1=(__m128i*) O1; _M0=(__m128*) M0; _M1=(__m128*) M1;
136 |   if( interpolate ) for( i=0; i<=n-4; i+=4 ) {
137 |     _o=MUL(LDu(O[i]),_oMult); _o0=CVT(_o); _od=SUB(_o,CVT(_o0));
138 |     _o0=CVT(MUL(CVT(_o0),_nbf)); _o0=AND(CMPGT(_oMax,_o0),_o0); *_O0++=_o0;
139 |     _o1=ADD(_o0,_nb); _o1=AND(CMPGT(_oMax,_o1),_o1); *_O1++=_o1;
140 |     _m=MUL(LDu(M[i]),_norm); *_M1=MUL(_od,_m); *_M0++=SUB(_m,*_M1); _M1++;
141 |   } else for( i=0; i<=n-4; i+=4 ) {
142 |     _o=MUL(LDu(O[i]),_oMult); _o0=CVT(ADD(_o,SET(.5f)));
143 |     _o0=CVT(MUL(CVT(_o0),_nbf)); _o0=AND(CMPGT(_oMax,_o0),_o0); *_O0++=_o0;
144 |     *_M0++=MUL(LDu(M[i]),_norm); *_M1++=SET(0.f); *_O1++=SET(0);
145 |   }
146 |   // compute trailing locations without sse
147 |   if( interpolate ) for(; i<n; i++ ) {
148 |     o=O[i]*oMult; o0=(int) o; od=o-o0;
149 |     o0*=nb; if(o0>=oMax) o0=0; O0[i]=o0;
150 |     o1=o0+nb; if(o1==oMax) o1=0; O1[i]=o1;
151 |     m=M[i]*norm; M1[i]=od*m; M0[i]=m-M1[i];
152 |   } else for(; i<n; i++ ) {
153 |     o=O[i]*oMult; o0=(int) (o+.5f);
154 |     o0*=nb; if(o0>=oMax) o0=0; O0[i]=o0;
155 |     M0[i]=M[i]*norm; M1[i]=0; O1[i]=0;
156 |   }
157 | }
158 | 
159 | // compute nOrients gradient histograms per bin x bin block of pixels
160 | void gradHist( float *M, float *O, float *H, int h, int w,
161 |   int bin, int nOrients, int softBin, bool full )
162 | {
163 |   const int hb=h/bin, wb=w/bin, h0=hb*bin, w0=wb*bin, nb=wb*hb;
164 |   const float s=(float)bin, sInv=1/s, sInv2=1/s/s;
165 |   float *H0, *H1, *M0, *M1; int x, y; int *O0, *O1; float xb, init;
166 |   O0=(int*)alMalloc(h*sizeof(int),16); M0=(float*) alMalloc(h*sizeof(float),16);
167 |   O1=(int*)alMalloc(h*sizeof(int),16); M1=(float*) alMalloc(h*sizeof(float),16);
168 |   // main loop
169 |   for( x=0; x<w0; x++ ) {
170 |     //debug("%p, %p", O+x*h,M+x*h);
171 |     // compute target orientation bins for entire column - very fast
172 |     gradQuantize(O+x*h,M+x*h,O0,O1,M0,M1,nb,h0,sInv2,nOrients,full,softBin>=0);
173 | /*
174 |     for (int i = 0; i < h; i++)
175 | 		{
176 | 			printf("%f ", *(O+i));
177 | 		}
178 |     printf("\nO end\n");
179 |     for (int i = 0; i < h; i++)
180 | 		{
181 | 			printf("%d ",*(O0+i));
182 | 		}
183 |     printf("\nO0 end\n");
184 |     for (int i = 0; i < h; i++)
185 | 		{
186 | 			printf("%f ", *(M0+i));
187 | 		}
188 |     printf("\nM0 end\n");
189 |   */
190 |     if( softBin<0 && softBin%2==0 ) { // softBin<0 and even
191 |       // no interpolation w.r.t. either orienation or spatial bin
192 |       H1=H+(x/bin)*hb;
193 |       #define GH H1[O0[y]]+=M0[y]; y++;
194 |       if( bin==1 )      for(y=0; y<h0;) { GH; H1++; }
195 |       else if( bin==2 ) for(y=0; y<h0;) { GH; GH; H1++; }
196 |       else if( bin==3 ) for(y=0; y<h0;) { GH; GH; GH; H1++; }
197 |       else if( bin==4 ) for(y=0; y<h0;) { GH; GH; GH; GH; H1++; }
198 |       else for( y=0; y<h0;) { for( int y1=0; y1<bin; y1++ ) { GH; } H1++; }
199 |       #undef GH
200 |     } else if( softBin%2==0 || bin==1 ) { // softBin>=0 and even
201 |       // interpolate w.r.t. orientation only, not spatial bin
202 |       H1=H+(x/bin)*hb;
203 |       #define GH H1[O0[y]]+=M0[y]; H1[O1[y]]+=M1[y]; y++;
204 |       if( bin==1 )      for(y=0; y<h0;) { GH; H1++; }
205 |       else if( bin==2 ) for(y=0; y<h0;) { GH; GH; H1++; }
206 |       else if( bin==3 ) for(y=0; y<h0;) { GH; GH; GH; H1++; }
207 |       else if( bin==4 ) for(y=0; y<h0;) { GH; GH; GH; GH; H1++; }
208 |       else for( y=0; y<h0;) { for( int y1=0; y1<bin; y1++ ) { GH; } H1++; }
209 |       #undef GH
210 |     } else { // softBin odd
211 |       // interpolate using trilinear interpolation
212 |       float ms[4], xyd, yb, xd, yd; //__m128 _m, _m0, _m1;
213 |       bool hasLf, hasRt; int xb0, yb0;
214 |       if( x==0 ) { init=(0+.5f)*sInv-0.5f; xb=init; }
215 |       hasLf = xb>=0; xb0 = hasLf?(int)xb:-1; hasRt = xb0 < wb-1;
216 |       xd=xb-xb0; xb+=sInv; yb=init; y=0;
217 |       // macros for code conciseness
218 |       #define GHinit yd=yb-yb0; yb+=sInv; H0=H+xb0*hb+yb0; xyd=xd*yd; \
219 |         ms[0]=1-xd-yd+xyd; ms[1]=yd-xyd; ms[2]=xd-xyd; ms[3]=xyd;
220 |       #define GH(H,ma,mb) H1=H; STRu(*H1,ADD(LDu(*H1),MUL(ma,mb)));
221 |       // leading rows, no top bin
222 |       for( ; y<bin/2; y++ ) {
223 |         //debug("%d", y);// 0, 1, 2
224 |         yb0=-1; GHinit;
225 |         if(hasLf) { H0[O0[y]+1]+=ms[1]*M0[y]; H0[O1[y]+1]+=ms[1]*M1[y]; }
226 |         if(hasRt) { H0[O0[y]+hb+1]+=ms[3]*M0[y]; H0[O1[y]+hb+1]+=ms[3]*M1[y]; }
227 |       }
228 |       // main rows, has top and bottom bins, use SSE for minor speedup
229 |       if( softBin<0 ) for( ; ; y++ ) { // softBin < 0 odd
230 |         yb0 = (int) yb; if(yb0>=hb-1) break; GHinit;
231 |         // none speedup version
232 |         if(hasLf) { H0[O0[y]+1]+=ms[1]*M0[y]; H0[O0[y]]+=ms[0]*M0[y]; }
233 |         if(hasRt) { H0[O0[y]+hb+1]+=ms[3]*M0[y]; H0[O0[y]+hb]+=ms[2]*M0[y]; }
234 |       // speedup version, have potential bugs 
235 |       //  _m0=SET(M0[y]);
236 |       //  if(hasLf) { _m=SET(0,0,ms[1],ms[0]); GH(H0+O0[y],_m,_m0); }
237 |       //  if(hasRt) { _m=SET(0,0,ms[3],ms[2]); GH(H0+O0[y]+hb,_m,_m0); }
238 |       } else for( ; ; y++ ) { //softBin > 0 odd
239 |         yb0 = (int) yb; if(yb0>=hb-1) break; GHinit;        
240 |         // none speedup version
241 |         if(hasLf) { 
242 |           H0[O0[y]+1]+=ms[1]*M0[y]; H0[O1[y]+1]+=ms[1]*M1[y]; 
243 |           H0[O0[y]]+=ms[0]*M0[y]; H0[O1[y]]+=ms[0]*M1[y]; }
244 |         if(hasRt) { 
245 |           H0[O0[y]+hb+1]+=ms[3]*M0[y]; H0[O1[y]+hb+1]+=ms[3]*M1[y];
246 |           H0[O0[y]+hb]+=ms[2]*M0[y]; H0[O1[y]+hb]+=ms[2]*M1[y];  }
247 |       // speedup version, have potential bugs 
248 |       /*  _m0=SET(M0[y]); _m1=SET(M1[y]);
249 |         if(hasLf) { _m=SET(0,0,ms[1],ms[0]);
250 |           GH(H0+O0[y],_m,_m0); GH(H0+O1[y],_m,_m1); }
251 |         if(hasRt) { _m=SET(0,0,ms[3],ms[2]);
252 |           GH(H0+O0[y]+hb,_m,_m0); GH(H0+O1[y]+hb,_m,_m1); }*/
253 |       }
254 |       // final rows, no bottom bin
255 |       for( ; y<h0; y++ ) {
256 |         //debug("%d", y); // 147, 148, 149
257 |         yb0 = (int) yb; GHinit;
258 |         if(hasLf) { H0[O0[y]]+=ms[0]*M0[y]; H0[O1[y]]+=ms[0]*M1[y]; }
259 |         if(hasRt) { H0[O0[y]+hb]+=ms[2]*M0[y]; H0[O1[y]+hb]+=ms[2]*M1[y]; }
260 |       }
261 |       #undef GHinit
262 |       #undef GH
263 |     }
264 |   }
265 |   alFree(O0); alFree(O1); alFree(M0); alFree(M1);
266 |   // normalize boundary bins which only get 7/8 of weight of interior bins
267 |   if( softBin%2!=0 ) for( int o=0; o<nOrients; o++ ) {
268 |     x=0; for( y=0; y<hb; y++ ) H[o*nb+x*hb+y]*=8.f/7.f;
269 |     y=0; for( x=0; x<wb; x++ ) H[o*nb+x*hb+y]*=8.f/7.f;
270 |     x=wb-1; for( y=0; y<hb; y++ ) H[o*nb+x*hb+y]*=8.f/7.f;
271 |     y=hb-1; for( x=0; x<wb; x++ ) H[o*nb+x*hb+y]*=8.f/7.f;
272 |   }
273 | }
274 | /******************************************************************************/
275 | 
276 | // HOG helper: compute 2x2 block normalization values (padded by 1 pixel)
277 | float* hogNormMatrix( float *H, int nOrients, int hb, int wb, int bin ) {
278 |   float *N, *N1, *n; int o, x, y, dx, dy, hb1=hb+1, wb1=wb+1;
279 |   float eps = 1e-4f/4/bin/bin/bin/bin; // precise backward equality
280 |   N = (float*) wrCalloc(hb1*wb1,sizeof(float)); N1=N+hb1+1;
281 |   for( o=0; o<nOrients; o++ ) for( x=0; x<wb; x++ ) for( y=0; y<hb; y++ )
282 |     N1[x*hb1+y] += H[o*wb*hb+x*hb+y]*H[o*wb*hb+x*hb+y];
283 |   for( x=0; x<wb-1; x++ ) for( y=0; y<hb-1; y++ ) {
284 |     n=N1+x*hb1+y; *n=1/float(sqrt(n[0]+n[1]+n[hb1]+n[hb1+1]+eps)); }
285 |   x=0;     dx= 1; dy= 1; y=0;                  N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
286 |   x=0;     dx= 1; dy= 0; for(y=0; y<hb1; y++)  N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
287 |   x=0;     dx= 1; dy=-1; y=hb1-1;              N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
288 |   x=wb1-1; dx=-1; dy= 1; y=0;                  N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
289 |   x=wb1-1; dx=-1; dy= 0; for( y=0; y<hb1; y++) N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
290 |   x=wb1-1; dx=-1; dy=-1; y=hb1-1;              N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
291 |   y=0;     dx= 0; dy= 1; for(x=0; x<wb1; x++)  N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
292 |   y=hb1-1; dx= 0; dy=-1; for(x=0; x<wb1; x++)  N[x*hb1+y]=N[(x+dx)*hb1+y+dy];
293 |   return N;
294 | }
295 | 
296 | // HOG helper: compute HOG or FHOG channels
297 | void hogChannels( float *H, const float *R, const float *N,
298 |   int hb, int wb, int nOrients, float clip, int type )
299 | {
300 |   #define GETT(blk) t=R1[y]*N1[y-(blk)]; if(t>clip) t=clip; c++;
301 |   const float r=.2357f; int o, x, y, c; float t;
302 |   const int nb=wb*hb, nbo=nOrients*nb, hb1=hb+1;
303 |   for( o=0; o<nOrients; o++ ) for( x=0; x<wb; x++ ) {
304 |     const float *R1=R+o*nb+x*hb, *N1=N+x*hb1+hb1+1;
305 |     float *H1 = (type<=1) ? (H+o*nb+x*hb) : (H+x*hb);
306 |     if( type==0) for( y=0; y<hb; y++ ) {
307 |       // store each orientation and normalization (nOrients*4 channels)
308 |       c=-1; GETT(0); H1[c*nbo+y]=t; GETT(1); H1[c*nbo+y]=t;
309 |       GETT(hb1); H1[c*nbo+y]=t; GETT(hb1+1); H1[c*nbo+y]=t;
310 |     } else if( type==1 ) for( y=0; y<hb; y++ ) {
311 |       // sum across all normalizations (nOrients channels)
312 |       c=-1; GETT(0); H1[y]+=t*.5f; GETT(1); H1[y]+=t*.5f;
313 |       GETT(hb1); H1[y]+=t*.5f; GETT(hb1+1); H1[y]+=t*.5f;
314 |     } else if( type==2 ) for( y=0; y<hb; y++ ) {
315 |       // sum across all orientations (4 channels)
316 |       c=-1; GETT(0); H1[c*nb+y]+=t*r; GETT(1); H1[c*nb+y]+=t*r;
317 |       GETT(hb1); H1[c*nb+y]+=t*r; GETT(hb1+1); H1[c*nb+y]+=t*r;
318 |     }
319 |   }
320 |   #undef GETT
321 | }
322 | 
323 | // compute HOG features
324 | void hog( float *M, float *O, float *H, int h, int w, int binSize,
325 |   int nOrients, int softBin, bool full, float clip )
326 | {
327 |   float *N, *R; const int hb=h/binSize, wb=w/binSize;//, nb=hb*wb;
328 |   // compute unnormalized gradient histograms
329 |   R = (float*) wrCalloc(wb*hb*nOrients,sizeof(float));
330 |   gradHist( M, O, R, h, w, binSize, nOrients, softBin, full );
331 |   // compute block normalization values
332 |   N = hogNormMatrix( R, nOrients, hb, wb, binSize );
333 |   // perform four normalizations per spatial block
334 |   hogChannels( H, R, N, hb, wb, nOrients, clip, 0 );
335 |   wrFree(N); wrFree(R);
336 | }
337 | 
338 | // compute FHOG features
339 | void fhog( float *M, float *O, float *H, int h, int w, int binSize,
340 |   int nOrients, int softBin, float clip )
341 | {
342 |   const int hb=h/binSize, wb=w/binSize, nb=hb*wb, nbo=nb*nOrients;
343 |   float *N, *R1, *R2; int o, x;
344 |   // compute unnormalized constrast sensitive histograms
345 |   R1 = (float*) wrCalloc(wb*hb*nOrients*2,sizeof(float));
346 |   gradHist( M, O, R1, h, w, binSize, nOrients*2, softBin, true );
347 |   // compute unnormalized contrast insensitive histograms
348 |   R2 = (float*) wrCalloc(wb*hb*nOrients,sizeof(float));
349 |   for( o=0; o<nOrients; o++ ) for( x=0; x<nb; x++ )
350 |     R2[o*nb+x] = R1[o*nb+x]+R1[(o+nOrients)*nb+x];
351 |   // compute block normalization values
352 |   N = hogNormMatrix( R2, nOrients, hb, wb, binSize );
353 |   // normalized histograms and texture channels
354 |   hogChannels( H+nbo*0, R1, N, hb, wb, nOrients*2, clip, 1 );
355 |   hogChannels( H+nbo*2, R2, N, hb, wb, nOrients*1, clip, 1 );
356 |   hogChannels( H+nbo*3, R1, N, hb, wb, nOrients*2, clip, 2 );		
357 |   wrFree(N); wrFree(R1); wrFree(R2);
358 | }
359 | 


--------------------------------------------------------------------------------
/ECO/ffttools.cc:
--------------------------------------------------------------------------------
  1 | 
  2 | #include "ffttools.hpp"
  3 | namespace eco
  4 | {
  5 | cv::Mat dft(const cv::Mat img_org, const bool backwards)
  6 | {
  7 | 	if (img_org.empty())
  8 | 		return cv::Mat();
  9 | 	cv::Mat img;
 10 | 	int type = img_org.type() & CV_MAT_DEPTH_MASK;
 11 | 	//debug("%d", type);//5:float;6:double
 12 | 	assert(((type == 5) || (type == 6)) && "error: input mat type error!");
 13 | 
 14 | #ifdef USE_FFTW
 15 | 	size_t cols = img_org.cols;
 16 | 	size_t rows = img_org.rows;
 17 | 	size_t size = cols * rows;
 18 | 	fftw_complex *in, *out;
 19 | 	fftw_plan p;
 20 | 	in = (fftw_complex *)fftw_malloc(sizeof(fftw_complex) * cols * rows);
 21 | 	out = (fftw_complex *)fftw_malloc(sizeof(fftw_complex) * cols * rows);
 22 | 	p = fftw_plan_dft_2d(rows, cols, in, out, backwards ? FFTW_BACKWARD : FFTW_FORWARD, FFTW_ESTIMATE);
 23 | 
 24 | 	if (img_org.channels() == 1)
 25 | 	{
 26 | 		for (size_t i = 0; i < rows; i++)
 27 | 			for (size_t j = 0; j < cols; j++)
 28 | 			{
 29 | 				if (type == 5)
 30 | 				{
 31 | 					in[i * cols + j][0] = img_org.at<float>(i, j);
 32 | 					in[i * cols + j][1] = 0;
 33 | 				}
 34 | 				else if (type == 6)
 35 | 				{
 36 | 					in[i * cols + j][0] = img_org.at<double>(i, j);
 37 | 					in[i * cols + j][1] = 0;
 38 | 				}
 39 | 			}
 40 | 	}
 41 | 	else if (img_org.channels() == 2)
 42 | 	{
 43 | 		for (size_t i = 0; i < rows; i++)
 44 | 			for (size_t j = 0; j < cols; j++)
 45 | 			{
 46 | 				if (type == 5)
 47 | 				{
 48 | 					in[i * cols + j][0] = img_org.at<cv::Vec2f>(i, j)[0];
 49 | 					in[i * cols + j][1] = img_org.at<cv::Vec2f>(i, j)[1];
 50 | 				}
 51 | 				else if (type == 6)
 52 | 				{
 53 | 					in[i * cols + j][0] = img_org.at<cv::Vec2d>(i, j)[0];
 54 | 					in[i * cols + j][1] = img_org.at<cv::Vec2d>(i, j)[1];
 55 | 				}
 56 | 			}
 57 | 	}
 58 | 	else
 59 | 	{
 60 | 		assert(0 && "error: dft input channels error!");
 61 | 	}
 62 | 
 63 | 	fftw_execute(p);
 64 | 	fftw_destroy_plan(p);
 65 | 
 66 | 	if (type == 5)
 67 | 	{
 68 | 		img = cv::Mat(rows, cols, CV_32FC2);
 69 | 	}
 70 | 	else if (type == 6)
 71 | 	{
 72 | 		img = cv::Mat(rows, cols, CV_64FC2);
 73 | 	}
 74 | 	for (size_t i = 0; i < rows; i++)
 75 | 		for (size_t j = 0; j < cols; j++)
 76 | 		{
 77 | 			if (type == 5)
 78 | 			{
 79 | 				img.at<cv::Vec2f>(i, j)[0] = out[i * cols + j][0];
 80 | 				img.at<cv::Vec2f>(i, j)[1] = out[i * cols + j][1];
 81 | 			}
 82 | 			else if (type == 6)
 83 | 			{
 84 | 				img.at<cv::Vec2d>(i, j)[0] = out[i * cols + j][0];
 85 | 				img.at<cv::Vec2d>(i, j)[1] = out[i * cols + j][1];
 86 | 			}
 87 | 		}
 88 | 	if (backwards) // FFTW computes an unnormalized transform
 89 | 	{
 90 | 		img = img / size;
 91 | 	}
 92 | 	fftw_free(in);
 93 | 	fftw_free(out);
 94 | #else
 95 | 	img_org.copyTo(img);
 96 | 	if (img.channels() == 1)
 97 | 	{
 98 | 		if (type == 5)
 99 | 		{
100 | 			cv::Mat planes[] = {cv::Mat_<float>(img),
101 | 								cv::Mat_<float>::zeros(img.size())};
102 | 			cv::merge(planes, 2, img);
103 | 		}
104 | 		else if (type == 6)
105 | 		{
106 | 			cv::Mat planes[] = {cv::Mat_<double>(img),
107 | 								cv::Mat_<double>::zeros(img.size())};
108 | 			cv::merge(planes, 2, img);
109 | 		}
110 | 	}
111 | 	cv::dft(img, img, backwards ? (cv::DFT_INVERSE + cv::DFT_SCALE) : 0);
112 | #endif
113 | 	return img;
114 | } // namespace eco
115 | 
116 | cv::Mat fftshift(const cv::Mat img_org,
117 | 				 const bool rowshift,
118 | 				 const bool colshift,
119 | 				 const bool reverse)
120 | {
121 | 	if (img_org.empty())
122 | 		return cv::Mat();
123 | 	int type = img_org.type() & CV_MAT_DEPTH_MASK;
124 | 	//debug("%d", type);//5:float;6:double
125 | 
126 | 	assert(((type == 5) || (type == 6)) && "error: input mat type error!");
127 | 
128 | 	cv::Mat temp(img_org.size(), img_org.type());
129 | 
130 | 	int w = img_org.cols, h = img_org.rows;
131 | 	int rshift = reverse ? h - h / 2 : h / 2,
132 | 		cshift = reverse ? w - w / 2 : w / 2;
133 | 
134 | 	for (int i = 0; i < img_org.rows; i++)
135 | 	{
136 | 		int ii = rowshift ? (i + rshift) % h : i;
137 | 		for (int j = 0; j < img_org.cols; j++)
138 | 		{
139 | 			int jj = colshift ? (j + cshift) % w : j;
140 | 			if (type == 5)
141 | 			{
142 | 				if (img_org.channels() == 2)
143 | 					temp.at<cv::Vec<float, 2>>(ii, jj) =
144 | 						img_org.at<cv::Vec<float, 2>>(i, j);
145 | 				else if (img_org.channels() == 1)
146 | 					temp.at<float>(ii, jj) = img_org.at<float>(i, j);
147 | 				else
148 | 					assert(0 && "error of image channels.");
149 | 			}
150 | 			else if (type == 6)
151 | 			{
152 | 				if (img_org.channels() == 2)
153 | 					temp.at<cv::Vec<double, 2>>(ii, jj) =
154 | 						img_org.at<cv::Vec<double, 2>>(i, j);
155 | 				else if (img_org.channels() == 1)
156 | 					temp.at<double>(ii, jj) = img_org.at<double>(i, j);
157 | 				else
158 | 					assert(0 && "error of image channels.");
159 | 			}
160 | 		}
161 | 	}
162 | 	return temp;
163 | }
164 | 
165 | // take the real part of a complex img
166 | cv::Mat real(const cv::Mat img)
167 | {
168 | 	std::vector<cv::Mat> planes;
169 | 	cv::split(img, planes);
170 | 	return planes[0];
171 | }
172 | // take the image part of a complex img
173 | cv::Mat imag(const cv::Mat img)
174 | {
175 | 	std::vector<cv::Mat> planes;
176 | 	cv::split(img, planes);
177 | 	return planes[1];
178 | }
179 | // calculate the magnitde of a complex img
180 | cv::Mat magnitude(const cv::Mat img)
181 | {
182 | 	cv::Mat res;
183 | 	std::vector<cv::Mat> planes;
184 | 	cv::split(img, planes);
185 | 	if (planes.size() == 1)
186 | 		res = cv::abs(img);
187 | 	else if (planes.size() == 2)
188 | 		cv::magnitude(planes[0], planes[1], res);
189 | 	else
190 | 		assert(0 && "error: img size error!");
191 | 	return res;
192 | }
193 | // complex element-wise multiplication for 32Float type
194 | cv::Mat complexDotMultiplication(const cv::Mat &a, const cv::Mat &b)
195 | {
196 | 	cv::Mat res;
197 | #ifdef USE_SIMD
198 | 	//res = complexDotMultiplicationCPU(a, b);
199 | 	res = complexDotMultiplicationSIMD(a, b);
200 | #else
201 | 	res = complexDotMultiplicationCPU(a, b);
202 | #endif
203 | 	/*
204 | #ifdef USE_CUDA
205 | 	res = complexDotMultiplicationGPU(a, b);
206 | #else
207 | 	res = complexDotMultiplicationCPU(a, b);
208 | #endif
209 | */
210 | 	return res;
211 | }
212 | 
213 | #ifdef USE_SIMD
214 | cv::Mat complexDotMultiplicationSIMD(const cv::Mat &a, const cv::Mat &b)
215 | {
216 | 	if (a.rows != b.rows || a.cols != b.cols)
217 | 	{
218 | 		assert(0 && "Error: Mat a and b different size!");
219 | 	}
220 | 	//debug("type a: %d, b: %d", a.type() & CV_MAT_DEPTH_MASK, b.type() & CV_MAT_DEPTH_MASK);
221 | 	if ((a.type() & CV_MAT_DEPTH_MASK) != 5 || (b.type() & CV_MAT_DEPTH_MASK) != 5)
222 | 	{
223 | 		assert(0 && "Error: Mat a or b type is not float!");
224 | 	}
225 | 	int h = a.rows;
226 | 	int w = a.cols;
227 | 	float *ar, *ai, *br, *bi, *rr, *ri;
228 | 	/*
229 | 	ar = (float *)wrCalloc(h * w, sizeof(float));
230 | 	ai = (float *)wrCalloc(h * w, sizeof(float));
231 | 	br = (float *)wrCalloc(h * w, sizeof(float));
232 | 	bi = (float *)wrCalloc(h * w, sizeof(float));
233 | 	rr = (float *)wrCalloc(h * w, sizeof(float));
234 | 	ri = (float *)wrCalloc(h * w, sizeof(float));
235 | 	*/
236 | 	ar = (float *)alMalloc(h * w * sizeof(float), 16);
237 | 	ai = (float *)alMalloc(h * w * sizeof(float), 16);
238 | 	br = (float *)alMalloc(h * w * sizeof(float), 16);
239 | 	bi = (float *)alMalloc(h * w * sizeof(float), 16);
240 | 	rr = (float *)alMalloc(h * w * sizeof(float), 16);
241 | 	ri = (float *)alMalloc(h * w * sizeof(float), 16); 
242 | 	//	debug("%p, %p, %p, %p, %p, %p", ar, ai, br, bi, rr, ri);
243 | 	if (a.channels() == 1)
244 | 	{
245 | 		for (int i = 0; i < h; i++)
246 | 			for (int j = 0; j < w; j++)
247 | 			{
248 | 				*(ar + i * w + j) = a.at<float>(i, j);
249 | 				*(ai + i * w + j) = 0;
250 | 			}
251 | 	}
252 | 	else if (a.channels() == 2)
253 | 	{
254 | 		for (int i = 0; i < h; i++)
255 | 			for (int j = 0; j < w; j++)
256 | 			{
257 | 				*(ar + i * w + j) = a.at<cv::Vec2f>(i, j)[0];
258 | 				*(ai + i * w + j) = a.at<cv::Vec2f>(i, j)[1];
259 | 			}
260 | 	}
261 | 	else
262 | 	{
263 | 		assert(0 && "Error: a.channels error!");
264 | 	}
265 | 
266 | 	if (b.channels() == 1)
267 | 	{
268 | 		for (int i = 0; i < h; i++)
269 | 			for (int j = 0; j < w; j++)
270 | 			{
271 | 				*(br + i * w + j) = b.at<float>(i, j);
272 | 				*(bi + i * w + j) = 0;
273 | 			}
274 | 	}
275 | 	else if (b.channels() == 2)
276 | 	{
277 | 		for (int i = 0; i < h; i++)
278 | 			for (int j = 0; j < w; j++)
279 | 			{
280 | 				*(br + i * w + j) = b.at<cv::Vec2f>(i, j)[0];
281 | 				*(bi + i * w + j) = b.at<cv::Vec2f>(i, j)[1];
282 | 			}
283 | 	}
284 | 	else
285 | 	{
286 | 		assert(0 && "Error: b.channels error!");
287 | 	}
288 | 
289 | 	//(a0+ia1)x(b0+ib1)=(a0b0-a1b1)+i(a0b1+a1b0)
290 | 	/* orininal implementation
291 | 	for (int i = 0; i < h; i++)
292 | 		for (int j = 0; j < w; j++)
293 | 		{
294 | 			*(rr + i * w + j) = *(ar + i * w + j) * *(br + i * w + j) - *(ai + i * w + j) * *(bi + i * w + j);
295 | 			*(ri + i * w + j) = *(ar + i * w + j) * *(bi + i * w + j) + *(ai + i * w + j) * *(br + i * w + j); 
296 | 		}
297 | */
298 | 	__m128 *_ar, *_ai, *_br, *_bi, *_rr, *_ri;
299 | 	_ar = (__m128 *)ar;
300 | 	_ai = (__m128 *)ai;
301 | 	_br = (__m128 *)br;
302 | 	_bi = (__m128 *)bi;
303 | 	_rr = (__m128 *)rr;
304 | 	_ri = (__m128 *)ri;
305 | 	int i = 0;
306 | 	for (; i < h * w - 4; i += 4)
307 | 	{
308 | 		*_rr++ = SUB(MUL(*_ar, *_br), MUL(*_ai, *_bi));
309 | 		*_ri++ = ADD(MUL(*_ar++, *_bi++), MUL(*_ai++, *_br++));
310 | 	}
311 | 	for (; i < h * w; i++)
312 | 	{
313 | 		*(rr + i) = *(ar + i) * *(br + i) - *(ai + i) * *(bi + i);
314 | 		*(ri + i) = *(ar + i) * *(bi + i) + *(ai + i) * *(br + i);
315 | 	}
316 | 
317 | 	cv::Mat res = cv::Mat::zeros(h, w, CV_32FC2);
318 | 	for (int i = 0; i < h; i++) // for each row
319 | 		for (int j = 0; j < w; j++) // for each col
320 | 		{
321 | 			res.at<cv::Vec2f>(i, j)[0] = *(rr + i * w + j);
322 | 			res.at<cv::Vec2f>(i, j)[1] = *(ri + i * w + j);
323 | 		}
324 | /*
325 | 	wrFree(ar);
326 | 	wrFree(ai);
327 | 	wrFree(br);
328 | 	wrFree(bi);
329 | 	wrFree(rr);
330 | 	wrFree(ri);
331 | 	*/
332 | 	alFree(ar);
333 | 	alFree(ai);
334 | 	alFree(br);
335 | 	alFree(bi);
336 | 	alFree(rr);
337 | 	alFree(ri);
338 | 
339 | 	return res;
340 | }
341 | #endif
342 | 
343 | cv::Mat complexDotMultiplicationCPU(const cv::Mat &a, const cv::Mat &b)
344 | {
345 | 	cv::Mat temp_a;
346 | 	cv::Mat temp_b;
347 | 	a.copyTo(temp_a);
348 | 	b.copyTo(temp_b);
349 | 
350 | 	if (a.channels() == 1) // for single channel image a
351 | 	{
352 | 		std::vector<cv::Mat> a_vector =
353 | 			{a, cv::Mat::zeros(a.size(), CV_32FC1)};
354 | 		cv::merge(a_vector, temp_a);
355 | 	}
356 | 	if (b.channels() == 1) // for single channel image b
357 | 	{
358 | 		std::vector<cv::Mat> b_vector =
359 | 			{b, cv::Mat::zeros(b.size(), CV_32FC1)};
360 | 		cv::merge(b_vector, temp_b);
361 | 	}
362 | 
363 | 	cv::Mat res = cv::Mat::zeros(temp_a.size(), CV_32FC2);
364 | 	//(a0+ia1)x(b0+ib1)=(a0b0-a1b1)+i(a0b1+a1b0)
365 | 	//#pragma omp parallel for collapse(2)
366 | 	for (int j = 0; j < temp_a.cols; j++)
367 | 	{
368 | 		for (int i = 0; i < temp_a.rows; i++)
369 | 		{
370 | 			res.at<cv::Vec2f>(i, j)[0] = temp_a.at<cv::Vec2f>(i, j)[0] * temp_b.at<cv::Vec2f>(i, j)[0] - temp_a.at<cv::Vec2f>(i, j)[1] * temp_b.at<cv::Vec2f>(i, j)[1];
371 | 			res.at<cv::Vec2f>(i, j)[1] = temp_a.at<cv::Vec2f>(i, j)[0] * temp_b.at<cv::Vec2f>(i, j)[1] + temp_a.at<cv::Vec2f>(i, j)[1] * temp_b.at<cv::Vec2f>(i, j)[0];
372 | 		}
373 | 	}
374 | 	return res;
375 | }
376 | /*
377 | // use .mul() much slower than for loop
378 | cv::Mat complexDotMultiplicationCPU(const cv::Mat &a, const cv::Mat &b)
379 | {
380 | 	cv::Mat res;
381 | 
382 | 	cv::Mat temp_a;
383 | 	cv::Mat temp_b;
384 | 	a.copyTo(temp_a);
385 | 	b.copyTo(temp_b);
386 | 
387 | 	if (a.channels() == 1) // for single channel image a
388 | 	{
389 | 		std::vector<cv::Mat> a_vector =
390 | 			{a, cv::Mat::zeros(a.size(), CV_32FC1)};
391 | 		cv::merge(a_vector, temp_a);
392 | 	}
393 | 	if (b.channels() == 1) // for single channel image b
394 | 	{
395 | 		std::vector<cv::Mat> b_vector =
396 | 			{b, cv::Mat::zeros(b.size(), CV_32FC1)};
397 | 		cv::merge(b_vector, temp_b);
398 | 	}
399 | 
400 | 	std::vector<cv::Mat> pa;
401 | 	std::vector<cv::Mat> pb;
402 | 	cv::split(temp_a, pa);
403 | 	cv::split(temp_b, pb);
404 | 
405 | 	std::vector<cv::Mat> pres;
406 | 	//(a0+ia1)x(b0+ib1)=(a0b0-a1b1)+i(a0b1+a1b0)
407 | 	pres.push_back(pa[0].mul(pb[0]) - pa[1].mul(pb[1]));
408 | 	pres.push_back(pa[0].mul(pb[1]) + pa[1].mul(pb[0]));
409 | 
410 | 	cv::merge(pres, res);
411 | 	return res;
412 | }
413 | // Only with quite large matrix, GPU is faster!!!!!
414 | #ifdef USE_CUDA
415 | cv::Mat complexDotMultiplicationGPU(const cv::Mat &a, const cv::Mat &b)
416 | {
417 | 	cv::Mat res;
418 | 
419 | 	cv::Mat temp_a;
420 | 	cv::Mat temp_b;
421 | 	a.copyTo(temp_a);
422 | 	b.copyTo(temp_b);
423 | 
424 | 	if (a.channels() == 1) // for single channel image a
425 | 	{
426 | 		std::vector<cv::Mat> a_vector =
427 | 			{a, cv::Mat::zeros(a.size(), CV_32FC1)};
428 | 		cv::merge(a_vector, temp_a);
429 | 	}
430 | 	if (b.channels() == 1) // for single channel image b
431 | 	{
432 | 		std::vector<cv::Mat> b_vector =
433 | 			{b, cv::Mat::zeros(b.size(), CV_32FC1)};
434 | 		cv::merge(b_vector, temp_b);
435 | 	}
436 | 
437 | 	cv::cuda::GpuMat temp_a_gpu;
438 | 	cv::cuda::GpuMat temp_b_gpu;
439 | 	temp_a_gpu.upload(temp_a);
440 | 	temp_b_gpu.upload(temp_b);
441 | 
442 | 	std::vector<cv::cuda::GpuMat> pa;
443 | 	std::vector<cv::cuda::GpuMat> pb;
444 | 	cv::cuda::split(temp_a_gpu, pa);
445 | 	cv::cuda::split(temp_b_gpu, pb);
446 | 
447 | 	std::vector<cv::cuda::GpuMat> pres;
448 | 	cv::cuda::GpuMat temp_a_gpu_split;
449 | 	cv::cuda::GpuMat temp_b_gpu_split;
450 | 
451 | 	//(a0+ia1)x(b0+ib1)=(a0b0-a1b1)+i(a0b1+a1b0)
452 | 	cv::cuda::multiply(pa[0], pb[0], temp_a_gpu);
453 | 	cv::cuda::multiply(pa[1], pb[1], temp_b_gpu);
454 | 	cv::cuda::subtract(temp_a_gpu, temp_b_gpu, temp_a_gpu_split);
455 | 	pres.push_back(temp_a_gpu_split);
456 | 
457 | 	cv::cuda::multiply(pa[0], pb[1], temp_a_gpu);
458 | 	cv::cuda::multiply(pa[1], pb[0], temp_b_gpu);
459 | 	cv::cuda::add(temp_a_gpu, temp_b_gpu, temp_b_gpu_split);
460 | 	pres.push_back(temp_b_gpu_split);
461 | 
462 | 	cv::cuda::GpuMat res_gpu;
463 | 	cv::cuda::merge(pres, res_gpu);
464 | 
465 | 	res_gpu.download(res);
466 | 
467 | 	return res;
468 | }
469 | #endif
470 | */
471 | // complex element-wise division
472 | cv::Mat complexDotDivision(const cv::Mat a, const cv::Mat b)
473 | {
474 | 	std::vector<cv::Mat> pa;
475 | 	std::vector<cv::Mat> pb;
476 | 	cv::split(a, pa);
477 | 	cv::split(b, pb);
478 | 	// Opencv if divide by ZERO, result is ZERO!
479 | 	cv::Mat divisor = 1. / (pb[0].mul(pb[0]) + pb[1].mul(pb[1]));
480 | 	//(a0+ia1)/(b0+ib1)=[(a0b0+a1b1)+i(a1b0-a0b1)] / divisor
481 | 	std::vector<cv::Mat> pres;
482 | 	pres.push_back((pa[0].mul(pb[0]) + pa[1].mul(pb[1])).mul(divisor));
483 | 	pres.push_back((pa[1].mul(pb[0]) - pa[0].mul(pb[1])).mul(divisor));
484 | 
485 | 	cv::Mat res;
486 | 	cv::merge(pres, res);
487 | 	return res;
488 | }
489 | // the mulitiplciation of two complex matrix
490 | cv::Mat complexMatrixMultiplication(const cv::Mat &a, const cv::Mat &b)
491 | {
492 | 	if (a.empty() || b.empty())
493 | 		return a;
494 | 
495 | 	if (a.cols != b.rows)
496 | 		assert(0 && "error: a and b size unmatched!");
497 | 
498 | 	cv::Mat res(a.rows, b.cols, CV_32FC2);
499 | 	for (size_t i = 0; i < (size_t)res.rows; i++)
500 | 	{
501 | 		for (size_t j = 0; j < (size_t)res.cols; j++)
502 | 		{
503 | 			cv::Complex<float> rest(0, 0);
504 | 			for (size_t k = 0; k < (size_t)a.cols; k++)
505 | 			{
506 | 				rest += cv::Complex<float>(a.at<cv::Vec<float, 2>>(i, k)[0],
507 | 										   a.at<cv::Vec<float, 2>>(i, k)[1]) *
508 | 						cv::Complex<float>(b.at<cv::Vec<float, 2>>(k, j)[0],
509 | 										   b.at<cv::Vec<float, 2>>(k, j)[1]);
510 | 			}
511 | 			res.at<cv::Vec<float, 2>>(i, j) =
512 | 				cv::Vec<float, 2>(rest.re, rest.im);
513 | 		}
514 | 	}
515 | 	return res;
516 | }
517 | // impliment matlab c = convn(a,b) and convn(a, b, 'valid')
518 | cv::Mat complexConvolution(const cv::Mat a_input,
519 | 						   const cv::Mat b_input,
520 | 						   const bool valid)
521 | {
522 | 	cv::Mat res;
523 | 	cv::Mat a_temp, a, b;
524 | 
525 | 	if (a_input.channels() == 1)
526 | 	{
527 | 		a_temp = real2complex(a_input);
528 | 	}
529 | 	else if (a_input.channels() == 2)
530 | 	{
531 | 		a_temp = a_input;
532 | 	}
533 | 	else if (a_input.channels() > 2)
534 | 	{
535 | 		assert(0 && "error: a_input's channel dimensions error!");
536 | 	}
537 | 
538 | 	if (b_input.channels() == 1)
539 | 	{
540 | 		b = real2complex(b_input);
541 | 	}
542 | 	else if (b_input.channels() == 2)
543 | 	{
544 | 		b = b_input;
545 | 	}
546 | 	else if (b_input.channels() > 2)
547 | 	{
548 | 		assert(0 && "error: b_input's channel dimensions error!");
549 | 	}
550 | 	// padding with zeros
551 | 	a = cv::Mat::zeros(a_input.rows + b_input.rows - 1,
552 | 					   a_input.cols + b_input.cols - 1,
553 | 					   CV_32FC2);
554 | 	cv::Point pos(b_input.cols / 2, b_input.rows / 2);
555 | 	// copy to coresoponding location of the matrix
556 | 	a_temp.copyTo(a(cv::Rect(b_input.cols - 1 - pos.x,
557 | 							 b_input.rows - 1 - pos.y,
558 | 							 a_input.cols,
559 | 							 a_input.rows)));
560 | 
561 | 	rot90(b, 3); // flip around x and y axis
562 | 
563 | 	std::vector<cv::Mat> va, vb;
564 | 	cv::split(a, va);
565 | 	cv::split(b, vb);
566 | 
567 | 	cv::Mat r, i, r1, r2, i1, i2;
568 | 	cv::filter2D(va[0], r1, -1, vb[0],
569 | 				 cv::Point(-1, -1), 0, cv::BORDER_ISOLATED);
570 | 	cv::filter2D(va[1], r2, -1, vb[1],
571 | 				 cv::Point(-1, -1), 0, cv::BORDER_ISOLATED);
572 | 	cv::filter2D(va[0], i1, -1, vb[1],
573 | 				 cv::Point(-1, -1), 0, cv::BORDER_ISOLATED);
574 | 	cv::filter2D(va[1], i2, -1, vb[0],
575 | 				 cv::Point(-1, -1), 0, cv::BORDER_ISOLATED);
576 | 
577 | 	//(a0+ia1)x(b0+ib1)=(a0b0-a1b1)+i(a0b1+a1b0)
578 | 	r = r1 - r2; // a0b0-a1b1
579 | 	i = i1 + i2; // a0b1+a1b0
580 | 
581 | 	cv::merge(std::vector<cv::Mat>({r, i}), res);
582 | 
583 | 	if (valid)
584 | 	{
585 | 		if (b_input.cols > a_input.cols || b_input.rows > a_input.rows)
586 | 		{
587 | 			return cv::Mat::zeros(0, 0, CV_32FC2);
588 | 		}
589 | 		else
590 | 		{
591 | 			return res(cv::Rect(b_input.cols - 1,
592 | 								b_input.rows - 1,
593 | 								a_input.cols - b_input.cols + 1,
594 | 								a_input.rows - b_input.rows + 1));
595 | 		}
596 | 	}
597 | 	else
598 | 	{
599 | 		return res;
600 | 	}
601 | }
602 | // change real mat to complex mat
603 | cv::Mat real2complex(const cv::Mat &x)
604 | {
605 | 	if (x.empty() || x.channels() == 2)
606 | 		return x;
607 | 	std::vector<cv::Mat> c = {x, cv::Mat::zeros(x.size(), CV_32FC1)};
608 | 	cv::Mat res;
609 | 	cv::merge(c, res);
610 | 	return res;
611 | }
612 | // mat conjugation
613 | cv::Mat mat_conj(const cv::Mat &org)
614 | {
615 | 	if (org.empty())
616 | 		return org;
617 | 	std::vector<cv::Mat_<float>> planes;
618 | 	cv::split(org, planes);
619 | 	planes[1] = -planes[1];
620 | 	cv::Mat result;
621 | 	cv::merge(planes, result);
622 | 	return result;
623 | }
624 | // sum up all the mat elements, just for float type.
625 | float mat_sum_f(const cv::Mat &org)
626 | {
627 | 	if (org.empty())
628 | 		return 0;
629 | 	float sum = 0;
630 | 	for (size_t r = 0; r < (size_t)org.rows; r++)
631 | 	{
632 | 		const float *orgPtr = org.ptr<float>(r);
633 | 		for (size_t c = 0; c < (size_t)org.cols; c++)
634 | 		{
635 | 			sum += orgPtr[c];
636 | 		}
637 | 	}
638 | 	return sum;
639 | }
640 | // double type version of mat_sum
641 | double mat_sum_d(const cv::Mat &org)
642 | {
643 | 	if (org.empty())
644 | 		return 0;
645 | 	double sum = 0;
646 | 	for (size_t r = 0; r < (size_t)org.rows; r++)
647 | 	{
648 | 		const double *orgPtr = org.ptr<double>(r);
649 | 		for (size_t c = 0; c < (size_t)org.cols; c++)
650 | 		{
651 | 			sum += orgPtr[c];
652 | 		}
653 | 	}
654 | 	return sum;
655 | }
656 | 
657 | } // namespace eco
658 | 


--------------------------------------------------------------------------------