├── CMakeLists.txt
├── README.md
├── src
    ├── connect.h
    ├── detector.h
    ├── nms.h
    └── text_proposal_layer.h
└── test
    └── test.cpp


/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8)
 2 | project( CTPN )
 3 | 
 4 | file( GLOB LIB_SRCS "src/*.cpp" )
 5 | file( GLOB TEST_SRCS ${LIB_SRCS} "test/*.cpp" )
 6 | #file( GLOB JNI_SRCS  ${LIB_SRCS} "./jni/*.cpp" )
 7 | 
 8 | SET( OPENCV_Path "/home/hzqiaohan/.local" ) 
 9 | SET( OPENCV_INCLUDE "${OPENCV_Path}/include" "${OPENCV_Path}/include/opencv" "${OPENCV_Path}/include/opencv2" ) 
10 | SET( OPENCV_LIB "${OPENCV_Path}/lib" ) 
11 | 
12 | SET( CAFFE_PATH "/home/hzqiaohan/OCR/CTPN/caffe")
13 | SET( CAFFE_INC "${CAFFE_PATH}/include")
14 | SET( CAFFE_LIB "${CAFFE_PATH}/build/lib")
15 | 
16 | INCLUDE_DIRECTORIES(${CAFFE_INC} ${OPENCV_INCLUDE} "src")
17 | LINK_DIRECTORIES( ${CAFFE_LIB} ${OPENCV_LIB} "/usr/local/cuda/lib64" )
18 | 
19 | SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2 -D__STDC_CONSTANT_MACROS" )
20 | 
21 | add_executable( test ${TEST_SRCS} )
22 | #target_link_libraries( IDrec -lopencv_core -lopencv_highgui -lopencv_imgproc -lopencv_features2d -lopencv_legacy -lboost_filesystem -lboost_system )
23 | target_link_libraries( test -lglog -lcaffe -lprotobuf -lopencv_core -lopencv_highgui -lopencv_imgproc -lopencv_features2d -lopencv_legacy -lboost_filesystem -lboost_system)
24 | #add_library( deskew SHARED ${LIB_SRCS}  )
25 | #add_library( deskewjni SHARED ${JNI_SRCS}  )
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # CTPN implement by C++
 2 | 
 3 | Ref: https://github.com/tianzhi0549/CTPN 
 4 | 
 5 | This project impliments the CTPN with C++
 6 | 
 7 | ## build
 8 | cd ctpn-cpp
 9 | mkdir build
10 | cd build
11 | cmake ..
12 | make
13 | 
14 | ## the finished and developing work
15 | We have finished the network till the TextDetector::detect() in src/detectors.py Line 51.
16 | The remained work is to connect the text patch after nms to form the textline, see:
17 | * TextDetector::detect() in src/detectors.py Line 52 -> text_lines=self.text_proposal_connector.get_text_lines(text_proposals, scores, im.shape[:2])
18 | * src/text_proposal_connector.py -> whole file and its dependences
19 | 


--------------------------------------------------------------------------------
/src/connect.h:
--------------------------------------------------------------------------------
 1 | /*************************************************************************
 2 |     > File Name: ../src/connect.h
 3 |     > Author: ma6174
 4 |     > Mail: ma6174@163.com 
 5 |     > Created Time: Fri 07 Apr 2017 04:48:11 PM CST
 6 |  ************************************************************************/
 7 | 
 8 | #include<iostream>
 9 | using namespace std;
10 | 
11 | vector< vector<float> > get_text_lines(vector< vector<float> >& text_proposals,
12 | 									   vector<float>& scores, int h, int w)
13 | {
14 | 	//graph builder
15 | 	
16 | }
17 | 


--------------------------------------------------------------------------------
/src/detector.h:
--------------------------------------------------------------------------------
  1 | /*************************************************************************
  2 |     > File Name: detector.h
  3 |     > Author: ma6174
  4 |     > Mail: ma6174@163.com 
  5 |     > Created Time: Thu 02 Mar 2017 10:23:47 AM CST
  6 |  ************************************************************************/
  7 | 
  8 | #include<iostream>
  9 | #include "opencv2/opencv.hpp"
 10 | #include "caffe/blob.hpp"
 11 | #include "caffe/common.hpp"
 12 | #include "caffe/net.hpp"
 13 | //#include "caffe/proto/caffe.pb.h"
 14 | #include "caffe/util/db.hpp"
 15 | //#include "caffe/util/format.hpp"
 16 | #include "caffe/util/io.hpp"
 17 | #include "text_proposal_layer.h"
 18 | #include "nms.h"
 19 | #include "connect.h"
 20 | #include <map>
 21 | 
 22 | using namespace std;
 23 | using namespace cv;
 24 | 
 25 | using caffe::Blob;
 26 | using caffe::Caffe;
 27 | using caffe::Datum;
 28 | using caffe::Net;
 29 | 
 30 | #define TEXT_PROPOSALS_MIN_SCORE 0.7
 31 | #define TEXT_PROPOSALS_NMS_THRESH 0.3
 32 | #define LINE_MIN_SCORE 0.7
 33 | 
 34 | class TextProposalDetector{
 35 | 	public:
 36 | 		TextProposalDetector(boost::shared_ptr< Net<float> > net){
 37 | 			m_caffenet = net;
 38 | 		}
 39 | 		bool detect(Mat img, vector< vector<float> >& rois, vector<float>& scores){
 40 | 			img.convertTo(img, CV_32FC3);
 41 | 			Mat mean(img.rows,img.cols,CV_32FC3,cv::Scalar(102.9801, 115.9465, 122.7717));
 42 | 			img -= mean;
 43 | 			int shorter = img.rows<img.cols?img.rows:img.cols;
 44 | 			float ff = 600.0/shorter;
 45 | 			int num=1,channel=3,height=img.rows*ff,width=img.cols*ff;
 46 | 			m_caffenet->input_blobs()[0]->Reshape(num, channel, height, width);
 47 | 			resize(img,img,cv::Size(width,height));
 48 | /*
 49 | 			boost::shared_ptr<caffe::Blob<float> > info_blob=Net->blob_by_name("im_info");
 50 | 			int * info_data = info_blob->mutable_cpu_data();
 51 | 			info_data[0] = height;
 52 | 			info_data[1] = width;
 53 | */			
 54 | 			boost::shared_ptr<caffe::Blob<float> > input_blob = m_caffenet->blob_by_name("data");
 55 | 			float * input_data = input_blob->mutable_cpu_data();
 56 | 			vector<Mat> mat_vec;
 57 | 			cv::split(img, mat_vec);//(224,224,CV_32FC3,input_data);
 58 | 			float * data_ptr = new float[input_blob->count()];		
 59 |   			for (int i = 0; i < 3; ++i) {
 60 | 	  			float* src_ptr = reinterpret_cast<float *>(mat_vec[i].data);
 61 | 				caffe::caffe_copy(img.total(), src_ptr, data_ptr+img.total()*i);
 62 | 				//memcpy(data_ptr+img.total()*i,src_ptr,img.total());
 63 | 			}
 64 | 			caffe::caffe_copy(input_blob->count(),data_ptr,input_data);
 65 | 			delete data_ptr;
 66 | 			
 67 | 			std::vector<caffe::Blob<float>* > input_vec;
 68 | 			m_caffenet->Forward(input_vec);
 69 | 			boost::shared_ptr<caffe::Blob<float> > rpn_cls_blob = m_caffenet->blob_by_name("rpn_cls_prob_reshape");
 70 | 			//const float * rpn_cls = rpn_cls_blob->cpu_data();
 71 | 			//for(int i=0;i<10;i++)
 72 | 			//	cout<<*(rpn_cls+i)<<endl;
 73 | 			//cout<<rpn_cls_blob->shape_string()<<endl;
 74 | 			boost::shared_ptr<caffe::Blob<float> > rpn_bbox_blob = m_caffenet->blob_by_name("rpn_bbox_pred");
 75 | 			//const float * rpn_bbox = rpn_bbox_blob->cpu_data();
 76 | 			//cout<<rpn_cls_blob->shape_string()<<endl;
 77 | 			
 78 | 			//vector<float> scores;
 79 | 			//vector< vector<int> > rois;
 80 | 			ProposalLayerForward(img.rows, img.cols, rpn_cls_blob, rpn_bbox_blob, rois, scores, TEXT_PROPOSALS_MIN_SCORE);
 81 | 		
 82 | 			return true;
 83 | 		}
 84 | 		~TextProposalDetector(){};
 85 | 	private:
 86 | 		boost::shared_ptr< Net<float> > m_caffenet;
 87 | };
 88 | 
 89 | vector< vector<int> > detect_tline(Mat img)
 90 | {
 91 | 	string deployfile = "models/deploy.prototxt";
 92 | 	string modelfile = "models/ctpn_trained_model.caffemodel";
 93 | 	boost::shared_ptr< Net<float> > net(new Net<float>(deployfile,caffe::TEST));
 94 | 	net->CopyTrainedLayersFrom(modelfile);
 95 | 	TextProposalDetector tpd(net);
 96 | 
 97 | 	vector< vector<int> > tline;
 98 | 	vector< vector<float> > rois;
 99 | 	vector<float> scores;
100 | 
101 | 	if(!tpd.detect(img,rois,scores))
102 | 		return tline;
103 | 
104 | 	map<float, int> scores_index;
105 | 	for(int i=0; i<scores.size(); i++)
106 | 		scores_index.insert(pair<float,int>(scores[i],i));
107 | 	
108 | 	vector< vector<float> > roi = rois;
109 | 	int i=scores.size()-1;
110 | 	for(auto it=scores_index.begin(); it!=scores_index.end(); it++)
111 | 	{
112 | 		scores[i] = it->first;
113 | 		rois[i] = roi[it->second];
114 | 		i--;
115 | 	}
116 | 		//cout<<i++<<','<<it->second<<endl;
117 | 	
118 | 	// nms for text proposals
119 | 	vector<int> keep_idx;
120 | 	keep_idx = nms(rois,scores,TEXT_PROPOSALS_NMS_THRESH);
121 | 	for(int i=0; i<keep_idx.size(); i++)
122 | 	{
123 | 		scores[i] = scores[keep_idx[i]];
124 | 		rois[i] = rois[keep_idx[i]];
125 | 	}
126 | 	rois.resize(keep_idx.size());
127 | 	scores.resize(keep_idx.size());
128 | 
129 | 	//normalize
130 | 	int maxidx=0,minidx=0;
131 | 	for(int i=0; i<scores.size(); i++)
132 | 	{
133 | 		if(scores[i] > scores[maxidx])
134 | 			maxidx = i;
135 | 		if(scores[i] < scores[minidx])
136 | 			minidx = i;
137 | 	}
138 | 	if(scores[maxidx] == scores[minidx])
139 | 	{
140 | 		for(int i=0; i<scores.size(); i++)
141 | 			scores[i] -= scores[minidx];
142 | 	}
143 | 	else
144 | 	{
145 | 		for(int i=0; i<scores.size(); i++)
146 | 			scores[i]=(scores[i]-scores[minidx])/(scores[maxidx]-scores[minidx]);
147 | 	}
148 | 	//connect
149 | 	vector< vector<float> >	tlines;
150 | 	tlines = get_text_lines(rois,scores,img.rows,img.cols);
151 | 	//nms for text lines
152 | 	
153 | 	return tline;
154 | }
155 | 


--------------------------------------------------------------------------------
/src/nms.h:
--------------------------------------------------------------------------------
 1 | /*************************************************************************
 2 |     > File Name: ../src/nms.h
 3 |     > Author: ma6174
 4 |     > Mail: ma6174@163.com 
 5 |     > Created Time: Fri 07 Apr 2017 03:56:33 PM CST
 6 |  ************************************************************************/
 7 | 
 8 | #include<iostream>
 9 | using namespace std;
10 | 
11 | vector<int> nms(vector< vector<float> >& rois, vector<float>& scores, float thresh)
12 | {
13 | 	vector<int> keep;
14 | 	vector<bool> suppressed;
15 | 	suppressed.resize(rois.size());
16 | 	for(int i=0; i<rois.size(); i++)
17 | 		suppressed[i] = false;
18 | 	for(int i=0; i<rois.size(); i++)
19 | 	{
20 | 		if(suppressed[i])
21 | 			continue;
22 | 		keep.push_back(i);
23 | 		float x1 = rois[i][0];
24 | 		float y1 = rois[i][1];
25 | 		float x2 = rois[i][2];
26 | 		float y2 = rois[i][3];
27 | 		float area = (x2 - x1 + 1) * (y2 - y1 + 1);
28 | 		for(int j=i+1; j<rois.size(); j++)
29 | 		{
30 | 			if(suppressed[j])
31 | 				continue;
32 | 			float xx1 = max(rois[j][0],x1);
33 | 			float yy1 = max(rois[j][1],y1);
34 | 			float xx2 = min(rois[j][2],x2);
35 | 			float yy2 = min(rois[j][3],y2);
36 | 			float areaj = (xx2 - xx1 + 1) * (yy2 - yy1 + 1);
37 | 			float w = xx2-xx1+1;
38 | 			w = w>0?w:0;
39 | 			float h = yy2-yy1+1;
40 | 			h = h>0?h:0;
41 | 			float inter = w*h;
42 | 			float ovr = inter / (area + areaj - inter);
43 | 			if(ovr>=thresh)
44 | 				suppressed[j]=true;
45 | 		}
46 | 	}
47 | 	//	cout<<scores[i]<<endl;
48 | 	return keep;
49 | }
50 | 


--------------------------------------------------------------------------------
/src/text_proposal_layer.h:
--------------------------------------------------------------------------------
  1 | /*************************************************************************
  2 |     > File Name: text_proposal_layer.cpp
  3 |     > Author: ma6174
  4 |     > Mail: ma6174@163.com 
  5 |     > Created Time: Fri 03 Mar 2017 02:22:58 PM CST
  6 |  ************************************************************************/
  7 | 
  8 | #include<iostream>
  9 | #include <vector>
 10 | using namespace std;
 11 | 
 12 | inline float threshold(float x, int min_, int max_)
 13 | {
 14 | 	float t = x>max_?max_:x;
 15 | 	return t>min_?t:min_;
 16 | }
 17 | 
 18 | bool apply_deltas_to_anchors(vector< vector<float> >& res, vector<float>& bb_deltas, vector<float>& scores, int stride, int height, int width, int imgh, int imgw, float min_score )
 19 | {
 20 | 	vector<int> heights={11, 16, 23, 33, 48, 68, 97, 139, 198, 283};
 21 | 	vector<int> widths={16};
 22 | 	vector<float> score = scores;
 23 | 	scores.clear();
 24 | 	int base_size = 16;
 25 | 	int num_anchors = heights.size()*widths.size();
 26 | 	//vector< vector<float> > res;
 27 | 	//locate_anchors: 10x[4,] per feature pixel
 28 | 	for(int hh=0; hh<height; hh++)
 29 | 		for(int ww=0; ww<width; ww++)
 30 | 		{
 31 | 			// 10 anchors
 32 | 			int x_ = ww*stride;
 33 | 			int y_ = hh*stride;
 34 | 			for(int h=0; h<heights.size(); h++)
 35 | 				for(int w=0; w<widths.size(); w++)
 36 | 				{
 37 | 					int x1,y1,x2,y2;
 38 | 					float x_ctr = base_size*0.5;
 39 | 					float y_ctr = base_size*0.5;
 40 | 					x1 = x_ctr - widths[w]/2 + x_;
 41 | 					x2 = x_ctr + widths[w]/2 -1 + x_;
 42 | 					y1 = y_ctr - heights[h]/2 + y_;
 43 | 					y2 = y_ctr + heights[h]/2 -1 + y_;
 44 | 					
 45 | 					int anchor_y_ctr = (y1+y2)/2;
 46 | 					int anchor_h = heights[h]+1;
 47 | 					/*
 48 | 					float delta0 = bb_deltas[ww*height*num_anchors+hh*num_anchors+h+w];
 49 | 					float delta1 = bb_deltas[ww*height*num_anchors+hh*num_anchors+h+w+1];
 50 | 					*/
 51 | 					float delta0 = bb_deltas[2*hh*width*num_anchors+2*ww*num_anchors+2*(h+w)];
 52 | 					float delta1 = bb_deltas[2*hh*width*num_anchors+2*ww*num_anchors+2*(h+w)+1];
 53 | 					float global_coords1 = anchor_h*exp(delta1);
 54 | 					float global_coords0 = delta0*anchor_h+anchor_y_ctr-global_coords1/2;
 55 | 					if(score[hh*width*num_anchors+ww*num_anchors+(h+w)]<min_score)
 56 | 						continue;
 57 | 					//cout<<hh*width*num_anchors+ww*num_anchors+(h+w)<<','<<scores[hh*width*num_anchors+ww*num_anchors+(h+w)]<<endl;
 58 | 					scores.push_back(score[hh*width*num_anchors+ww*num_anchors+(h+w)]);
 59 | 					vector<float> rect;
 60 | 					rect.push_back(threshold(x1,0,imgw-1));
 61 | 					rect.push_back(threshold(global_coords0,0,imgh-1));
 62 | 					rect.push_back(threshold(x2,0,imgw-1));
 63 | 					rect.push_back(threshold(global_coords0+global_coords1,0,imgh-1));
 64 | 					res.push_back(rect);
 65 | 					//cout<<delta0<<','<<delta1<<endl;
 66 | 					//cout<<rect[0]<<','<<rect[1]<<','<<rect[2]<<','<<rect[3]<<endl;	
 67 | 					//cout<<x1<<','<<y1<<','<<x2<<','<<y2<<endl;	
 68 | 				}
 69 | 		}
 70 | 
 71 | 	return true;
 72 | }
 73 | 
 74 | bool ProposalLayerForward(int originimgh, int originimgw,
 75 | 		boost::shared_ptr<caffe::Blob<float> > rpn_cls_prob,
 76 | 		boost::shared_ptr<caffe::Blob<float> > rpn_bbox_pred,
 77 | 		vector< vector<float> >& rois,
 78 | 		vector<float>& scores, float min_score)
 79 | {
 80 | 	int _feat_stride = 16;
 81 | 	int _num_anchors = 10;
 82 | 	if( rpn_cls_prob->shape(0) != 1 )
 83 | 		return false;
 84 | 	
 85 | 	const float* cls = rpn_cls_prob->cpu_data();
 86 | 	const float* bbox = rpn_bbox_pred->cpu_data();
 87 | 	const int imgw = rpn_cls_prob->shape(3);
 88 | 	const int imgh = rpn_cls_prob->shape(2);
 89 | 
 90 | 	const int ss = rpn_cls_prob->shape(1)-_num_anchors;	
 91 | 	scores.resize( ss*imgh*imgw );
 92 | 	for(int k=0; k<ss; k++)
 93 | 		for(int i=0; i<imgh; i++)	
 94 | 			for(int j=0; j<imgw; j++)
 95 | 	{
 96 | 		*( scores.data()+i*ss*imgw+j*ss+k ) = *( cls+(k+_num_anchors)*imgh*imgw+i*imgw+j );
 97 | 	}
 98 | /*
 99 | 	for(int i=0; i<10; i++)
100 | 		cout<<*(cls+imgh*imgw*_num_anchors+i)<<';';
101 | 	cout<<endl;
102 | 	for(int i=0; i<scores.size(); i++)
103 | 		cout<<scores[i]<<',';
104 | */
105 | 	cout<<endl;
106 | 	const int sb = rpn_bbox_pred->shape(1);
107 | 	vector<float> bb;
108 | 	bb.resize( sb*imgh*imgw );
109 | 	for(int k=0; k<sb; k++)
110 | 		for(int i=0; i<imgh; i++)	
111 | 			for(int j=0; j<imgw; j++)
112 | 	{
113 | 		*( bb.data()+i*sb*imgw+j*sb+k ) = *( bbox+k*imgh*imgw+i*imgw+j );
114 | 	}
115 | 	
116 | 	
117 | 	//for(int i=0; i<bb.size(); i++)
118 | 		//cout<<bb[i*2]<<','<<bb[i*2+1]<<endl;
119 | 		//cout<<bbox[i]<<endl;
120 | 		//cout<<bb[i]<<endl;
121 | 	
122 | 	
123 | 	return apply_deltas_to_anchors(rois,bb,scores,_feat_stride,imgh,imgw,originimgh,originimgw,min_score);
124 | 	//cout<<roi.size()<<endl;
125 | /*
126 | 	for(int i=0; i<10; i++)
127 | 	{
128 | 		for(int j=0; j<roi[0].size(); j++)
129 | 			cout<<roi[i][j]<<',';
130 | 		cout<<endl;
131 | 		
132 | 	}
133 | */
134 | 	//return true;
135 | }
136 | 


--------------------------------------------------------------------------------
/test/test.cpp:
--------------------------------------------------------------------------------
 1 | /*************************************************************************
 2 |     > File Name: test.cpp
 3 |     > Author: ma6174
 4 |     > Mail: ma6174@163.com 
 5 |     > Created Time: Thu 02 Mar 2017 09:47:58 AM CST
 6 |  ************************************************************************/
 7 | 
 8 | #include<iostream>
 9 | #include "caffe/blob.hpp"
10 | #include "caffe/common.hpp"
11 | #include "caffe/net.hpp"
12 | //#include "caffe/proto/caffe.pb.h"
13 | #include "caffe/util/db.hpp"
14 | //#include "caffe/util/format.hpp"
15 | #include "caffe/util/io.hpp"
16 | 
17 | #include "detector.h"
18 | 
19 | using caffe::Blob;
20 | using caffe::Caffe;
21 | using caffe::Datum;
22 | using caffe::Net;
23 | 
24 | using namespace cv;
25 | using namespace std;
26 | 
27 | 
28 | int main(){
29 | 	
30 | 	Mat img = imread("test.png");
31 | 	vector< vector<int> > tline = detect_tline(img);
32 | 	return 0;
33 | }
34 | 


--------------------------------------------------------------------------------