├── patchmatch
    ├── data
    │   ├── multiple_oreo.jpg
    │   └── single_oreo.jpg
    ├── metric.h
    ├── patchmatch.cpp
    ├── pch.cpp
    ├── pch.h
    └── util.h
└── readme.md


/patchmatch/data/multiple_oreo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/stormouse/patchmatch/4f055d62204ee9b99dff7a84fa08f9769511ccf9/patchmatch/data/multiple_oreo.jpg


--------------------------------------------------------------------------------
/patchmatch/data/single_oreo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/stormouse/patchmatch/4f055d62204ee9b99dff7a84fa08f9769511ccf9/patchmatch/data/single_oreo.jpg


--------------------------------------------------------------------------------
/patchmatch/metric.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "pch.h"
 3 | 
 4 | namespace metric {
 5 | 
 6 | 	using namespace cv;
 7 | 	using namespace std;
 8 | 
 9 | 	constexpr double uc2f = 1.0 / 255.0;
10 | 
11 | 	double sim_abs_diff(Mat& a, Mat& b, int _) {
12 | 		Mat tmp;
13 | 		absdiff(a, b, tmp);
14 | 		return -sum(sum(tmp))[0];
15 | 	}
16 | 
17 | 	template <typename T>
18 | 	double sim_diff_dxy(Mat& a, Mat& b, int _) {
19 | 		double ndiff = sim_abs_diff(a, b, _);
20 | 		double dxy = 0.0;
21 | 		for (int i = 1; i < a.rows - 1; i++)
22 | 		{
23 | 			for (int j = 1; j < a.cols - 1; j++)
24 | 			{
25 | 				for (int k = 0; k < 3; k++)
26 | 				{
27 | 					dxy += abs((a.at<T>(i + 1, j)[k] * uc2f - a.at<T>(i - 1, j)[k] * uc2f)
28 | 						- (b.at<T>(i + 1, j)[k] * uc2f - b.at<T>(i - 1, j)[k] * uc2f)) * 0.5;
29 | 					dxy += abs((a.at<T>(i, j + 1)[k] * uc2f - a.at<T>(i, j - 1)[k] * uc2f)
30 | 						- (b.at<T>(i, j + 1)[k] * uc2f - b.at<T>(i, j - 1)[k] * uc2f)) * 0.5;
31 | 				}
32 | 			}
33 | 		}
34 | 		return ndiff - dxy * 255.0 / 6.0;
35 | 	}
36 | 
37 | 	double sigma(Mat& m, int i, int j, int block_size) {
38 | 		double sd = 0;
39 | 
40 | 		Mat m_tmp = m(Range(i, i + block_size), Range(j, j + block_size));
41 | 		Mat m_squared(block_size, block_size, CV_64F);
42 | 
43 | 		multiply(m_tmp, m_tmp, m_squared);
44 | 
45 | 		double avg = mean(m_tmp)[0];
46 | 		double avg_2 = mean(m_squared)[0];
47 | 
48 | 		sd = sqrt(avg_2 - avg*avg);
49 | 
50 | 		return sd;
51 | 	}
52 | 
53 | 	double cov(Mat& m1, Mat &m2, int i, int j, int block_size) {
54 | 		Mat m3 = Mat::zeros(block_size, block_size, m1.depth());
55 | 		Mat m1_tmp = m1(Range(i, i + block_size), Range(j, j + block_size));
56 | 		Mat m2_tmp = m2(Range(i, i + block_size), Range(j, j + block_size));
57 | 
58 | 		multiply(m1_tmp, m2_tmp, m3);
59 | 
60 | 		double avg_ro = mean(m3)[0];		// E(XY)
61 | 		double avg_r = mean(m1_tmp)[0];		// E(X)
62 | 		double avg_o = mean(m2_tmp)[0];		// E(Y)
63 | 
64 | 		double sd_ro = avg_ro - avg_o * avg_r;	// E(XY) - E(X)E(Y)
65 | 
66 | 		return sd_ro;
67 | 	}
68 | 
69 | 	double ssim(Mat& img_src, Mat& img_compressed, int block_size) {
70 | 		double ssim = 0;
71 | 		int nBlockPerHeight = img_src.rows / block_size;
72 | 		int nBlockPerWidth = img_src.cols / block_size;
73 | 
74 | 		double C1 = 0.01, C2 = 0.03;
75 | 
76 | 		for (int i = 0; i < nBlockPerHeight; i++) {
77 | 			for (int j = 0; j < nBlockPerWidth; j++) {
78 | 				int m = i * block_size, n = j * block_size;
79 | 
80 | 				double avg_o = mean(img_src(Range(i, i + block_size), Range(j, j + block_size)))[0];
81 | 				double avg_r = mean(img_compressed(Range(i, i + block_size), Range(j, j + block_size)))[0];
82 | 				double sigma_o = sigma(img_src, m, n, block_size);
83 | 				double sigma_r = sigma(img_compressed, m, n, block_size);
84 | 				double sigma_ro = cov(img_src, img_compressed, m, n, block_size);
85 | 
86 | 				ssim += ((2 * avg_o *avg_r*C1) * (2 * sigma_ro + C2))
87 | 					/ ((avg_o*avg_o + avg_r*avg_r + C1) * (sigma_o*sigma_o + sigma_r*sigma_r + C2));
88 | 			}
89 | 		}
90 | 
91 | 		ssim /= nBlockPerHeight * nBlockPerWidth;
92 | 
93 | 		return ssim;
94 | 	}
95 | }


--------------------------------------------------------------------------------
/patchmatch/patchmatch.cpp:
--------------------------------------------------------------------------------
  1 | #include "pch.h"
  2 | #include "metric.h"
  3 | #include "util.h"
  4 | 
  5 | using namespace cv;
  6 | using namespace std;
  7 | 
  8 | #define CHANNEL_TYPE Vec3b
  9 | 
 10 | // double(*sim)(Mat&, Mat&, int) = metric::sim_abs_diff;  //metric::ssim;
 11 | double(*sim)(Mat&, Mat&, int) = metric::sim_diff_dxy<CHANNEL_TYPE>;  //metric::ssim;
 12 | 
 13 | typedef vector < vector < vector<int> > >	Vector3i;
 14 | typedef vector < vector <int> >				Vector2i;
 15 | 
 16 | 
 17 | void reconstruct(Vector3i& f, Mat& dst, Mat& ref, int patch_size);
 18 | 
 19 | Mat pick_patch(Mat& mat, int r, int c, int r_offset, int c_offset, int patch_size) {
 20 | 	int rr = r + r_offset, rc = c + c_offset;
 21 | 	return mat(Range(rr - patch_size / 2, rr + patch_size / 2), Range(rc - patch_size / 2, rc + patch_size / 2));
 22 | }
 23 | 
 24 | // initialize offset store
 25 | void initialize(Vector3i& f, int n_rows_dst, int n_cols_dst, int n_rows_ref, int n_cols_ref, int patch_size) {
 26 | 
 27 | 	f.resize(n_rows_dst);
 28 | 	for (int i = 0; i < n_rows_dst; i++) {
 29 | 		f[i].resize(n_cols_dst);
 30 | 		for (int j = 0; j < n_cols_dst; j++) 
 31 | 			f[i][j].resize(2);
 32 | 	}
 33 | 
 34 | 	for (int i = 0; i < n_rows_dst; i++) {
 35 | 		for (int j = 0; j < n_cols_dst; j++) {
 36 | 			f[i][j][0] = int(util::random_range(0, n_rows_ref - patch_size)) - i + patch_size / 2;
 37 | 			f[i][j][1] = int(util::random_range(0, n_cols_ref - patch_size)) - j + patch_size / 2;
 38 | 		}
 39 | 	}
 40 | 
 41 | }
 42 | 
 43 | 
 44 | // iterate
 45 | void patchmatch(Vector3i& f, Mat& img_dst, Mat& img_ref, int patch_size = 3, int n_iterations = 5) {
 46 | 
 47 | 	const int n_rows_dst = img_dst.rows, n_cols_dst = img_dst.cols;
 48 | 	const int n_rows_ref = img_ref.rows, n_cols_ref = img_ref.cols;
 49 | 	
 50 | 	/* initialize */
 51 | 	cout << "initializing..." << endl;
 52 | 	initialize(f, n_rows_dst, n_cols_dst, n_rows_ref, n_cols_ref, patch_size);
 53 | 
 54 | 	/* iterate */
 55 | 	int row_start, row_end, col_start, col_end, step;
 56 | 
 57 | 	Vector2i v;  // current similarity compared with current patch offset
 58 | 	v.resize(n_rows_dst); for (int i = 0; i < n_rows_dst; i++) { v[i].resize(n_cols_dst); }
 59 | 
 60 | 	for (int i = patch_size / 2; i < n_rows_dst - patch_size / 2; i++)
 61 | 		for (int j = patch_size / 2; j < n_rows_dst - patch_size / 2; j++)
 62 | 		{
 63 | 			auto p1 = pick_patch(img_dst, i, j, 0, 0, patch_size);
 64 | 			auto p2 = pick_patch(img_ref, i, j, f[i][j][0], f[i][j][1], patch_size);
 65 | 			v[i][j] = sim(p1, p2, 1);
 66 | 		}
 67 | 
 68 | 
 69 | 	bool reverse = false;
 70 | 
 71 | 	for (int t = 0; t < n_iterations; t++) {
 72 | 		
 73 | 		//Mat progress(img_dst.rows, img_dst.cols, img_dst.type());
 74 | 		//reconstruct(f, progress, img_ref, patch_size);
 75 | 		//imshow("progress", progress);
 76 | 		//cvWaitKey(0);
 77 | 
 78 | 		/* propagate */
 79 | 		cout << "iteration " << t + 1<< endl;
 80 | 
 81 | 		if (reverse) {
 82 | 			row_start = n_rows_dst - patch_size / 2;
 83 | 			row_end = patch_size / 2;
 84 | 			col_start = n_cols_dst - patch_size / 2;
 85 | 			col_end = patch_size / 2;
 86 | 			step = -1;
 87 | 		}
 88 | 		else {
 89 | 			row_start = patch_size / 2;
 90 | 			row_end = n_rows_dst - patch_size / 2;
 91 | 			col_start = patch_size / 2;
 92 | 			col_end = n_cols_dst - patch_size / 2;
 93 | 			step = 1;
 94 | 		}
 95 | 
 96 | 
 97 | 		auto checkvalid_ref = [patch_size, n_rows_ref, n_cols_ref](int r, int c, int ro, int co)
 98 | 		{
 99 | 			if (r + ro < patch_size / 2) return false;
100 | 			if (c + co < patch_size / 2) return false;
101 | 			if (r + ro + patch_size / 2 >= n_rows_ref) return false;
102 | 			if (c + co + patch_size / 2 >= n_cols_ref) return false;
103 | 			return true;
104 | 		};
105 | 
106 | 		for (int i = row_start; i != row_end; i += step) {
107 | 			for (int j = col_start; j != col_end; j += step) {
108 | 				double sm[3];
109 | 				Mat patch = pick_patch(img_dst, i, j, 0, 0, patch_size); 
110 | 
111 | 				sm[0] = v[i][j];
112 | 				if (checkvalid_ref(i, j, f[i + step][j][0], f[i + step][j][1])) {
113 | 					Mat xpatch = pick_patch(img_ref, i, j, f[i + step][j][0], f[i + step][j][1], patch_size);
114 | 					sm[1] = sim(patch, xpatch, 1);
115 | 				}
116 | 				else sm[1] = -1e16f;
117 | 				
118 | 				if (checkvalid_ref(i, j, f[i][j + step][0], f[i][j + step][1])) {
119 | 					Mat ypatch = pick_patch(img_ref, i, j, f[i][j + step][0], f[i][j + step][1], patch_size);
120 | 					sm[2] = sim(patch, ypatch, 1);
121 | 				}
122 | 				else sm[2] = -1e16f;
123 | 				
124 | 				int k = util::argmax(sm, 3);
125 | 				v[i][j] = sm[k];
126 | 
127 | 				switch (k) {
128 | 				case 1: f[i][j][0] = f[i + step][j][0]; f[i][j][1] = f[i + step][j][1]; break;
129 | 				case 2: f[i][j][0] = f[i][j + step][0]; f[i][j][1] = f[i][j + step][1]; break;
130 | 				}
131 | 			}
132 | 		}
133 | 		reverse = !reverse;
134 | 	
135 | 		/* random search */
136 | 	
137 | 		for (int i = row_start; i != row_end; i += step){
138 | 			for (int j = col_start; j != col_end; j += step) {
139 | 				int r_ws = n_rows_ref, c_ws = n_cols_ref;
140 | 				float alpha = 0.5f, exp = 0.5f;
141 | 				while (r_ws*alpha > 1 && c_ws*alpha > 1) {
142 | 					int rmin = util::max(0, int(i + f[i][j][0] - r_ws*alpha)) + patch_size / 2;
143 | 					int rmax = util::min(int(i + f[i][j][0] + r_ws*alpha), n_rows_ref - patch_size) + patch_size / 2;
144 | 					int cmin = util::max(0, int(j + f[i][j][1] - c_ws*alpha)) + patch_size / 2;
145 | 					int cmax = util::min(int(j + f[i][j][1] + c_ws*alpha), n_cols_ref - patch_size) + patch_size / 2;
146 | 
147 | 					if (rmin > rmax) rmin = rmax = f[i][j][0] + i;
148 | 					if (cmin > cmax) cmin = cmax = f[i][j][1] + j;
149 | 
150 | 					int r_offset = int(util::random_range(rmin, rmax)) - i;
151 | 					int c_offset = int(util::random_range(cmin, cmax)) - j;
152 | 					
153 | 					Mat patch = pick_patch(img_dst, i, j, 0, 0, patch_size);
154 | 					Mat cand = pick_patch(img_ref, i, j, r_offset, c_offset, patch_size);
155 | 
156 | 					float similarity = sim(patch, cand, 1);
157 | 					if (similarity > v[i][j]) {
158 | 						v[i][j] = similarity;
159 | 						f[i][j][0] = r_offset; f[i][j][1] = c_offset;
160 | 					}
161 | 					
162 | 					alpha *= exp;
163 | 				}
164 | 			}
165 | 		}
166 | 
167 | 	}
168 | }
169 | 
170 | 
171 | void reconstruct(Vector3i& f, Mat& dst, Mat& ref, int patch_size) {
172 | 	int n_rows = dst.rows, n_cols = dst.cols;
173 | 	for (int i = 0; i < n_rows; i++) {
174 | 		for (int j = 0; j < n_cols; j++) {
175 | 			int r = f[i][j][0], c = f[i][j][1];
176 | 			dst.at<CHANNEL_TYPE>(i, j) = ref.at<CHANNEL_TYPE>(i + r, j + c);
177 | 		}
178 | 	}
179 | }
180 | 
181 | 
182 | void testImage(const char *img_dst_path, const char *img_ref_path) {
183 | 	srand((unsigned int)time(NULL));
184 | 	
185 | 	Mat dst = imread(img_dst_path);
186 | 	Mat ref = imread(img_ref_path);
187 | 	Mat res = dst.clone();
188 | 
189 | 	assert(dst.type() == CV_8UC3);
190 | 	
191 | 	Vector3i mapping;
192 | 	patchmatch(mapping, res, ref, 3, 5);
193 | 	reconstruct(mapping, res, ref, 5);
194 | 
195 | 	cv::imwrite("data\\output.jpg", res);
196 | 
197 | 	imshow("result", res);
198 | 	cv::waitKey(0);
199 | 
200 | }
201 | 
202 | 
203 | int main() {
204 | 	const char *img_dst = "data\\multiple_oreo.jpg";
205 | 	const char *img_ref = "data\\single_oreo.jpg";
206 | 
207 | 	testImage(img_dst, img_ref);
208 | 
209 | 	return 0;
210 | }
211 | 


--------------------------------------------------------------------------------
/patchmatch/pch.cpp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/stormouse/patchmatch/4f055d62204ee9b99dff7a84fa08f9769511ccf9/patchmatch/pch.cpp


--------------------------------------------------------------------------------
/patchmatch/pch.h:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/stormouse/patchmatch/4f055d62204ee9b99dff7a84fa08f9769511ccf9/patchmatch/pch.h


--------------------------------------------------------------------------------
/patchmatch/util.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | namespace util {
 4 | 	using namespace std;
 5 | 
 6 | 	float random_range(float min, float max) {
 7 | 		if (min > max) { int t = min; min = max; max = t; }
 8 | 		float f = (rand() % 65537) * 1.0f / 65537.0f;
 9 | 		return f * (max - min) + min;
10 | 	}
11 | 
12 | 	int argmax(double v[], int n) {
13 | 		int max_i = 0;
14 | 		for (int i = 1; i < n; i++)
15 | 			if (v[i] > v[max_i]) max_i = i;
16 | 
17 | 		return max_i;
18 | 	}
19 | 
20 | 	int argmin(double v[], int n) {
21 | 		int min_i = 0;
22 | 		for (int i = 1; i < n; i++)
23 | 			if (v[i] < v[min_i])min_i = i;
24 | 		return min_i;
25 | 	}
26 | 
27 | 	int max(int a, int b) {
28 | 		return a > b ? a : b;
29 | 	}
30 | 
31 | 	int min(int a, int b) { return a < b ? a : b; }
32 | }


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | # PatchMatch
 2 | 
 3 | This project is a pretty old attempt to implement PatchMatch and is **not** robost and **not** even complete. 
 4 | 
 5 | I am writing this because several people have forked this repo. I wanna thank whoever did it but please notice the code is not usable right now.
 6 | 
 7 | Currently the project:
 8 | 
 9 |  - requires opencv2 to run
10 |  - doesn't support masked image, which means you can't use it for inpainting right now
11 | 
12 | Frankly, I feel sorry when I look back at the quality of it. But I'd still like dump it here for (possible) future improvement. If you want to find a good reference for implementing PatchMatch there are lots of good repos on Github, please get out of this repo and explore further. 
13 | 
14 | Thank you for reading this.


--------------------------------------------------------------------------------