├── CMakeLists.txt
├── HPM.cpp
├── HPM.cu
├── HPM.h
├── README.md
├── colmap2mvsnet_acm.py
├── main.cpp
└── main.h


/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required (VERSION 2.8)
 2 | project (HPM-MVS_plusplus)
 3 | 
 4 | find_package(CUDA 6.0 REQUIRED ) # For Cuda Managed Memory and c++11
 5 | find_package(OpenCV REQUIRED )
 6 | 
 7 | include_directories(${OpenCV_INCLUDE_DIRS})
 8 | include_directories(.)
 9 | 
10 | set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-O3 --use_fast_math --maxrregcount=128 --ptxas-options=-v -std=c++11 --compiler-options -Wall -gencode arch=compute_86,code=sm_86)
11 | if(CMAKE_COMPILER_IS_GNUCXX)
12 |     add_definitions(-std=c++11)
13 |     add_definitions(-pthread)
14 |     add_definitions(-Wall)
15 |     add_definitions(-Wextra)
16 |     add_definitions(-pedantic)
17 |     add_definitions(-Wno-unused-function)
18 |     add_definitions(-Wno-switch)
19 |     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -ffast-math -march=native") # extend release-profile with fast-math
20 | endif()
21 | 
22 | find_package(OpenMP)
23 | if (OPENMP_FOUND)
24 |     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
25 | endif()
26 | 
27 | 
28 | # For compilation ...
29 | # Specify target & source files to compile it from
30 | cuda_add_executable(
31 |     HPM-MVS_plusplus
32 |     main.h
33 |     HPM.h
34 |     HPM.cpp
35 |     HPM.cu
36 |     main.cpp
37 |     )
38 | 
39 | # For linking ...
40 | # Specify target & libraries to link it with
41 | target_link_libraries(HPM-MVS_plusplus
42 |     ${OpenCV_LIBS}
43 |     )
44 | 


--------------------------------------------------------------------------------
/HPM.cpp:
--------------------------------------------------------------------------------
   1 | #include "HPM.h"
   2 | 
   3 | #include <cstdarg>
   4 | 
   5 | void StringAppendV(std::string* dst, const char* format, va_list ap) {
   6 | 	// First try with a small fixed size buffer.
   7 | 	static const int kFixedBufferSize = 1024;
   8 | 	char fixed_buffer[kFixedBufferSize];
   9 | 
  10 | 	// It is possible for methods that use a va_list to invalidate
  11 | 	// the data in it upon use.  The fix is to make a copy
  12 | 	// of the structure before using it and use that copy instead.
  13 | 	va_list backup_ap;
  14 | 	va_copy(backup_ap, ap);
  15 | 	int result = vsnprintf(fixed_buffer, kFixedBufferSize, format, backup_ap);
  16 | 	va_end(backup_ap);
  17 | 
  18 | 	if (result < kFixedBufferSize) {
  19 | 		if (result >= 0) {
  20 | 			// Normal case - everything fits.
  21 | 			dst->append(fixed_buffer, result);
  22 | 			return;
  23 | 		}
  24 | 
  25 | #ifdef _MSC_VER
  26 | 		// Error or MSVC running out of space.  MSVC 8.0 and higher
  27 | 		// can be asked about space needed with the special idiom below:
  28 | 		va_copy(backup_ap, ap);
  29 | 		result = vsnprintf(nullptr, 0, format, backup_ap);
  30 | 		va_end(backup_ap);
  31 | #endif
  32 | 
  33 | 		if (result < 0) {
  34 | 			// Just an error.
  35 | 			return;
  36 | 		}
  37 | 	}
  38 | 
  39 | 	// Increase the buffer size to the size requested by vsnprintf,
  40 | 	// plus one for the closing \0.
  41 | 	const int variable_buffer_size = result + 1;
  42 | 	std::unique_ptr<char> variable_buffer(new char[variable_buffer_size]);
  43 | 
  44 | 	// Restore the va_list before we use it again.
  45 | 	va_copy(backup_ap, ap);
  46 | 	result =
  47 | 		vsnprintf(variable_buffer.get(), variable_buffer_size, format, backup_ap);
  48 | 	va_end(backup_ap);
  49 | 
  50 | 	if (result >= 0 && result < variable_buffer_size) {
  51 | 		dst->append(variable_buffer.get(), result);
  52 | 	}
  53 | }
  54 | 
  55 | std::string StringPrintf(const char* format, ...) {
  56 | 	va_list ap;
  57 | 	va_start(ap, format);
  58 | 	std::string result;
  59 | 	StringAppendV(&result, format, ap);
  60 | 	va_end(ap);
  61 | 	return result;
  62 | }
  63 | 
  64 | void CudaSafeCall(const cudaError_t error, const std::string& file,
  65 | 	const int line) {
  66 | 	if (error != cudaSuccess) {
  67 | 		std::cerr << StringPrintf("%s in %s at line %i", cudaGetErrorString(error),
  68 | 			file.c_str(), line)
  69 | 			<< std::endl;
  70 | 		exit(EXIT_FAILURE);
  71 | 	}
  72 | }
  73 | 
  74 | void CudaCheckError(const char* file, const int line) {
  75 | 	cudaError error = cudaGetLastError();
  76 | 	if (error != cudaSuccess) {
  77 | 		std::cerr << StringPrintf("cudaCheckError() failed at %s:%i : %s", file,
  78 | 			line, cudaGetErrorString(error))
  79 | 			<< std::endl;
  80 | 		exit(EXIT_FAILURE);
  81 | 	}
  82 | 
  83 | 	// More careful checking. However, this will affect performance.
  84 | 	// Comment away if needed.
  85 | 	error = cudaDeviceSynchronize();
  86 | 	if (cudaSuccess != error) {
  87 | 		std::cerr << StringPrintf("cudaCheckError() with sync failed at %s:%i : %s",
  88 | 			file, line, cudaGetErrorString(error))
  89 | 			<< std::endl;
  90 | 		std::cerr
  91 | 			<< "This error is likely caused by the graphics card timeout "
  92 | 			"detection mechanism of your operating system. Please refer to "
  93 | 			"the FAQ in the documentation on how to solve this problem."
  94 | 			<< std::endl;
  95 | 		exit(EXIT_FAILURE);
  96 | 	}
  97 | }
  98 | 
  99 | HPM::HPM() {}
 100 | 
 101 | HPM::~HPM()
 102 | {
 103 | 	delete[] plane_hypotheses_host;
 104 | 	delete[] costs_host;
 105 | 
 106 | 	for (int i = 0; i < num_images; ++i) {
 107 | 		cudaDestroyTextureObject(texture_objects_host.images[i]);
 108 | 		cudaFreeArray(cuArray[i]);
 109 | 	}
 110 | 	cudaFree(texture_objects_cuda);
 111 | 	cudaFree(cameras_cuda);
 112 | 	cudaFree(plane_hypotheses_cuda);
 113 | 	cudaFree(costs_cuda);
 114 | 	cudaFree(pre_costs_cuda);
 115 | 	cudaFree(rand_states_cuda);
 116 | 	cudaFree(selected_views_cuda);
 117 | 	cudaFree(depths_cuda);
 118 | 
 119 | 	if (params.geom_consistency) {
 120 | 		for (int i = 0; i < num_images; ++i) {
 121 | 			cudaDestroyTextureObject(texture_depths_host.images[i]);
 122 | 			cudaFreeArray(cuDepthArray[i]);
 123 | 		}
 124 | 		cudaFree(texture_depths_cuda);
 125 | 	}
 126 | 
 127 | 	if (params.hierarchy) {
 128 | 		delete[] scaled_plane_hypotheses_host;
 129 | 		delete[] pre_costs_host;
 130 | 
 131 | 		cudaFree(scaled_plane_hypotheses_cuda);
 132 | 		cudaFree(pre_costs_cuda);
 133 | 	}
 134 | 
 135 | 	if (params.prior_consistency) {
 136 | 		delete[] prior_planes_host;
 137 | 		delete[] plane_masks_host;
 138 | 
 139 | 		cudaFree(prior_planes_cuda);
 140 | 		cudaFree(plane_masks_cuda);
 141 | 	}
 142 | 
 143 | }
 144 | 
 145 | Camera ReadCamera(const std::string& cam_path)
 146 | {
 147 | 	Camera camera;
 148 | 	std::ifstream file(cam_path);
 149 | 
 150 | 	std::string line;
 151 | 	file >> line;
 152 | 
 153 | 	for (int i = 0; i < 3; ++i) {
 154 | 		file >> camera.R[3 * i + 0] >> camera.R[3 * i + 1] >> camera.R[3 * i + 2] >> camera.t[i];
 155 | 	}
 156 | 
 157 | 	float tmp[4];
 158 | 	file >> tmp[0] >> tmp[1] >> tmp[2] >> tmp[3];
 159 | 	file >> line;
 160 | 
 161 | 	for (int i = 0; i < 3; ++i) {
 162 | 		file >> camera.K[3 * i + 0] >> camera.K[3 * i + 1] >> camera.K[3 * i + 2];
 163 | 	}
 164 | 
 165 | 	float depth_num;
 166 | 	float interval;
 167 | 	file >> camera.depth_min >> interval >> depth_num >> camera.depth_max;
 168 | 
 169 | 	return camera;
 170 | }
 171 | 
 172 | void  RescaleImageAndCamera(cv::Mat_<cv::Vec3b>& src, cv::Mat_<cv::Vec3b>& dst, cv::Mat_<float>& depth, Camera& camera)
 173 | {
 174 | 	const int cols = depth.cols;
 175 | 	const int rows = depth.rows;
 176 | 
 177 | 	if (cols == src.cols && rows == src.rows) {
 178 | 		dst = src.clone();
 179 | 		return;
 180 | 	}
 181 | 
 182 | 	const float scale_x = cols / static_cast<float>(src.cols);
 183 | 	const float scale_y = rows / static_cast<float>(src.rows);
 184 | 
 185 | 	cv::resize(src, dst, cv::Size(cols, rows), 0, 0, cv::INTER_LINEAR);
 186 | 
 187 | 	camera.K[0] *= scale_x;
 188 | 	camera.K[2] *= scale_x;
 189 | 	camera.K[4] *= scale_y;
 190 | 	camera.K[5] *= scale_y;
 191 | 	camera.width = cols;
 192 | 	camera.height = rows;
 193 | }
 194 | 
 195 | void RescaleMask(cv::Mat_<cv::Vec3b>& src, cv::Mat_<cv::Vec3b>& dst, cv::Mat_<float>& depth) 
 196 | {
 197 | 	const int cols = depth.cols;
 198 | 	const int rows = depth.rows;
 199 | 
 200 | 	if (cols == src.cols && rows == src.rows) {
 201 | 		dst = src.clone();
 202 | 	}
 203 | 
 204 | 	cv::resize(src, dst, cv::Size(cols, rows), 0, 0, cv::INTER_LINEAR);
 205 | }
 206 | 
 207 | float3 Get3DPointonWorld(const int x, const int y, const float depth, const Camera camera)
 208 | {
 209 | 	float3 pointX;
 210 | 	float3 tmpX;
 211 | 	// Reprojection
 212 | 	pointX.x = depth * (x - camera.K[2]) / camera.K[0];
 213 | 	pointX.y = depth * (y - camera.K[5]) / camera.K[4];
 214 | 	pointX.z = depth;
 215 | 
 216 | 	// Rotation
 217 | 	tmpX.x = camera.R[0] * pointX.x + camera.R[3] * pointX.y + camera.R[6] * pointX.z;
 218 | 	tmpX.y = camera.R[1] * pointX.x + camera.R[4] * pointX.y + camera.R[7] * pointX.z;
 219 | 	tmpX.z = camera.R[2] * pointX.x + camera.R[5] * pointX.y + camera.R[8] * pointX.z;
 220 | 
 221 | 	// Transformation
 222 | 	float3 C;
 223 | 	C.x = -(camera.R[0] * camera.t[0] + camera.R[3] * camera.t[1] + camera.R[6] * camera.t[2]);
 224 | 	C.y = -(camera.R[1] * camera.t[0] + camera.R[4] * camera.t[1] + camera.R[7] * camera.t[2]);
 225 | 	C.z = -(camera.R[2] * camera.t[0] + camera.R[5] * camera.t[1] + camera.R[8] * camera.t[2]);
 226 | 	pointX.x = tmpX.x + C.x;
 227 | 	pointX.y = tmpX.y + C.y;
 228 | 	pointX.z = tmpX.z + C.z;
 229 | 
 230 | 	return pointX;
 231 | }
 232 | 
 233 | float3 Get3DPointonRefCam_factor(const int x, const int y, const float depth, const Camera camera, float factor)
 234 | {
 235 | 	float3 pointX;
 236 | 	// Reprojection
 237 | 	pointX.x = depth * (x - camera.K[2] * factor) / (camera.K[0] * factor);
 238 | 	pointX.y = depth * (y - camera.K[5] * factor) / (camera.K[4] * factor);
 239 | 	pointX.z = depth;
 240 | 
 241 | 	return pointX;
 242 | }
 243 | 
 244 | float3 Get3DPointonRefCam(const int x, const int y, const float depth, const Camera camera)
 245 | {
 246 | 	float3 pointX;
 247 | 	// Reprojection
 248 | 	pointX.x = depth * (x - camera.K[2]) / camera.K[0];
 249 | 	pointX.y = depth * (y - camera.K[5]) / camera.K[4];
 250 | 	pointX.z = depth;
 251 | 
 252 | 	return pointX;
 253 | }
 254 | 
 255 | void ProjectonCamera(const float3 PointX, const Camera camera, float2& point, float& depth)
 256 | {
 257 | 	float3 tmp;
 258 | 	tmp.x = camera.R[0] * PointX.x + camera.R[1] * PointX.y + camera.R[2] * PointX.z + camera.t[0];
 259 | 	tmp.y = camera.R[3] * PointX.x + camera.R[4] * PointX.y + camera.R[5] * PointX.z + camera.t[1];
 260 | 	tmp.z = camera.R[6] * PointX.x + camera.R[7] * PointX.y + camera.R[8] * PointX.z + camera.t[2];
 261 | 
 262 | 	depth = camera.K[6] * tmp.x + camera.K[7] * tmp.y + camera.K[8] * tmp.z;
 263 | 	point.x = (camera.K[0] * tmp.x + camera.K[1] * tmp.y + camera.K[2] * tmp.z) / depth;
 264 | 	point.y = (camera.K[3] * tmp.x + camera.K[4] * tmp.y + camera.K[5] * tmp.z) / depth;
 265 | }
 266 | 
 267 | float GetAngle(const cv::Vec3f& v1, const cv::Vec3f& v2)
 268 | {
 269 | 	float dot_product = v1[0] * v2[0] + v1[1] * v2[1] + v1[2] * v2[2];
 270 | 	float angle = acosf(dot_product);
 271 | 	//if angle is not a number the dot product was 1 and thus the two vectors should be identical --> return 0
 272 | 	if (angle != angle)
 273 | 		return 0.0f;
 274 | 
 275 | 	return angle;
 276 | }
 277 | int readDepthDmb(const std::string file_path, cv::Mat_<float>& depth)
 278 | {
 279 | 	FILE* inimage;
 280 | 	inimage = fopen(file_path.c_str(), "rb");
 281 | 	if (!inimage) {
 282 | 		std::cout << "Error opening file " << file_path << std::endl;
 283 | 		return -1;
 284 | 	}
 285 | 
 286 | 	int32_t type, h, w, nb;
 287 | 
 288 | 	type = -1;
 289 | 
 290 | 	fread(&type, sizeof(int32_t), 1, inimage);
 291 | 	fread(&h, sizeof(int32_t), 1, inimage);
 292 | 	fread(&w, sizeof(int32_t), 1, inimage);
 293 | 	fread(&nb, sizeof(int32_t), 1, inimage);
 294 | 
 295 | 	if (type != 1) {
 296 | 		fclose(inimage);
 297 | 		return -1;
 298 | 	}
 299 | 
 300 | 	int32_t dataSize = h * w * nb;
 301 | 
 302 | 	depth = cv::Mat::zeros(h, w, CV_32F);
 303 | 	fread(depth.data, sizeof(float), dataSize, inimage);
 304 | 
 305 | 	fclose(inimage);
 306 | 	return 0;
 307 | }
 308 | 
 309 | int writeDepthDmb(const std::string file_path, const cv::Mat_<float> depth)
 310 | {
 311 | 	FILE* outimage;
 312 | 	outimage = fopen(file_path.c_str(), "wb");
 313 | 	if (!outimage) {
 314 | 		std::cout << "Error opening file " << file_path << std::endl;
 315 | 	}
 316 | 
 317 | 	int32_t type = 1;
 318 | 	int32_t h = depth.rows;
 319 | 	int32_t w = depth.cols;
 320 | 	int32_t nb = 1;
 321 | 
 322 | 	fwrite(&type, sizeof(int32_t), 1, outimage);
 323 | 	fwrite(&h, sizeof(int32_t), 1, outimage);
 324 | 	fwrite(&w, sizeof(int32_t), 1, outimage);
 325 | 	fwrite(&nb, sizeof(int32_t), 1, outimage);
 326 | 
 327 | 	float* data = (float*)depth.data;
 328 | 
 329 | 	int32_t datasize = w * h * nb;
 330 | 	fwrite(data, sizeof(float), datasize, outimage);
 331 | 
 332 | 	fclose(outimage);
 333 | 	return 0;
 334 | }
 335 | 
 336 | int readNormalDmb(const std::string file_path, cv::Mat_<cv::Vec3f>& normal)
 337 | {
 338 | 	FILE* inimage;
 339 | 	inimage = fopen(file_path.c_str(), "rb");
 340 | 	if (!inimage) {
 341 | 		std::cout << "Error opening file " << file_path << std::endl;
 342 | 		return -1;
 343 | 	}
 344 | 
 345 | 	int32_t type, h, w, nb;
 346 | 
 347 | 	type = -1;
 348 | 
 349 | 	fread(&type, sizeof(int32_t), 1, inimage);
 350 | 	fread(&h, sizeof(int32_t), 1, inimage);
 351 | 	fread(&w, sizeof(int32_t), 1, inimage);
 352 | 	fread(&nb, sizeof(int32_t), 1, inimage);
 353 | 
 354 | 	if (type != 1) {
 355 | 		fclose(inimage);
 356 | 		return -1;
 357 | 	}
 358 | 
 359 | 	int32_t dataSize = h * w * nb;
 360 | 
 361 | 	normal = cv::Mat::zeros(h, w, CV_32FC3);
 362 | 	fread(normal.data, sizeof(float), dataSize, inimage);
 363 | 
 364 | 	fclose(inimage);
 365 | 	return 0;
 366 | }
 367 | 
 368 | int writeNormalDmb(const std::string file_path, const cv::Mat_<cv::Vec3f> normal)
 369 | {
 370 | 	FILE* outimage;
 371 | 	outimage = fopen(file_path.c_str(), "wb");
 372 | 	if (!outimage) {
 373 | 		std::cout << "Error opening file " << file_path << std::endl;
 374 | 	}
 375 | 
 376 | 	int32_t type = 1; //float
 377 | 	int32_t h = normal.rows;
 378 | 	int32_t w = normal.cols;
 379 | 	int32_t nb = 3;
 380 | 
 381 | 	fwrite(&type, sizeof(int32_t), 1, outimage);
 382 | 	fwrite(&h, sizeof(int32_t), 1, outimage);
 383 | 	fwrite(&w, sizeof(int32_t), 1, outimage);
 384 | 	fwrite(&nb, sizeof(int32_t), 1, outimage);
 385 | 
 386 | 	float* data = (float*)normal.data;
 387 | 
 388 | 	int32_t datasize = w * h * nb;
 389 | 	fwrite(data, sizeof(float), datasize, outimage);
 390 | 
 391 | 	fclose(outimage);
 392 | 	return 0;
 393 | }
 394 | void ExportPointCloud(const std::string& plyFilePath, const std::vector<PointList>& pc)
 395 | {
 396 | 	std::cout << "store 3D points to ply file" << std::endl;
 397 | 
 398 | 	FILE* outputPly;
 399 | 	outputPly = fopen(plyFilePath.c_str(), "wb");
 400 | 
 401 | 	/*write header*/
 402 | 	fprintf(outputPly, "ply\n");
 403 | 	fprintf(outputPly, "format binary_little_endian 1.0\n");
 404 | 	fprintf(outputPly, "element vertex %d\n", pc.size());
 405 | 	fprintf(outputPly, "property float x\n");
 406 | 	fprintf(outputPly, "property float y\n");
 407 | 	fprintf(outputPly, "property float z\n");
 408 | 	fprintf(outputPly, "property uchar red\n");
 409 | 	fprintf(outputPly, "property uchar green\n");
 410 | 	fprintf(outputPly, "property uchar blue\n");
 411 | 	fprintf(outputPly, "end_header\n");
 412 | 
 413 | 	//write data
 414 | #pragma omp parallel for
 415 | 	for (int i = 0; i < pc.size(); i++) {
 416 | 		const PointList& p = pc[i];
 417 | 		float3 X = p.coord;
 418 | 		//const float3 normal = p.normal;
 419 | 		const float3 color = p.color;
 420 | 		const char b_color = (int)color.x;
 421 | 		const char g_color = (int)color.y;
 422 | 		const char r_color = (int)color.z;
 423 | 
 424 | 		//if ((int)color.z == 55 && (int)color.y == 235 && (int)color.x == 234) {
 425 | 		//	continue;
 426 | 		//}
 427 | 
 428 | 		if (!(X.x < FLT_MAX && X.x > -FLT_MAX) || !(X.y < FLT_MAX && X.y > -FLT_MAX) || !(X.z < FLT_MAX && X.z >= -FLT_MAX)) {
 429 | 			X.x = 0.0f;
 430 | 			X.y = 0.0f;
 431 | 			X.z = 0.0f;
 432 | 		}
 433 | #pragma omp critical
 434 | 		{
 435 | 			fwrite(&X.x, sizeof(X.x), 1, outputPly);
 436 | 			fwrite(&X.y, sizeof(X.y), 1, outputPly);
 437 | 			fwrite(&X.z, sizeof(X.z), 1, outputPly);
 438 | 			//fwrite(&normal.x, sizeof(normal.x), 1, outputPly);
 439 | 			//fwrite(&normal.y, sizeof(normal.y), 1, outputPly);
 440 | 			//fwrite(&normal.z, sizeof(normal.z), 1, outputPly);
 441 | 			fwrite(&r_color, sizeof(char), 1, outputPly);
 442 | 			fwrite(&g_color, sizeof(char), 1, outputPly);
 443 | 			fwrite(&b_color, sizeof(char), 1, outputPly);
 444 | 		}
 445 | 
 446 | 	}
 447 | 	fclose(outputPly);
 448 | }
 449 | 
 450 | void StoreColorPlyFileBinaryPointCloud(const std::string& plyFilePath, const std::vector<PointList>& pc)
 451 | {
 452 | 	std::cout << "store 3D points to ply file" << std::endl;
 453 | 
 454 | 	FILE* outputPly;
 455 | 	outputPly = fopen(plyFilePath.c_str(), "wb");
 456 | 
 457 | 	/*write header*/
 458 | 	fprintf(outputPly, "ply\n");
 459 | 	fprintf(outputPly, "format binary_little_endian 1.0\n");
 460 | 	fprintf(outputPly, "element vertex %d\n", pc.size());
 461 | 	fprintf(outputPly, "property float x\n");
 462 | 	fprintf(outputPly, "property float y\n");
 463 | 	fprintf(outputPly, "property float z\n");
 464 | 	fprintf(outputPly, "property float nx\n");
 465 | 	fprintf(outputPly, "property float ny\n");
 466 | 	fprintf(outputPly, "property float nz\n");
 467 | 	fprintf(outputPly, "property uchar red\n");
 468 | 	fprintf(outputPly, "property uchar green\n");
 469 | 	fprintf(outputPly, "property uchar blue\n");
 470 | 	fprintf(outputPly, "end_header\n");
 471 | 
 472 | 	//write data
 473 | #pragma omp parallel for
 474 | 	for (int i = 0; i < pc.size(); i++) {
 475 | 		const PointList& p = pc[i];
 476 | 		float3 X = p.coord;
 477 | 		const float3 normal = p.normal;
 478 | 		const float3 color = p.color;
 479 | 		const char b_color = (int)color.x;
 480 | 		const char g_color = (int)color.y;
 481 | 		const char r_color = (int)color.z;
 482 | 
 483 | 		if (!(X.x < FLT_MAX && X.x > -FLT_MAX) || !(X.y < FLT_MAX && X.y > -FLT_MAX) || !(X.z < FLT_MAX && X.z >= -FLT_MAX)) {
 484 | 			X.x = 0.0f;
 485 | 			X.y = 0.0f;
 486 | 			X.z = 0.0f;
 487 | 		}
 488 | #pragma omp critical
 489 | 		{
 490 | 			fwrite(&X.x, sizeof(X.x), 1, outputPly);
 491 | 			fwrite(&X.y, sizeof(X.y), 1, outputPly);
 492 | 			fwrite(&X.z, sizeof(X.z), 1, outputPly);
 493 | 			fwrite(&normal.x, sizeof(normal.x), 1, outputPly);
 494 | 			fwrite(&normal.y, sizeof(normal.y), 1, outputPly);
 495 | 			fwrite(&normal.z, sizeof(normal.z), 1, outputPly);
 496 | 			fwrite(&r_color, sizeof(char), 1, outputPly);
 497 | 			fwrite(&g_color, sizeof(char), 1, outputPly);
 498 | 			fwrite(&b_color, sizeof(char), 1, outputPly);
 499 | 		}
 500 | 
 501 | 	}
 502 | 	fclose(outputPly);
 503 | }
 504 | 
 505 | static float GetDisparity(const Camera& camera, const int2& p, const float& depth)
 506 | {
 507 | 	float point3D[3];
 508 | 	point3D[0] = depth * (p.x - camera.K[2]) / camera.K[0];
 509 | 	point3D[1] = depth * (p.y - camera.K[5]) / camera.K[4];
 510 | 	point3D[3] = depth;
 511 | 
 512 | 	return std::sqrt(point3D[0] * point3D[0] + point3D[1] * point3D[1] + point3D[2] * point3D[2]);
 513 | }
 514 | 
 515 | void HPM::SetGeomConsistencyParams(bool multi_geometry = false)
 516 | {
 517 | 	params.geom_consistency = true;
 518 | 	params.max_iterations = 2;
 519 | 	if (multi_geometry) {
 520 | 		params.multi_geometry = true;
 521 | 	}
 522 | }
 523 | 
 524 | void HPM::SetHierarchyParams()
 525 | {
 526 | 	params.hierarchy = true;
 527 | }
 528 | 
 529 | void HPM::SetPlanarPriorParams()
 530 | {
 531 | 	params.prior_consistency = true;
 532 | }
 533 | 
 534 | void HPM::SetMandConsistencyParams(bool flag)
 535 | {
 536 | 	params.mand_consistency = flag;
 537 | }
 538 | 
 539 | void HPM::CudaPlanarPriorRelease() {
 540 | 	cudaFree(prior_planes_cuda);
 541 | 	cudaFree(plane_masks_cuda);
 542 | 	cudaFree(Canny_cuda);
 543 | 	//updated by ChunLin Ren 2023-3-30
 544 | }
 545 | 
 546 | void HPM::CudaSpaceRelease(bool geom_consistency)
 547 | {
 548 | 	cudaFree(texture_objects_cuda);
 549 | 	cudaFree(cameras_cuda);
 550 | 	cudaFree(plane_hypotheses_cuda);
 551 | 	cudaFree(costs_cuda);
 552 | 	cudaFree(rand_states_cuda);
 553 | 	cudaFree(selected_views_cuda);
 554 | 	cudaFree(depths_cuda);
 555 | 	cudaFree(texture_cuda);
 556 | 
 557 | 	if (geom_consistency) {
 558 | 		cudaFree(texture_depths_cuda);
 559 | 	}
 560 | }
 561 | 
 562 | void HPM::ReleaseProblemHostMemory() {
 563 | 	//delete(plane_hypotheses_host);
 564 | 	//delete(costs_host);
 565 | 	images.swap(std::vector<cv::Mat>());
 566 | 	cameras.swap(std::vector<Camera>());
 567 | 	depths.swap(std::vector<cv::Mat>());
 568 | 	std::cout << "Releasing Host memory..." << std::endl;
 569 | }
 570 | 
 571 | void HPM::InuputInitialization(const std::string& dense_folder, const std::vector<Problem>& problems, const int idx)
 572 | {
 573 | 	images.clear();
 574 | 	cameras.clear();
 575 | 	const Problem problem = problems[idx];
 576 | 
 577 | 	std::string image_folder = dense_folder + std::string("/images");
 578 | 	std::string cam_folder = dense_folder + std::string("/cams");
 579 | 
 580 | 	std::stringstream image_path;
 581 | 	image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problem.ref_image_id << ".jpg";
 582 | 	cv::Mat_<uint8_t> image_uint = cv::imread(image_path.str(), cv::IMREAD_GRAYSCALE);
 583 | 	cv::Mat image_float;
 584 | 	image_uint.convertTo(image_float, CV_32FC1);
 585 | 	images.push_back(image_float);
 586 | 	std::stringstream cam_path;
 587 | 	cam_path << cam_folder << "/" << std::setw(8) << std::setfill('0') << problem.ref_image_id << "_cam.txt";
 588 | 	Camera camera = ReadCamera(cam_path.str());
 589 | 	camera.height = image_float.rows;
 590 | 	camera.width = image_float.cols;
 591 | 	cameras.push_back(camera);
 592 | 
 593 | 	size_t num_src_images = problem.src_image_ids.size();
 594 | 	for (size_t i = 0; i < num_src_images; ++i) {
 595 | 		std::stringstream image_path;
 596 | 		image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problem.src_image_ids[i] << ".jpg";
 597 | 		cv::Mat_<uint8_t> image_uint = cv::imread(image_path.str(), cv::IMREAD_GRAYSCALE);
 598 | 		cv::Mat image_float;
 599 | 		image_uint.convertTo(image_float, CV_32FC1);
 600 | 		images.push_back(image_float);
 601 | 		std::stringstream cam_path;
 602 | 		cam_path << cam_folder << "/" << std::setw(8) << std::setfill('0') << problem.src_image_ids[i] << "_cam.txt";
 603 | 		Camera camera = ReadCamera(cam_path.str());
 604 | 		camera.height = image_float.rows;
 605 | 		camera.width = image_float.cols;
 606 | 		cameras.push_back(camera);
 607 | 	}
 608 | 
 609 | 	// Scale cameras and images
 610 | 	int max_image_size = problems[idx].cur_image_size;
 611 | 	for (size_t i = 0; i < images.size(); ++i) {
 612 | 		if (i > 0) {
 613 | 			max_image_size = problems[problem.src_image_ids[i - 1]].cur_image_size;
 614 | 		}
 615 | 
 616 | 		if (images[i].cols <= max_image_size && images[i].rows <= max_image_size) {
 617 | 			continue;
 618 | 		}
 619 | 
 620 | 		const float factor_x = static_cast<float>(max_image_size) / images[i].cols;
 621 | 		const float factor_y = static_cast<float>(max_image_size) / images[i].rows;
 622 | 		const float factor = std::min(factor_x, factor_y);
 623 | 
 624 | 		const int new_cols = std::round(images[i].cols * factor);
 625 | 		const int new_rows = std::round(images[i].rows * factor);
 626 | 
 627 | 		const float scale_x = new_cols / static_cast<float>(images[i].cols);
 628 | 		const float scale_y = new_rows / static_cast<float>(images[i].rows);
 629 | 
 630 | 		cv::Mat_<float> scaled_image_float;
 631 | 		cv::resize(images[i], scaled_image_float, cv::Size(new_cols, new_rows), 0, 0, cv::INTER_LINEAR);
 632 | 		images[i] = scaled_image_float.clone();
 633 | 
 634 | 		cameras[i].K[0] *= scale_x;
 635 | 		cameras[i].K[2] *= scale_x;
 636 | 		cameras[i].K[4] *= scale_y;
 637 | 		cameras[i].K[5] *= scale_y;
 638 | 		cameras[i].height = scaled_image_float.rows;
 639 | 		cameras[i].width = scaled_image_float.cols;
 640 | 	}
 641 | 
 642 | 	params.depth_min = cameras[0].depth_min * 0.6f;
 643 | 	params.depth_max = cameras[0].depth_max * 1.2f;
 644 | 	std::cout << "depthe range: " << params.depth_min << " " << params.depth_max << std::endl;
 645 | 	params.num_images = (int)images.size();
 646 | 	std::cout << "num images: " << params.num_images << std::endl;
 647 | 	params.disparity_min = cameras[0].K[0] * params.baseline / params.depth_max;
 648 | 	params.disparity_max = cameras[0].K[0] * params.baseline / params.depth_min;
 649 | 
 650 | 	if (params.geom_consistency) {
 651 | 		depths.clear();
 652 | 
 653 | 		std::stringstream result_path;
 654 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 655 | 		std::string result_folder = result_path.str();
 656 | 		std::string suffix = "/depths.dmb";
 657 | 		if (params.multi_geometry) {
 658 | 			suffix = "/depths_geom.dmb";
 659 | 		}
 660 | 		std::string depth_path = result_folder + suffix;
 661 | 		cv::Mat_<float> ref_depth;
 662 | 		readDepthDmb(depth_path, ref_depth);
 663 | 		depths.push_back(ref_depth);
 664 | 
 665 | 		size_t num_src_images = problem.src_image_ids.size();
 666 | 		for (size_t i = 0; i < num_src_images; ++i) {
 667 | 			std::stringstream result_path;
 668 | 			result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.src_image_ids[i];
 669 | 			std::string result_folder = result_path.str();
 670 | 			//if (!params.multi_geometry) {
 671 | 			//    if (params.geom_consistency) {
 672 | 			//        if (problem.src_image_ids[i] < problem.ref_image_id) {
 673 | 			//            suffix = "/depths_geom.dmb";
 674 | 			//        }
 675 | 			//        else {
 676 | 			//            suffix = "/depths.dmb";
 677 | 			//        }
 678 | 			//    }
 679 | 			//    else {
 680 | 			//        suffix = "/depths.dmb";
 681 | 			//    }
 682 | 			//}
 683 | 			std::string depth_path = result_folder + suffix;
 684 | 			//std::cout << depth_path << std::endl;
 685 | 			cv::Mat_<float> depth;
 686 | 			readDepthDmb(depth_path, depth);
 687 | 			depths.push_back(depth);
 688 | 		}
 689 | 	}
 690 | }
 691 | 
 692 | void HPM::TextureInformationInitialization()
 693 | {
 694 | 	texture_host = new float[cameras[0].height * cameras[0].width];
 695 | 	cudaMalloc((void**)&texture_cuda, sizeof(float) * (cameras[0].height * cameras[0].width));
 696 | }
 697 | 
 698 | void HPM::CudaSpaceInitialization(const std::string& dense_folder, const Problem& problem)
 699 | {
 700 | 	num_images = (int)images.size();
 701 | 
 702 | 	for (int i = 0; i < num_images; ++i) {
 703 | 		int rows = images[i].rows;
 704 | 		int cols = images[i].cols;
 705 | 
 706 | 		cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat);
 707 | 		cudaMallocArray(&cuArray[i], &channelDesc, cols, rows);
 708 | 		cudaMemcpy2DToArray(cuArray[i], 0, 0, images[i].ptr<float>(), images[i].step[0], cols * sizeof(float), rows, cudaMemcpyHostToDevice);
 709 | 
 710 | 		struct cudaResourceDesc resDesc;
 711 | 		memset(&resDesc, 0, sizeof(cudaResourceDesc));
 712 | 		resDesc.resType = cudaResourceTypeArray;
 713 | 		resDesc.res.array.array = cuArray[i];
 714 | 
 715 | 		struct cudaTextureDesc texDesc;
 716 | 		memset(&texDesc, 0, sizeof(cudaTextureDesc));
 717 | 		texDesc.addressMode[0] = cudaAddressModeWrap;
 718 | 		texDesc.addressMode[1] = cudaAddressModeWrap;
 719 | 		texDesc.filterMode = cudaFilterModeLinear;
 720 | 		texDesc.readMode = cudaReadModeElementType;
 721 | 		texDesc.normalizedCoords = 0;
 722 | 
 723 | 		cudaCreateTextureObject(&(texture_objects_host.images[i]), &resDesc, &texDesc, NULL);
 724 | 	}
 725 | 	cudaMalloc((void**)&texture_objects_cuda, sizeof(cudaTextureObjects));
 726 | 	cudaMemcpy(texture_objects_cuda, &texture_objects_host, sizeof(cudaTextureObjects), cudaMemcpyHostToDevice);
 727 | 
 728 | 	cudaMalloc((void**)&cameras_cuda, sizeof(Camera) * (num_images));
 729 | 	cudaMemcpy(cameras_cuda, &cameras[0], sizeof(Camera) * (num_images), cudaMemcpyHostToDevice);
 730 | 
 731 | 	plane_hypotheses_host = new float4[cameras[0].height * cameras[0].width];
 732 | 	cudaMalloc((void**)&plane_hypotheses_cuda, sizeof(float4) * (cameras[0].height * cameras[0].width));
 733 | 
 734 | 	costs_host = new float[cameras[0].height * cameras[0].width];
 735 | 	cudaMalloc((void**)&costs_cuda, sizeof(float) * (cameras[0].height * cameras[0].width));
 736 | 	cudaMalloc((void**)&pre_costs_cuda, sizeof(float) * (cameras[0].height * cameras[0].width));
 737 | 
 738 | 	cudaMalloc((void**)&rand_states_cuda, sizeof(curandState) * (cameras[0].height * cameras[0].width));
 739 | 	cudaMalloc((void**)&selected_views_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width));
 740 | 
 741 | 	cudaMalloc((void**)&depths_cuda, sizeof(float) * (cameras[0].height * cameras[0].width));
 742 | 
 743 | 	if (params.geom_consistency) {
 744 | 		for (int i = 0; i < num_images; ++i) {
 745 | 			int rows = depths[i].rows;
 746 | 			int cols = depths[i].cols;
 747 | 
 748 | 			cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat);
 749 | 			cudaMallocArray(&cuDepthArray[i], &channelDesc, cols, rows);
 750 | 			cudaMemcpy2DToArray(cuDepthArray[i], 0, 0, depths[i].ptr<float>(), depths[i].step[0], cols * sizeof(float), rows, cudaMemcpyHostToDevice);
 751 | 
 752 | 			struct cudaResourceDesc resDesc;
 753 | 			memset(&resDesc, 0, sizeof(cudaResourceDesc));
 754 | 			resDesc.resType = cudaResourceTypeArray;
 755 | 			resDesc.res.array.array = cuDepthArray[i];
 756 | 
 757 | 			struct cudaTextureDesc texDesc;
 758 | 			memset(&texDesc, 0, sizeof(cudaTextureDesc));
 759 | 			texDesc.addressMode[0] = cudaAddressModeWrap;
 760 | 			texDesc.addressMode[1] = cudaAddressModeWrap;
 761 | 			texDesc.filterMode = cudaFilterModeLinear;
 762 | 			texDesc.readMode = cudaReadModeElementType;
 763 | 			texDesc.normalizedCoords = 0;
 764 | 
 765 | 			cudaCreateTextureObject(&(texture_depths_host.images[i]), &resDesc, &texDesc, NULL);
 766 | 		}
 767 | 		cudaMalloc((void**)&texture_depths_cuda, sizeof(cudaTextureObjects));
 768 | 		cudaMemcpy(texture_depths_cuda, &texture_depths_host, sizeof(cudaTextureObjects), cudaMemcpyHostToDevice);
 769 | 
 770 | 		std::stringstream result_path;
 771 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 772 | 		std::string result_folder = result_path.str();
 773 | 		std::string suffix = "/depths.dmb";
 774 | 		if (params.multi_geometry) {
 775 | 			suffix = "/depths_geom.dmb";
 776 | 		}
 777 | 
 778 | 		std::string depth_path = result_folder + suffix;
 779 | 		std::string normal_path = result_folder + "/normals.dmb";
 780 | 		std::string cost_path = result_folder + "/costs.dmb";
 781 | 		cv::Mat_<float> ref_depth;
 782 | 		cv::Mat_<cv::Vec3f> ref_normal;
 783 | 		cv::Mat_<float> ref_cost;
 784 | 		readDepthDmb(depth_path, ref_depth);
 785 | 		depths.push_back(ref_depth);
 786 | 		readNormalDmb(normal_path, ref_normal);
 787 | 		readDepthDmb(cost_path, ref_cost);
 788 | 		int width = ref_depth.cols;
 789 | 		int height = ref_depth.rows;
 790 | 		for (int col = 0; col < width; ++col) {
 791 | 			for (int row = 0; row < height; ++row) {
 792 | 				int center = row * width + col;
 793 | 				float4 plane_hypothesis;
 794 | 				plane_hypothesis.x = ref_normal(row, col)[0];
 795 | 				plane_hypothesis.y = ref_normal(row, col)[1];
 796 | 				plane_hypothesis.z = ref_normal(row, col)[2];
 797 | 				plane_hypothesis.w = ref_depth(row, col);
 798 | 				plane_hypotheses_host[center] = plane_hypothesis;
 799 | 				costs_host[center] = ref_cost(row, col);
 800 | 			}
 801 | 		}
 802 | 		cudaMemcpy(plane_hypotheses_cuda, plane_hypotheses_host, sizeof(float4) * width * height, cudaMemcpyHostToDevice);
 803 | 		cudaMemcpy(costs_cuda, costs_host, sizeof(float) * width * height, cudaMemcpyHostToDevice);
 804 | 	}
 805 | 
 806 | 	if (params.hierarchy) {
 807 | 		std::stringstream result_path;
 808 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 809 | 		std::string result_folder = result_path.str();
 810 | 		std::string depth_path = result_folder + "/depths.dmb";
 811 | 		std::string normal_path = result_folder + "/normals.dmb";
 812 | 		std::string cost_path = result_folder + "/costs.dmb";
 813 | 		cv::Mat_<float> ref_depth;
 814 | 		cv::Mat_<cv::Vec3f> ref_normal;
 815 | 		cv::Mat_<float> ref_cost;
 816 | 		readDepthDmb(depth_path, ref_depth);
 817 | 		depths.push_back(ref_depth);
 818 | 		readNormalDmb(normal_path, ref_normal);
 819 | 		readDepthDmb(cost_path, ref_cost);
 820 | 		int width = ref_normal.cols;
 821 | 		int height = ref_normal.rows;
 822 | 		scaled_plane_hypotheses_host = new float4[height * width];
 823 | 		cudaMalloc((void**)&scaled_plane_hypotheses_cuda, sizeof(float4) * height * width);
 824 | 		pre_costs_host = new float[height * width];
 825 | 		cudaMalloc((void**)&pre_costs_cuda, sizeof(float) * cameras[0].height * cameras[0].width);
 826 | 		if (width != images[0].rows || height != images[0].cols) {
 827 | 			params.upsample = true;
 828 | 			params.scaled_cols = width;
 829 | 			params.scaled_rows = height;
 830 | 		}
 831 | 		else {
 832 | 			params.upsample = false;
 833 | 		}
 834 | 		for (int col = 0; col < width; ++col) {
 835 | 			for (int row = 0; row < height; ++row) {
 836 | 				int center = row * width + col;
 837 | 				float4 plane_hypothesis;
 838 | 				plane_hypothesis.x = ref_normal(row, col)[0];
 839 | 				plane_hypothesis.y = ref_normal(row, col)[1];
 840 | 				plane_hypothesis.z = ref_normal(row, col)[2];
 841 | 				if (params.upsample) {
 842 | 					plane_hypothesis.w = ref_cost(row, col);
 843 | 				}
 844 | 				else {
 845 | 					plane_hypothesis.w = ref_depth(row, col);
 846 | 				}
 847 | 				scaled_plane_hypotheses_host[center] = plane_hypothesis;
 848 | 			}
 849 | 		}
 850 | 
 851 | 		for (int col = 0; col < cameras[0].width; ++col) {
 852 | 			for (int row = 0; row < cameras[0].height; ++row) {
 853 | 				int center = row * cameras[0].width + col;
 854 | 				float4 plane_hypothesis;
 855 | 				plane_hypothesis.w = ref_depth(row, col);
 856 | 				plane_hypotheses_host[center] = plane_hypothesis;
 857 | 			}
 858 | 		}
 859 | 
 860 | 		cudaMemcpy(scaled_plane_hypotheses_cuda, scaled_plane_hypotheses_host, sizeof(float4) * height * width, cudaMemcpyHostToDevice);
 861 | 		cudaMemcpy(plane_hypotheses_cuda, plane_hypotheses_host, sizeof(float4) * cameras[0].width * cameras[0].height, cudaMemcpyHostToDevice);
 862 | 	}
 863 | }
 864 | 
 865 | void HPM::CudaCannyInitialization(const cv::Mat_<int>& Canny) {
 866 | 	unsigned int* Canny_host = new unsigned int[cameras[0].height * cameras[0].width];
 867 | 	cudaMalloc((void**)&Canny_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width));
 868 | 	for (int i = 0; i < cameras[0].width; ++i) {
 869 | 		for (int j = 0; j < cameras[0].height; ++j) {
 870 | 			int center = j * cameras[0].width + i;
 871 | 			Canny_host[center] = (unsigned int)Canny(j, i);
 872 | 		}
 873 | 	}
 874 | 	cudaMemcpy(Canny_cuda, Canny_host, sizeof(unsigned int) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice);
 875 | 	delete(Canny_host);
 876 | 	Canny_host = NULL;
 877 | }
 878 | 
 879 | void HPM::CudaConfidenceInitialization(const std::string& dense_folder, const std::vector<Problem>& problems, const int idx) {
 880 | 	const Problem problem = problems[idx];
 881 | 	std::stringstream result_path;
 882 | 	result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 883 | 	std::string result_folder = result_path.str();
 884 | 	std::string confidence_path = result_folder + "/Confidence.dmb";
 885 | 	cv::Mat_<float>confidences;
 886 | 	confidences_host = new float[cameras[0].height * cameras[0].width];
 887 | 	readDepthDmb(confidence_path, confidences);
 888 | 	for (int i = 0; i < cameras[0].width; ++i) {
 889 | 		for (int j = 0; j < cameras[0].height; ++j) {
 890 | 			int center = j * cameras[0].width + i;
 891 | 			confidences_host[center] = confidences(j, i);
 892 | 		}
 893 | 	}
 894 | 	cudaMalloc((void**)&confidences_cuda, sizeof(float) * (cameras[0].height * cameras[0].width));
 895 | 	cudaMemcpy(confidences_cuda, confidences_host, sizeof(float) * cameras[0].width * cameras[0].height, cudaMemcpyHostToDevice);
 896 | 	confidences.release();
 897 | }
 898 | 
 899 | void HPM::CudaHypothesesReload(cv::Mat_ <float>depths, cv::Mat_<float>costs, cv::Mat_<cv::Vec3f>normals) {
 900 | 	int width = cameras[0].width;
 901 | 	int height = cameras[0].height;
 902 | 	for (int col = 0; col < width; col++) {
 903 | 		for (int row = 0; row < height; row++) {
 904 | 			int center = row * width + col;
 905 | 			float4 plane_hypotheses;
 906 | 			plane_hypotheses.x = normals(row, col)[0];
 907 | 			plane_hypotheses.y = normals(row, col)[1];
 908 | 			plane_hypotheses.z = normals(row, col)[2];
 909 | 			plane_hypotheses.w = depths(row, col);
 910 | 			plane_hypotheses_host[center] = plane_hypotheses;
 911 | 			costs_host[center] = costs(row, col);
 912 | 			//if (costs_host[center] < 0.1) {
 913 | 			//    std::cout << costs_host[center] << std::endl;
 914 | 			//}
 915 | 		}
 916 | 	}
 917 | 	cudaMemcpy(plane_hypotheses_cuda, plane_hypotheses_host, sizeof(float4) * width * height, cudaMemcpyHostToDevice);
 918 | 	cudaMemcpy(costs_cuda, costs_host, sizeof(float) * width * height, cudaMemcpyHostToDevice);
 919 | }
 920 | 
 921 | void HPM::CudaPlanarPriorInitialization(const std::vector<float4>& PlaneParams, const cv::Mat_<float>& masks)
 922 | {
 923 | 	prior_planes_host = new float4[cameras[0].height * cameras[0].width];
 924 | 	cudaMalloc((void**)&prior_planes_cuda, sizeof(float4) * (cameras[0].height * cameras[0].width));
 925 | 
 926 | 	plane_masks_host = new unsigned int[cameras[0].height * cameras[0].width];
 927 | 	cudaMalloc((void**)&plane_masks_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width));
 928 | 
 929 | 	for (int i = 0; i < cameras[0].width; ++i) {
 930 | 		for (int j = 0; j < cameras[0].height; ++j) {
 931 | 			int center = j * cameras[0].width + i;
 932 | 			plane_masks_host[center] = (unsigned int)masks(j, i);
 933 | 			if (masks(j, i) > 0) {
 934 | 				prior_planes_host[center] = PlaneParams[masks(j, i) - 1];
 935 | 			}
 936 | 		}
 937 | 	}
 938 | 
 939 | 	cudaMemcpy(prior_planes_cuda, prior_planes_host, sizeof(float4) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice);
 940 | 	cudaMemcpy(plane_masks_cuda, plane_masks_host, sizeof(unsigned int) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice);
 941 | }
 942 | 
 943 | int HPM::GetReferenceImageWidth()
 944 | {
 945 | 	return cameras[0].width;
 946 | }
 947 | 
 948 | int HPM::GetReferenceImageHeight()
 949 | {
 950 | 	return cameras[0].height;
 951 | }
 952 | 
 953 | cv::Mat HPM::GetReferenceImage()
 954 | {
 955 | 	return images[0];
 956 | }
 957 | 
 958 | float4 HPM::GetPlaneHypothesis(const int index)
 959 | {
 960 | 	return plane_hypotheses_host[index];
 961 | }
 962 | 
 963 | float HPM::GetTexture(const int index)
 964 | {
 965 | 	return texture_host[index];
 966 | }
 967 | 
 968 | float HPM::GetCost(const int index)
 969 | {
 970 | 	return costs_host[index];
 971 | }
 972 | 
 973 | float HPM::GetMinDepth()
 974 | {
 975 | 	return params.depth_min;
 976 | }
 977 | 
 978 | float HPM::GetMaxDepth()
 979 | {
 980 | 	return params.depth_max;
 981 | }
 982 | 
 983 | void HPM::GetSupportPoints_Classify_Check(std::vector<cv::Point>& support2DPoints, const cv::Mat_<float>& costs, const cv::Mat_<float>& confidences, const cv::Mat_<float>& texture, float hpm_factor)
 984 | {
 985 | 	support2DPoints.clear();
 986 | 	const int step_size = 5;
 987 | 	const int width = GetReferenceImageWidth() * hpm_factor;
 988 | 	const int height = GetReferenceImageHeight() * hpm_factor;
 989 | 
 990 | 	for (int col = 0; col < width; col += step_size) {
 991 | 		for (int row = 0; row < height; row += step_size) {
 992 | 			float min_cost_no_texture = 2.0f;
 993 | 			float min_cost_texture = 2.0f;
 994 | 			cv::Point temp_point_no_texture;
 995 | 			cv::Point temp_point_texture;
 996 | 			int c_bound = std::min(width, col + step_size);
 997 | 			int r_bound = std::min(height, row + step_size);
 998 | 			int texture_flag = false;
 999 | 			for (int c = col; c < c_bound; ++c) {
1000 | 				for (int r = row; r < r_bound; ++r) {
1001 | 
1002 | 					int center = r * width + c;
1003 | 					if (texture(r, c) > 0.5f) {
1004 | 						texture_flag = true;
1005 | 					}
1006 | 					float photometric_cost = costs(r, c);
1007 | 					float confidence = confidences(r, c);
1008 | 					if (photometric_cost < 2.0f) {
1009 | 						float final_cost_no_texture = photometric_cost - confidence;
1010 | 						float final_cost_texture = photometric_cost + 0.2 - confidence;
1011 | 						if (min_cost_no_texture > final_cost_no_texture) {
1012 | 							temp_point_no_texture = cv::Point(c, r);
1013 | 							min_cost_no_texture = final_cost_no_texture;
1014 | 						}
1015 | 						if (min_cost_texture > final_cost_texture) {
1016 | 							temp_point_texture = cv::Point(c, r);
1017 | 							min_cost_texture = final_cost_texture;
1018 | 						}
1019 | 					}
1020 | 				}
1021 | 			}
1022 | 			if (min_cost_texture < 0.1) {
1023 | 				support2DPoints.push_back(temp_point_texture);
1024 | 			}
1025 | 			if (min_cost_no_texture < 0.1) {
1026 | 				support2DPoints.push_back(temp_point_no_texture);
1027 | 			}
1028 | 		}
1029 | 	}
1030 | }
1031 | 
1032 | std::vector<Triangle> HPM::DelaunayTriangulation(const cv::Rect boundRC, const std::vector<cv::Point>& points)
1033 | {
1034 | 	if (points.empty()) {
1035 | 		return std::vector<Triangle>();
1036 | 	}
1037 | 
1038 | 	std::vector<Triangle> results;
1039 | 
1040 | 	std::vector<cv::Vec6f> temp_results;
1041 | 	cv::Subdiv2D subdiv2d(boundRC);
1042 | 	for (const auto point : points) {
1043 | 		subdiv2d.insert(cv::Point2f((float)point.x, (float)point.y));
1044 | 	}
1045 | 	subdiv2d.getTriangleList(temp_results);
1046 | 
1047 | 	for (const auto temp_vec : temp_results) {
1048 | 		cv::Point pt1((int)temp_vec[0], (int)temp_vec[1]);
1049 | 		cv::Point pt2((int)temp_vec[2], (int)temp_vec[3]);
1050 | 		cv::Point pt3((int)temp_vec[4], (int)temp_vec[5]);
1051 | 		results.push_back(Triangle(pt1, pt2, pt3));
1052 | 	}
1053 | 	return results;
1054 | }
1055 | 
1056 | float4 HPM::GetPriorPlaneParams_factor(const Triangle triangle, const cv::Mat_<float> depths, float factor)
1057 | {
1058 | 	cv::Mat A(3, 4, CV_32FC1);
1059 | 	cv::Mat B(4, 1, CV_32FC1);
1060 | 
1061 | 	float3 ptX1 = Get3DPointonRefCam_factor(triangle.pt1.x, triangle.pt1.y, depths(triangle.pt1.y, triangle.pt1.x), cameras[0], factor);
1062 | 	float3 ptX2 = Get3DPointonRefCam_factor(triangle.pt2.x, triangle.pt2.y, depths(triangle.pt2.y, triangle.pt2.x), cameras[0], factor);
1063 | 	float3 ptX3 = Get3DPointonRefCam_factor(triangle.pt3.x, triangle.pt3.y, depths(triangle.pt3.y, triangle.pt3.x), cameras[0], factor);
1064 | 
1065 | 	A.at<float>(0, 0) = ptX1.x;
1066 | 	A.at<float>(0, 1) = ptX1.y;
1067 | 	A.at<float>(0, 2) = ptX1.z;
1068 | 	A.at<float>(0, 3) = 1.0;
1069 | 	A.at<float>(1, 0) = ptX2.x;
1070 | 	A.at<float>(1, 1) = ptX2.y;
1071 | 	A.at<float>(1, 2) = ptX2.z;
1072 | 	A.at<float>(1, 3) = 1.0;
1073 | 	A.at<float>(2, 0) = ptX3.x;
1074 | 	A.at<float>(2, 1) = ptX3.y;
1075 | 	A.at<float>(2, 2) = ptX3.z;
1076 | 	A.at<float>(2, 3) = 1.0;
1077 | 	cv::SVD::solveZ(A, B);
1078 | 	float4 n4 = make_float4(B.at<float>(0, 0), B.at<float>(1, 0), B.at<float>(2, 0), B.at<float>(3, 0));
1079 | 	float norm2 = sqrt(pow(n4.x, 2) + pow(n4.y, 2) + pow(n4.z, 2));
1080 | 	if (n4.w < 0) {
1081 | 		norm2 *= -1;
1082 | 	}
1083 | 	n4.x /= norm2;
1084 | 	n4.y /= norm2;
1085 | 	n4.z /= norm2;
1086 | 	n4.w /= norm2;
1087 | 
1088 | 	return n4;
1089 | }
1090 | 
1091 | float4 HPM::GetPriorPlaneParams(const Triangle triangle, const cv::Mat_<float> depths)
1092 | {
1093 | 	cv::Mat A(3, 4, CV_32FC1);
1094 | 	cv::Mat B(4, 1, CV_32FC1);
1095 | 
1096 | 	float3 ptX1 = Get3DPointonRefCam(triangle.pt1.x, triangle.pt1.y, depths(triangle.pt1.y, triangle.pt1.x), cameras[0]);
1097 | 	float3 ptX2 = Get3DPointonRefCam(triangle.pt2.x, triangle.pt2.y, depths(triangle.pt2.y, triangle.pt2.x), cameras[0]);
1098 | 	float3 ptX3 = Get3DPointonRefCam(triangle.pt3.x, triangle.pt3.y, depths(triangle.pt3.y, triangle.pt3.x), cameras[0]);
1099 | 
1100 | 	A.at<float>(0, 0) = ptX1.x;
1101 | 	A.at<float>(0, 1) = ptX1.y;
1102 | 	A.at<float>(0, 2) = ptX1.z;
1103 | 	A.at<float>(0, 3) = 1.0;
1104 | 	A.at<float>(1, 0) = ptX2.x;
1105 | 	A.at<float>(1, 1) = ptX2.y;
1106 | 	A.at<float>(1, 2) = ptX2.z;
1107 | 	A.at<float>(1, 3) = 1.0;
1108 | 	A.at<float>(2, 0) = ptX3.x;
1109 | 	A.at<float>(2, 1) = ptX3.y;
1110 | 	A.at<float>(2, 2) = ptX3.z;
1111 | 	A.at<float>(2, 3) = 1.0;
1112 | 	cv::SVD::solveZ(A, B);
1113 | 	float4 n4 = make_float4(B.at<float>(0, 0), B.at<float>(1, 0), B.at<float>(2, 0), B.at<float>(3, 0));
1114 | 	float norm2 = sqrt(pow(n4.x, 2) + pow(n4.y, 2) + pow(n4.z, 2));
1115 | 	if (n4.w < 0) {
1116 | 		norm2 *= -1;
1117 | 	}
1118 | 	n4.x /= norm2;
1119 | 	n4.y /= norm2;
1120 | 	n4.z /= norm2;
1121 | 	n4.w /= norm2;
1122 | 
1123 | 	return n4;
1124 | }
1125 | 
1126 | float HPM::GetDepthFromPlaneParam(const float4 plane_hypothesis, const int x, const int y)
1127 | {
1128 | 	return -plane_hypothesis.w * cameras[0].K[0] / ((x - cameras[0].K[2]) * plane_hypothesis.x + (cameras[0].K[0] / cameras[0].K[4]) * (y - cameras[0].K[5]) * plane_hypothesis.y + cameras[0].K[0] * plane_hypothesis.z);
1129 | }
1130 | 
1131 | float HPM::GetDepthFromPlaneParam_factor(const float4 plane_hypothesis, const int x, const int y, float factor)
1132 | {
1133 | 	return -plane_hypothesis.w * (cameras[0].K[0] * factor) / ((x - cameras[0].K[2] * factor) * plane_hypothesis.x + (cameras[0].K[0] / cameras[0].K[4]) * (y - cameras[0].K[5] * factor) * plane_hypothesis.y + cameras[0].K[0] * factor * plane_hypothesis.z);
1134 | }
1135 | 
1136 | float4 HPM::TransformNormal(float4 plane_hypothesis)
1137 | {
1138 | 	float4 transformed_normal;
1139 | 	transformed_normal.x = cameras[0].R[0] * plane_hypothesis.x + cameras[0].R[3] * plane_hypothesis.y + cameras[0].R[6] * plane_hypothesis.z;
1140 | 	transformed_normal.y = cameras[0].R[1] * plane_hypothesis.x + cameras[0].R[4] * plane_hypothesis.y + cameras[0].R[7] * plane_hypothesis.z;
1141 | 	transformed_normal.z = cameras[0].R[2] * plane_hypothesis.x + cameras[0].R[5] * plane_hypothesis.y + cameras[0].R[8] * plane_hypothesis.z;
1142 | 	transformed_normal.w = plane_hypothesis.w;
1143 | 	return transformed_normal;
1144 | }
1145 | 
1146 | float4 HPM::TransformNormal2RefCam(float4 plane_hypothesis)
1147 | {
1148 | 	float4 transformed_normal;
1149 | 	transformed_normal.x = cameras[0].R[0] * plane_hypothesis.x + cameras[0].R[1] * plane_hypothesis.y + cameras[0].R[2] * plane_hypothesis.z;
1150 | 	transformed_normal.y = cameras[0].R[3] * plane_hypothesis.x + cameras[0].R[4] * plane_hypothesis.y + cameras[0].R[5] * plane_hypothesis.z;
1151 | 	transformed_normal.z = cameras[0].R[6] * plane_hypothesis.x + cameras[0].R[7] * plane_hypothesis.y + cameras[0].R[8] * plane_hypothesis.z;
1152 | 	transformed_normal.w = plane_hypothesis.w;
1153 | 	return transformed_normal;
1154 | }
1155 | 
1156 | float HPM::GetDistance2Origin(const int2 p, const float depth, const float4 normal)
1157 | {
1158 | 	float X[3];
1159 | 	X[0] = depth * (p.x - cameras[0].K[2]) / (cameras[0].K[0]);
1160 | 	X[1] = depth * (p.y - cameras[0].K[5]) / (cameras[0].K[4]);
1161 | 	X[2] = depth;
1162 | 	return -(normal.x * X[0] + normal.y * X[1] + normal.z * X[2]);
1163 | }
1164 | 
1165 | void JBUAddImageToTextureFloatGray(std::vector<cv::Mat_<float>>& imgs, cudaTextureObject_t texs[], cudaArray* cuArray[], const int& numSelViews)
1166 | {
1167 | 	for (int i = 0; i < numSelViews; i++) {
1168 | 		int index = i;
1169 | 		int rows = imgs[index].rows;
1170 | 		int cols = imgs[index].cols;
1171 | 		// Create channel with floating point type
1172 | 		cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat);
1173 | 		// Allocate array with correct size and number of channels
1174 | 		cudaMallocArray(&cuArray[i], &channelDesc, cols, rows);
1175 | 		cudaMemcpy2DToArray(cuArray[i], 0, 0, imgs[index].ptr<float>(), imgs[index].step[0], cols * sizeof(float), rows, cudaMemcpyHostToDevice);
1176 | 
1177 | 		// Specify texture
1178 | 		struct cudaResourceDesc resDesc;
1179 | 		memset(&resDesc, 0, sizeof(cudaResourceDesc));
1180 | 		resDesc.resType = cudaResourceTypeArray;
1181 | 		resDesc.res.array.array = cuArray[i];
1182 | 
1183 | 		// Specify texture object parameters
1184 | 		struct cudaTextureDesc texDesc;
1185 | 		memset(&texDesc, 0, sizeof(cudaTextureDesc));
1186 | 		texDesc.addressMode[0] = cudaAddressModeWrap;
1187 | 		texDesc.addressMode[1] = cudaAddressModeWrap;
1188 | 		texDesc.filterMode = cudaFilterModeLinear;
1189 | 		texDesc.readMode = cudaReadModeElementType;
1190 | 		texDesc.normalizedCoords = 0;
1191 | 
1192 | 		// Create texture object
1193 | 		cudaCreateTextureObject(&(texs[i]), &resDesc, &texDesc, NULL);
1194 | 	}
1195 | 	return;
1196 | }
1197 | 
1198 | JBU::JBU() {}
1199 | 
1200 | JBU::~JBU()
1201 | {
1202 | 	free(depth_h);
1203 | 
1204 | 	cudaFree(depth_d);
1205 | 	cudaFree(jp_d);
1206 | 	cudaFree(jt_d);
1207 | }
1208 | 
1209 | void JBU::InitializeParameters(int n)
1210 | {
1211 | 	depth_h = (float*)malloc(sizeof(float) * n);
1212 | 
1213 | 	cudaMalloc((void**)&depth_d, sizeof(float) * n);
1214 | 
1215 | 	cudaMalloc((void**)&jp_d, sizeof(JBUParameters) * 1);
1216 | 	cudaMemcpy(jp_d, &jp_h, sizeof(JBUParameters) * 1, cudaMemcpyHostToDevice);
1217 | 
1218 | 	cudaMalloc((void**)&jt_d, sizeof(JBUTexObj) * 1);
1219 | 	cudaMemcpy(jt_d, &jt_h, sizeof(JBUTexObj) * 1, cudaMemcpyHostToDevice);
1220 | 	cudaDeviceSynchronize();
1221 | }
1222 | 
1223 | void RunJBU(const cv::Mat_<float>& scaled_image_float, const cv::Mat_<float>& src_depthmap, const std::string& dense_folder, const Problem& problem)
1224 | {
1225 | 	uint32_t rows = scaled_image_float.rows;
1226 | 	uint32_t cols = scaled_image_float.cols;
1227 | 	int Imagescale = std::max(scaled_image_float.rows / src_depthmap.rows, scaled_image_float.cols / src_depthmap.cols);
1228 | 
1229 | 	if (Imagescale == 1) {
1230 | 		std::cout << "Image.rows = Depthmap.rows" << std::endl;
1231 | 		return;
1232 | 	}
1233 | 
1234 | 	std::vector<cv::Mat_<float> > imgs(JBU_NUM);
1235 | 	imgs[0] = scaled_image_float.clone();
1236 | 	imgs[1] = src_depthmap.clone();
1237 | 
1238 | 	JBU jbu;
1239 | 	jbu.jp_h.height = rows;
1240 | 	jbu.jp_h.width = cols;
1241 | 	jbu.jp_h.s_height = src_depthmap.rows;
1242 | 	jbu.jp_h.s_width = src_depthmap.cols;
1243 | 	jbu.jp_h.Imagescale = Imagescale;
1244 | 	JBUAddImageToTextureFloatGray(imgs, jbu.jt_h.imgs, jbu.cuArray, JBU_NUM);
1245 | 
1246 | 	jbu.InitializeParameters(rows * cols);
1247 | 	jbu.CudaRun();
1248 | 
1249 | 	cv::Mat_<float> depthmap = cv::Mat::zeros(rows, cols, CV_32FC1);
1250 | 
1251 | 	for (uint32_t i = 0; i < cols; ++i) {
1252 | 		for (uint32_t j = 0; j < rows; ++j) {
1253 | 			int center = i + cols * j;
1254 | 			if (jbu.depth_h[center] != jbu.depth_h[center]) {
1255 | 				//std::cout << "wrong!" << std::endl;
1256 | 				jbu.depth_h[center] = src_depthmap(int(j / 2), int(i / 2));
1257 | 			}
1258 | 			depthmap(j, i) = jbu.depth_h[center];
1259 | 		}
1260 | 	}
1261 | 
1262 | 	cv::Mat_<float> disp0 = depthmap.clone();
1263 | 	std::stringstream result_path;
1264 | 	result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
1265 | 	std::string result_folder = result_path.str();
1266 | 	mkdir(result_folder.c_str());
1267 | 	std::string depth_path = result_folder + "/depths.dmb";
1268 | 	writeDepthDmb(depth_path, disp0);
1269 | 
1270 | 	for (int i = 0; i < JBU_NUM; i++) {
1271 | 		CUDA_SAFE_CALL(cudaDestroyTextureObject(jbu.jt_h.imgs[i]));
1272 | 		CUDA_SAFE_CALL(cudaFreeArray(jbu.cuArray[i]));
1273 | 	}
1274 | 	cudaDeviceSynchronize();
1275 | }
1276 | 
1277 | 
1278 | 
1279 | void HPM::JointBilateralUpsampling_prior(const cv::Mat_<float>& scaled_image_float, const cv::Mat_<float>& src_depthmap, cv::Mat_<float>& upsample_depthmap, const cv::Mat_<cv::Vec3f>& src_normal, cv::Mat_<cv::Vec3f>& upsample_normal)
1280 | {
1281 | 	uint32_t rows = scaled_image_float.rows;
1282 | 	uint32_t cols = scaled_image_float.cols;
1283 | 	int Imagescale = std::max(scaled_image_float.rows / src_depthmap.rows, scaled_image_float.cols / src_depthmap.cols);
1284 | 	if (Imagescale == 1) {
1285 | 		std::cout << "Image.rows = Depthmap.rows" << std::endl;
1286 | 		return;
1287 | 	}
1288 | 	std::vector<cv::Mat_<float> > imgs(JBU_NUM);
1289 | 	imgs[0] = scaled_image_float.clone();
1290 | 	imgs[1] = src_depthmap.clone();
1291 | 
1292 | 	JBU_prior jbu_prior;
1293 | 	jbu_prior.jp_h.height = rows;
1294 | 	jbu_prior.jp_h.width = cols;
1295 | 	jbu_prior.jp_h.s_height = src_depthmap.rows;
1296 | 	jbu_prior.jp_h.s_width = src_depthmap.cols;
1297 | 	jbu_prior.jp_h.Imagescale = Imagescale;
1298 | 
1299 | 	JBUAddImageToTextureFloatGray(imgs, jbu_prior.jt_h.imgs, jbu_prior.cuArray, JBU_NUM);
1300 | 	jbu_prior.normal_origin_host = new float4[src_depthmap.rows * src_depthmap.cols];
1301 | 	for (int i = 0; i < src_depthmap.rows; i++) {
1302 | 		for (int j = 0; j < src_depthmap.cols; j++) {
1303 | 			int center = i * src_depthmap.cols + j;
1304 | 			jbu_prior.normal_origin_host[center].x = src_normal(i, j)[0];
1305 | 			jbu_prior.normal_origin_host[center].y = src_normal(i, j)[1];
1306 | 			jbu_prior.normal_origin_host[center].z = src_normal(i, j)[2];
1307 | 			jbu_prior.normal_origin_host[center].w = src_depthmap(i, j);
1308 | 			//std::cout << jbu.normal_origin_host[center].x << " " << jbu.normal_origin_host[center].y << " " << jbu.normal_origin_host[center].z << " " << jbu.normal_origin_host[center].w << std::endl;
1309 | 		}
1310 | 	}
1311 | 	jbu_prior.InitializeParameters_prior(rows * cols, src_depthmap.rows * src_depthmap.cols);
1312 | 	jbu_prior.CudaRun_prior();
1313 | 
1314 | 
1315 | 	for (uint32_t i = 0; i < cols; ++i) {
1316 | 		for (uint32_t j = 0; j < rows; ++j) {
1317 | 			int center = i + cols * j;
1318 | 			if (jbu_prior.depth_h[center] != jbu_prior.depth_h[center]) {
1319 | 				//std::cout << "wrong!" << std::endl;
1320 | 				upsample_depthmap(j, i) = src_depthmap(j / 2, i / 2);
1321 | 				upsample_normal(j, i)[0] = src_normal(j / 2, i / 2)[0];
1322 | 				upsample_normal(j, i)[1] = src_normal(j / 2, i / 2)[1];
1323 | 				upsample_normal(j, i)[2] = src_normal(j / 2, i / 2)[2];
1324 | 			}
1325 | 			upsample_depthmap(j, i) = jbu_prior.depth_h[center];
1326 | 			upsample_normal(j, i)[0] = jbu_prior.normal_h[center].x;
1327 | 			upsample_normal(j, i)[1] = jbu_prior.normal_h[center].y;
1328 | 			upsample_normal(j, i)[2] = jbu_prior.normal_h[center].z;
1329 | 		}
1330 | 	}
1331 | 
1332 | 	for (int i = 0; i < JBU_NUM; i++) {
1333 | 		CUDA_SAFE_CALL(cudaDestroyTextureObject(jbu_prior.jt_h.imgs[i]));
1334 | 		CUDA_SAFE_CALL(cudaFreeArray(jbu_prior.cuArray[i]));
1335 | 	}
1336 | 	//jbu.~JBU();
1337 | 	jbu_prior.ReleaseJBUCudaMemory_prior();
1338 | 	imgs[0].release();
1339 | 	imgs[1].release();
1340 | 	//delete(jbu.depth_h);
1341 | 	//delete(jbu.normal_h);
1342 | 	//free(jbu.depth_h);
1343 | 	//free(jbu.normal_h);
1344 | 	imgs.clear();
1345 | 	imgs.shrink_to_fit();
1346 | 	delete(jbu_prior.normal_origin_host);
1347 | 	cudaDeviceSynchronize();
1348 | }
1349 | 
1350 | 
1351 | JBU_prior::JBU_prior() {}
1352 | 
1353 | JBU_prior::~JBU_prior()
1354 | {
1355 | 	free(depth_h);
1356 | 
1357 | 	cudaFree(depth_d);
1358 | 	cudaFree(jp_d);
1359 | 	cudaFree(jt_d);
1360 | }
1361 | 
1362 | void JBU_prior::InitializeParameters_prior(int n, int origin_n)
1363 | {
1364 | 	depth_h = (float*)malloc(sizeof(float) * n);
1365 | 	normal_h = new float4[n];
1366 | 
1367 | 
1368 | 	cudaMalloc((void**)&depth_d, sizeof(float) * n);
1369 | 	cudaMalloc((void**)&normal_d, sizeof(float4) * n);
1370 | 	cudaMalloc((void**)&normal_origin_cuda, sizeof(float4) * origin_n);
1371 | 	cudaMemcpy(normal_origin_cuda, normal_origin_host, sizeof(float4) * origin_n, cudaMemcpyHostToDevice);
1372 | 
1373 | 	cudaMalloc((void**)&jp_d, sizeof(JBUParameters) * 1);
1374 | 	cudaMemcpy(jp_d, &jp_h, sizeof(JBUParameters) * 1, cudaMemcpyHostToDevice);
1375 | 
1376 | 	cudaMalloc((void**)&jt_d, sizeof(JBUTexObj) * 1);
1377 | 	cudaMemcpy(jt_d, &jt_h, sizeof(JBUTexObj) * 1, cudaMemcpyHostToDevice);
1378 | 	cudaDeviceSynchronize();
1379 | }
1380 | 
1381 | void JBU_prior::ReleaseJBUCudaMemory_prior() {
1382 | 	cudaFree(depth_d);
1383 | 	cudaFree(normal_d);
1384 | 	cudaFree(normal_origin_cuda);
1385 | 	cudaFree(jp_d);
1386 | 	cudaFree(jt_d);
1387 | }
1388 | 
1389 | void HPM::ReloadPlanarPriorInitialization(const cv::Mat_<float>& masks, float4* prior_plane_parameters)
1390 | {
1391 | 	prior_planes_host = new float4[cameras[0].height * cameras[0].width];
1392 | 	cudaMalloc((void**)&prior_planes_cuda, sizeof(float4) * (cameras[0].height * cameras[0].width));
1393 | 
1394 | 	plane_masks_host = new unsigned int[cameras[0].height * cameras[0].width];
1395 | 	cudaMalloc((void**)&plane_masks_cuda, sizeof(unsigned int) * (cameras[0].height * cameras[0].width));
1396 | 
1397 | 
1398 | 	for (int i = 0; i < cameras[0].width; ++i) {
1399 | 		for (int j = 0; j < cameras[0].height; ++j) {
1400 | 			int center = j * cameras[0].width + i;
1401 | 			plane_masks_host[center] = (unsigned int)masks(j, i);
1402 | 			if (masks(j, i) > 0) {
1403 | 				prior_planes_host[center] = prior_plane_parameters[center];
1404 | 			}
1405 | 		}
1406 | 	}
1407 | 	cudaMemcpy(prior_planes_cuda, prior_planes_host, sizeof(float4) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice);
1408 | 	cudaMemcpy(plane_masks_cuda, plane_masks_host, sizeof(unsigned int) * (cameras[0].height * cameras[0].width), cudaMemcpyHostToDevice);
1409 | }


--------------------------------------------------------------------------------
/HPM.h:
--------------------------------------------------------------------------------
  1 | #ifndef _HPM_H_
  2 | #define _HPM_H_
  3 | 
  4 | #include "main.h"
  5 | 
  6 | int readDepthDmb(const std::string file_path, cv::Mat_<float> &depth);
  7 | int readNormalDmb(const std::string file_path, cv::Mat_<cv::Vec3f> &normal);
  8 | int writeDepthDmb(const std::string file_path, const cv::Mat_<float> depth);
  9 | int writeNormalDmb(const std::string file_path, const cv::Mat_<cv::Vec3f> normal);
 10 | 
 11 | Camera ReadCamera(const std::string &cam_path);
 12 | void  RescaleImageAndCamera(cv::Mat_<cv::Vec3b> &src, cv::Mat_<cv::Vec3b> &dst, cv::Mat_<float> &depth, Camera &camera);
 13 | void RescaleMask(cv::Mat_<cv::Vec3b>& src, cv::Mat_<cv::Vec3b>& dst, cv::Mat_<float>& depth);
 14 | float3 Get3DPointonWorld(const int x, const int y, const float depth, const Camera camera);
 15 | void ProjectonCamera(const float3 PointX, const Camera camera, float2 &point, float &depth);
 16 | float GetAngle(const cv::Vec3f &v1, const cv::Vec3f &v2);
 17 | void StoreColorPlyFileBinaryPointCloud (const std::string &plyFilePath, const std::vector<PointList> &pc);
 18 | void ExportPointCloud(const std::string& plyFilePath, const std::vector<PointList>& pc);
 19 | void RunJBU(const cv::Mat_<float>  &scaled_image_float, const cv::Mat_<float> &src_depthmap, const std::string &dense_folder , const Problem &problem);
 20 | 
 21 | #define CUDA_SAFE_CALL(error) CudaSafeCall(error, __FILE__, __LINE__)
 22 | #define CUDA_CHECK_ERROR() CudaCheckError(__FILE__, __LINE__)
 23 | 
 24 | void CudaSafeCall(const cudaError_t error, const std::string& file, const int line);
 25 | void CudaCheckError(const char* file, const int line);
 26 | 
 27 | struct cudaTextureObjects {
 28 |     cudaTextureObject_t images[MAX_IMAGES];
 29 | };
 30 | 
 31 | struct PatchMatchParams {
 32 |     int max_iterations = 3;
 33 |     int patch_size = 11;
 34 |     int num_images = 5;
 35 |     int max_image_size=3200;
 36 |     int radius_increment = 2;
 37 |     float sigma_spatial = 5.0f;
 38 |     float sigma_color = 3.0f;
 39 |     int top_k = 4;
 40 |     float baseline = 0.54f;
 41 |     float depth_min = 0.0f;
 42 |     float depth_max = 1.0f;
 43 |     float disparity_min = 0.0f;
 44 |     float disparity_max = 1.0f;
 45 | 
 46 |     float scaled_cols;
 47 |     float scaled_rows;
 48 | 
 49 |     bool geom_consistency = false;
 50 |     bool prior_consistency = false;
 51 |     bool multi_geometry = false;
 52 |     bool hierarchy = false;
 53 |     bool upsample = false;
 54 |     bool mand_consistency = false;
 55 | };
 56 | 
 57 | class HPM {
 58 | public:
 59 |     HPM();
 60 |     ~HPM();
 61 | 
 62 |     void InuputInitialization(const std::string &dense_folder, const std::vector<Problem> &problem, const int idx);
 63 |     void Colmap2MVS(const std::string &dense_folder, std::vector<Problem> &problems);
 64 |     void CudaSpaceInitialization(const std::string &dense_folder, const Problem &problem);
 65 |     void RunPatchMatch();
 66 |     void SetGeomConsistencyParams(bool multi_geometry);
 67 |     void SetPlanarPriorParams();
 68 |     void SetHierarchyParams();
 69 |     void SetMandConsistencyParams(bool flag);
 70 | 
 71 |     int GetReferenceImageWidth();
 72 |     int GetReferenceImageHeight();
 73 |     cv::Mat GetReferenceImage();
 74 |     float4 GetPlaneHypothesis(const int index);
 75 |     float GetCost(const int index);
 76 |     float GetTexture(const int index);
 77 |     void GetSupportPoints(std::vector<cv::Point>& support2DPoints);
 78 |     void GetSupportPoints_Double_Check(std::vector<cv::Point>& support2DPoints, const cv::Mat_<float>& costs, const cv::Mat_<float>& mand_consistency, float hpm_factor);
 79 |     void GetSupportPoints_Simple_Check(std::vector<cv::Point>& support2DPoints, const cv::Mat_<float>& costs, const cv::Mat_<float>& mand_consistency, float hpm_factor);
 80 |     void GetSupportPoints_Classify_Check(std::vector<cv::Point>& support2DPoints, const cv::Mat_<float>& costs, const cv::Mat_<float>& mand_consistency, const cv::Mat_<float>& texture, float hpm_factor);
 81 |     std::vector<Triangle> DelaunayTriangulation(const cv::Rect boundRC, const std::vector<cv::Point>& points);
 82 |     float4 GetPriorPlaneParams(const Triangle triangle, const cv::Mat_<float> depths);
 83 |     float4 GetPriorPlaneParams_factor(const Triangle triangle, const cv::Mat_<float> depths, float factor);
 84 |     float GetDepthFromPlaneParam(const float4 plane_hypothesis, const int x, const int y);
 85 |     float GetMinDepth();
 86 |     float GetMaxDepth();
 87 |     void CudaPlanarPriorInitialization(const std::vector<float4> &PlaneParams, const cv::Mat_<float> &masks);
 88 |     void CudaHypothesesReload(cv::Mat_ <float>depths, cv::Mat_<float>costs, cv::Mat_<cv::Vec3f>normals);
 89 |     void CudaConfidenceInitialization(const std::string& dense_folder, const std::vector<Problem>& problems, const int idx);
 90 |     void CudaCannyInitialization(const cv::Mat_<int>& Canny);
 91 | 
 92 |     void CudaPlanarPriorRelease();
 93 |     void CudaSpaceRelease(bool geom_consistency);
 94 |     void ReleaseProblemHostMemory();
 95 | 
 96 |     float4 TransformNormal(float4 plane_hypothesis);
 97 |     float GetDepthFromPlaneParam_factor(const float4 plane_hypothesis, const int x, const int y, float factor);
 98 |     void JointBilateralUpsampling_prior(const cv::Mat_<float>& scaled_image_float, const cv::Mat_<float>& src_depthmap, cv::Mat_<float>& upsample_depthmap, const cv::Mat_<cv::Vec3f>& src_normal, cv::Mat_<cv::Vec3f>& upsample_normal);
 99 |     float4 TransformNormal2RefCam(float4 plane_hypothesis);
100 |     float GetDistance2Origin(const int2 p, const float depth, const float4 normal);
101 |     void ReloadPlanarPriorInitialization(const cv::Mat_<float>& masks, float4* prior_plane_parameters);
102 | 
103 |     void TextureInformationInitialization();
104 | 
105 | private:
106 |     int num_images;
107 |     std::vector<cv::Mat> images;
108 |     std::vector<cv::Mat> depths;
109 |     std::vector<Camera> cameras;
110 |     cudaTextureObjects texture_objects_host;
111 |     cudaTextureObjects texture_depths_host;
112 |     float4 *plane_hypotheses_host;
113 |     float4 *scaled_plane_hypotheses_host;
114 |     float *costs_host;
115 |     float *pre_costs_host;
116 |     float4 *prior_planes_host;
117 |     unsigned int *plane_masks_host;
118 |     PatchMatchParams params;
119 |     float* confidences_host;
120 |     float* texture_host;
121 | 
122 | 
123 |     Camera *cameras_cuda;
124 |     cudaArray *cuArray[MAX_IMAGES];
125 |     cudaArray *cuDepthArray[MAX_IMAGES];
126 |     cudaTextureObjects *texture_objects_cuda;
127 |     cudaTextureObjects *texture_depths_cuda;
128 |     float4 *plane_hypotheses_cuda;
129 |     float4 *scaled_plane_hypotheses_cuda;
130 |     float *costs_cuda;
131 |     float *pre_costs_cuda;
132 |     curandState *rand_states_cuda;
133 |     unsigned int *selected_views_cuda;
134 |     float* depths_cuda;
135 |     float4* prior_planes_cuda;
136 |     unsigned int* plane_masks_cuda;
137 |     float* confidences_cuda;
138 |     unsigned int* Canny_cuda;
139 |     float* texture_cuda;
140 | };
141 | 
142 | struct TexObj {
143 |     cudaTextureObject_t imgs[MAX_IMAGES];
144 | };
145 | 
146 | struct JBUParameters {
147 |     int height;
148 |     int width;
149 |     int s_height;
150 |     int s_width;
151 |     int Imagescale;
152 | };
153 | 
154 | struct JBUTexObj {
155 |     cudaTextureObject_t imgs[JBU_NUM];
156 | };
157 | 
158 | class JBU {
159 | public:
160 |     JBU();
161 |     ~JBU();
162 | 
163 |     // Host Parameters
164 |     float *depth_h;
165 |     JBUTexObj jt_h;
166 |     JBUParameters jp_h;
167 | 
168 |     // Device Parameters
169 |     float *depth_d;
170 |     cudaArray *cuArray[JBU_NUM]; // The first for reference image, and the second for stereo depth image
171 |     JBUTexObj *jt_d;
172 |     JBUParameters *jp_d;
173 | 
174 |     void InitializeParameters(int n);
175 |     void CudaRun();
176 | };
177 | 
178 | class JBU_prior {
179 | public:
180 |     JBU_prior();
181 |     ~JBU_prior();
182 | 
183 |     // Host Parameters
184 |     float* depth_h;
185 |     float4* normal_h;
186 |     JBUTexObj jt_h;
187 |     JBUParameters jp_h;
188 |     float4* normal_origin_host;
189 | 
190 |     // Device Parameters
191 |     float* depth_d;
192 |     cudaArray* cuArray[JBU_NUM]; // The first for reference image, and the second for stereo depth image
193 |     JBUTexObj* jt_d;
194 |     JBUParameters* jp_d;
195 |     float4* normal_d;
196 |     float4* normal_origin_cuda;
197 | 
198 |     void InitializeParameters_prior(int n, int origin_n);
199 |     void CudaRun_prior();
200 |     void ReleaseJBUCudaMemory_prior();
201 |     void ReleaseJBUHostMemory_prior();
202 | };
203 | 
204 | 
205 | #endif // _ACMMP_H_
206 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # HPM-MVS++
 2 | * an enhanced version of HPM-MVS (HPM-MVS with Prior Consistency and Mandatory Consistency)
 3 | ## News
 4 | * The code for [HPM-MVS](https://github.com/CLinvx/HPM-MVS) has been released.
 5 | 
 6 | ## Dependencies
 7 | The code has been tested on Windows 10 with RTX 3070.<br />
 8 |  [cmake](https://cmake.org/)<br />
 9 |  [CUDA](https://developer.nvidia.com/cuda-toolkit) >= 6.0<br />
10 |  [OpenCV](https://opencv.org/) >= 2.4
11 | 
12 | ## Useage
13 | * Compile
14 | ```
15 | mkdir build
16 | cd build
17 | cmake ..
18 | make
19 | ```
20 | * Test 
21 | ``` 
22 | Use script colmap2mvsnet_acm.py to convert COLMAP SfM result to MVS input   
23 | Run ./HPM-MVS_plusplus $data_folder true/flase(semantic segmentation masks for filtering sky area) to get reconstruction results 
24 | ```
25 | 
26 | ## Citation
27 | If you find our work useful in your research, please consider citing:
28 | ```
29 | @InProceedings{Ren_2023_ICCV,
30 |     author    = {Ren, Chunlin and Xu, Qingshan and Zhang, Shikun and Yang, Jiaqi},
31 |     title     = {Hierarchical Prior Mining for Non-local Multi-View Stereo},
32 |     booktitle = {Proc. IEEE/CVF International Conference on Computer Vision},
33 |     month     = {October},
34 |     year      = {2023},
35 |     pages     = {3611-3620}
36 | }
37 | ```
38 | 
39 | ## Acknowledgemets
40 | This code largely benefits from the following repositories: [ACMH](https://github.com/GhiXu/ACMH), [ACMP](https://github.com/GhiXu/ACMP), [ACMMP](https://github.com/GhiXu/ACMMP). Thanks to their authors for opening source of their excellent works.
41 | 


--------------------------------------------------------------------------------
/colmap2mvsnet_acm.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | """
  3 | Copyright 2019, Jingyang Zhang and Yao Yao, HKUST. Model reading is provided by COLMAP.
  4 | Preprocess script.
  5 | View selection is modified according to COLMAP's strategy, Qingshan Xu
  6 | """
  7 | 
  8 | from __future__ import print_function
  9 | import collections
 10 | import struct
 11 | import numpy as np
 12 | import multiprocessing as mp
 13 | from functools import partial
 14 | import os
 15 | import argparse
 16 | import shutil
 17 | import cv2
 18 | from tqdm.contrib.concurrent import process_map
 19 | 
 20 | #============================ read_model.py ============================#
 21 | CameraModel = collections.namedtuple(
 22 |     "CameraModel", ["model_id", "model_name", "num_params"])
 23 | Camera = collections.namedtuple(
 24 |     "Camera", ["id", "model", "width", "height", "params"])
 25 | BaseImage = collections.namedtuple(
 26 |     "Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"])
 27 | Point3D = collections.namedtuple(
 28 |     "Point3D", ["id", "xyz", "rgb", "error", "image_ids", "point2D_idxs"])
 29 | 
 30 | class Image(BaseImage):
 31 |     def qvec2rotmat(self):
 32 |         return qvec2rotmat(self.qvec)
 33 | 
 34 | CAMERA_MODELS = {
 35 |     CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3),
 36 |     CameraModel(model_id=1, model_name="PINHOLE", num_params=4),
 37 |     CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4),
 38 |     CameraModel(model_id=3, model_name="RADIAL", num_params=5),
 39 |     CameraModel(model_id=4, model_name="OPENCV", num_params=8),
 40 |     CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8),
 41 |     CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12),
 42 |     CameraModel(model_id=7, model_name="FOV", num_params=5),
 43 |     CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4),
 44 |     CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5),
 45 |     CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12)
 46 | }
 47 | CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) \
 48 |                          for camera_model in CAMERA_MODELS])
 49 | 
 50 | 
 51 | def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"):
 52 |     """Read and unpack the next bytes from a binary file.
 53 |     :param fid:
 54 |     :param num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc.
 55 |     :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}.
 56 |     :param endian_character: Any of {@, =, <, >, !}
 57 |     :return: Tuple of read and unpacked values.
 58 |     """
 59 |     data = fid.read(num_bytes)
 60 |     return struct.unpack(endian_character + format_char_sequence, data)
 61 | 
 62 | 
 63 | def read_cameras_text(path):
 64 |     """
 65 |     see: src/base/reconstruction.cc
 66 |         void Reconstruction::WriteCamerasText(const std::string& path)
 67 |         void Reconstruction::ReadCamerasText(const std::string& path)
 68 |     """
 69 |     cameras = {}
 70 |     with open(path, "r") as fid:
 71 |         while True:
 72 |             line = fid.readline()
 73 |             if not line:
 74 |                 break
 75 |             line = line.strip()
 76 |             if len(line) > 0 and line[0] != "#":
 77 |                 elems = line.split()
 78 |                 camera_id = int(elems[0])
 79 |                 model = elems[1]
 80 |                 width = int(elems[2])
 81 |                 height = int(elems[3])
 82 |                 params = np.array(tuple(map(float, elems[4:])))
 83 |                 cameras[camera_id] = Camera(id=camera_id, model=model,
 84 |                                             width=width, height=height,
 85 |                                             params=params)
 86 |     return cameras
 87 | 
 88 | 
 89 | def read_cameras_binary(path_to_model_file):
 90 |     """
 91 |     see: src/base/reconstruction.cc
 92 |         void Reconstruction::WriteCamerasBinary(const std::string& path)
 93 |         void Reconstruction::ReadCamerasBinary(const std::string& path)
 94 |     """
 95 |     cameras = {}
 96 |     with open(path_to_model_file, "rb") as fid:
 97 |         num_cameras = read_next_bytes(fid, 8, "Q")[0]
 98 |         for camera_line_index in range(num_cameras):
 99 |             camera_properties = read_next_bytes(
100 |                 fid, num_bytes=24, format_char_sequence="iiQQ")
101 |             camera_id = camera_properties[0]
102 |             model_id = camera_properties[1]
103 |             model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name
104 |             width = camera_properties[2]
105 |             height = camera_properties[3]
106 |             num_params = CAMERA_MODEL_IDS[model_id].num_params
107 |             params = read_next_bytes(fid, num_bytes=8*num_params,
108 |                                      format_char_sequence="d"*num_params)
109 |             cameras[camera_id] = Camera(id=camera_id,
110 |                                         model=model_name,
111 |                                         width=width,
112 |                                         height=height,
113 |                                         params=np.array(params))
114 |         assert len(cameras) == num_cameras
115 |     return cameras
116 | 
117 | 
118 | def read_images_text(path):
119 |     """
120 |     see: src/base/reconstruction.cc
121 |         void Reconstruction::ReadImagesText(const std::string& path)
122 |         void Reconstruction::WriteImagesText(const std::string& path)
123 |     """
124 |     images = {}
125 |     with open(path, "r") as fid:
126 |         while True:
127 |             line = fid.readline()
128 |             if not line:
129 |                 break
130 |             line = line.strip()
131 |             if len(line) > 0 and line[0] != "#":
132 |                 elems = line.split()
133 |                 image_id = int(elems[0])
134 |                 qvec = np.array(tuple(map(float, elems[1:5])))
135 |                 tvec = np.array(tuple(map(float, elems[5:8])))
136 |                 camera_id = int(elems[8])
137 |                 image_name = elems[9]
138 |                 elems = fid.readline().split()
139 |                 xys = np.column_stack([tuple(map(float, elems[0::3])),
140 |                                        tuple(map(float, elems[1::3]))])
141 |                 point3D_ids = np.array(tuple(map(int, elems[2::3])))
142 |                 images[image_id] = Image(
143 |                     id=image_id, qvec=qvec, tvec=tvec,
144 |                     camera_id=camera_id, name=image_name,
145 |                     xys=xys, point3D_ids=point3D_ids)
146 |     return images
147 | 
148 | 
149 | def read_images_binary(path_to_model_file):
150 |     """
151 |     see: src/base/reconstruction.cc
152 |         void Reconstruction::ReadImagesBinary(const std::string& path)
153 |         void Reconstruction::WriteImagesBinary(const std::string& path)
154 |     """
155 |     images = {}
156 |     with open(path_to_model_file, "rb") as fid:
157 |         num_reg_images = read_next_bytes(fid, 8, "Q")[0]
158 |         for image_index in range(num_reg_images):
159 |             binary_image_properties = read_next_bytes(
160 |                 fid, num_bytes=64, format_char_sequence="idddddddi")
161 |             image_id = binary_image_properties[0]
162 |             qvec = np.array(binary_image_properties[1:5])
163 |             tvec = np.array(binary_image_properties[5:8])
164 |             camera_id = binary_image_properties[8]
165 |             image_name = ""
166 |             current_char = read_next_bytes(fid, 1, "c")[0]
167 |             while current_char != b"\x00":   # look for the ASCII 0 entry
168 |                 image_name += current_char.decode("utf-8")
169 |                 current_char = read_next_bytes(fid, 1, "c")[0]
170 |             num_points2D = read_next_bytes(fid, num_bytes=8,
171 |                                            format_char_sequence="Q")[0]
172 |             x_y_id_s = read_next_bytes(fid, num_bytes=24*num_points2D,
173 |                                        format_char_sequence="ddq"*num_points2D)
174 |             xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])),
175 |                                    tuple(map(float, x_y_id_s[1::3]))])
176 |             point3D_ids = np.array(tuple(map(int, x_y_id_s[2::3])))
177 |             images[image_id] = Image(
178 |                 id=image_id, qvec=qvec, tvec=tvec,
179 |                 camera_id=camera_id, name=image_name,
180 |                 xys=xys, point3D_ids=point3D_ids)
181 |     return images
182 | 
183 | 
184 | def read_points3D_text(path):
185 |     """
186 |     see: src/base/reconstruction.cc
187 |         void Reconstruction::ReadPoints3DText(const std::string& path)
188 |         void Reconstruction::WritePoints3DText(const std::string& path)
189 |     """
190 |     points3D = {}
191 |     with open(path, "r") as fid:
192 |         while True:
193 |             line = fid.readline()
194 |             if not line:
195 |                 break
196 |             line = line.strip()
197 |             if len(line) > 0 and line[0] != "#":
198 |                 elems = line.split()
199 |                 point3D_id = int(elems[0])
200 |                 xyz = np.array(tuple(map(float, elems[1:4])))
201 |                 rgb = np.array(tuple(map(int, elems[4:7])))
202 |                 error = float(elems[7])
203 |                 image_ids = np.array(tuple(map(int, elems[8::2])))
204 |                 point2D_idxs = np.array(tuple(map(int, elems[9::2])))
205 |                 points3D[point3D_id] = Point3D(id=point3D_id, xyz=xyz, rgb=rgb,
206 |                                                error=error, image_ids=image_ids,
207 |                                                point2D_idxs=point2D_idxs)
208 |     return points3D
209 | 
210 | 
211 | def read_points3d_binary(path_to_model_file):
212 |     """
213 |     see: src/base/reconstruction.cc
214 |         void Reconstruction::ReadPoints3DBinary(const std::string& path)
215 |         void Reconstruction::WritePoints3DBinary(const std::string& path)
216 |     """
217 |     points3D = {}
218 |     with open(path_to_model_file, "rb") as fid:
219 |         num_points = read_next_bytes(fid, 8, "Q")[0]
220 |         for point_line_index in range(num_points):
221 |             binary_point_line_properties = read_next_bytes(
222 |                 fid, num_bytes=43, format_char_sequence="QdddBBBd")
223 |             point3D_id = binary_point_line_properties[0]
224 |             xyz = np.array(binary_point_line_properties[1:4])
225 |             rgb = np.array(binary_point_line_properties[4:7])
226 |             error = np.array(binary_point_line_properties[7])
227 |             track_length = read_next_bytes(
228 |                 fid, num_bytes=8, format_char_sequence="Q")[0]
229 |             track_elems = read_next_bytes(
230 |                 fid, num_bytes=8*track_length,
231 |                 format_char_sequence="ii"*track_length)
232 |             image_ids = np.array(tuple(map(int, track_elems[0::2])))
233 |             point2D_idxs = np.array(tuple(map(int, track_elems[1::2])))
234 |             points3D[point3D_id] = Point3D(
235 |                 id=point3D_id, xyz=xyz, rgb=rgb,
236 |                 error=error, image_ids=image_ids,
237 |                 point2D_idxs=point2D_idxs)
238 |     return points3D
239 | 
240 | 
241 | def read_model(path, ext):
242 |     if ext == ".txt":
243 |         cameras = read_cameras_text(os.path.join(path, "cameras" + ext))
244 |         images = read_images_text(os.path.join(path, "images" + ext))
245 |         points3D = read_points3D_text(os.path.join(path, "points3D") + ext)
246 |     else:
247 |         cameras = read_cameras_binary(os.path.join(path, "cameras" + ext))
248 |         images = read_images_binary(os.path.join(path, "images" + ext))
249 |         points3D = read_points3d_binary(os.path.join(path, "points3D") + ext)
250 |     return cameras, images, points3D
251 | 
252 | 
253 | def qvec2rotmat(qvec):
254 |     return np.array([
255 |         [1 - 2 * qvec[2]**2 - 2 * qvec[3]**2,
256 |          2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3],
257 |          2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]],
258 |         [2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3],
259 |          1 - 2 * qvec[1]**2 - 2 * qvec[3]**2,
260 |          2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]],
261 |         [2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2],
262 |          2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1],
263 |          1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]])
264 | 
265 | 
266 | def rotmat2qvec(R):
267 |     Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
268 |     K = np.array([
269 |         [Rxx - Ryy - Rzz, 0, 0, 0],
270 |         [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
271 |         [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
272 |         [Ryz - Rzy, Rzx - Rxz, Rxy - Ryx, Rxx + Ryy + Rzz]]) / 3.0
273 |     eigvals, eigvecs = np.linalg.eigh(K)
274 |     qvec = eigvecs[[3, 0, 1, 2], np.argmax(eigvals)]
275 |     if qvec[0] < 0:
276 |         qvec *= -1
277 |     return qvec
278 | 
279 | 
280 | 
281 | def calc_score(inputs, images, points3d, extrinsic, args):
282 |     i, j = inputs
283 |     id_i = images[i+1].point3D_ids
284 |     id_j = images[j+1].point3D_ids
285 |     id_intersect = [it for it in id_i if it in id_j]
286 |     cam_center_i = -np.matmul(extrinsic[i+1][:3, :3].transpose(), extrinsic[i+1][:3, 3:4])[:, 0]
287 |     cam_center_j = -np.matmul(extrinsic[j+1][:3, :3].transpose(), extrinsic[j+1][:3, 3:4])[:, 0]
288 |     score = 0
289 |     angles = []
290 |     for pid in id_intersect:
291 |         if pid == -1:
292 |             continue
293 |         p = points3d[pid].xyz
294 |         theta = (180 / np.pi) * np.arccos(np.dot(cam_center_i - p, cam_center_j - p) / np.linalg.norm(cam_center_i - p) / np.linalg.norm(cam_center_j - p)) # triangulation angle
295 |         # score += np.exp(-(theta - args.theta0) * (theta - args.theta0) / (2 * (args.sigma1 if theta <= args.theta0 else args.sigma2) ** 2))
296 |         angles.append(theta)
297 |         score += 1
298 |     if len(angles) > 0:
299 |         angles_sorted = sorted(angles)
300 |         triangulationangle = angles_sorted[int(len(angles_sorted) * 0.75)]
301 |         if triangulationangle < 1:
302 |             score = 0.0
303 |     return i, j, score
304 | 
305 | def processing_single_scene(args):
306 | 
307 |     image_dir = os.path.join(args.dense_folder, 'images')
308 |     model_dir = os.path.join(args.dense_folder, 'sparse')
309 |     cam_dir = os.path.join(args.save_folder, 'cams')
310 |     image_converted_dir = os.path.join(args.save_folder, 'images')
311 | 
312 |     if os.path.exists(image_converted_dir):
313 |         print("remove:{}".format(image_converted_dir))
314 |         shutil.rmtree(image_converted_dir)
315 |     os.makedirs(image_converted_dir)
316 |     if os.path.exists(cam_dir):
317 |         print("remove:{}".format(cam_dir))
318 |         shutil.rmtree(cam_dir)
319 | 
320 |     cameras, images, points3d = read_model(model_dir, args.model_ext)
321 |     num_images = len(list(images.items()))
322 | 
323 |     param_type = {
324 |         'SIMPLE_PINHOLE': ['f', 'cx', 'cy'],
325 |         'PINHOLE': ['fx', 'fy', 'cx', 'cy'],
326 |         'SIMPLE_RADIAL': ['f', 'cx', 'cy', 'k'],
327 |         'SIMPLE_RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k'],
328 |         'RADIAL': ['f', 'cx', 'cy', 'k1', 'k2'],
329 |         'RADIAL_FISHEYE': ['f', 'cx', 'cy', 'k1', 'k2'],
330 |         'OPENCV': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2'],
331 |         'OPENCV_FISHEYE': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'k3', 'k4'],
332 |         'FULL_OPENCV': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'k5', 'k6'],
333 |         'FOV': ['fx', 'fy', 'cx', 'cy', 'omega'],
334 |         'THIN_PRISM_FISHEYE': ['fx', 'fy', 'cx', 'cy', 'k1', 'k2', 'p1', 'p2', 'k3', 'k4', 'sx1', 'sy1']
335 |     }
336 | 
337 |     # intrinsic
338 |     intrinsic = {}
339 |     for camera_id, cam in cameras.items():
340 |         params_dict = {key: value for key, value in zip(param_type[cam.model], cam.params)}
341 |         if 'f' in param_type[cam.model]:
342 |             params_dict['fx'] = params_dict['f']
343 |             params_dict['fy'] = params_dict['f']
344 |         i = np.array([
345 |             [params_dict['fx'], 0, params_dict['cx']],
346 |             [0, params_dict['fy'], params_dict['cy']],
347 |             [0, 0, 1]
348 |         ])
349 |         intrinsic[camera_id] = i
350 |     print('intrinsic\n', intrinsic, end='\n\n')
351 | 
352 |     new_images = {}
353 |     for i, image_id in enumerate(sorted(images.keys())):
354 |         new_images[i+1] = images[image_id]
355 |     images = new_images
356 | 
357 |     # extrinsic
358 |     extrinsic = {}
359 |     for image_id, image in images.items():
360 |         e = np.zeros((4, 4))
361 |         e[:3, :3] = qvec2rotmat(image.qvec)
362 |         e[:3, 3] = image.tvec
363 |         e[3, 3] = 1
364 |         extrinsic[image_id] = e
365 |     print('extrinsic[1]\n', extrinsic[1], end='\n\n')
366 | 
367 |     # depth range and interval
368 |     depth_ranges = {}
369 |     for i in range(num_images):
370 |         zs = []
371 |         for p3d_id in images[i+1].point3D_ids:
372 |             if p3d_id == -1:
373 |                 continue
374 |             transformed = np.matmul(extrinsic[i+1], [points3d[p3d_id].xyz[0], points3d[p3d_id].xyz[1], points3d[p3d_id].xyz[2], 1])
375 |             zs.append(np.asscalar(transformed[2]))
376 |         zs_sorted = sorted(zs)
377 |         # relaxed depth range
378 |         depth_min = zs_sorted[int(len(zs) * .01)] * 0.75
379 |         depth_max = zs_sorted[int(len(zs) * .99)] * 1.25
380 |         # determine depth number by inverse depth setting, see supplementary material
381 |         if args.max_d == 0:
382 |             image_int = intrinsic[images[i+1].camera_id]
383 |             image_ext = extrinsic[i+1]
384 |             image_r = image_ext[0:3, 0:3]
385 |             image_t = image_ext[0:3, 3]
386 |             p1 = [image_int[0, 2], image_int[1, 2], 1]
387 |             p2 = [image_int[0, 2] + 1, image_int[1, 2], 1]
388 |             P1 = np.matmul(np.linalg.inv(image_int), p1) * depth_min
389 |             P1 = np.matmul(np.linalg.inv(image_r), (P1 - image_t))
390 |             P2 = np.matmul(np.linalg.inv(image_int), p2) * depth_min
391 |             P2 = np.matmul(np.linalg.inv(image_r), (P2 - image_t))
392 |             depth_num = (1 / depth_min - 1 / depth_max) / (1 / depth_min - 1 / (depth_min + np.linalg.norm(P2 - P1)))
393 |         else:
394 |             depth_num = args.max_d
395 |         depth_interval = (depth_max - depth_min) / (depth_num - 1) / args.interval_scale
396 |         depth_ranges[i+1] = (depth_min, depth_interval, depth_num, depth_max)
397 |     print('depth_ranges[1]\n', depth_ranges[1], end='\n\n')
398 | 
399 |     # view selection
400 |     score = np.zeros((len(images), len(images)))
401 |     queue = []
402 |     for i in range(len(images)):
403 |         for j in range(i + 1, len(images)):
404 |             queue.append((i, j))
405 | 
406 |     func = partial(calc_score, images=images, points3d=points3d, args=args, extrinsic=extrinsic)
407 |     result = process_map(func, queue, max_workers=8, chunksize=512)
408 |     for i, j, s in result:
409 |         score[i, j] = s
410 |         score[j, i] = s
411 |     view_sel = []
412 |     num_view = min(20, len(images) - 1)
413 |     for i in range(len(images)):
414 |         sorted_score = np.argsort(score[i])[::-1]
415 |         view_sel.append([(k, score[i, k]) for k in sorted_score[:num_view]])
416 |     print('view_sel[0]\n', view_sel[0], end='\n\n')
417 | 
418 |     # write
419 |     try:
420 |         os.makedirs(cam_dir)
421 |     except os.error:
422 |         print(cam_dir + ' already exist.')
423 |     for i in range(num_images):
424 |         with open(os.path.join(cam_dir, '%08d_cam.txt' % i), 'w') as f:
425 |             f.write('extrinsic\n')
426 |             for j in range(4):
427 |                 for k in range(4):
428 |                     f.write(str(extrinsic[i+1][j, k]) + ' ')
429 |                 f.write('\n')
430 |             f.write('\nintrinsic\n')
431 |             for j in range(3):
432 |                 for k in range(3):
433 |                     f.write(str(intrinsic[images[i+1].camera_id][j, k]) + ' ')
434 |                 f.write('\n')
435 |             f.write('\n%f %f %f %f\n' % (depth_ranges[i+1][0], depth_ranges[i+1][1], depth_ranges[i+1][2], depth_ranges[i+1][3]))
436 |     with open(os.path.join(args.save_folder, 'pair.txt'), 'w') as f:
437 |         f.write('%d\n' % len(images))
438 |         for i, sorted_score in enumerate(view_sel):
439 |             f.write('%d\n%d ' % (i, len(sorted_score)))
440 |             for image_id, s in sorted_score:
441 |                 f.write('%d %d ' % (image_id, s))
442 |             f.write('\n')
443 | 
444 |     #convert to jpg
445 |     for i in range(num_images):
446 |         img_path = os.path.join(image_dir, images[i + 1].name)
447 |         if not img_path.endswith(".jpg"):
448 |             img = cv2.imread(img_path)
449 |             cv2.imwrite(os.path.join(image_converted_dir, '%08d.jpg' % i), img)
450 |         else:
451 |             shutil.copyfile(os.path.join(image_dir, images[i+1].name), os.path.join(image_converted_dir, '%08d.jpg' % i))
452 | 
453 | 
454 | if __name__ == '__main__':
455 |     parser = argparse.ArgumentParser(description='Convert colmap camera')
456 | 
457 |     parser.add_argument('--dense_folder', required=True, type=str, help='dense_folder.')
458 |     parser.add_argument('--save_folder', required=True, type=str, help='save_folder.')
459 | 
460 |     parser.add_argument('--max_d', type=int, default=192)
461 |     parser.add_argument('--interval_scale', type=float, default=1)
462 | 
463 |     parser.add_argument('--theta0', type=float, default=5)
464 |     parser.add_argument('--sigma1', type=float, default=1)
465 |     parser.add_argument('--sigma2', type=float, default=10)
466 |     parser.add_argument('--model_ext', type=str, default=".txt",  choices=[".txt", ".bin"], help='sparse model ext')
467 | 
468 |     args = parser.parse_args()
469 | 
470 |     os.makedirs(os.path.join(args.save_folder), exist_ok=True)
471 |     processing_single_scene(args)
472 | 


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
   1 | #include "main.h"
   2 | #include "HPM.h"
   3 | 
   4 | void GenerateSampleList(const std::string& dense_folder, std::vector<Problem>& problems)
   5 | {
   6 | 	std::string cluster_list_path = dense_folder + std::string("/pair.txt");
   7 | 
   8 | 	problems.clear();
   9 | 
  10 | 	std::ifstream file(cluster_list_path);
  11 | 
  12 | 	int num_images;
  13 | 	file >> num_images;
  14 | 
  15 | 	for (int i = 0; i < num_images; ++i) {
  16 | 		Problem problem;
  17 | 		problem.src_image_ids.clear();
  18 | 		file >> problem.ref_image_id;
  19 | 
  20 | 		int num_src_images;
  21 | 		file >> num_src_images;
  22 | 		for (int j = 0; j < num_src_images; ++j) {
  23 | 			int id;
  24 | 			float score;
  25 | 			file >> id >> score;
  26 | 			if (score <= 0.0f) {
  27 | 				continue;
  28 | 			}
  29 | 			problem.src_image_ids.push_back(id);
  30 | 		}
  31 | 		problems.push_back(problem);
  32 | 	}
  33 | }
  34 | 
  35 | int ComputeMultiScaleSettings(const std::string& dense_folder, std::vector<Problem>& problems)
  36 | {
  37 | 	int max_num_downscale = -1;
  38 | 	int size_bound = 1000;
  39 | 	PatchMatchParams pmp;
  40 | 	std::string image_folder = dense_folder + std::string("/images");
  41 | 
  42 | 	size_t num_images = problems.size();
  43 | 
  44 | 	for (size_t i = 0; i < num_images; ++i) {
  45 | 		std::stringstream image_path;
  46 | 		image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << ".jpg";
  47 | 		cv::Mat_<uint8_t> image_uint = cv::imread(image_path.str(), cv::IMREAD_GRAYSCALE);
  48 | 
  49 | 		int rows = image_uint.rows;
  50 | 		int cols = image_uint.cols;
  51 | 		int max_size = std::max(rows, cols);
  52 | 		if (max_size > pmp.max_image_size) {
  53 | 			max_size = pmp.max_image_size;
  54 | 		}
  55 | 		problems[i].max_image_size = max_size;
  56 | 
  57 | 		int k = 0;
  58 | 		while (max_size > size_bound) {
  59 | 			max_size /= 2;
  60 | 			k++;
  61 | 		}
  62 | 
  63 | 		if (k > max_num_downscale) {
  64 | 			max_num_downscale = k;
  65 | 		}
  66 | 
  67 | 		problems[i].num_downscale = k;
  68 | 	}
  69 | 
  70 | 	return max_num_downscale;
  71 | }
  72 | 
  73 | void ProcessProblem(const std::string& dense_folder, const std::vector<Problem>& problems, const int idx, bool geom_consistency, bool prior_consistency, bool hierarchy, bool mand_consistency, int image_scale, bool multi_geometrty = false, int hpm_scale_distance = 0)
  74 | {
  75 | 	const Problem problem = problems[idx];
  76 | 	std::cout << "Processing image " << std::setw(8) << std::setfill('0') << problem.ref_image_id << "..." << std::endl;
  77 | 	cudaSetDevice(0);
  78 | 	std::stringstream result_path;
  79 | 	result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
  80 | 	std::string result_folder = result_path.str();
  81 | 	mkdir(result_folder.c_str());
  82 | 
  83 | 	HPM hpm;
  84 | 	if (geom_consistency) {
  85 | 		hpm.SetGeomConsistencyParams(multi_geometrty);
  86 | 	}
  87 | 	if (hierarchy) {
  88 | 		hpm.SetHierarchyParams();
  89 | 	}
  90 | 
  91 | 	hpm.InuputInitialization(dense_folder, problems, idx);
  92 | 	hpm.CudaSpaceInitialization(dense_folder, problem);
  93 | 
  94 | 	const int width = hpm.GetReferenceImageWidth();
  95 | 	const int height = hpm.GetReferenceImageHeight();
  96 | 
  97 | 	cv::Mat_<float> depths = cv::Mat::zeros(height, width, CV_32FC1);
  98 | 	cv::Mat_<cv::Vec3f> normals = cv::Mat::zeros(height, width, CV_32FC3);
  99 | 	cv::Mat_<float> costs = cv::Mat::zeros(height, width, CV_32FC1);
 100 | 	cv::Mat_<float>texture = cv::Mat::zeros(height, width, CV_32FC1);
 101 | 
 102 | 	hpm.SetMandConsistencyParams(mand_consistency);
 103 | 
 104 | 	std::stringstream canny_image_path;
 105 | 	cv::Mat Image_grey;
 106 | 	cv::Mat Canny_edge;
 107 | 	std::string image_folder = dense_folder + std::string("/images");
 108 | 	canny_image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problem.ref_image_id << ".jpg";
 109 | 	cv::resize(cv::imread(canny_image_path.str(), 1), Image_grey, cv::Size(width, height), 0, 0, cv::INTER_LINEAR);
 110 | 	cv::Canny(Image_grey, Canny_edge, 50, 150);
 111 | 	std::cout << "Get Canny egdes down!" << std::endl;
 112 | 	hpm.CudaCannyInitialization(Canny_edge);
 113 | 	Image_grey.release();
 114 | 	Canny_edge.release();
 115 | 
 116 | 	if (mand_consistency || prior_consistency) {
 117 | 		hpm.CudaConfidenceInitialization(dense_folder, problems, idx);
 118 | 	}
 119 | 	if (!prior_consistency && !mand_consistency) {
 120 | 		hpm.TextureInformationInitialization();
 121 | 	}
 122 | 
 123 | 	if (!prior_consistency) {
 124 | 		if (mand_consistency) {
 125 | 			std::cout << "Run Mandatory Consistency ..." << std::endl;
 126 | 		}
 127 | 		else if (geom_consistency) {
 128 | 			std::cout << "Run Geometric Consistency ..." << std::endl;
 129 | 		}
 130 | 		else {
 131 | 			std::cout << "Run Photometric Consistency ..." << std::endl;
 132 | 		}
 133 | 		hpm.RunPatchMatch();
 134 | 		for (int col = 0; col < width; ++col) {
 135 | 			for (int row = 0; row < height; ++row) {
 136 | 				int center = row * width + col;
 137 | 				float4 plane_hypothesis = hpm.GetPlaneHypothesis(center);
 138 | 				depths(row, col) = plane_hypothesis.w;
 139 | 				normals(row, col) = cv::Vec3f(plane_hypothesis.x, plane_hypothesis.y, plane_hypothesis.z);
 140 | 				costs(row, col) = hpm.GetCost(center);
 141 | 				if (!mand_consistency) {
 142 | 					texture(row, col) = hpm.GetTexture(center);
 143 | 				}
 144 | 			}
 145 | 		}
 146 | 	}
 147 | 	else if (prior_consistency) {
 148 | 		std::cout << "Run Prior Consistency ..." << std::endl;
 149 | 		hpm.SetPlanarPriorParams();
 150 | 
 151 | 		std::stringstream result_path;
 152 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 153 | 		std::string result_folder = result_path.str();
 154 | 
 155 | 		std::string suffix = "/depths.dmb";
 156 | 		if (multi_geometrty) {
 157 | 			suffix = "/depths_geom.dmb";
 158 | 		}
 159 | 		std::string depth_path = result_folder + suffix;
 160 | 		std::string normal_path = result_folder + "/normals.dmb";
 161 | 		std::string conf_path = result_folder + "/confidence.dmb";
 162 | 		std::string cost_path = result_folder + "/costs.dmb";
 163 | 
 164 | 		cv::Mat_<float>confidences;
 165 | 
 166 | 		readDepthDmb(depth_path, depths);
 167 | 		readNormalDmb(normal_path, normals);
 168 | 		readDepthDmb(conf_path, confidences);
 169 | 		readDepthDmb(cost_path, costs);
 170 | 
 171 | 
 172 | 		if (hpm_scale_distance == 0) {
 173 | 			const cv::Rect imageRC(0, 0, width, height);
 174 | 			std::vector<cv::Point> support2DPoints;
 175 | 
 176 | 			std::string texture_path = result_folder + "/texture" + std::to_string(image_scale) + ".dmb";
 177 | 			cv::Mat_<float>textures;
 178 | 			readDepthDmb(texture_path, textures);
 179 | 
 180 | 			hpm.GetSupportPoints_Classify_Check(support2DPoints, costs, confidences, textures, 1);
 181 | 			const auto triangles = hpm.DelaunayTriangulation(imageRC, support2DPoints);
 182 | 
 183 | 			cv::Mat refImage = hpm.GetReferenceImage().clone();
 184 | 			std::vector<cv::Mat> mbgr(3);
 185 | 			mbgr[0] = refImage.clone();
 186 | 			mbgr[1] = refImage.clone();
 187 | 			mbgr[2] = refImage.clone();
 188 | 			cv::Mat srcImage;
 189 | 			cv::merge(mbgr, srcImage);
 190 | 			for (const auto triangle : triangles) {
 191 | 				if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) {
 192 | 					cv::line(srcImage, triangle.pt1, triangle.pt2, cv::Scalar(0, 0, 255));
 193 | 					cv::line(srcImage, triangle.pt1, triangle.pt3, cv::Scalar(0, 0, 255));
 194 | 					cv::line(srcImage, triangle.pt2, triangle.pt3, cv::Scalar(0, 0, 255));
 195 | 				}
 196 | 			}
 197 | 			std::string triangulation_path = result_folder + "/triangulation0.png";
 198 | 			cv::imwrite(triangulation_path, srcImage);
 199 | 
 200 | 			refImage.release();
 201 | 			mbgr.clear();
 202 | 			mbgr.shrink_to_fit();
 203 | 			srcImage.release();
 204 | 
 205 | 			cv::Mat_<float> mask_tri = cv::Mat::zeros(height, width, CV_32FC1);
 206 | 			std::vector<float4> planeParams_tri;
 207 | 			planeParams_tri.clear();
 208 | 
 209 | 			uint32_t idx = 0;
 210 | 			for (const auto triangle : triangles) {
 211 | 				if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) {
 212 | 					float L01 = sqrt(pow(triangle.pt1.x - triangle.pt2.x, 2) + pow(triangle.pt1.y - triangle.pt2.y, 2));
 213 | 					float L02 = sqrt(pow(triangle.pt1.x - triangle.pt3.x, 2) + pow(triangle.pt1.y - triangle.pt3.y, 2));
 214 | 					float L12 = sqrt(pow(triangle.pt2.x - triangle.pt3.x, 2) + pow(triangle.pt2.y - triangle.pt3.y, 2));
 215 | 
 216 | 					float max_edge_length = std::max(L01, std::max(L02, L12));
 217 | 					float step = 1.0 / max_edge_length;
 218 | 
 219 | 					for (float p = 0; p < 1.0; p += step) {
 220 | 						for (float q = 0; q < 1.0 - p; q += step) {
 221 | 							int x = p * triangle.pt1.x + q * triangle.pt2.x + (1.0 - p - q) * triangle.pt3.x;
 222 | 							int y = p * triangle.pt1.y + q * triangle.pt2.y + (1.0 - p - q) * triangle.pt3.y;
 223 | 							mask_tri(y, x) = idx + 1.0; // To distinguish from the label of non-triangulated areas
 224 | 						}
 225 | 					}
 226 | 					float4 n4 = hpm.GetPriorPlaneParams(triangle, depths);
 227 | 					planeParams_tri.push_back(n4);
 228 | 					idx++;
 229 | 				}
 230 | 			}
 231 | 
 232 | 			cv::Mat_<float> priordepths = cv::Mat::zeros(height, width, CV_32FC1);
 233 | 			for (int i = 0; i < width; ++i) {
 234 | 				for (int j = 0; j < height; ++j) {
 235 | 					if (mask_tri(j, i) > 0) {
 236 | 						float d = hpm.GetDepthFromPlaneParam(planeParams_tri[mask_tri(j, i) - 1], i, j);
 237 | 						if (d <= hpm.GetMaxDepth() && d >= hpm.GetMinDepth()) {
 238 | 							priordepths(j, i) = d;
 239 | 						}
 240 | 						else {
 241 | 							mask_tri(j, i) = 0;
 242 | 						}
 243 | 					}
 244 | 				}
 245 | 			}
 246 | 			std::string prior_path = result_folder + "/depths_prior0.dmb";
 247 | 			writeDepthDmb(prior_path, priordepths);
 248 | 			priordepths.release();
 249 | 
 250 | 			hpm.CudaPlanarPriorInitialization(planeParams_tri, mask_tri);
 251 | 			hpm.CudaHypothesesReload(depths, costs, normals);
 252 | 			hpm.RunPatchMatch();
 253 | 			textures.release();
 254 | 			mask_tri.release();
 255 | 			planeParams_tri.clear();
 256 | 			planeParams_tri.shrink_to_fit();
 257 | 			support2DPoints.clear();
 258 | 			support2DPoints.shrink_to_fit();
 259 | 			planeParams_tri.clear();
 260 | 			planeParams_tri.shrink_to_fit();
 261 | 		}
 262 | 		else if (hpm_scale_distance == 1 || hpm_scale_distance == 2) {
 263 | 			float hpm_factor = 1.0 / (hpm_scale_distance * 2);
 264 | 			int hpm_width = std::round(width * hpm_factor);
 265 | 			int hpm_height = std::round(height * hpm_factor);
 266 | 
 267 | 			cv::Mat_<float>depths_downsample;
 268 | 			cv::Mat_<float>costs_downsample;
 269 | 			cv::Mat_<float>confidences_downsample;
 270 | 
 271 | 			cv::resize(depths, depths_downsample, cv::Size(hpm_width, hpm_height), 0, 0, cv::INTER_LINEAR);
 272 | 			cv::resize(costs, costs_downsample, cv::Size(hpm_width, hpm_height), 0, 0, cv::INTER_LINEAR);
 273 | 			cv::resize(confidences, confidences_downsample, cv::Size(hpm_width, hpm_height), 0, 0, cv::INTER_LINEAR);
 274 | 
 275 | 
 276 | 
 277 | 			std::string texture_path = result_folder + "/texture" + std::to_string(image_scale + hpm_scale_distance) + ".dmb";
 278 | 			cv::Mat_<float>textures;
 279 | 			readDepthDmb(texture_path, textures);
 280 | 
 281 | 			const cv::Rect imageRC(0, 0, hpm_width, hpm_height);
 282 | 			std::vector<cv::Point> support2DPoints;
 283 | 			support2DPoints.clear();
 284 | 			hpm.GetSupportPoints_Classify_Check(support2DPoints, costs_downsample, confidences_downsample, textures, hpm_factor);
 285 | 
 286 | 			const auto triangles = hpm.DelaunayTriangulation(imageRC, support2DPoints);
 287 | 
 288 | 			cv::Mat refImage;
 289 | 			cv::resize(hpm.GetReferenceImage().clone(), refImage, cv::Size(hpm_width, hpm_height), 0, 0, cv::INTER_LINEAR);
 290 | 			std::vector<cv::Mat> mbgr(3);
 291 | 			mbgr[0] = refImage.clone();
 292 | 			mbgr[1] = refImage.clone();
 293 | 			mbgr[2] = refImage.clone();
 294 | 			cv::Mat srcImage;
 295 | 			cv::merge(mbgr, srcImage);
 296 | 			for (const auto triangle : triangles) {
 297 | 				if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) {
 298 | 					cv::line(srcImage, triangle.pt1, triangle.pt2, cv::Scalar(0, 0, 255));
 299 | 					cv::line(srcImage, triangle.pt1, triangle.pt3, cv::Scalar(0, 0, 255));
 300 | 					cv::line(srcImage, triangle.pt2, triangle.pt3, cv::Scalar(0, 0, 255));
 301 | 				}
 302 | 			}
 303 | 			std::string triangulation_path = result_folder + "/triangulation" + std::to_string(hpm_scale_distance) + ".png";
 304 | 			cv::imwrite(triangulation_path, srcImage);
 305 | 
 306 | 			std::vector<float4>planeParams_tri;
 307 | 			cv::Mat_<float> mask_tri = cv::Mat::zeros(hpm_height, hpm_width, CV_32FC1);
 308 | 			planeParams_tri.clear();
 309 | 			uint32_t idx = 0;
 310 | 			for (const auto triangle : triangles) {
 311 | 				if (imageRC.contains(triangle.pt1) && imageRC.contains(triangle.pt2) && imageRC.contains(triangle.pt3)) {
 312 | 					float L01 = sqrt(pow(triangle.pt1.x - triangle.pt2.x, 2) + pow(triangle.pt1.y - triangle.pt2.y, 2));
 313 | 					float L02 = sqrt(pow(triangle.pt1.x - triangle.pt3.x, 2) + pow(triangle.pt1.y - triangle.pt3.y, 2));
 314 | 					float L12 = sqrt(pow(triangle.pt2.x - triangle.pt3.x, 2) + pow(triangle.pt2.y - triangle.pt3.y, 2));
 315 | 
 316 | 					float max_edge_length = std::max(L01, std::max(L02, L12));
 317 | 					float step = 1.0 / max_edge_length;
 318 | 
 319 | 					for (float p = 0; p < 1.0; p += step) {
 320 | 						for (float q = 0; q < 1.0 - p; q += step) {
 321 | 							int x = p * triangle.pt1.x + q * triangle.pt2.x + (1.0 - p - q) * triangle.pt3.x;
 322 | 							int y = p * triangle.pt1.y + q * triangle.pt2.y + (1.0 - p - q) * triangle.pt3.y;
 323 | 							mask_tri(y, x) = idx + 1.0; // To distinguish from the label of non-triangulated areas
 324 | 						}
 325 | 					}
 326 | 					//renew the camera's parameters 
 327 | 					float4 n4 = hpm.GetPriorPlaneParams_factor(triangle, depths_downsample, hpm_factor);
 328 | 					planeParams_tri.push_back(n4);
 329 | 					idx++;
 330 | 				}
 331 | 			}
 332 | 
 333 | 			cv::Mat_<float>priordepths = cv::Mat::zeros(hpm_height, hpm_width, CV_32FC1);
 334 | 			cv::Mat_<cv::Vec3f>priornormals = cv::Mat::zeros(hpm_height, hpm_width, CV_32FC3);
 335 | 
 336 | 			for (int i = 0; i < hpm_width; ++i) {
 337 | 				for (int j = 0; j < hpm_height; ++j) {
 338 | 					if (mask_tri(j, i) > 0) {
 339 | 						float d = hpm.GetDepthFromPlaneParam_factor(planeParams_tri[mask_tri(j, i) - 1], i, j, hpm_factor);
 340 | 						if (d <= hpm.GetMaxDepth() * 1.2f && d >= hpm.GetMinDepth() * 0.6f) {
 341 | 							priordepths(j, i) = d;
 342 | 							float4 tmp_n4 = hpm.TransformNormal(planeParams_tri[mask_tri(j, i) - 1]);
 343 | 							priornormals(j, i)[0] = tmp_n4.x;
 344 | 							priornormals(j, i)[1] = tmp_n4.y;
 345 | 							priornormals(j, i)[2] = tmp_n4.z;
 346 | 						}
 347 | 						else {
 348 | 							mask_tri(j, i) = 0;
 349 | 						}
 350 | 					}
 351 | 				}
 352 | 			}
 353 | 
 354 | 			std::string depths_prior_path = result_folder + "/depths_prior" + std::to_string(hpm_scale_distance) + ".dmb";
 355 | 			writeDepthDmb(depths_prior_path, priordepths);
 356 | 
 357 | 			std::stringstream image_path;
 358 | 			image_path << dense_folder << "/images" << "/" << std::setw(8) << std::setfill('0') << problem.ref_image_id << ".jpg";
 359 | 			cv::Mat_<uint8_t> image_uint;
 360 | 			cv::resize(cv::imread(image_path.str(), cv::IMREAD_GRAYSCALE), image_uint, cv::Size(width, height), 0, 0, cv::INTER_LINEAR);
 361 | 			cv::Mat image_float;
 362 | 			image_uint.convertTo(image_float, CV_32FC1);
 363 | 			cv::Mat_<float>priordepths_upsample = cv::Mat::zeros(height, width, CV_32FC1);
 364 | 			cv::Mat_ < cv::Vec3f >priornormals_upsample = cv::Mat::zeros(height, width, CV_32FC3);
 365 | 			std::cout << "Running JBU..." << std::endl;
 366 | 			hpm.JointBilateralUpsampling_prior(image_float, priordepths, priordepths_upsample, priornormals, priornormals_upsample);
 367 | 
 368 | 			cv::Mat_<float>mask_tri_new = cv::Mat::zeros(height, width, CV_32FC1);
 369 | 			float4* prior_planeParams = new float4[height * width];
 370 | 			for (int i = 0; i < width; i++) {
 371 | 				for (int j = 0; j < height; j++) {
 372 | 					if (priordepths_upsample(j, i) <= hpm.GetMaxDepth() && priordepths_upsample(j, i) >= hpm.GetMinDepth()) {
 373 | 						mask_tri_new(j, i) = 1;
 374 | 						int center = j * width + i;
 375 | 						float4 tmp_reload;
 376 | 						tmp_reload.x = priornormals_upsample(j, i)[0];
 377 | 						tmp_reload.y = priornormals_upsample(j, i)[1];
 378 | 						tmp_reload.z = priornormals_upsample(j, i)[2];
 379 | 						tmp_reload.w = priordepths_upsample(j, i);
 380 | 						tmp_reload = hpm.TransformNormal2RefCam(tmp_reload);
 381 | 						float depth_now = tmp_reload.w;
 382 | 						int2 p = make_int2(i, j);
 383 | 						tmp_reload.w = hpm.GetDistance2Origin(p, depth_now, tmp_reload);
 384 | 						prior_planeParams[center] = tmp_reload;
 385 | 					}
 386 | 					else {
 387 | 						mask_tri_new(j, i) = 0;
 388 | 					}
 389 | 				}
 390 | 			}
 391 | 
 392 | 			depths_prior_path = result_folder + "/depths_prior" + std::to_string(hpm_scale_distance) + "_upsample.dmb";
 393 | 			writeDepthDmb(depths_prior_path, priordepths_upsample);
 394 | 			hpm.ReloadPlanarPriorInitialization(mask_tri_new, prior_planeParams);
 395 | 			hpm.CudaHypothesesReload(depths, costs, normals);
 396 | 			hpm.RunPatchMatch();
 397 | 
 398 | 			refImage.release();
 399 | 			mbgr.clear();
 400 | 			mbgr.shrink_to_fit();
 401 | 			srcImage.release();
 402 | 			textures.release();
 403 | 			support2DPoints.clear();
 404 | 			support2DPoints.shrink_to_fit();
 405 | 			depths_downsample.release();
 406 | 			costs_downsample.release();
 407 | 			confidences_downsample.release();
 408 | 			priordepths.release();
 409 | 			priornormals.release();
 410 | 			image_uint.release();
 411 | 			image_float.release();
 412 | 			priordepths_upsample.release();
 413 | 			priornormals_upsample.release();
 414 | 			mask_tri_new.release();
 415 | 			delete(prior_planeParams);
 416 | 		}
 417 | 
 418 | 		for (int col = 0; col < width; ++col) {
 419 | 			for (int row = 0; row < height; ++row) {
 420 | 				int center = row * width + col;
 421 | 				float4 plane_hypothesis = hpm.GetPlaneHypothesis(center);
 422 | 				depths(row, col) = plane_hypothesis.w;
 423 | 				normals(row, col) = cv::Vec3f(plane_hypothesis.x, plane_hypothesis.y, plane_hypothesis.z);
 424 | 				costs(row, col) = hpm.GetCost(center);
 425 | 			}
 426 | 		}
 427 | 
 428 | 		hpm.CudaPlanarPriorRelease();
 429 | 	}
 430 | 
 431 | 	std::string suffix = "/depths.dmb";
 432 | 	if (geom_consistency) {
 433 | 		suffix = "/depths_geom.dmb";
 434 | 	}
 435 | 	std::string depth_path = result_folder + suffix;
 436 | 	std::string normal_path = result_folder + "/normals.dmb";
 437 | 	std::string cost_path = result_folder + "/costs.dmb";
 438 | 
 439 | 	writeDepthDmb(depth_path, depths);
 440 | 	writeNormalDmb(normal_path, normals);
 441 | 	writeDepthDmb(cost_path, costs);
 442 | 	if (!mand_consistency && !prior_consistency) {
 443 | 		std::string texture_path = result_folder + "/texture" + std::to_string(image_scale) + ".dmb";
 444 | 		writeDepthDmb(texture_path, texture);
 445 | 	}
 446 | 	texture.release();
 447 | 	depths.release();
 448 | 	normals.release();
 449 | 	costs.release();
 450 | 	hpm.CudaSpaceRelease(geom_consistency);
 451 | 	hpm.ReleaseProblemHostMemory();
 452 | 	std::cout << "Processing image " << std::setw(8) << std::setfill('0') << problem.ref_image_id << " done!" << std::endl;
 453 | }
 454 | 
 455 | void JointBilateralUpsampling(const std::string& dense_folder, const Problem& problem, int acmmp_size)
 456 | {
 457 | 	std::stringstream result_path;
 458 | 	result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problem.ref_image_id;
 459 | 	std::string result_folder = result_path.str();
 460 | 	std::string depth_path = result_folder + "/depths_geom.dmb";
 461 | 	cv::Mat_<float> ref_depth;
 462 | 	readDepthDmb(depth_path, ref_depth);
 463 | 
 464 | 	std::string image_folder = dense_folder + std::string("/images");
 465 | 	std::stringstream image_path;
 466 | 	image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problem.ref_image_id << ".jpg";
 467 | 	cv::Mat_<uint8_t> image_uint = cv::imread(image_path.str(), cv::IMREAD_GRAYSCALE);
 468 | 	cv::Mat image_float;
 469 | 	image_uint.convertTo(image_float, CV_32FC1);
 470 | 	const float factor_x = static_cast<float>(acmmp_size) / image_float.cols;
 471 | 	const float factor_y = static_cast<float>(acmmp_size) / image_float.rows;
 472 | 	const float factor = std::min(factor_x, factor_y);
 473 | 
 474 | 	const int new_cols = std::round(image_float.cols * factor);
 475 | 	const int new_rows = std::round(image_float.rows * factor);
 476 | 	cv::Mat scaled_image_float;
 477 | 	cv::resize(image_float, scaled_image_float, cv::Size(new_cols, new_rows), 0, 0, cv::INTER_LINEAR);
 478 | 
 479 | 	std::cout << "Run JBU for image " << problem.ref_image_id << ".jpg" << std::endl;
 480 | 	RunJBU(scaled_image_float, ref_depth, dense_folder, problem);
 481 | }
 482 | 
 483 | void RunFusion_Sky_Strict(std::string& dense_folder, const std::vector<Problem>& problems, bool geom_consistency)
 484 | {
 485 | 	size_t num_images = problems.size();
 486 | 	std::string image_folder = dense_folder + std::string("/images");
 487 | 	std::string cam_folder = dense_folder + std::string("/cams");
 488 | 	std::string mask_folder = dense_folder + std::string("/masks");
 489 | 
 490 | 	std::vector<cv::Mat> images;
 491 | 	std::vector<Camera> cameras;
 492 | 	std::vector<cv::Mat_<float>> depths;
 493 | 	std::vector<cv::Mat_<cv::Vec3f>> normals;
 494 | 	std::vector<cv::Mat> masks;
 495 | 	std::vector<cv::Mat> sky_masks;
 496 | 	images.clear();
 497 | 	cameras.clear();
 498 | 	depths.clear();
 499 | 	normals.clear();
 500 | 	masks.clear();
 501 | 	sky_masks.clear();
 502 | 
 503 | 	for (size_t i = 0; i < num_images; ++i) {
 504 | 		std::cout << "Reading image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 505 | 
 506 | 		std::stringstream image_path;
 507 | 		image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << ".jpg";
 508 | 		cv::Mat_<cv::Vec3b> image = cv::imread(image_path.str(), cv::IMREAD_COLOR);
 509 | 
 510 | 		std::stringstream sky_mask_path;
 511 | 		sky_mask_path << mask_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << ".jpg";
 512 | 		cv::Mat_<cv::Vec3b> sky_mask = cv::imread(sky_mask_path.str(), cv::IMREAD_COLOR);
 513 | 
 514 | 		std::stringstream cam_path;
 515 | 		cam_path << cam_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << "_cam.txt";
 516 | 		Camera camera = ReadCamera(cam_path.str());
 517 | 
 518 | 		std::stringstream result_path;
 519 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id;
 520 | 		std::string result_folder = result_path.str();
 521 | 		std::string suffix = "/depths.dmb";
 522 | 		if (geom_consistency) {
 523 | 			suffix = "/depths_geom.dmb";
 524 | 		}
 525 | 		std::string depth_path = result_folder + suffix;
 526 | 		std::string normal_path = result_folder + "/normals.dmb";
 527 | 		cv::Mat_<float> depth;
 528 | 		cv::Mat_<cv::Vec3f> normal;
 529 | 		readDepthDmb(depth_path, depth);
 530 | 		readNormalDmb(normal_path, normal);
 531 | 
 532 | 		cv::Mat_<cv::Vec3b> scaled_image;
 533 | 		RescaleImageAndCamera(image, scaled_image, depth, camera);
 534 | 		cv::Mat_<cv::Vec3b>scaled_sky_mask;
 535 | 		RescaleMask(sky_mask, scaled_sky_mask, depth);
 536 | 		
 537 | 		cv::Mat mask = cv::Mat::zeros(depth.rows, depth.cols, CV_8UC1);
 538 | 
 539 | 		images.push_back(scaled_image);
 540 | 		cameras.push_back(camera);
 541 | 		depths.push_back(depth);
 542 | 		normals.push_back(normal);
 543 | 		masks.push_back(mask);
 544 | 		sky_masks.push_back(scaled_sky_mask);
 545 | 		image.release();
 546 | 		sky_mask.release();
 547 | 		depth.release();
 548 | 		normal.release();
 549 | 		scaled_image.release();
 550 | 		mask.release();
 551 | 	}
 552 | 
 553 | 	std::vector<PointList> PointCloud;
 554 | 	PointCloud.clear();
 555 | 
 556 | 	for (size_t i = 0; i < num_images; ++i) {
 557 | 		std::cout << "Fusing image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 558 | 		const int cols = depths[i].cols;
 559 | 		const int rows = depths[i].rows;
 560 | 		int num_ngb = problems[i].src_image_ids.size();
 561 | 		std::vector<int2> used_list(num_ngb, make_int2(-1, -1));
 562 | 		for (int r = 0; r < rows; ++r) {
 563 | 			for (int c = 0; c < cols; ++c) {
 564 | 				bool view_strict = false;
 565 | 				if (masks[i].at<uchar>(r, c) == 1)
 566 | 					continue;
 567 | 				float ref_depth = depths[i].at<float>(r, c);
 568 | 				cv::Vec3f ref_normal = normals[i].at<cv::Vec3f>(r, c);
 569 | 
 570 | 				if (ref_depth <= 0.0)
 571 | 					continue;
 572 | 
 573 | 				float3 PointX = Get3DPointonWorld(c, r, ref_depth, cameras[i]);
 574 | 				float3 consistent_Point = PointX;
 575 | 				cv::Vec3f consistent_normal = ref_normal;
 576 | 				float consistent_Color[3] = { (float)images[i].at<cv::Vec3b>(r, c)[0], (float)images[i].at<cv::Vec3b>(r, c)[1], (float)images[i].at<cv::Vec3b>(r, c)[2] };
 577 | 				float segment_Color[3] = { (float)sky_masks[i].at<cv::Vec3b>(r, c)[0], (float)sky_masks[i].at<cv::Vec3b>(r, c)[1], (float)sky_masks[i].at<cv::Vec3b>(r, c)[2] };
 578 | 				int num_consistent = 0;
 579 | 				float dynamic_consistency = 0;
 580 | 
 581 | 				for (int j = 0; j < num_ngb; ++j) {
 582 | 					int src_id = problems[i].src_image_ids[j];
 583 | 					const int src_cols = depths[src_id].cols;
 584 | 					const int src_rows = depths[src_id].rows;
 585 | 					float2 point;
 586 | 					float proj_depth;
 587 | 					ProjectonCamera(PointX, cameras[src_id], point, proj_depth);
 588 | 					int src_r = int(point.y + 0.5f);
 589 | 					int src_c = int(point.x + 0.5f);
 590 | 					if (src_c >= 0 && src_c < src_cols && src_r >= 0 && src_r < src_rows) {
 591 | 						if (masks[src_id].at<uchar>(src_r, src_c) == 1)
 592 | 							continue;
 593 | 
 594 | 						float src_depth = depths[src_id].at<float>(src_r, src_c);
 595 | 						cv::Vec3f src_normal = normals[src_id].at<cv::Vec3f>(src_r, src_c);
 596 | 						if (src_depth <= 0.0)
 597 | 							continue;
 598 | 
 599 | 						float3 tmp_X = Get3DPointonWorld(src_c, src_r, src_depth, cameras[src_id]);
 600 | 						float2 tmp_pt;
 601 | 						ProjectonCamera(tmp_X, cameras[i], tmp_pt, proj_depth);
 602 | 						float reproj_error = sqrt(pow(c - tmp_pt.x, 2) + pow(r - tmp_pt.y, 2));
 603 | 						float relative_depth_diff = fabs(proj_depth - ref_depth) / ref_depth;
 604 | 						float angle = GetAngle(ref_normal, src_normal);
 605 | 
 606 | 						if (reproj_error < 2.0f && relative_depth_diff < 0.01f && angle < 0.174533f) {
 607 | 							used_list[j].x = src_c;
 608 | 							used_list[j].y = src_r;
 609 | 
 610 | 							float tmp_index = reproj_error + 200 * relative_depth_diff + angle * 10;
 611 | 							float cons = exp(-tmp_index);
 612 | 							dynamic_consistency += exp(-tmp_index);
 613 | 							num_consistent++;
 614 | 						}
 615 | 					}
 616 | 				}
 617 | 
 618 | 				int view_num = 1;
 619 | 				float factor = 0.3;
 620 | 
 621 | 
 622 | 				if (num_consistent >= view_num && (dynamic_consistency > factor * num_consistent)) {
 623 | 					PointList point3D;
 624 | 					point3D.coord = consistent_Point;
 625 | 					point3D.color = make_float3(consistent_Color[0], consistent_Color[1], consistent_Color[2]);
 626 | 					
 627 | 					float3 seg_Color = make_float3(segment_Color[0], segment_Color[1], segment_Color[2]);
 628 | 					if ((int)seg_Color.x != 234 && (int)seg_Color.y != 235 && (int)seg_Color.z != 55) {
 629 | 						PointCloud.push_back(point3D);
 630 | 					}
 631 | 					for (int j = 0; j < num_ngb; ++j) {
 632 | 						if (used_list[j].x == -1)
 633 | 							continue;
 634 | 						masks[problems[i].src_image_ids[j]].at<uchar>(used_list[j].y, used_list[j].x) = 1;
 635 | 					}
 636 | 				}
 637 | 			}
 638 | 		}
 639 | 	}
 640 | 
 641 | 	std::string ply_path = dense_folder + "/HPM_MVS_plusplus/HPM_MVS_plusplus_mask.ply";
 642 | 	ExportPointCloud(ply_path, PointCloud);
 643 | }
 644 | 
 645 | void RunFusion(std::string& dense_folder, const std::vector<Problem>& problems, bool geom_consistency)
 646 | {
 647 | 	size_t num_images = problems.size();
 648 | 	std::string image_folder = dense_folder + std::string("/images");
 649 | 	std::string cam_folder = dense_folder + std::string("/cams");
 650 | 
 651 | 	std::vector<cv::Mat> images;
 652 | 	std::vector<Camera> cameras;
 653 | 	std::vector<cv::Mat_<float>> depths;
 654 | 	std::vector<cv::Mat_<cv::Vec3f>> normals;
 655 | 	std::vector<cv::Mat> masks;
 656 | 	images.clear();
 657 | 	cameras.clear();
 658 | 	depths.clear();
 659 | 	normals.clear();
 660 | 	masks.clear();
 661 | 
 662 | 	for (size_t i = 0; i < num_images; ++i) {
 663 | 		std::cout << "Reading image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 664 | 		std::stringstream image_path;
 665 | 		image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << ".jpg";
 666 | 		cv::Mat_<cv::Vec3b> image = cv::imread(image_path.str(), cv::IMREAD_COLOR);
 667 | 		std::stringstream cam_path;
 668 | 		cam_path << cam_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << "_cam.txt";
 669 | 		Camera camera = ReadCamera(cam_path.str());
 670 | 
 671 | 		std::stringstream result_path;
 672 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id;
 673 | 		std::string result_folder = result_path.str();
 674 | 		std::string suffix = "/depths.dmb";
 675 | 		if (geom_consistency) {
 676 | 			suffix = "/depths_geom.dmb";
 677 | 		}
 678 | 		std::string depth_path = result_folder + suffix;
 679 | 		std::string normal_path = result_folder + "/normals.dmb";
 680 | 		cv::Mat_<float> depth;
 681 | 		cv::Mat_<cv::Vec3f> normal;
 682 | 		readDepthDmb(depth_path, depth);
 683 | 		readNormalDmb(normal_path, normal);
 684 | 
 685 | 		cv::Mat_<cv::Vec3b> scaled_image;
 686 | 		RescaleImageAndCamera(image, scaled_image, depth, camera);
 687 | 		images.push_back(scaled_image);
 688 | 		cameras.push_back(camera);
 689 | 		depths.push_back(depth);
 690 | 		normals.push_back(normal);
 691 | 		cv::Mat mask = cv::Mat::zeros(depth.rows, depth.cols, CV_8UC1);
 692 | 		masks.push_back(mask);
 693 | 
 694 | 
 695 | 	}
 696 | 
 697 | 	std::vector<PointList> PointCloud;
 698 | 	PointCloud.clear();
 699 | 
 700 | 	for (size_t i = 0; i < num_images; ++i) {
 701 | 		std::cout << "Fusing image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 702 | 		const int cols = depths[i].cols;
 703 | 		const int rows = depths[i].rows;
 704 | 		int num_ngb = problems[i].src_image_ids.size();
 705 | 		std::vector<int2> used_list(num_ngb, make_int2(-1, -1));
 706 | 		for (int r = 0; r < rows; ++r) {
 707 | 			for (int c = 0; c < cols; ++c) {
 708 | 				if (masks[i].at<uchar>(r, c) == 1)
 709 | 					continue;
 710 | 				float ref_depth = depths[i].at<float>(r, c);
 711 | 				cv::Vec3f ref_normal = normals[i].at<cv::Vec3f>(r, c);
 712 | 
 713 | 				if (ref_depth <= 0.0)
 714 | 					continue;
 715 | 
 716 | 				float3 PointX = Get3DPointonWorld(c, r, ref_depth, cameras[i]);
 717 | 				float3 consistent_Point = PointX;
 718 | 				cv::Vec3f consistent_normal = ref_normal;
 719 | 				float consistent_Color[3] = { (float)images[i].at<cv::Vec3b>(r, c)[0], (float)images[i].at<cv::Vec3b>(r, c)[1], (float)images[i].at<cv::Vec3b>(r, c)[2] };
 720 | 				int num_consistent = 0;
 721 | 				float dynamic_consistency = 0;
 722 | 
 723 | 				for (int j = 0; j < num_ngb; ++j) {
 724 | 					int src_id = problems[i].src_image_ids[j];
 725 | 					const int src_cols = depths[src_id].cols;
 726 | 					const int src_rows = depths[src_id].rows;
 727 | 					float2 point;
 728 | 					float proj_depth;
 729 | 					ProjectonCamera(PointX, cameras[src_id], point, proj_depth);
 730 | 					int src_r = int(point.y + 0.5f);
 731 | 					int src_c = int(point.x + 0.5f);
 732 | 					if (src_c >= 0 && src_c < src_cols && src_r >= 0 && src_r < src_rows) {
 733 | 						if (masks[src_id].at<uchar>(src_r, src_c) == 1)
 734 | 							continue;
 735 | 
 736 | 						float src_depth = depths[src_id].at<float>(src_r, src_c);
 737 | 						cv::Vec3f src_normal = normals[src_id].at<cv::Vec3f>(src_r, src_c);
 738 | 						if (src_depth <= 0.0)
 739 | 							continue;
 740 | 
 741 | 						float3 tmp_X = Get3DPointonWorld(src_c, src_r, src_depth, cameras[src_id]);
 742 | 						float2 tmp_pt;
 743 | 						ProjectonCamera(tmp_X, cameras[i], tmp_pt, proj_depth);
 744 | 						float reproj_error = sqrt(pow(c - tmp_pt.x, 2) + pow(r - tmp_pt.y, 2));
 745 | 						float relative_depth_diff = fabs(proj_depth - ref_depth) / ref_depth;
 746 | 						float angle = GetAngle(ref_normal, src_normal);
 747 | 
 748 | 						if (reproj_error < 2.0f && relative_depth_diff < 0.01f && angle < 0.174533f) {
 749 | 							used_list[j].x = src_c;
 750 | 							used_list[j].y = src_r;
 751 | 
 752 | 							float tmp_index = reproj_error + 200 * relative_depth_diff + angle * 10;
 753 | 							float cons = exp(-tmp_index);
 754 | 							dynamic_consistency += exp(-tmp_index);
 755 | 							num_consistent++;
 756 | 						}
 757 | 					}
 758 | 				}
 759 | 
 760 | 				if (num_consistent >= 1 && (dynamic_consistency > 0.3 * num_consistent)) {
 761 | 					PointList point3D;
 762 | 					point3D.coord = consistent_Point;
 763 | 					point3D.normal = make_float3(consistent_normal[0], consistent_normal[1], consistent_normal[2]);
 764 | 					point3D.color = make_float3(consistent_Color[0], consistent_Color[1], consistent_Color[2]);
 765 | 					PointCloud.push_back(point3D);
 766 | 					for (int j = 0; j < num_ngb; ++j) {
 767 | 						if (used_list[j].x == -1)
 768 | 							continue;
 769 | 						masks[problems[i].src_image_ids[j]].at<uchar>(used_list[j].y, used_list[j].x) = 1;
 770 | 					}
 771 | 				}
 772 | 			}
 773 | 		}
 774 | 	}
 775 | 
 776 | 	std::string ply_path = dense_folder + "/HPM_MVS_plusplus/HPM_MVS_plusplus.ply";
 777 | 	ExportPointCloud(ply_path, PointCloud);
 778 | }
 779 | 
 780 | void ConfidenceEvaluation(std::string& dense_folder, const std::vector<Problem>& problems, bool geom_consistency) {
 781 | 	size_t num_images = problems.size();
 782 | 	std::string image_folder = dense_folder + std::string("/images");
 783 | 	std::string cam_folder = dense_folder + std::string("/cams");
 784 | 
 785 | 	std::vector<Camera> cameras;
 786 | 	std::vector<cv::Mat_<float>> depths;
 787 | 	std::vector<cv::Mat_<cv::Vec3f>> normals;
 788 | 	std::vector<cv::Mat> masks;
 789 | 	std::vector<cv::Mat_<float>>consistency;
 790 | 	cameras.clear();
 791 | 	depths.clear();
 792 | 	normals.clear();
 793 | 	masks.clear();
 794 | 	consistency.clear();
 795 | 	for (size_t i = 0; i < num_images; ++i) {
 796 | 		std::cout << "Reading image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 797 | 		std::stringstream image_path;
 798 | 		image_path << image_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << ".jpg";
 799 | 		cv::Mat_<cv::Vec3b> image = cv::imread(image_path.str(), cv::IMREAD_COLOR);
 800 | 		std::stringstream cam_path;
 801 | 		cam_path << cam_folder << "/" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id << "_cam.txt";
 802 | 		Camera camera = ReadCamera(cam_path.str());
 803 | 		std::stringstream result_path;
 804 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id;
 805 | 		std::string result_folder = result_path.str();
 806 | 		std::string suffix = "/depths.dmb";
 807 | 		if (geom_consistency) {
 808 | 			suffix = "/depths_geom.dmb";
 809 | 		}
 810 | 		std::string depth_path = result_folder + suffix;
 811 | 		std::string normal_path = result_folder + "/normals.dmb";
 812 | 		cv::Mat_<float> depth;
 813 | 		cv::Mat_<cv::Vec3f> normal;
 814 | 		readDepthDmb(depth_path, depth);
 815 | 		readNormalDmb(normal_path, normal);
 816 | 
 817 | 		cv::Mat_<cv::Vec3b> scaled_image;
 818 | 		RescaleImageAndCamera(image, scaled_image, depth, camera);
 819 | 		cameras.push_back(camera);
 820 | 		depths.push_back(depth);
 821 | 		normals.push_back(normal);
 822 | 		cv::Mat mask = cv::Mat::zeros(depth.rows, depth.cols, CV_8UC1);
 823 | 		masks.push_back(mask);
 824 | 		cv::Mat consist = cv::Mat::zeros(depth.rows, depth.cols, CV_32FC1);
 825 | 		consistency.push_back(consist);
 826 | 		depth.release();
 827 | 		normal.release();
 828 | 		mask.release();
 829 | 		consist.release();
 830 | 		image.release();
 831 | 		scaled_image.release();
 832 | 
 833 | 	}
 834 | 	for (size_t i = 0; i < num_images; ++i) {
 835 | 		std::cout << "Hypothesis Confidence Evaluating Image " << std::setw(8) << std::setfill('0') << i << "..." << std::endl;
 836 | 		const int cols = depths[i].cols;
 837 | 		const int rows = depths[i].rows;
 838 | 		int num_ngb = problems[i].src_image_ids.size();
 839 | 		std::vector<int2> used_list(num_ngb, make_int2(-1, -1));
 840 | 		for (int r = 0; r < rows; ++r) {
 841 | 			for (int c = 0; c < cols; ++c) {
 842 | 				if (masks[i].at<uchar>(r, c) == 1)
 843 | 					continue;
 844 | 				float ref_depth = depths[i].at<float>(r, c);
 845 | 				cv::Vec3f ref_normal = normals[i].at<cv::Vec3f>(r, c);
 846 | 
 847 | 				if (ref_depth <= 0.0) {
 848 | 					continue;
 849 | 				}
 850 | 				if (ref_depth < cameras[i].depth_min || ref_depth > cameras[i].depth_max) {
 851 | 					continue;
 852 | 				}
 853 | 
 854 | 				float3 PointX = Get3DPointonWorld(c, r, ref_depth, cameras[i]);
 855 | 				float3 consistent_Point = PointX;
 856 | 				cv::Vec3f consistent_normal = ref_normal;
 857 | 				int num_consistent = 0;
 858 | 				float dynamic_consistency = 0;
 859 | 
 860 | 				for (int j = 0; j < num_ngb; ++j) {
 861 | 					int src_id = problems[i].src_image_ids[j];
 862 | 					const int src_cols = depths[src_id].cols;
 863 | 					const int src_rows = depths[src_id].rows;
 864 | 					float2 point;
 865 | 					float proj_depth;
 866 | 					ProjectonCamera(PointX, cameras[src_id], point, proj_depth);
 867 | 					int src_r = int(point.y + 0.5f);
 868 | 					int src_c = int(point.x + 0.5f);
 869 | 					if (src_c >= 0 && src_c < src_cols && src_r >= 0 && src_r < src_rows) {
 870 | 						if (masks[src_id].at<uchar>(src_r, src_c) == 1)
 871 | 							continue;
 872 | 
 873 | 						float src_depth = depths[src_id].at<float>(src_r, src_c);
 874 | 						cv::Vec3f src_normal = normals[src_id].at<cv::Vec3f>(src_r, src_c);
 875 | 						if (src_depth <= 0.0) {
 876 | 							continue;
 877 | 						}
 878 | 						if (src_depth < cameras[i].depth_min || src_depth > cameras[i].depth_max) {
 879 | 							continue;
 880 | 						}
 881 | 
 882 | 						float3 tmp_X = Get3DPointonWorld(src_c, src_r, src_depth, cameras[src_id]);
 883 | 						float2 tmp_pt;
 884 | 						ProjectonCamera(tmp_X, cameras[i], tmp_pt, proj_depth);
 885 | 						float reproj_error = sqrt(pow(c - tmp_pt.x, 2) + pow(r - tmp_pt.y, 2));
 886 | 						float relative_depth_diff = fabs(proj_depth - ref_depth) / ref_depth;
 887 | 						float angle = GetAngle(ref_normal, src_normal);
 888 | 
 889 | 						if (reproj_error < 2.0f && relative_depth_diff < 0.01f && angle < 0.174533f) {
 890 | 							used_list[j].x = src_c;
 891 | 							used_list[j].y = src_r;
 892 | 
 893 | 							float tmp_index = reproj_error + 200 * relative_depth_diff + angle * 10;
 894 | 							dynamic_consistency += exp(-tmp_index);
 895 | 							num_consistent++;
 896 | 						}
 897 | 					}
 898 | 				}
 899 | 
 900 | 				if (num_consistent >= 1 && (dynamic_consistency > 0.3 * num_consistent)) {
 901 | 					consistency[i](r, c) = dynamic_consistency;
 902 | 
 903 | 					for (int j = 0; j < num_ngb; ++j) {
 904 | 						if (used_list[j].x == -1)
 905 | 							continue;
 906 | 						masks[problems[i].src_image_ids[j]].at<uchar>(used_list[j].y, used_list[j].x) = 1;
 907 | 						consistency[problems[i].src_image_ids[j]](used_list[j].y, used_list[j].x) = dynamic_consistency;
 908 | 					}
 909 | 				}
 910 | 			}
 911 | 		}
 912 | 		std::stringstream result_path;
 913 | 		result_path << dense_folder << "/HPM_MVS_plusplus" << "/2333_" << std::setw(8) << std::setfill('0') << problems[i].ref_image_id;
 914 | 		std::string result_folder = result_path.str();
 915 | 		std::string mask_path = result_folder + "/confidence.dmb";
 916 | 		writeDepthDmb(mask_path, consistency[i]);
 917 | 	}
 918 | 	cameras.clear();
 919 | 	depths.clear();
 920 | 	normals.clear();
 921 | 	masks.clear();
 922 | 	consistency.clear();
 923 | 
 924 | 	cameras.shrink_to_fit();
 925 | 	depths.shrink_to_fit();
 926 | 	normals.shrink_to_fit();
 927 | 	masks.shrink_to_fit();
 928 | 	consistency.shrink_to_fit();
 929 | 	std::cout << "Hypotheses Confidence Evaluating Over..." << std::endl;
 930 | }
 931 | 
 932 | int main(int argc, char** argv)
 933 | {
 934 | 	if (argc < 2) {
 935 | 		std::cout << "USAGE: HPM-MVS_plusplus dense_folder true/flase(mask defualt: flase)" << std::endl;
 936 | 		return -1;
 937 | 	}
 938 | 
 939 | 	std::string dense_folder = argv[1];
 940 | 	std::string mask = argv[2];
 941 | 	bool mask_flag = false;
 942 | 	if (mask == "true") {
 943 | 		mask_flag = true;
 944 | 	}
 945 | 
 946 | 	std::vector<Problem> problems;
 947 | 	GenerateSampleList(dense_folder, problems);
 948 | 
 949 | 	std::string output_folder = dense_folder + std::string("/HPM_MVS_plusplus");
 950 | 	mkdir(output_folder.c_str());
 951 | 
 952 | 	size_t num_images = problems.size();
 953 | 	std::cout << "There are " << num_images << " problems needed to be processed!" << std::endl;
 954 | 
 955 | 	int max_num_downscale = ComputeMultiScaleSettings(dense_folder, problems);
 956 | 
 957 | 	int flag = 0;
 958 | 	int geom_iterations;
 959 | 	bool geom_consistency = false;
 960 | 	bool prior_consistency = false;
 961 | 	bool hierarchy = false;
 962 | 	bool multi_geometry = false;
 963 | 	bool mand_consistency = false;
 964 | 	int max_hpm_scale = max_num_downscale;
 965 | 	while (max_num_downscale >= 0) {
 966 | 		geom_iterations = 3;
 967 | 		std::cout << "Scale: " << max_num_downscale << std::endl;
 968 | 
 969 | 		for (size_t i = 0; i < num_images; ++i) {
 970 | 			if (problems[i].num_downscale >= 0) {
 971 | 				problems[i].cur_image_size = problems[i].max_image_size / pow(2, problems[i].num_downscale);
 972 | 				problems[i].num_downscale--;
 973 | 			}
 974 | 		}
 975 | 
 976 | 		if (flag == 0) {
 977 | 			flag = 1;
 978 | 			geom_consistency = false;
 979 | 			prior_consistency = false;
 980 | 			for (size_t i = 0; i < num_images; ++i) {
 981 | 				ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale);
 982 | 			}
 983 | 			prior_consistency = true;
 984 | 			geom_consistency = false;
 985 | 			for (int hpm_scale = max_hpm_scale; hpm_scale >= max_num_downscale; hpm_scale--) {
 986 | 				ConfidenceEvaluation(dense_folder, problems, geom_consistency);
 987 | 				for (size_t i = 0; i < num_images; ++i) {
 988 | 					std::cout << "HPM Scale: " << hpm_scale << std::endl;
 989 | 					int hpm_scale_distance = hpm_scale - problems[i].num_downscale - 1;
 990 | 					ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale, multi_geometry, hpm_scale_distance);
 991 | 				}
 992 | 			}
 993 | 
 994 | 			geom_consistency = false;
 995 | 			prior_consistency = false;
 996 | 			for (int geom_iter = 0; geom_iter < geom_iterations; ++geom_iter) {
 997 | 				if (geom_iter == 0) {
 998 | 					multi_geometry = false;
 999 | 				}
1000 | 				else {
1001 | 					multi_geometry = true;
1002 | 				}
1003 | 
1004 | 				if (geom_iter > 0) {
1005 | 					mand_consistency = true;
1006 | 					ConfidenceEvaluation(dense_folder, problems, geom_consistency);
1007 | 					geom_consistency = true;
1008 | 				}
1009 | 				else {
1010 | 					geom_consistency = true;
1011 | 					mand_consistency = false;
1012 | 				}
1013 | 
1014 | 				for (size_t i = 0; i < num_images; ++i) {
1015 | 					ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale, multi_geometry);
1016 | 				}
1017 | 			}
1018 | 		}
1019 | 		else {
1020 | 			for (size_t i = 0; i < num_images; ++i) {
1021 | 				JointBilateralUpsampling(dense_folder, problems[i], problems[i].cur_image_size);
1022 | 			}
1023 | 
1024 | 			hierarchy = true;
1025 | 			mand_consistency = false;
1026 | 			geom_consistency = false;
1027 | 			prior_consistency = false;
1028 | 			for (size_t i = 0; i < num_images; ++i) {
1029 | 				ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale);
1030 | 			}
1031 | 			hierarchy = false;
1032 | 			prior_consistency = true;
1033 | 			geom_consistency = false;
1034 | 			multi_geometry = false;
1035 | 			for (int hpm_scale = max_hpm_scale; hpm_scale >= max_num_downscale; hpm_scale--) {
1036 | 				ConfidenceEvaluation(dense_folder, problems, geom_consistency);
1037 | 				for (size_t i = 0; i < num_images; ++i) {
1038 | 					std::cout << "HPM Scale: " << hpm_scale << std::endl;
1039 | 					int hpm_scale_distance = hpm_scale - problems[i].num_downscale - 1;
1040 | 					ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale, multi_geometry, hpm_scale_distance);
1041 | 				}
1042 | 			}
1043 | 
1044 | 			geom_consistency = false;
1045 | 			prior_consistency = false;
1046 | 			for (int geom_iter = 0; geom_iter < geom_iterations; ++geom_iter) {
1047 | 				if (geom_iter == 0) {
1048 | 					multi_geometry = false;
1049 | 				}
1050 | 				else {
1051 | 					multi_geometry = true;
1052 | 				}
1053 | 
1054 | 				if (geom_iter > 0) {
1055 | 					mand_consistency = true;
1056 | 					ConfidenceEvaluation(dense_folder, problems, geom_consistency);
1057 | 					geom_consistency = true;
1058 | 				}
1059 | 				else {
1060 | 					geom_consistency = true;
1061 | 					mand_consistency = false;
1062 | 				}
1063 | 
1064 | 				for (size_t i = 0; i < num_images; ++i) {
1065 | 					ProcessProblem(dense_folder, problems, i, geom_consistency, prior_consistency, hierarchy, mand_consistency, max_num_downscale, multi_geometry);
1066 | 				}
1067 | 			}
1068 | 		}
1069 | 
1070 | 		max_num_downscale--;
1071 | 	}
1072 | 	geom_consistency = true;
1073 | 	if (mask_flag) {
1074 | 		RunFusion_Sky_Strict(dense_folder, problems, geom_consistency);
1075 | 	}
1076 | 	else {
1077 | 		RunFusion(dense_folder, problems, geom_consistency);
1078 | 	}
1079 | 	return 0;
1080 | }
1081 | 


--------------------------------------------------------------------------------
/main.h:
--------------------------------------------------------------------------------
 1 | #ifndef _MAIN_H_
 2 | #define _MAIN_H_
 3 | 
 4 | #include "opencv2/calib3d/calib3d.hpp"
 5 | #include "opencv2/imgproc/imgproc.hpp"
 6 | #include "opencv2/core/core.hpp"
 7 | #include "opencv2/highgui/highgui.hpp"
 8 | #include "opencv2/highgui/highgui.hpp"
 9 | #include "opencv2/opencv.hpp"
10 | 
11 | // Includes CUDA
12 | #include <cuda_runtime.h>
13 | #include <cuda.h>
14 | #include <cuda_runtime_api.h>
15 | #include <cuda_texture_types.h>
16 | #include <curand_kernel.h>
17 | #include <vector_types.h>
18 | 
19 | #include <vector>
20 | #include <string>
21 | #include <iostream>
22 | #include <fstream>
23 | #include <sstream>
24 | #include <algorithm>
25 | #include <map>
26 | #include <memory>
27 | #include "iomanip"
28 | 
29 | #include<direct.h>
30 | #include <sys/stat.h> // mkdir
31 | #include <sys/types.h> // mkdir
32 | 
33 | 
34 | #define MAX_IMAGES 256
35 | #define JBU_NUM 2
36 | 
37 | struct Camera {
38 |     float K[9];
39 |     float R[9];
40 |     float t[3];
41 |     int height;
42 |     int width;
43 |     float depth_min;
44 |     float depth_max;
45 | };
46 | 
47 | struct Problem {
48 |     int ref_image_id;
49 |     std::vector<int> src_image_ids;
50 |     int max_image_size = 3200;
51 |     int num_downscale = 0;
52 |     int cur_image_size = 3200;
53 | };
54 | 
55 | struct Triangle {
56 |     cv::Point pt1, pt2, pt3;
57 |     Triangle (const cv::Point _pt1, const cv::Point _pt2, const cv::Point _pt3) : pt1(_pt1) , pt2(_pt2), pt3(_pt3) {}
58 | };
59 | 
60 | struct PointList {
61 |     float3 coord;
62 |     float3 normal;
63 |     float3 color;
64 | };
65 | 
66 | #define M_PI 3.14159265358979323846
67 | 
68 | #endif // _MAIN_H_
69 | 


--------------------------------------------------------------------------------