├── Input ├── DUC129_1.png ├── cse013_0.png ├── cse013_5.png ├── image_01.bmp ├── Olin071_5.png └── image_163.bmp ├── TRW_S ├── errorFn.cpp ├── instances.inc ├── instances.h ├── treeProbabilities.def ├── ordering.def ├── MRFEnergy.def ├── MRFEnergy.h ├── typeBinaryFast.h ├── typeBinary.h ├── typePotts.h ├── typeTruncatedLinear.h └── typeTruncatedLinear2D.h ├── cv_utils ├── ProposalGenerator.h ├── CostFunctor.h ├── CMakeLists.txt ├── MatrixOperations.cpp ├── FusionSpaceSolver.h ├── Histogram.h ├── PointCloudOperations.cpp ├── CommonOperations.cpp ├── ImageMask.h ├── StatisticsCalculations.cpp ├── FusionSpaceSolver.cpp ├── cv_utils.h └── ImageOperations.cpp ├── CMakeLists.txt ├── DataStructure.h ├── StructureFinder.h ├── README.md ├── ConcaveHullFinder.h ├── BSplineSurface.h ├── TRWSFusion.h ├── utils.h ├── LayerDepthRepresenter.h ├── main.cpp ├── ProposalDesigner.h ├── Segment.h ├── StructureFinder.cpp ├── BSplineSurface.cpp └── ConcaveHullFinder.cpp /Input/DUC129_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/DUC129_1.png -------------------------------------------------------------------------------- /Input/cse013_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/cse013_0.png -------------------------------------------------------------------------------- /Input/cse013_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/cse013_5.png -------------------------------------------------------------------------------- /Input/image_01.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/image_01.bmp -------------------------------------------------------------------------------- /Input/Olin071_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/Olin071_5.png -------------------------------------------------------------------------------- /Input/image_163.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/art-programmer/LayeredSceneDecomposition/HEAD/Input/image_163.bmp -------------------------------------------------------------------------------- /TRW_S/errorFn.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "MRFEnergy.h" 6 | 7 | #include "instances.inc" 8 | 9 | using namespace std; 10 | 11 | inline void DefaultErrorFn(char* msg) 12 | { 13 | fprintf(stderr, "%s\n", msg); 14 | exit(1); 15 | } 16 | -------------------------------------------------------------------------------- /cv_utils/ProposalGenerator.h: -------------------------------------------------------------------------------- 1 | #ifndef PROPOSAL_GENERATOR_H__ 2 | #define PROPOSAL_GENERATOR_H__ 3 | 4 | #include 5 | 6 | 7 | class ProposalGenerator 8 | { 9 | public: 10 | virtual void setCurrentSolution(const std::vector ¤t_solution) = 0; 11 | virtual std::vector > getProposal() const = 0; 12 | 13 | protected: 14 | std::vector current_solution_; 15 | }; 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /TRW_S/instances.inc: -------------------------------------------------------------------------------- 1 | 2 | #include "MRFEnergy.h" 3 | 4 | //template class MRFEnergy; 5 | //template class MRFEnergy; 6 | //template class MRFEnergy; 7 | template class MRFEnergy; 8 | //template class MRFEnergy; 9 | //template class MRFEnergy; 10 | //template class MRFEnergy; 11 | //template class MRFEnergy; 12 | 13 | -------------------------------------------------------------------------------- /cv_utils/CostFunctor.h: -------------------------------------------------------------------------------- 1 | #ifndef COST_FUNCTOR_H__ 2 | #define COST_FUNCTOR_H__ 3 | 4 | class CostFunctor 5 | { 6 | public: 7 | virtual double operator()(const int node_index, const int label) const = 0; 8 | virtual double operator()(const int node_index_1, const int node_index_2, const int label_1, const int label_2) const = 0; 9 | virtual void setCurrentSolution(const std::vector ¤t_solution) {}; 10 | virtual double getLabelCost() const { return 0; }; 11 | virtual double getLabelIndicatorConflictCost() const { return 0; }; 12 | }; 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /TRW_S/instances.h: -------------------------------------------------------------------------------- 1 | #ifndef __INSTANCES_H__ 2 | #define __INSTANCES_H__ 3 | 4 | 5 | #if defined(_MSC_VER) 6 | 7 | // C4661: '...' : no suitable definition provided for explicit template instantiation request 8 | #pragma warning(disable: 4661) 9 | 10 | #endif 11 | 12 | #include "typeBinary.h" 13 | #include "typeBinaryFast.h" 14 | #include "typePotts.h" 15 | #include "typeGeneral.h" 16 | #include "typeTruncatedLinear.h" 17 | #include "typeTruncatedQuadratic.h" 18 | #include "typeTruncatedLinear2D.h" 19 | #include "typeTruncatedQuadratic2D.h" 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | project (LayeredSceneDecomposition) 3 | find_package(OpenCV REQUIRED) 4 | find_package(gflags REQUIRED) 5 | 6 | set(CMAKE_BUILD_TYPE Release) 7 | set(CMAKE_CXX_FLAGS "-std=c++0x -w") 8 | 9 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 10 | add_subdirectory(cv_utils) 11 | 12 | #link_directories(cv_utils) 13 | #} /opt/gurobi650/linux64/include) 14 | file(GLOB SOURCES "*.cpp") 15 | add_executable(LayeredSceneDecomposition ${SOURCES}) 16 | target_link_libraries(LayeredSceneDecomposition ${OpenCV_LIBS} ${GFlags_LIBS} gflags cv_utils) 17 | -------------------------------------------------------------------------------- /cv_utils/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | project (cv_utils) 3 | set(CMAKE_BUILD_TYPE Release) 4 | set(CMAKE_CXX_FLAGS "-std=c++0x") 5 | #add_definitions(-std=c++11) 6 | #set(CMAKE_CXX_STANDARD 11) 7 | find_package(OpenCV REQUIRED) 8 | find_package(PCL REQUIRED COMPONENTS features kdtree) 9 | include_directories(${PCL_LIBRARY_DIRS}) 10 | link_directories(${PCL_LIBRARY_DIRS}) 11 | add_definitions(${PCL_DEFINITIONS}) 12 | file(GLOB SOURCES "*.cpp") 13 | add_library(cv_utils SHARED ${SOURCES} TRW_S/errorFn.cpp) 14 | target_link_libraries(cv_utils ${OpenCV_LIBS}) 15 | target_link_libraries(cv_utils ${PCL_FEATURES_LIBRARIES} ${PCL_KDTREE_LIBRARIES}) 16 | -------------------------------------------------------------------------------- /cv_utils/MatrixOperations.cpp: -------------------------------------------------------------------------------- 1 | #include "cv_utils.h" 2 | 3 | using namespace std; 4 | using namespace Eigen; 5 | 6 | namespace cv_utils 7 | { 8 | vector > calcInverse(const vector > &matrix) 9 | { 10 | assert(matrix.size() > 0 && matrix.size() == matrix.begin()->size()); 11 | const int NUM_DIMENSIONS = matrix.size(); 12 | MatrixXd matrix_eigen(NUM_DIMENSIONS, NUM_DIMENSIONS); 13 | for (int c = 0; c < NUM_DIMENSIONS; c++) 14 | for (int d = 0; d < NUM_DIMENSIONS; d++) 15 | matrix_eigen(c, d) = matrix[c][d]; 16 | MatrixXd inverse_matrix_eigen = matrix_eigen.inverse(); 17 | vector > inverse_matrix(NUM_DIMENSIONS, vector(NUM_DIMENSIONS)); 18 | for (int c = 0; c < NUM_DIMENSIONS; c++) 19 | for (int d = 0; d < NUM_DIMENSIONS; d++) 20 | inverse_matrix[c][d] = inverse_matrix_eigen(c, d); 21 | return inverse_matrix; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cv_utils/FusionSpaceSolver.h: -------------------------------------------------------------------------------- 1 | #ifndef FUSION_SPACE_SOLVER_H__ 2 | #define FUSION_SPACE_SOLVER_H__ 3 | 4 | #include 5 | 6 | #include "CostFunctor.h" 7 | #include "ProposalGenerator.h" 8 | 9 | 10 | class FusionSpaceSolver 11 | { 12 | public: 13 | 14 | FusionSpaceSolver(const int NUM_NODES, const std::vector > &node_neighbors, CostFunctor &cost_functor, ProposalGenerator &proposal_generator, const int NUM_ITERATIONS = 1000, const bool CONSIDER_LABEL_COST = false); 15 | 16 | // void setNeighbors(); 17 | //void setNeighbors(const int width, const int height, const int neighbor_system = 8); 18 | 19 | std::vector solve(const int NUM_ITERATIONS, const std::vector &initial_solution); 20 | 21 | private: 22 | const int NUM_NODES_; 23 | const int NUM_ITERATIONS_; 24 | const bool CONSIDER_LABEL_COST_; 25 | 26 | const std::vector > node_neighbors_; 27 | CostFunctor &cost_functor_; 28 | ProposalGenerator &proposal_generator_; 29 | 30 | std::vector fuse(const std::vector > &proposal_labels, std::vector &energy_info); 31 | }; 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /DataStructure.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__DataStructure__ 2 | #define __LayerDepthMap__DataStructure__ 3 | 4 | #include 5 | #include 6 | 7 | struct RepresenterPenalties { 8 | double data_depth_pen; 9 | double data_color_pen; 10 | double data_normal_pen; 11 | double data_non_plane_pen; 12 | 13 | double surface_pen; 14 | 15 | double smoothness_pen; 16 | double smoothness_small_constant_pen; 17 | double smoothness_concave_shape_pen; 18 | double smoothness_anisotropic_diffusion_pen; 19 | 20 | double other_viewpoint_smoothness_pen; 21 | double other_viewpoint_depth_conflict_pen; 22 | 23 | double smoothness_empty_non_empty_ratio; 24 | 25 | double huge_pen; 26 | }; 27 | 28 | struct DataStatistics { 29 | double pixel_fitting_distance_threshold; 30 | double pixel_fitting_angle_threshold; 31 | double pixel_fitting_color_likelihood_threshold; 32 | double depth_diff_var; 33 | double similar_angle_threshold; 34 | double depth_conflict_tolerance; 35 | double depth_change_smoothness_threshold; 36 | double viewpoint_movement; 37 | double bspline_surface_num_pixels_threshold; 38 | double background_depth_diff_tolerance; 39 | }; 40 | 41 | #endif /* defined(__LayerDepthMap__DataStructure__) */ 42 | -------------------------------------------------------------------------------- /TRW_S/treeProbabilities.def: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "MRFEnergy.h" 7 | 8 | //#include "instances.inc" 9 | 10 | /////////////////////////////////////////////////////////////////////////// 11 | /////////////////////////////////////////////////////////////////////////// 12 | /////////////////////////////////////////////////////////////////////////// 13 | 14 | template void MRFEnergy::SetMonotonicTrees() 15 | { 16 | Node* i; 17 | MRFEdge* e; 18 | 19 | if (!m_isEnergyConstructionCompleted) 20 | { 21 | CompleteGraphConstruction(); 22 | } 23 | 24 | for (i=m_nodeFirst; i; i=i->m_next) 25 | { 26 | REAL mu; 27 | 28 | int nForward = 0, nBackward = 0; 29 | for (e=i->m_firstForward; e; e=e->m_nextForward) 30 | { 31 | nForward ++; 32 | } 33 | for (e=i->m_firstBackward; e; e=e->m_nextBackward) 34 | { 35 | nBackward ++; 36 | } 37 | int ni = (nForward > nBackward) ? nForward : nBackward; 38 | 39 | mu = (REAL)1 / ni; 40 | for (e=i->m_firstBackward; e; e=e->m_nextBackward) 41 | { 42 | e->m_gammaBackward = mu; 43 | } 44 | for (e=i->m_firstForward; e; e=e->m_nextForward) 45 | { 46 | e->m_gammaForward = mu; 47 | } 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /StructureFinder.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__StructureFinder__ 2 | #define __LayerDepthMap__StructureFinder__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "DataStructure.h" 10 | #include "TRW_S/MRFEnergy.h" 11 | #include "Segment.h" 12 | 13 | class StructureFinder{ 14 | 15 | public: 16 | StructureFinder(const int image_width, const int image_height, const std::map &segments, const std::vector &candidate_segment_mask, const std::vector visible_segmentation, const std::vector &visible_depths, const std::vector &background_depths, const std::vector &segment_backmost_layer_index_map, const RepresenterPenalties penalties, const DataStatistics statistics); 17 | 18 | std::vector > > getStructures() const; 19 | 20 | private: 21 | const int IMAGE_WIDTH_; 22 | const int IMAGE_HEIGHT_; 23 | 24 | const int NUM_SURFACES_; 25 | const int NUM_PIXELS_; 26 | 27 | const RepresenterPenalties penalties_; 28 | const DataStatistics statistics_; 29 | 30 | const std::map segments_; 31 | const std::vector visible_segmentation_; 32 | const std::vector candidate_segment_mask_; 33 | const std::vector visible_depths_; 34 | const std::vector background_depths_; 35 | const std::vector &segment_backmost_layer_index_map_; 36 | 37 | std::vector > > structure_score_surface_ids_pairs_; 38 | 39 | 40 | void findTwoOrthogonalSurfaceStructures(); 41 | }; 42 | 43 | #endif /* defined(__LayerDepthMap__StructureFinder__) */ 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Layered Scene Decomposition via the Occlusion-CRF 2 | 3 | By Chen Liu, Yasutaka Furukawa, and Pushmeet Kohli 4 | 5 | ### Introduction 6 | 7 | This paper proposes a novel layered depth map representation and its inference algorithm which is able to infer invisible surfaces behind occlusions. To learn more, please see our CVPR 2016 [paper](http://www.cse.wustl.edu/~furukawa/papers/2016-cvpr-layer.pdf) or visit our [project website](http://sites.wustl.edu/chenliu/layered-scene) 8 | 9 | This code implements the algorithm described in our paper in C++. 10 | 11 | ### Requirements 12 | 13 | 0. OpenCV 14 | 1. PCL 15 | 2. gflags 16 | 17 | ### Usages 18 | 19 | To compile the program: 20 | 21 | 0. mkdir build 22 | 1. cd build 23 | 2. cmake .. 24 | 3. make 25 | 26 | To run the program on your own data: 27 | 28 | ./LayeredSceneDecomposition --image_path=*"your image path"* --point_cloud_path=*"your point cloud path"* --result_folder=*"where you want to save results"* --cache_folder=*"where you want to save cache"* 29 | 30 | To run the program on the demo data: 31 | 32 | ./LayeredSceneDecomposition --image_path=../Input/image_01.txt --point_cloud_path=../Input/point_cloud_01.txt --result_folder=../Result --cache_folder=../Cache 33 | 34 | Point cloud file format: 35 | 36 | The point cloud file stores a 3D point cloud, each of which corresponds to one image pixel. 37 | The number in the first row equals to image_width * image_height. 38 | Then, each row stores 3D coordinates for a point which corresponds to a pixel (indexed by y * image_width + x). 39 | 40 | ### Contact 41 | 42 | If you have any questions, please contact me at chenliu@wustl.edu. 43 | -------------------------------------------------------------------------------- /ConcaveHullFinder.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__ConcaveHullFinder__ 2 | #define __LayerDepthMap__ConcaveHullFinder__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "DataStructure.h" 10 | #include "TRW_S/MRFEnergy.h" 11 | #include "Segment.h" 12 | 13 | class ConcaveHullFinder{ 14 | 15 | public: 16 | ConcaveHullFinder(const int image_width, const int image_height, const std::vector &point_cloud, const std::vector &segmentation, const std::map &segments, const std::vector &ROI_mask, const RepresenterPenalties penalties, const DataStatistics statistics, const bool consider_background); 17 | 18 | ~ConcaveHullFinder(); 19 | 20 | //get concave hull 21 | std::vector getConcaveHull(); 22 | 23 | 24 | private: 25 | const std::vector segmentation_; 26 | const std::vector point_cloud_; 27 | const int IMAGE_WIDTH_; 28 | const int IMAGE_HEIGHT_; 29 | 30 | 31 | std::map > surface_point_clouds_; 32 | std::map > surface_depths_; 33 | std::map > surface_relations_; 34 | std::map segment_type_map_; 35 | std::map segment_direction_map_; 36 | std::vector surface_normals_angles_; 37 | 38 | const std::vector ROI_mask_; 39 | const int NUM_SURFACES_; 40 | const int NUM_PIXELS_; 41 | 42 | const RepresenterPenalties penalties_; 43 | const DataStatistics statistics_; 44 | 45 | std::vector concave_hull_labels_; 46 | std::set concave_hull_surfaces_; 47 | 48 | 49 | //calculate concave hull 50 | void calcConcaveHull(); 51 | }; 52 | 53 | #endif /* defined(__LayerDepthMap__ConcaveHullFinder__) */ 54 | -------------------------------------------------------------------------------- /BSplineSurface.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__BSplineSurface__ 2 | #define __LayerDepthMap__BSplineSurface__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "DataStructure.h" 12 | //#include "BSpline.h" 13 | 14 | 15 | class BSplineSurface{ 16 | 17 | public: 18 | BSplineSurface(const std::vector &point_cloud, const std::vector &pixels, const int image_width, const int image_heigth, const double stride_x, const double stride_y, const int bspline_order); 19 | 20 | //get depth map 21 | std::vector getDepthMap() const; 22 | 23 | private: 24 | const int IMAGE_WIDTH_; 25 | const int IMAGE_HEIGHT_; 26 | const int NUM_PIXELS_; 27 | const double STRIDE_X_; 28 | const double STRIDE_Y_; 29 | const int BSPLINE_ORDER_; 30 | 31 | std::vector segment_pixels_; 32 | std::vector control_point_xs_; 33 | std::vector control_point_ys_; 34 | 35 | std::vector depth_map_; 36 | 37 | 38 | //initialize control points 39 | void initControlPoints(); 40 | 41 | //fit b-spline surface 42 | void fitBSplineSurface(const std::vector &point_cloud, const std::vector &pixels); 43 | 44 | //calculate 2D basis function value 45 | double calcBasisFunctionValue2D(const double x, const double y, const double control_point_x, const double control_point_y, const double stride_x, const double stride_y, const int order); 46 | 47 | //calculate 1D basis function value 48 | double calcBasisFunctionValue1D(const double &x, const double &control_point_x, const double &stride_x, const int &order); 49 | }; 50 | 51 | #endif /* defined(__LayerDepthMap__BSplineSurface__) */ 52 | -------------------------------------------------------------------------------- /cv_utils/Histogram.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace cv_utils 5 | { 6 | template class Histogram 7 | { 8 | public: 9 | Histogram(const int NUM_GRAMS, const T MIN_VALUE, const T MAX_VALUE, const std::vector &values = std::vector()) : NUM_GRAMS_(NUM_GRAMS), MIN_VALUE_(MIN_VALUE), MAX_VALUE_(MAX_VALUE), num_values_(values.size()), histos_(std::vector(NUM_GRAMS, 0)) 10 | { 11 | //std::cout << MIN_VALUE_ << '\t' << MAX_VALUE_ << std::endl; 12 | assert(MIN_VALUE_ < MAX_VALUE_); 13 | //std::cout << NUM_GRAMS_ << std::endl; 14 | for (typename std::vector::const_iterator value_it = values.begin(); value_it != values.end(); value_it++) { 15 | if (*value_it >= MIN_VALUE_ && *value_it <= MAX_VALUE) { 16 | histos_[calcHistoIndex(*value_it)]++; 17 | //std::cout << calcHistoIndex(*value_it) << std::endl; 18 | } 19 | } 20 | }; 21 | double getEntropy() 22 | { 23 | double entropy = 0; 24 | for (std::vector::const_iterator histo_it = histos_.begin(); histo_it != histos_.end(); histo_it++) 25 | entropy += -1.0 * *histo_it / num_values_ * log(1.0 * *histo_it / num_values_); 26 | return entropy; 27 | }; 28 | double getProbability(const T &value) 29 | { 30 | if (value < MIN_VALUE_ || value > MAX_VALUE_) 31 | return 0; 32 | return histos_[calcHistoIndex(value)]; 33 | }; 34 | 35 | private: 36 | const int NUM_GRAMS_; 37 | const T MIN_VALUE_, MAX_VALUE_; 38 | int num_values_; 39 | std::vector histos_; 40 | 41 | 42 | double calcHistoIndex(const T &value) 43 | { 44 | return std::min(std::max(static_cast(1.0 * (value - MIN_VALUE_) / (MAX_VALUE_ - MIN_VALUE_) * NUM_GRAMS_), 0), NUM_GRAMS_ - 1); 45 | }; 46 | }; 47 | } 48 | -------------------------------------------------------------------------------- /cv_utils/PointCloudOperations.cpp: -------------------------------------------------------------------------------- 1 | #include "cv_utils.h" 2 | 3 | #include 4 | 5 | using namespace std; 6 | 7 | namespace cv_utils 8 | { 9 | // void segmentPointCloudRansac(const vector &point_cloud, const vector > &neighbors, vector > &planes, vector &assignment, const double DENOTED_FITTING_ERROR_THRESHOLD, const int DENOTED_NUM_PLANES_THRESHOLD, const double DENOTED_FITTING_RATIO_THRESHOLD) 10 | // { 11 | // const double FITTING_ERROR_THRESHOLD = DENOTED_FITTING_ERROR_THRESHOLD; 12 | // const int NUM_PLANES_THRESHOLD = DENOTED_NUM_PLANES_THRESHOLD > 0 ? DENOTED_NUM_PLANES_THRESHOLD : point_cloud.size() / 3; 13 | // for (int plane_index = 0; plane_index < NUM_PLANES_THRESHOLD; plane_index++) { 14 | // } 15 | // } 16 | 17 | 18 | bool writePointCloud(const string &filename, const vector &point_cloud, const int IMAGE_WIDTH, const int IMAGE_HEIGHT) 19 | { 20 | ofstream out_str(filename); 21 | if (!out_str) 22 | return false; 23 | out_str << IMAGE_WIDTH << '\t' << IMAGE_HEIGHT << endl; 24 | for (int pixel = 0; pixel < IMAGE_WIDTH * IMAGE_HEIGHT; pixel++) 25 | out_str << point_cloud[pixel * 3 + 0] << '\t' << point_cloud[pixel * 3 + 1] << '\t' << point_cloud[pixel * 3 + 2] << endl; 26 | out_str.close(); 27 | return true; 28 | } 29 | 30 | bool readPointCloud(const string &filename, vector &point_cloud) 31 | { 32 | ifstream in_str(filename); 33 | if (!in_str) 34 | return false; 35 | int image_width, image_height; 36 | in_str >> image_width >> image_height; 37 | point_cloud.assign(image_width * image_height * 3, 0); 38 | for (int pixel = 0; pixel < image_width * image_height; pixel++) 39 | in_str >> point_cloud[pixel * 3 + 0] >> point_cloud[pixel * 3 + 1] >> point_cloud[pixel * 3 + 2]; 40 | in_str.close(); 41 | return true; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /TRWSFusion.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__TRWSFusion__ 2 | #define __LayerDepthMap__TRWSFusion__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "DataStructure.h" 12 | #include "TRW_S/MRFEnergy.h" 13 | #include "Segment.h" 14 | 15 | using namespace std; 16 | 17 | class TRWSFusion 18 | { 19 | public: 20 | 21 | TRWSFusion(const cv::Mat &image, const vector &point_cloud, const vector &normals, const RepresenterPenalties &penalties, const DataStatistics &statistics, const bool consider_surface_cost = true); 22 | 23 | // Destructor 24 | ~TRWSFusion(); 25 | 26 | 27 | //find the best configuration in all proposals and use it to update the current solution 28 | vector fuse(const vector > &proposal_labels, const int proposal_num_surfaces, const int proposal_num_layers, const map &proposal_segments, const vector &previous_solution_indices, const vector &proposal_ROI_mask = vector()); 29 | 30 | //get information about optimization 31 | std::vector getEnergyInfo(); 32 | 33 | 34 | private: 35 | const int IMAGE_WIDTH_, IMAGE_HEIGHT_, NUM_PIXELS_; 36 | const cv::Mat image_; 37 | cv::Mat blurred_hsv_image_; 38 | const vector point_cloud_; 39 | const vector normals_; 40 | const RepresenterPenalties penalties_; 41 | const DataStatistics statistics_; 42 | const bool consider_surface_cost_; 43 | 44 | int proposal_num_surfaces_; 45 | int proposal_num_layers_; 46 | map proposal_segments_; 47 | map > proposal_surface_depths_; 48 | vector proposal_ROI_mask_; 49 | vector proposal_distance_to_boundaries_; 50 | 51 | 52 | double energy_; 53 | double lower_bound_; 54 | 55 | vector solution_; 56 | vector ori_labels_; 57 | 58 | 59 | double color_diff_var_; 60 | 61 | 62 | //calculate unary cost of a specific label a specific pixel 63 | double calcUnaryCost(const int pixel, const int label); 64 | 65 | //calculate pairwise cost 66 | double calcPairwiseCost(const int pixel_1, const int pixel_2, const int label_1, const int label_2); 67 | 68 | //check solution energy (mainly for debug purpose) 69 | double checkSolutionEnergy(const vector &solution_for_check); 70 | 71 | //calculate the variance of color difference 72 | void calcColorDiffVar(); 73 | 74 | //calculate color difference 75 | double calcColorDiff(const int pixel_1, const int pixel_2); 76 | 77 | //calculate overlap region after the viewpoint is moved to either left, right, up or down 78 | std::vector > > calcOverlapPixels(const vector > &proposal_labels); 79 | }; 80 | 81 | #endif /* defined(__LayerDepthMap__TRWSFusion__) */ 82 | -------------------------------------------------------------------------------- /cv_utils/CommonOperations.cpp: -------------------------------------------------------------------------------- 1 | #include "cv_utils.h" 2 | 3 | using namespace std; 4 | 5 | namespace cv_utils 6 | { 7 | std::vector findNeighbors(const int pixel, const int WIDTH, const int HEIGHT, const bool USE_PANORAMA, const int NEIGHBOR_SYSTEM) 8 | { 9 | int x = pixel % WIDTH; 10 | int y = pixel / WIDTH; 11 | std::vector neighbors; 12 | if (x > 0) 13 | neighbors.push_back(pixel - 1); 14 | if (x < WIDTH - 1) 15 | neighbors.push_back(pixel + 1); 16 | if (y > 0) 17 | neighbors.push_back(pixel - WIDTH); 18 | if (y < HEIGHT - 1) 19 | neighbors.push_back(pixel + WIDTH); 20 | 21 | if (USE_PANORAMA && x == 0) 22 | neighbors.push_back(pixel + (WIDTH - 1)); 23 | if (USE_PANORAMA && x == WIDTH - 1) 24 | neighbors.push_back(pixel - (WIDTH - 1)); 25 | 26 | if (NEIGHBOR_SYSTEM == 8) { 27 | if (x > 0 && y > 0) 28 | neighbors.push_back(pixel - 1 - WIDTH); 29 | if (x > 0 && y < HEIGHT - 1) 30 | neighbors.push_back(pixel - 1 + WIDTH); 31 | if (x < WIDTH - 1 && y > 0) 32 | neighbors.push_back(pixel + 1 - WIDTH); 33 | if (x < WIDTH - 1 && y < HEIGHT - 1) 34 | neighbors.push_back(pixel + 1 + WIDTH); 35 | 36 | if (USE_PANORAMA && x == 0) { 37 | if (y > 0) 38 | neighbors.push_back(pixel + (WIDTH - 1) - WIDTH); 39 | if (y < HEIGHT - 1) 40 | neighbors.push_back(pixel + (WIDTH - 1) + WIDTH); 41 | } 42 | if (USE_PANORAMA && x == WIDTH - 1) { 43 | if (y > 0) 44 | neighbors.push_back(pixel - (WIDTH - 1) - WIDTH); 45 | if (y < HEIGHT - 1) 46 | neighbors.push_back(pixel - (WIDTH - 1) + WIDTH); 47 | } 48 | } 49 | return neighbors; 50 | } 51 | 52 | std::vector > findNeighborsForAllPixels(const int WIDTH, const int HEIGHT, const int NEIGHBOR_SYSTEM) 53 | { 54 | vector > pixel_neighbors(WIDTH * HEIGHT); 55 | for (int pixel = 0; pixel < WIDTH * HEIGHT; pixel++) 56 | pixel_neighbors[pixel] = findNeighbors(pixel, WIDTH, HEIGHT, NEIGHBOR_SYSTEM); 57 | return pixel_neighbors; 58 | } 59 | 60 | std::vector findWindowPixels(const int pixel, const int WIDTH, const int HEIGHT, const int WINDOW_SIZE, const bool USE_PANORAMA) 61 | { 62 | vector window_pixels; 63 | int x = pixel % WIDTH; 64 | int y = pixel / WIDTH; 65 | for (int offset_x = -(WINDOW_SIZE - 1) / 2; offset_x <= (WINDOW_SIZE - 1) / 2; offset_x++) { 66 | for (int offset_y = -(WINDOW_SIZE - 1) / 2; offset_y <= (WINDOW_SIZE - 1) / 2; offset_y++) { 67 | if (x + offset_x >= 0 && x + offset_x < WIDTH && y + offset_y >= 0 && y + offset_y < HEIGHT) 68 | window_pixels.push_back((y + offset_y) * WIDTH + (x + offset_x)); 69 | if (USE_PANORAMA && (x + offset_x < 0 || x + offset_x >= WIDTH) && (y + offset_y >= 0 && y + offset_y < HEIGHT)) 70 | window_pixels.push_back((y + offset_y) * WIDTH + (x + offset_x + WIDTH) % WIDTH); 71 | } 72 | } 73 | return window_pixels; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /utils.h: -------------------------------------------------------------------------------- 1 | // utils.h 2 | // SurfaceStereo 3 | // 4 | // Created by Chen Liu on 9/30/14. 5 | // Copyright (c) 2014 Chen Liu. All rights reserved. 6 | // 7 | 8 | #ifndef SurfaceStereo_utils_h 9 | #define SurfaceStereo_utils_h 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | #include "Segment.h" 22 | 23 | 24 | using namespace std; 25 | using cv::Mat; 26 | using Eigen::MatrixXd; 27 | using Eigen::Matrix3d; 28 | using Eigen::VectorXd; 29 | using Eigen::Vector3d; 30 | 31 | 32 | //read point cloud from a .obj file 33 | vector readPointCloudFromObj(const string filename, const int image_width, const int image_height, const double rotation_angle); 34 | 35 | //save point cloud in a .ply file 36 | void savePointCloudAsPly(const cv::Mat &image, const vector &point_cloud, const char *filename); 37 | 38 | //save point cloud as a mesh in a .ply file 39 | void savePointCloudAsMesh(const vector &point_cloud, const char *filename); 40 | 41 | //load point cloud from a text file 42 | vector loadPointCloud(const string &filename); 43 | 44 | //save point cloud to a text file 45 | void savePointCloud(const vector &point_cloud, const char *filename); 46 | 47 | //draw a disp (inverse depth) image based on point cloud 48 | Mat drawDispImage(const vector &point_cloud, const int width, const MatrixXd &projection_matrix); 49 | 50 | //draw a disp (inverse depth) image based on point cloud 51 | Mat drawDispImage(const vector &point_cloud, const int width, const int height); 52 | 53 | //normalize point cloud on depth direction 54 | vector normalizePointCloudByZ(const vector &point_cloud); 55 | 56 | //zoom image and point cloud 57 | void zoomScene(Mat &image, vector &point_cloud, const double scale_x, const double scale_y); 58 | 59 | //crop image and point cloud 60 | void cropScene(Mat &image, vector &point_cloud, const int start_x, const int start_y, const int end_x, const int end_y); 61 | 62 | //inpaint empty point in a point cloud 63 | vector inpaintPointCloud(const vector &point_cloud, const int image_width, const int image_height); 64 | 65 | //read point cloud from a .ptx file 66 | bool readPtxFile(const string &filename, cv::Mat &image, vector &point_cloud, vector &camera_parameters); 67 | 68 | //unproject a pixel to 3D given depth 69 | vector unprojectPixel(const int pixel, const double depth, const int IMAGE_WIDTH, const int IMAGE_HEIGHT, const vector &CAMERA_PARAMETERS, const bool USE_PANORAMA); 70 | 71 | //project a 3D point to image domain 72 | int projectPoint(const vector &point, const int IMAGE_WIDTH, const int IMAGE_HEIGHT, const vector &CAMERA_PARAMETERS, const bool USE_PANORAMA); 73 | 74 | //calculate plane depth at pixel given plane parameters 75 | double calcPlaneDepthAtPixel(const vector &plane, const int pixel, const int IMAGE_WIDTH, const int IMAGE_HEIGHT, const vector &CAMERA_PARAMETERS, const bool USE_PANORAMA); 76 | 77 | //normalize values based on mean and svar 78 | double normalizeStatistically(const double value, const double mean, const double svar, const double normalized_value_for_mean, const double scale_factor); 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /cv_utils/ImageMask.h: -------------------------------------------------------------------------------- 1 | #ifndef IMAGE_MASK_H__ 2 | #define IMAGE_MASK_H__ 3 | 4 | #include 5 | #include 6 | 7 | namespace cv_utils 8 | { 9 | class ImageMask 10 | { 11 | public: 12 | ImageMask(); 13 | ImageMask(const std::vector &mask, const int width, const int height); 14 | ImageMask(const bool value, const int width, const int height); 15 | ImageMask(const std::vector &pixels, const int width, const int height); 16 | ImageMask(const cv::Mat &image); 17 | 18 | ImageMask &operator = (const ImageMask &image_mask); 19 | 20 | // ImageMask clone(); 21 | 22 | void setMask(const std::vector &mask, const int width, const int height); 23 | 24 | void resize(const int new_width, const int new_height); 25 | void resizeByRatio(const double x_ratio, const double y_ratio); 26 | void resizeWithBias(const int new_width, const int new_height, const bool desired_value); 27 | 28 | void dilate(const int num_iterations = 1, const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8); 29 | void erode(const int num_iterations = 1, const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8); 30 | 31 | void smooth(const std::string type, const int window_size, const double sigma = 0); 32 | 33 | bool at(const int &pixel) const; 34 | void set(const int &pixel, const bool value); 35 | std::vector getPixels() const; 36 | int getNumPixels() const; 37 | std::vector getCenter() const; 38 | 39 | cv::Mat drawMaskImage(const int num_channels = 1) const; 40 | cv::Mat drawImageWithMask(const cv::Mat &image, const bool use_mask_color = true, const cv::Vec3b mask_color = cv::Vec3b(255, 255, 255), const bool use_outside_color = false, const cv::Vec3b outside_color = cv::Vec3b(0, 0, 0)) const; 41 | void readMaskImage(const cv::Mat &mask_image); 42 | 43 | std::vector calcDistanceMapOutside(const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8) const; 44 | std::vector calcDistanceMapInside(const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8) const; 45 | 46 | void calcBoundaryDistanceMap(std::vector &boundary_map, std::vector &distance_map, const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8) const; 47 | 48 | std::vector > findConnectedComponents(const bool USE_PANORAMA = false, const int NEIGHBOR_SYSTEM = 8); 49 | 50 | void printMask(); 51 | 52 | void addPixels(const std::vector &pixels); 53 | void subtractPixels(const std::vector &pixels); 54 | 55 | ImageMask &operator +=(const ImageMask &image_mask); 56 | ImageMask &operator -=(const ImageMask &image_mask); 57 | friend ImageMask operator +(const ImageMask &image_mask_1, const ImageMask &image_mask_2); 58 | friend ImageMask operator -(const ImageMask &image_mask_1, const ImageMask &image_mask_2); 59 | friend std::ostream & operator <<(std::ostream &out_str, const ImageMask &image_mask); 60 | friend std::istream & operator >>(std::istream &in_str, ImageMask &image_mask); 61 | 62 | std::vector findMaskWindowPixels(const int pixel, const int WINDOW_SIZE, const int USE_PANORAMA = false) const; 63 | 64 | private: 65 | std::vector mask_; 66 | int width_; 67 | int height_; 68 | }; 69 | } 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /LayerDepthRepresenter.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__LayerDepthRepresenter__ 2 | #define __LayerDepthMap__LayerDepthRepresenter__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | //#include "ProposalGenerator.h" 9 | #include "Segment.h" 10 | 11 | #include 12 | 13 | 14 | class LayerDepthRepresenter { 15 | 16 | public: 17 | LayerDepthRepresenter(const cv::Mat &image, const std::vector &point_cloud, const RepresenterPenalties &penalties, const DataStatistics &statistics, const cv::Mat &ori_image, const std::vector &ori_point_cloud, const int num_layers); 18 | 19 | ~LayerDepthRepresenter(); 20 | 21 | 22 | private: 23 | const cv::Mat image_; 24 | const cv::Mat ori_image_; 25 | const std::vector point_cloud_; 26 | const std::vector ori_point_cloud_; 27 | std::vector normals_; 28 | const int IMAGE_WIDTH_; 29 | const int IMAGE_HEIGHT_; 30 | const int NUM_PIXELS_; 31 | 32 | const RepresenterPenalties PENALTIES_; 33 | const DataStatistics STATISTICS_; 34 | 35 | 36 | std::map > surface_depths_; 37 | std::map surface_colors_; 38 | double max_depth_; 39 | 40 | // vector labels_; 41 | 42 | std::vector ROI_mask_; 43 | int num_surfaces_; 44 | int num_layers_; 45 | 46 | //unique_ptr proposal_generator_; 47 | 48 | std::vector > layers_; 49 | 50 | std::map > layer_surfaces_; 51 | std::map > layer_front_surfaces_; 52 | std::map > layer_back_surfaces_; 53 | 54 | std::vector camera_parameters_; 55 | 56 | double disp_image_numerator_; 57 | 58 | 59 | //optimize layer representation 60 | void optimizeLayerRepresentation(); 61 | 62 | //write rendering information 63 | void writeRenderingInfo(const std::vector &solution, const int solution_num_surfaces, const std::map &solution_segments); 64 | 65 | //generate a HTML page containing result images 66 | void generateLayerImageHTML(const std::map > &iteration_statistics_map, const std::map &iteration_proposal_type_map); 67 | 68 | //upsample results to original resolution 69 | void upsampleSolution(const std::vector &solution_labels, const int solution_num_surfaces, const std::map &solution_segments, std::vector &upsampled_solution_labels, int &upsampled_solution_num_surfaces, std::map &upsampled_solution_segments); 70 | }; 71 | 72 | //write intermediate results to cache 73 | void writeLayers(const cv::Mat &image, const int image_width, const int image_height, const std::vector &point_cloud, const std::vector &camera_parameters, const int num_layers, const std::vector &solution, const int solution_num_surfaces, const std::map &solution_segments, const int result_index, const cv::Mat &ori_image, const std::vector &ori_point_cloud); 74 | 75 | //read intermediate results from cache 76 | bool readLayers(const int image_width, const int image_height, const std::vector &camera_parameters, const RepresenterPenalties &penalties, const DataStatistics &statistics, const int num_layers, std::vector &solution, int &solution_num_surfaces, std::map &solution_segments, const int result_index); 77 | 78 | 79 | #endif /* defined(__LayerDepthMap__LayerDepthRepresenter__) */ 80 | -------------------------------------------------------------------------------- /cv_utils/StatisticsCalculations.cpp: -------------------------------------------------------------------------------- 1 | #include "cv_utils.h" 2 | 3 | using namespace std; 4 | 5 | 6 | namespace cv_utils 7 | { 8 | vector calcMeanAndSVar(const vector &values) 9 | { 10 | double sum = 0, sum2 = 0; 11 | for (std::vector::const_iterator value_it = values.begin(); value_it != values.end(); value_it++) { 12 | sum += *value_it; 13 | sum2 += pow(*value_it, 2); 14 | } 15 | double mean = sum / values.size(); 16 | double svar = sqrt(sum2 / values.size() - pow(mean, 2)); 17 | vector mean_and_svar; 18 | mean_and_svar.push_back(mean); 19 | mean_and_svar.push_back(svar); 20 | return mean_and_svar; 21 | } 22 | 23 | void calcMeanAndSVar(const vector > &values, vector &mean, vector > &var) 24 | { 25 | assert(values.size() > 0); 26 | const int NUM_DIMENSIONS = values.begin()->size(); 27 | vector sums(NUM_DIMENSIONS, 0); 28 | vector > sum2s(NUM_DIMENSIONS, vector(NUM_DIMENSIONS, 0)); 29 | for (std::vector >::const_iterator value_it = values.begin(); value_it != values.end(); value_it++) { 30 | for (std::vector::const_iterator c_it = value_it->begin(); c_it != value_it->end(); c_it++) { 31 | sums[c_it - value_it->begin()] += *c_it; 32 | for (std::vector::const_iterator d_it = value_it->begin(); d_it != value_it->end(); d_it++) 33 | sum2s[c_it - value_it->begin()][d_it - value_it->begin()] += *c_it * *d_it; 34 | } 35 | } 36 | mean.assign(NUM_DIMENSIONS, 0); 37 | for (int c = 0; c < NUM_DIMENSIONS; c++) 38 | mean[c] = sums[c] / values.size(); 39 | var.assign(NUM_DIMENSIONS, vector(NUM_DIMENSIONS)); 40 | for (int c = 0; c < NUM_DIMENSIONS; c++) 41 | for (int d = 0; d < NUM_DIMENSIONS; d++) 42 | var[c][d] = sum2s[c][d] / values.size() - mean[c] * mean[d]; 43 | } 44 | 45 | vector > findAllCombinations(const vector &candidates, const int num_elements) 46 | { 47 | if (num_elements == 0) 48 | return vector >(1, vector()); 49 | vector > combinations; 50 | int num_candidates = candidates.size(); 51 | if (num_candidates < num_elements) 52 | return combinations; 53 | 54 | vector selected_element_mask(num_candidates, false); 55 | for (int index = num_candidates - num_elements; index < num_candidates; index++) 56 | selected_element_mask[index] = true; 57 | while (true) { 58 | vector combination; 59 | for (int index = 0; index < num_candidates; index++) { 60 | if (selected_element_mask[index] == true) { 61 | combination.push_back(candidates[index]); 62 | } 63 | } 64 | combinations.push_back(combination); 65 | if (next_permutation(selected_element_mask.begin(), selected_element_mask.end()) == false) 66 | break; 67 | } 68 | return combinations; 69 | 70 | for (int configuration = 0; configuration < pow(2, num_candidates); configuration++) { 71 | vector selected_element_mask(num_candidates, false); 72 | int num_selected_elements = 0; 73 | int configuration_temp = configuration; 74 | for (int j = 0; j < num_candidates; j++) { 75 | if (configuration_temp % 2 == 1) { 76 | selected_element_mask[j] = true; 77 | num_selected_elements++; 78 | if (num_selected_elements > num_elements) 79 | break; 80 | } 81 | configuration_temp /= 2; 82 | } 83 | if (num_selected_elements != num_elements) 84 | continue; 85 | vector combination; 86 | for (int j = 0; j < num_candidates; j++) 87 | if (selected_element_mask[j] == true) 88 | combination.push_back(candidates[j]); 89 | combinations.push_back(combination); 90 | } 91 | return combinations; 92 | } 93 | 94 | int calcNumDistinctValues(const vector &values) 95 | { 96 | vector sorted_values = values; 97 | sort(sorted_values.begin(), sorted_values.end()); 98 | vector::iterator unique_end = unique(sorted_values.begin(), sorted_values.end()); 99 | return distance(sorted_values.begin(), unique_end); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | #include "LayerDepthRepresenter.h" 13 | #include "utils.h" 14 | #include "TRW_S/MRFEnergy.h" 15 | 16 | 17 | using namespace std; 18 | using namespace cv; 19 | 20 | DEFINE_string(image_path, "", "The path to the RGB image."); 21 | //Point cloud format: 22 | //The first number indicates the number equals to image_width * image_height. 23 | //Each row stores a 3D point for each pixel (ordered by y * image_width + x). 24 | DEFINE_string(point_cloud_path, "", "The path to the point cloud."); 25 | DEFINE_int32(num_layers, 4, "The number of layers."); 26 | 27 | 28 | int main(int argc, char* argv[]) { 29 | 30 | google::ParseCommandLineFlags(&argc, &argv, true); 31 | 32 | srand(time(0)); 33 | 34 | Mat ori_image = imread(FLAGS_image_path, 1); 35 | assert(ori_image); 36 | vector ori_point_cloud = loadPointCloud(FLAGS_point_cloud_path); 37 | assert(ori_point_cloud.size() == ori_image.cols * ori_image.rows * 3); 38 | 39 | double zoom_scale = min(200.0 / max(ori_image.cols, ori_image.rows), 1.0); 40 | Mat image = ori_image.clone(); 41 | vector point_cloud = ori_point_cloud; 42 | zoomScene(image, point_cloud, zoom_scale, zoom_scale); 43 | 44 | 45 | RepresenterPenalties penalties; 46 | 47 | penalties.data_depth_pen = 2000; //data cost for depth difference 48 | penalties.data_normal_pen = 200; //data cost for color difference 49 | penalties.data_color_pen = 10; //data cost for normal difference 50 | penalties.data_non_plane_pen = 100; //parameter in data cost 51 | 52 | 53 | penalties.smoothness_pen = 10000; //smoothness cost for depth change 54 | penalties.smoothness_small_constant_pen = 1; //small constant smoothness cost for smooth boundary (with label changes) 55 | penalties.smoothness_concave_shape_pen = 5000; //smoothness cost for concave shape 56 | penalties.smoothness_anisotropic_diffusion_pen = 500; //smoothness cost based on color difference (anisotropic diffusion) 57 | 58 | penalties.other_viewpoint_smoothness_pen = 2000; //smoothness cost for neighboring pixels from other viewpoint 59 | penalties.other_viewpoint_depth_conflict_pen = 200000; //depth conflict penalty for different layers at the same pixel from other viewpoint 60 | 61 | penalties.surface_pen = 20000; //label cost for the occurrence of a surface 62 | 63 | penalties.smoothness_empty_non_empty_ratio = 0.05; //the ratio of smoothness cost between a empty pixel and a non-empty pixel over the smoothness cost for a depth change of statistics.depth_change_smoothness_threshold 64 | 65 | penalties.huge_pen = 1000000; //a huge penalty for cases with conflicts 66 | 67 | 68 | DataStatistics statistics; 69 | 70 | statistics.pixel_fitting_distance_threshold = 0.03; //the 3D distance threshold when fitting a surface model 71 | statistics.pixel_fitting_angle_threshold = 30 * M_PI / 180; //the angle threshold when fitting a surface model 72 | statistics.pixel_fitting_color_likelihood_threshold = -20; //the color likelihood threshold when fitting a surface model 73 | statistics.depth_diff_var = 0.01; //the variance of depth difference for unary cost calculation 74 | statistics.similar_angle_threshold = 20 * M_PI / 180; //the angle threshold for two vectors to be regarded as parallel or vertical 75 | 76 | statistics.viewpoint_movement = 0.1; //the amount of viewpoint movement for parallex term calculation 77 | 78 | statistics.depth_conflict_tolerance = 0.03; //the tolerance for depth conflict between two layers at the same pixel 79 | statistics.depth_change_smoothness_threshold = 0.02; //depth change threshold for two surfaces to be regarded as smooth at the intersection 80 | statistics.bspline_surface_num_pixels_threshold = image.cols * image.rows / 50; //the number pixels allowed to appear in a b-spline surface 81 | statistics.background_depth_diff_tolerance = 0.05; //small amount of depth difference allowed without penalty in the background layer 82 | 83 | LayerDepthRepresenter representer(image, point_cloud, penalties, statistics, ori_image, ori_point_cloud, FLAGS_num_layers); 84 | } 85 | -------------------------------------------------------------------------------- /TRW_S/ordering.def: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "MRFEnergy.h" 6 | 7 | //#include "instances.inc" 8 | 9 | template void MRFEnergy::SetAutomaticOrdering() 10 | { 11 | int dMin; 12 | Node* i; 13 | Node* iMin; 14 | Node* list; 15 | Node* listBoundary; 16 | MRFEdge* e; 17 | 18 | if (m_isEnergyConstructionCompleted) 19 | { 20 | m_errorFn("Error in SetAutomaticOrdering(): function cannot be called after graph construction is completed"); 21 | } 22 | 23 | printf("Setting automatic ordering... "); 24 | 25 | list = m_nodeFirst; 26 | listBoundary = NULL; 27 | m_nodeFirst = m_nodeLast = NULL; 28 | for (i=list; i; i=i->m_next) 29 | { 30 | i->m_ordering = 2*m_nodeNum; // will contain remaining degree mod m_nodeNum (i.e. number of edges connecting to nodes in 'listBoundary' and 'list') 31 | // if i->m_ordering \in [2*m_nodeNum; 3*m_nodeNum) - not assigned yet, belongs to 'list' 32 | // if i->m_ordering \in [m_nodeNum; 2*m_nodeNum) - not assigned yet, belongs to 'listBoundary' 33 | // if i->m_ordering \in [0; m_nodeNum ) - assigned, belongs to 'm_nodeFirst' 34 | for (e=i->m_firstForward; e; e=e->m_nextForward) 35 | { 36 | i->m_ordering ++; 37 | } 38 | for (e=i->m_firstBackward; e; e=e->m_nextBackward) 39 | { 40 | i->m_ordering ++; 41 | } 42 | } 43 | 44 | while (list) 45 | { 46 | // find node with the smallest remaining degree in list 47 | dMin = m_nodeNum; 48 | for (i=list; i; i=i->m_next) 49 | { 50 | assert(i->m_ordering >= 2*m_nodeNum); 51 | if (dMin > i->m_ordering - 2*m_nodeNum) 52 | { 53 | dMin = i->m_ordering - 2*m_nodeNum; 54 | iMin = i; 55 | } 56 | } 57 | i = iMin; 58 | 59 | // remove i from list 60 | if (i->m_prev) i->m_prev->m_next = i->m_next; 61 | else list = i->m_next; 62 | if (i->m_next) i->m_next->m_prev = i->m_prev; 63 | 64 | // add i to listBoundary 65 | listBoundary = i; 66 | i->m_prev = NULL; 67 | i->m_next = NULL; 68 | i->m_ordering -= m_nodeNum; 69 | 70 | while (listBoundary) 71 | { 72 | // find node with the smallest remaining degree in listBoundary 73 | dMin = m_nodeNum; 74 | for (i=listBoundary; i; i=i->m_next) 75 | { 76 | assert(i->m_ordering >= m_nodeNum && i->m_ordering < 2*m_nodeNum); 77 | if (dMin > i->m_ordering - m_nodeNum) 78 | { 79 | dMin = i->m_ordering - m_nodeNum; 80 | iMin = i; 81 | } 82 | } 83 | i = iMin; 84 | 85 | // remove i from listBoundary 86 | if (i->m_prev) i->m_prev->m_next = i->m_next; 87 | else listBoundary = i->m_next; 88 | if (i->m_next) i->m_next->m_prev = i->m_prev; 89 | 90 | // add i to m_nodeFirst 91 | if (m_nodeLast) 92 | { 93 | m_nodeLast->m_next = i; 94 | i->m_ordering = m_nodeLast->m_ordering + 1; 95 | } 96 | else 97 | { 98 | m_nodeFirst = i; 99 | i->m_ordering = 0; 100 | } 101 | i->m_prev = m_nodeLast; 102 | m_nodeLast = i; 103 | i->m_next = NULL; 104 | 105 | // process neighbors of i=m_nodeLast: decrease their remaining degree, 106 | // put them into listBoundary (if they are in list) 107 | for (e=m_nodeLast->m_firstForward; e; e=e->m_nextForward) 108 | { 109 | assert(m_nodeLast == e->m_tail); 110 | i = e->m_head; 111 | if (i->m_ordering >= m_nodeNum) 112 | { 113 | i->m_ordering --; // decrease remaining degree of i 114 | if (i->m_ordering >= 2*m_nodeNum) 115 | { 116 | // remove i from list 117 | if (i->m_prev) i->m_prev->m_next = i->m_next; 118 | else list = i->m_next; 119 | if (i->m_next) i->m_next->m_prev = i->m_prev; 120 | 121 | // add i to listBoundary 122 | if (listBoundary) listBoundary->m_prev = i; 123 | i->m_prev = NULL; 124 | i->m_next = listBoundary; 125 | listBoundary = i; 126 | i->m_ordering -= m_nodeNum; 127 | } 128 | } 129 | } 130 | for (e=m_nodeLast->m_firstBackward; e; e=e->m_nextBackward) 131 | { 132 | assert(m_nodeLast == e->m_head); 133 | i = e->m_tail; 134 | if (i->m_ordering >= m_nodeNum) 135 | { 136 | i->m_ordering --; // decrease remaining degree of i 137 | if (i->m_ordering >= 2*m_nodeNum) 138 | { 139 | // remove i from list 140 | if (i->m_prev) i->m_prev->m_next = i->m_next; 141 | else list = i->m_next; 142 | if (i->m_next) i->m_next->m_prev = i->m_prev; 143 | 144 | // add i to listBoundary 145 | if (listBoundary) listBoundary->m_prev = i; 146 | i->m_prev = NULL; 147 | i->m_next = listBoundary; 148 | listBoundary = i; 149 | i->m_ordering -= m_nodeNum; 150 | } 151 | } 152 | } 153 | } 154 | } 155 | 156 | printf("done\n"); 157 | 158 | CompleteGraphConstruction(); 159 | } 160 | -------------------------------------------------------------------------------- /ProposalDesigner.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__ProposalDesigner__ 2 | #define __LayerDepthMap__ProposalDesigner__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | //#include "LayerInpainter.h" 12 | //#include "GraphRepresenter.h" 13 | //#include "LayerEstimator.h" 14 | #include "Segment.h" 15 | 16 | //using namespace cv; 17 | //using namespace Eigen; 18 | using Eigen::MatrixXd; 19 | using Eigen::Matrix3d; 20 | using Eigen::VectorXd; 21 | using Eigen::Vector3d; 22 | 23 | 24 | struct MeanshiftParams { 25 | double spatial_bandwidth; 26 | double range_bandwidth; 27 | int minimum_regions_area; 28 | }; 29 | 30 | const std::string EDISON_PATH = "edison"; 31 | const std::string EDISON_EXE = "edison/edison edison/config.txt"; 32 | 33 | class ProposalDesigner{ 34 | 35 | public: 36 | ProposalDesigner(const cv::Mat &image, const std::vector &point_cloud, const std::vector &normals, const std::vector &camera_parameters, const int num_layers, const RepresenterPenalties penalties, const DataStatistics statistics); 37 | 38 | ~ProposalDesigner(); 39 | 40 | //generate a proposal and return 41 | bool getProposal(int &iteration, std::vector > &proposal_labels, int &proposal_num_surfaces, std::map &proposal_segments, std::string &proposal_type); 42 | 43 | //set current solution which might be used to generate new proposals 44 | void setCurrentSolution(const std::vector ¤t_solution_labels, const int current_solution_num_surfaces, const std::map ¤t_solution_segments); 45 | 46 | //initialize current solution which might be used to generate new proposals 47 | void initializeCurrentSolution(); 48 | 49 | //get the indices (pixel-wise) of the current solution inside all proposals 50 | std::vector getCurrentSolutionIndices(); 51 | 52 | 53 | private: 54 | const cv::Mat image_; 55 | const std::vector point_cloud_; 56 | const std::vector normals_; 57 | const Eigen::MatrixXd projection_matrix_; 58 | const int IMAGE_WIDTH_; 59 | const int IMAGE_HEIGHT_; 60 | const std::vector CAMERA_PARAMETERS_; 61 | const RepresenterPenalties penalties_; 62 | const DataStatistics statistics_; 63 | 64 | cv::Mat blurred_hsv_image_; 65 | 66 | std::vector ROI_mask_; 67 | int NUM_LAYERS_; 68 | const int NUM_PIXELS_; 69 | 70 | std::vector current_solution_labels_; 71 | int current_solution_num_surfaces_; 72 | std::map current_solution_segments_; 73 | 74 | std::vector > proposal_labels_; 75 | int proposal_num_surfaces_; 76 | std::map proposal_segments_; 77 | std::string proposal_type_; 78 | 79 | int num_confident_segments_threshold_; 80 | double segment_confidence_threshold_; 81 | 82 | 83 | std::vector > segmentations_; 84 | 85 | std::set > used_confident_segment_layer_maps_; 86 | 87 | std::vector current_solution_indices_; 88 | 89 | std::vector single_surface_candidate_pixels_; 90 | 91 | std::vector proposal_type_indices_; 92 | int proposal_type_index_ptr_; 93 | int all_proposal_iteration_; 94 | const int NUM_ALL_PROPOSAL_ITERATIONS_; 95 | 96 | //generate segment refitting proposal 97 | bool generateSegmentRefittingProposal(); 98 | 99 | //generate single surface expansion proposal (provide segment_id to expand specific segment) 100 | bool generateSingleSurfaceExpansionProposal(const int segment_id = -1); 101 | 102 | //generate single surface expansion proposal 103 | bool generateLayerSwapProposal(); 104 | 105 | //generate concave hull proposal 106 | bool generateConcaveHullProposal(const bool consider_background = true); 107 | 108 | //generate segment adding proposal 109 | bool generateSegmentAddingProposal(const int denoted_segment_adding_type = -1); 110 | 111 | //generate structure expansion proposal 112 | bool generateStructureExpansionProposal(const int layer_index = -1, const int pixel = -1); 113 | 114 | //generate backward merging proposal 115 | bool generateBackwardMergingProposal(const int denoted_target_layer_index = -1); 116 | 117 | //generate desired proposal (for debug) 118 | bool generateDesiredProposal(); 119 | 120 | //generate a proposal identical with current solution (for debug) 121 | bool generateSingleProposal(); 122 | 123 | //calculate possible proposals for a pixel given which surfaces will appear in which layers 124 | std::vector calcPixelProposals(const int num_surfaces, const std::map > &pixel_layer_surfaces_map); 125 | 126 | //add surface indicator variables (for formulating label cost) 127 | void addIndicatorVariables(const int num_indicator_variables = -1); 128 | 129 | //check the validity of a label 130 | bool checkLabelValidity(const int pixel, const int label, const int num_surfaces, const std::map &segments); 131 | 132 | //convert a current solution label to a proposal label 133 | int convertToProposalLabel(const int current_solution_label); 134 | }; 135 | 136 | 137 | #endif /* defined(__LayerDepthMap__ProposalDesigner__) */ 138 | -------------------------------------------------------------------------------- /Segment.h: -------------------------------------------------------------------------------- 1 | #ifndef __LayerDepthMap__Segment__ 2 | #define __LayerDepthMap__Segment__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "DataStructure.h" 13 | //#include "BSpline.h" 14 | 15 | 16 | class Segment{ 17 | 18 | public: 19 | Segment(const cv::Mat &image, const std::vector &point_cloud, const std::vector &normals, const std::vector &camera_parameters, const std::vector &pixels, const RepresenterPenalties &penalties, const DataStatistics &input_statistics = DataStatistics(), const int segment_type = 0); 20 | Segment(const int image_width, const int image_height, const std::vector &camera_parameters, const RepresenterPenalties &penalties, const DataStatistics &statistics); 21 | //Segment(const Segment &segment); 22 | Segment(); 23 | 24 | //~Segment(); 25 | 26 | //write segment to file 27 | friend std::ostream & operator <<(std::ostream &out_str, const Segment &segment); 28 | 29 | //read segment from file 30 | friend std::istream & operator >>(std::istream &in_str, Segment &segment); 31 | 32 | //segment assignment 33 | Segment &operator = (const Segment &segment); 34 | 35 | 36 | //predict color likelihood based on the GMM model 37 | double predictColorLikelihood(const int pixel, const cv::Vec3f hsv_color) const; 38 | 39 | //set GMM model based on saved file. 40 | void setGMM(const cv::FileNode GMM_file_node); 41 | 42 | //get GMM model in order to save to file. 43 | cv::Ptr getGMM() const; 44 | 45 | //get depth map 46 | std::vector getDepthMap() const; 47 | 48 | //get depth at specific pixel 49 | double getDepth(const int pixel) const; 50 | 51 | //get depth at pixel specified by ratios 52 | double getDepth(const double x_ratio, const double y_ratio) const; 53 | 54 | //get plane parameters 55 | std::vector getDepthPlane() const; 56 | 57 | //get segment pixels 58 | std::vector getSegmentPixels() const; 59 | 60 | //get segment type 61 | int getType() const; 62 | 63 | //check whether the segment fits the specific pixel or not 64 | bool checkPixelFitting(const cv::Mat &hsv_image, const std::vector &point_cloud, const std::vector &normals, const int pixel) const; 65 | 66 | //calculate the angle between normals of the input visible surface and the segment 67 | double calcAngle(const std::vector &normals, const int pixel); 68 | 69 | //calculate the difference of the distance between two pixels and the segment. 70 | int calcDistanceOffset(const int pixel_1, const int pixel_2); 71 | 72 | //bool buildSubSegment(const cv::Mat &image, const std::vector &point_cloud, const std::vector &normals, const std::vector &visible_pixels); 73 | 74 | //project the segmnet to different viewpoints 75 | std::vector projectToOtherViewpoints(const int pixel, const double viewpoint_movement); 76 | 77 | //get the segment type 78 | int getSegmentType() const; 79 | 80 | //calculate 3D distance between the 3D point at specific pixel and the segment 81 | double calcDistance(const std::vector &point_cloud, const int pixel); 82 | 83 | 84 | private: 85 | int IMAGE_WIDTH_; 86 | int IMAGE_HEIGHT_; 87 | 88 | int NUM_PIXELS_; 89 | std::vector CAMERA_PARAMETERS_; 90 | 91 | 92 | RepresenterPenalties penalties_; 93 | DataStatistics input_statistics_; 94 | 95 | std::vector segment_pixels_; 96 | std::vector disp_plane_; 97 | std::vector depth_plane_; 98 | std::vector depth_map_; 99 | std::vector normals_; 100 | 101 | int segment_type_; 102 | 103 | cv::Ptr GMM_; 104 | 105 | double segment_confidence_; 106 | 107 | std::vector segment_mask_; 108 | double segment_radius_; 109 | double segment_center_x_; 110 | double segment_center_y_; 111 | 112 | std::vector distance_map_; 113 | 114 | 115 | //fit a plane segment 116 | void fitDepthPlane(const cv::Mat &image, const std::vector &point_cloud, const std::vector &normals, const std::vector &pixels); 117 | 118 | //fit a b-spline segment 119 | void fitBSplineSurface(const cv::Mat &image, const std::vector &point_cloud, const std::vector &normals, const std::vector &pixels); 120 | 121 | //fit a plane segment which is parallel to the image plane 122 | void fitParallelSurface(const std::vector &point_cloud, const std::vector &normals, const std::vector &pixels); 123 | 124 | //calculate depth map for this segment 125 | void calcDepthMap(const std::vector &point_cloud = std::vector(), const std::vector &fitted_pixels = std::vector()); 126 | 127 | //calculate color statistics for this segment 128 | void calcColorStatistics(const cv::Mat &image, const std::vector &pixels); 129 | 130 | //calculate mask info for this segment 131 | void calcSegmentMaskInfo(); 132 | 133 | //the distance map stores the distance between any pixel and the segment on image domain 134 | void calcDistanceMap(); 135 | 136 | //find largest connected component 137 | std::vector findLargestConnectedComponent(const std::vector &point_cloud, const std::vector &pixels); 138 | 139 | //calculate point cloud map for this segment. 140 | std::vector calcPointCloud(); 141 | 142 | //delete invalid pixels which have no depth values 143 | std::vector deleteInvalidPixels(const std::vector &point_cloud, const std::vector &pixels); 144 | }; 145 | 146 | #endif /* defined(__LayerDepthMap__Segment__) */ 147 | -------------------------------------------------------------------------------- /cv_utils/FusionSpaceSolver.cpp: -------------------------------------------------------------------------------- 1 | #include "FusionSpaceSolver.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "TRW_S/MRFEnergy.h" 9 | 10 | using namespace std; 11 | 12 | FusionSpaceSolver::FusionSpaceSolver(const int NUM_NODES, const std::vector > &node_neighbors, CostFunctor &cost_functor, ProposalGenerator &proposal_generator, const int NUM_ITERATIONS, const bool CONSIDER_LABEL_COST) : NUM_NODES_(NUM_NODES), node_neighbors_(node_neighbors), cost_functor_(cost_functor), proposal_generator_(proposal_generator), NUM_ITERATIONS_(NUM_ITERATIONS), CONSIDER_LABEL_COST_(CONSIDER_LABEL_COST) 13 | { 14 | } 15 | 16 | vector FusionSpaceSolver::fuse(const std::vector > &node_labels, std::vector &energy_info) 17 | { 18 | cout << "fuse" << endl; 19 | 20 | unique_ptr > energy(new MRFEnergy(TypeGeneral::GlobalSize())); 21 | map label_indicator_index_map; 22 | if (CONSIDER_LABEL_COST_) { 23 | int label_indicator_index = NUM_NODES_; 24 | for (vector >::const_iterator node_it = node_labels.begin(); node_it != node_labels.end(); node_it++) 25 | for (vector::const_iterator label_it = node_it->begin(); label_it != node_it->end(); label_it++) 26 | if (label_indicator_index_map.count(*label_it) == 0) 27 | label_indicator_index_map[*label_it] = label_indicator_index++; 28 | } 29 | int NUM_LABEL_INDICATORS = label_indicator_index_map.size(); 30 | vector::NodeId> nodes(NUM_NODES_ + NUM_LABEL_INDICATORS); 31 | 32 | //add unary cost 33 | for (int node_index = 0; node_index < NUM_NODES_; node_index++) { 34 | vector labels = node_labels[node_index]; 35 | const int NUM_LABELS = labels.size(); 36 | if (NUM_LABELS == 0) { 37 | cout << "empty proposal error: " << node_index << endl; 38 | exit(1); 39 | } 40 | vector unary_cost(NUM_LABELS); 41 | for (int label_index = 0; label_index < NUM_LABELS; label_index++) 42 | unary_cost[label_index] = cost_functor_(node_index, labels[label_index]); 43 | nodes[node_index] = energy->AddNode(TypeGeneral::LocalSize(NUM_LABELS), TypeGeneral::NodeData(&unary_cost[0])); 44 | } 45 | 46 | //add label indicator cost 47 | if (CONSIDER_LABEL_COST_ == true) { 48 | for (int label_indicator_index = 0; label_indicator_index < NUM_LABEL_INDICATORS; label_indicator_index++) { 49 | vector label_cost(2, 0); 50 | label_cost[1] = cost_functor_.getLabelCost(); 51 | nodes[label_indicator_index + NUM_NODES_] = energy->AddNode(TypeGeneral::LocalSize(2), TypeGeneral::NodeData(&label_cost[0])); 52 | } 53 | } 54 | 55 | //add pairwise cost 56 | for (int node_index = 0; node_index < NUM_NODES_; node_index++) { 57 | vector labels = node_labels[node_index]; 58 | vector neighbors = node_neighbors_[node_index]; 59 | for (vector::const_iterator neighbor_it = neighbors.begin(); neighbor_it != neighbors.end(); neighbor_it++) { 60 | vector neighbor_labels = node_labels[*neighbor_it]; 61 | vector pairwise_cost(labels.size() * neighbor_labels.size(), 0); 62 | for (int label_index = 0; label_index < labels.size(); label_index++) 63 | for (int neighbor_label_index = 0; neighbor_label_index < neighbor_labels.size(); neighbor_label_index++) 64 | pairwise_cost[label_index + neighbor_label_index * labels.size()] = cost_functor_(node_index, *neighbor_it, labels[label_index], neighbor_labels[neighbor_label_index]); 65 | bool has_non_zero_cost = false; 66 | for (int i = 0; i < pairwise_cost.size(); i++) 67 | if (pairwise_cost[i] > 0) 68 | has_non_zero_cost = true; 69 | if (has_non_zero_cost == true) 70 | energy->AddEdge(nodes[node_index], nodes[*neighbor_it], TypeGeneral::EdgeData(TypeGeneral::GENERAL, &pairwise_cost[0])); 71 | } 72 | } 73 | 74 | //add label indicator constraints 75 | if (CONSIDER_LABEL_COST_) { 76 | for (int node_index = 0; node_index < NUM_NODES_; node_index++) { 77 | vector labels = node_labels[node_index]; 78 | for (int label_index = 0; label_index < labels.size(); label_index++) { 79 | int label = labels[label_index]; 80 | int label_indicator_index = label_indicator_index_map[label]; 81 | vector label_indicator_conflict_cost(labels.size() * 2, 0); 82 | label_indicator_conflict_cost[label_index] = cost_functor_.getLabelIndicatorConflictCost(); 83 | 84 | energy->AddEdge(nodes[node_index], nodes[label_indicator_index + NUM_NODES_], TypeGeneral::EdgeData(TypeGeneral::GENERAL, &label_indicator_conflict_cost[0])); 85 | } 86 | } 87 | } 88 | 89 | MRFEnergy::Options options; 90 | options.m_iterMax = NUM_ITERATIONS_; 91 | options.m_printIter = NUM_ITERATIONS_ / 5; 92 | options.m_printMinIter = 0; 93 | options.m_eps = 0.1; 94 | 95 | double lower_bound, solution_energy; 96 | energy->Minimize_TRW_S(options, lower_bound, solution_energy); 97 | 98 | vector fused_labels(NUM_NODES_); 99 | for (int node_index = 0; node_index < NUM_NODES_; node_index++) { 100 | int label = energy->GetSolution(nodes[node_index]); 101 | fused_labels[node_index] = node_labels[node_index][label]; 102 | } 103 | energy_info.assign(2, 0); 104 | energy_info[0] = solution_energy; 105 | energy_info[1] = lower_bound; 106 | return fused_labels; 107 | } 108 | 109 | vector FusionSpaceSolver::solve(const int NUM_ITERATIONS, const vector &initial_solution) 110 | { 111 | vector current_solution = initial_solution; 112 | double current_solution_energy = numeric_limits::max(); 113 | proposal_generator_.setCurrentSolution(current_solution); 114 | cost_functor_.setCurrentSolution(current_solution); 115 | 116 | for (int iteration = 0; iteration < NUM_ITERATIONS; iteration++) { 117 | vector > pixel_labels = proposal_generator_.getProposal(); 118 | vector energy_info; 119 | vector solution = fuse(pixel_labels, energy_info); 120 | if (energy_info[0] >= current_solution_energy) 121 | continue; 122 | current_solution = solution; 123 | current_solution_energy = energy_info[0]; 124 | if (iteration < NUM_ITERATIONS - 1) { 125 | proposal_generator_.setCurrentSolution(current_solution); 126 | cost_functor_.setCurrentSolution(current_solution); 127 | } 128 | } 129 | return current_solution; 130 | } 131 | 132 | -------------------------------------------------------------------------------- /TRW_S/MRFEnergy.def: -------------------------------------------------------------------------------- 1 | template MRFEnergy::MRFEnergy(GlobalSize Kglobal, ErrorFunction errorFn) 2 | : m_errorFn(errorFn ? errorFn : DefaultErrorFn), 3 | m_mallocBlockFirst(NULL), 4 | m_nodeFirst(NULL), 5 | m_nodeLast(NULL), 6 | m_nodeNum(0), 7 | m_edgeNum(0), 8 | m_Kglobal(Kglobal), 9 | m_vectorMaxSizeInBytes(0), 10 | m_isEnergyConstructionCompleted(false), 11 | m_buf(NULL) 12 | { 13 | } 14 | 15 | template MRFEnergy::~MRFEnergy() 16 | { 17 | while (m_mallocBlockFirst) 18 | { 19 | MallocBlock* next = m_mallocBlockFirst->m_next; 20 | delete m_mallocBlockFirst; 21 | m_mallocBlockFirst = next; 22 | } 23 | } 24 | 25 | template typename MRFEnergy::NodeId MRFEnergy::AddNode(LocalSize K, NodeData data) 26 | { 27 | if (m_isEnergyConstructionCompleted) 28 | { 29 | m_errorFn("Error in AddNode(): graph construction completed - nodes cannot be added"); 30 | } 31 | 32 | int actualVectorSize = Vector::GetSizeInBytes(m_Kglobal, K); 33 | if (actualVectorSize < 0) 34 | { 35 | m_errorFn("Error in AddNode() (invalid parameter?)"); 36 | } 37 | if (m_vectorMaxSizeInBytes < actualVectorSize) 38 | { 39 | m_vectorMaxSizeInBytes = actualVectorSize; 40 | } 41 | int nodeSize = sizeof(Node) - sizeof(Vector) + actualVectorSize; 42 | Node* i = (Node *) Malloc(nodeSize); 43 | 44 | i->m_K = K; 45 | i->m_D.Initialize(m_Kglobal, K, data); 46 | 47 | i->m_firstForward = NULL; 48 | i->m_firstBackward = NULL; 49 | i->m_prev = m_nodeLast; 50 | if (m_nodeLast) 51 | { 52 | m_nodeLast->m_next = i; 53 | } 54 | else 55 | { 56 | m_nodeFirst = i; 57 | } 58 | m_nodeLast = i; 59 | i->m_next = NULL; 60 | 61 | i->m_ordering = m_nodeNum ++; 62 | return i; 63 | } 64 | 65 | template void MRFEnergy::AddNodeData(NodeId i, NodeData data) 66 | { 67 | i->m_D.Add(m_Kglobal, i->m_K, data); 68 | } 69 | 70 | template void MRFEnergy::AddEdge(NodeId i, NodeId j, EdgeData data) 71 | { 72 | if (m_isEnergyConstructionCompleted) 73 | { 74 | m_errorFn("Error in AddNode(): graph construction completed - nodes cannot be added"); 75 | } 76 | 77 | MRFEdge* e; 78 | 79 | int actualEdgeSize = Edge::GetSizeInBytes(m_Kglobal, i->m_K, j->m_K, data); 80 | if (actualEdgeSize < 0) 81 | { 82 | m_errorFn("Error in AddEdge() (invalid parameter?)"); 83 | } 84 | int MRFedgeSize = sizeof(MRFEdge) - sizeof(Edge) + actualEdgeSize; 85 | e = (MRFEdge*) Malloc(MRFedgeSize); 86 | 87 | e->m_message.Initialize(m_Kglobal, i->m_K, j->m_K, data, &i->m_D, &j->m_D); 88 | 89 | e->m_tail = i; 90 | e->m_nextForward = i->m_firstForward; 91 | i->m_firstForward = e; 92 | 93 | e->m_head = j; 94 | e->m_nextBackward = j->m_firstBackward; 95 | j->m_firstBackward = e; 96 | 97 | m_edgeNum ++; 98 | } 99 | 100 | ///////////////////////////////////////////////////////////////////////////////// 101 | 102 | template void MRFEnergy::ZeroMessages() 103 | { 104 | Node* i; 105 | MRFEdge* e; 106 | 107 | if (!m_isEnergyConstructionCompleted) 108 | { 109 | CompleteGraphConstruction(); 110 | } 111 | 112 | for (i=m_nodeFirst; i; i=i->m_next) 113 | { 114 | for (e=i->m_firstForward; e; e=e->m_nextForward) 115 | { 116 | e->m_message.GetMessagePtr()->SetZero(m_Kglobal, i->m_K); 117 | } 118 | } 119 | } 120 | 121 | template void MRFEnergy::AddRandomMessages(unsigned int random_seed, REAL min_value, REAL max_value) 122 | { 123 | Node* i; 124 | MRFEdge* e; 125 | int k; 126 | 127 | if (!m_isEnergyConstructionCompleted) 128 | { 129 | CompleteGraphConstruction(); 130 | } 131 | 132 | srand(random_seed); 133 | 134 | for (i=m_nodeFirst; i; i=i->m_next) 135 | { 136 | for (e=i->m_firstForward; e; e=e->m_nextForward) 137 | { 138 | Vector* M = e->m_message.GetMessagePtr(); 139 | for (k=0; kGetArraySize(m_Kglobal, i->m_K); k++) 140 | { 141 | REAL x = (REAL)( min_value + rand()/((double)RAND_MAX) * (max_value - min_value) ); 142 | x += M->GetArrayValue(m_Kglobal, i->m_K, k); 143 | M->SetArrayValue(m_Kglobal, i->m_K, k, x); 144 | } 145 | } 146 | } 147 | } 148 | 149 | ///////////////////////////////////////////////////////////////////////////////// 150 | 151 | template void MRFEnergy::CompleteGraphConstruction() 152 | { 153 | Node* i; 154 | Node* j; 155 | MRFEdge* e; 156 | MRFEdge* ePrev; 157 | 158 | if (m_isEnergyConstructionCompleted) 159 | { 160 | m_errorFn("Fatal error in CompleteGraphConstruction"); 161 | } 162 | 163 | printf("Completing graph construction... "); 164 | 165 | if (m_buf) 166 | { 167 | m_errorFn("CompleteGraphConstruction(): fatal error"); 168 | } 169 | 170 | m_buf = (char *) Malloc(m_vectorMaxSizeInBytes + 171 | ( m_vectorMaxSizeInBytes > Edge::GetBufSizeInBytes(m_vectorMaxSizeInBytes) ? 172 | m_vectorMaxSizeInBytes : Edge::GetBufSizeInBytes(m_vectorMaxSizeInBytes) ) ); 173 | 174 | // set forward and backward edges properly 175 | #ifdef _DEBUG 176 | int ordering; 177 | for (i=m_nodeFirst, ordering=0; i; i=i->m_next, ordering++) 178 | { 179 | if ( (i->m_ordering != ordering) 180 | || (i->m_ordering == 0 && i->m_prev) 181 | || (i->m_ordering != 0 && i->m_prev->m_ordering != ordering-1) ) 182 | { 183 | m_errorFn("CompleteGraphConstruction(): fatal error (wrong ordering)"); 184 | } 185 | } 186 | if (ordering != m_nodeNum) 187 | { 188 | m_errorFn("CompleteGraphConstruction(): fatal error"); 189 | } 190 | #endif 191 | for (i=m_nodeFirst; i; i=i->m_next) 192 | { 193 | i->m_firstBackward = NULL; 194 | } 195 | for (i=m_nodeFirst; i; i=i->m_next) 196 | { 197 | ePrev = NULL; 198 | for (e=i->m_firstForward; e; ) 199 | { 200 | assert(i == e->m_tail); 201 | j = e->m_head; 202 | 203 | if (i->m_ordering < j->m_ordering) 204 | { 205 | e->m_nextBackward = j->m_firstBackward; 206 | j->m_firstBackward = e; 207 | 208 | ePrev = e; 209 | e = e->m_nextForward; 210 | } 211 | else 212 | { 213 | e->m_message.Swap(m_Kglobal, i->m_K, j->m_K); 214 | e->m_tail = j; 215 | e->m_head = i; 216 | 217 | MRFEdge* eNext = e->m_nextForward; 218 | 219 | if (ePrev) 220 | { 221 | ePrev->m_nextForward = e->m_nextForward; 222 | } 223 | else 224 | { 225 | i->m_firstForward = e->m_nextForward; 226 | } 227 | 228 | e->m_nextForward = j->m_firstForward; 229 | j->m_firstForward = e; 230 | 231 | e->m_nextBackward = i->m_firstBackward; 232 | i->m_firstBackward = e; 233 | 234 | e = eNext; 235 | } 236 | } 237 | } 238 | 239 | m_isEnergyConstructionCompleted = true; 240 | 241 | // ZeroMessages(); 242 | 243 | printf("done\n"); 244 | } 245 | -------------------------------------------------------------------------------- /StructureFinder.cpp: -------------------------------------------------------------------------------- 1 | #include "StructureFinder.h" 2 | 3 | #include 4 | #include