├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LoGMatrix.m ├── README.md ├── ambient_occlusion.cpp ├── bruce_model.jpg ├── common.h ├── cost_functions.h ├── defs.h ├── explicit.cpp ├── faceshapefromshading.cpp ├── faceshapefromshading_exp.cpp ├── intel-compiler.cmake ├── ptrender.cpp ├── refine_mesh_with_normal.cpp ├── refine_mesh_with_normal_exp.cpp ├── settings.txt ├── tests ├── CMakeLists.txt ├── test_fbo.cpp └── test_transfer_color.cpp └── utils.h /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "MultilinearReconstruction"] 2 | path = MultilinearReconstruction 3 | url = https://github.com/phg1024/MultilinearReconstruction.git 4 | [submodule "Catch"] 5 | path = Catch 6 | url = https://github.com/philsquared/Catch.git 7 | [submodule "json"] 8 | path = json 9 | url = ../../nlohmann/json.git 10 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | 3 | project(FaceShapeFromShading) 4 | 5 | # Find includes in corresponding build directories 6 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 7 | # Instruct CMake to run moc automatically when needed. 8 | set(CMAKE_AUTOMOC ON) 9 | 10 | include(CheckCXXCompilerFlag) 11 | CHECK_CXX_COMPILER_FLAG("-std=c++14" COMPILER_SUPPORTS_CXX14) 12 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 13 | CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) 14 | if(COMPILER_SUPPORTS_CXX14) 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") 16 | message("Using c++11") 17 | elseif(COMPILER_SUPPORTS_CXX11) 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 19 | message("Using c++11") 20 | elseif(COMPILER_SUPPORTS_CXX0X) 21 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") 22 | message("Using c++0x") 23 | else() 24 | message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 25 | endif() 26 | 27 | # Boost 28 | find_package(Boost COMPONENTS filesystem timer program_options REQUIRED) 29 | include_directories(${Boost_INCLUDE_DIRS}) 30 | link_libraries(${Boost_LIBRARIES} -lboost_filesystem -lboost_system) 31 | 32 | # OpenMP 33 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") 34 | 35 | find_package( OpenCV REQUIRED ) 36 | link_libraries(${OpenCV_LIBS}) 37 | 38 | # OpenGL 39 | find_package(OpenGL REQUIRED) 40 | find_package(GLUT REQUIRED) 41 | include_directories( ${OPENGL_INCLUDE_DIRS} ${GLUT_INCLUDE_DIRS} ) 42 | link_libraries(${OPENGL_LIBRARIES} ${GLUT_LIBRARY}) 43 | 44 | find_package(GLEW REQUIRED) 45 | if (GLEW_FOUND) 46 | include_directories(${GLEW_INCLUDE_DIRS}) 47 | link_libraries(${GLEW_LIBRARIES}) 48 | endif() 49 | 50 | # GLM 51 | find_package(glm REQUIRED) 52 | if (GLM_FOUND) 53 | message([] ${GLM_INCLUDE_DIRS}) 54 | include_directories(${GLM_INCLUDE_DIRS}) 55 | endif() 56 | 57 | # GLI 58 | find_package(gli REQUIRED) 59 | if (GLI_FOUND) 60 | message([] ${GLI_INCLUDE_DIRS}) 61 | include_directories(${GLI_INCLUDE_DIRS}) 62 | endif() 63 | 64 | # SuiteSparse 65 | set(SUITESPARSE_INCLUDE_DIR /usr/include/suitesparse) 66 | include_directories(${SUITESPARSE_INCLUDE_DIR}) 67 | 68 | # Eigen 69 | find_package(Eigen) 70 | include_directories(${EIGEN_INCLUDE_DIR}) 71 | 72 | # MKL 73 | set(MKL_INCLUDE_DIRS "/opt/intel/mkl/include") 74 | message([] ${MKL_INCLUDE_DIRS} ) 75 | include_directories(${MKL_INCLUDE_DIRS}) 76 | set(MKLROOT "/opt/intel/mkl") 77 | set(MKLLIBS_DIRS "${MKLROOT}/lib/intel64/" "/opt/intel/lib/intel64_lin") 78 | link_directories(${MKLLIBS_DIRS}) 79 | set(MKLLIBS "-Wl,--start-group -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -Wl,--end-group -liomp5 -ldl -lpthread -lm") 80 | 81 | # PhGLib 82 | include_directories("$ENV{HOME}/SDKs/PhGLib/include") 83 | link_directories("$ENV{HOME}/SDKs/PhGLib/lib") 84 | set(PhGLib "-lPhGLib") 85 | 86 | # Qt5 87 | find_package(Qt5Core) 88 | find_package(Qt5Widgets) 89 | find_package(Qt5OpenGL) 90 | 91 | # Ceres solver 92 | find_package(Ceres REQUIRED) 93 | include_directories(${CERES_INCLUDE_DIRS}) 94 | link_libraries(${CERES_LIBRARIES}) 95 | 96 | # embree 97 | include_directories("/usr/local/include") 98 | link_libraries("/usr/local/lib/libembree.so.2.13.0") 99 | 100 | # json 101 | message(STATUS "Found json cpp") 102 | message(STATUS "${CMAKE_CURRENT_LIST_DIR}/json/include") 103 | include_directories("${CMAKE_CURRENT_LIST_DIR}/json/include") 104 | 105 | # Face shape from shading program 106 | add_executable(FaceShapeFromShading faceshapefromshading.cpp common.h MultilinearReconstruction/OffscreenMeshVisualizer.cpp MultilinearReconstruction/OffscreenMeshVisualizer.h utils.h) 107 | target_link_libraries(FaceShapeFromShading 108 | multilinearmodel 109 | basicmesh 110 | tensor 111 | ioutilities 112 | Qt5::Core 113 | Qt5::Widgets 114 | Qt5::OpenGL 115 | ${MKLLIBS} 116 | ${PhGLib}) 117 | 118 | # Face shape from shading using blendshapes program 119 | add_executable(FaceShapeFromShading_exp faceshapefromshading_exp.cpp common.h MultilinearReconstruction/OffscreenMeshVisualizer.cpp MultilinearReconstruction/OffscreenMeshVisualizer.h utils.h) 120 | target_link_libraries(FaceShapeFromShading_exp 121 | multilinearmodel 122 | basicmesh 123 | tensor 124 | ioutilities 125 | Qt5::Core 126 | Qt5::Widgets 127 | Qt5::OpenGL 128 | ${MKLLIBS} 129 | ${PhGLib}) 130 | 131 | add_executable(refine_mesh_with_normal refine_mesh_with_normal.cpp common.h MultilinearReconstruction/OffscreenMeshVisualizer.cpp MultilinearReconstruction/OffscreenMeshVisualizer.h) 132 | target_link_libraries(refine_mesh_with_normal 133 | multilinearmodel 134 | basicmesh 135 | tensor 136 | ioutilities 137 | Qt5::Core 138 | Qt5::Widgets 139 | Qt5::OpenGL 140 | ${MKLLIBS} 141 | ${PhGLib}) 142 | 143 | add_executable(refine_mesh_with_normal_exp refine_mesh_with_normal_exp.cpp common.h MultilinearReconstruction/OffscreenMeshVisualizer.cpp MultilinearReconstruction/OffscreenMeshVisualizer.h) 144 | target_link_libraries(refine_mesh_with_normal_exp 145 | multilinearmodel 146 | basicmesh 147 | tensor 148 | ioutilities 149 | Qt5::Core 150 | Qt5::Widgets 151 | Qt5::OpenGL 152 | ${MKLLIBS} 153 | ${PhGLib}) 154 | add_executable(ptrender ptrender.cpp) 155 | 156 | add_executable(explicit explicit.cpp) 157 | target_link_libraries(explicit CGAL basicmesh ioutilities ${PhGLib}) 158 | 159 | add_executable(ambient_occlusion ambient_occlusion.cpp) 160 | target_link_libraries(ambient_occlusion 161 | basicmesh 162 | offscreenmeshvisualizer 163 | ioutilities 164 | Qt5::Core 165 | Qt5::Widgets 166 | Qt5::OpenGL 167 | ${PhGLib}) 168 | 169 | link_directories(MultilinearReconstruction) 170 | 171 | add_subdirectory(MultilinearReconstruction) 172 | add_subdirectory(tests) 173 | -------------------------------------------------------------------------------- /LoGMatrix.m: -------------------------------------------------------------------------------- 1 | function M = LoGMatrix(w, sigma) 2 | M = zeros(w*2+1, w*2+1); 3 | for y=-w:w 4 | yy = y+w+1; 5 | for x=-w:w 6 | xx = x+w+1; 7 | val = exp(-(x*x+y*y)/(2*sigma*sigma)); 8 | M(yy, xx) = val; 9 | end 10 | end 11 | 12 | M = M ./ sum(sum(M)); 13 | 14 | for y=-w:w 15 | yy = y+w+1; 16 | for x=-w:w 17 | xx = x+w+1; 18 | M(yy, xx) = M(yy, xx) * (x*x+y*y - 2*sigma*sigma); 19 | end 20 | end 21 | M = M - sum(sum(M))/(size(M, 1)*size(M,2)); 22 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face Shape from Shading 2 | Yet another implementation of shape from shading for human face. 3 | 4 | ## Example 5 | ![Example](https://github.com/phg1024/FaceShapeFromShading/blob/master/bruce_model.jpg) 6 | 7 | ## Dependencies 8 | * Boost 1.63 9 | * freeglut 10 | * GLEW 11 | * glm 12 | * gli 13 | * SuiteSparse 4.5.3 14 | * Eigen 3.3.3 15 | * Intel MKL 16 | * Qt5 17 | * ceres solver 1.12.0 18 | * Intel embree 19 | * PhGLib 20 | 21 | ## Compile 22 | ```bash 23 | git clone --recursive https://github.com/phg1024/FaceShapeFromShading.git 24 | cd FaceShapeFromShading 25 | mkdir build 26 | cd build 27 | cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=icc -DCMAKE_CXX_COMPILER=icpc 28 | make -j8 29 | ``` 30 | -------------------------------------------------------------------------------- /ambient_occlusion.cpp: -------------------------------------------------------------------------------- 1 | #include "Geometry/geometryutils.hpp" 2 | #include "Utils/utility.hpp" 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include "common.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #ifdef __llvm__ 23 | double omp_get_wtime() { return 1; } 24 | int omp_get_max_threads() { return 1; } 25 | int omp_get_thread_num() { return 1; } 26 | #else 27 | #include 28 | #endif 29 | 30 | using namespace std; 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include "defs.h" 41 | #include "utils.h" 42 | 43 | // http://www.altdevblogaday.com/2012/05/03/generating-uniformly-distributed-points-on-sphere/ 44 | void random_direction(float* result) 45 | { 46 | float z = 2.0f * rand() / static_cast(RAND_MAX) - 1.0f; 47 | float t = 2.0f * rand() / static_cast(RAND_MAX) * 3.14f; 48 | float r = sqrt(1.0f - z * z); 49 | result[0] = r * cos(t); 50 | result[1] = r * sin(t); 51 | result[2] = z; 52 | 53 | //cout << result[0] << ' ' << result[1] << ' ' << result[2] << endl; 54 | } 55 | 56 | void raytrace(const char* meshobj, const char* resultpng, 57 | int nsamples = 128, int tex_size_in = 2048) 58 | { 59 | // Intel says to do this, so we're doing it. 60 | _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); 61 | _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); 62 | 63 | // Load the mesh. 64 | BasicMesh mesh; 65 | mesh.LoadOBJMesh(meshobj); 66 | mesh.ComputeNormals(); 67 | mesh.BuildHalfEdgeMesh(); 68 | mesh.Subdivide(); 69 | mesh.ComputeNormals(); 70 | 71 | // Create the embree device and scene. 72 | RTCDevice device = rtcNewDevice(NULL); 73 | assert(device && "Unable to create embree device."); 74 | RTCScene scene = rtcDeviceNewScene(device, RTC_SCENE_STATIC | RTC_SCENE_HIGH_QUALITY, 75 | RTC_INTERSECT1); 76 | assert(scene); 77 | 78 | // Populate the embree mesh. 79 | uint32_t gid = rtcNewTriangleMesh(scene, RTC_GEOMETRY_STATIC, 80 | mesh.NumFaces(), mesh.NumVertices()); 81 | float* vertices = (float*) rtcMapBuffer(scene, gid, RTC_VERTEX_BUFFER); 82 | for (size_t i = 0; i < mesh.NumVertices(); i++) { 83 | *vertices++ = mesh.vertex(i)[0]; 84 | *vertices++ = mesh.vertex(i)[1]; 85 | *vertices++ = mesh.vertex(i)[2]; 86 | vertices++; 87 | } 88 | rtcUnmapBuffer(scene, gid, RTC_VERTEX_BUFFER); 89 | 90 | uint32_t* triangles = (uint32_t*) rtcMapBuffer(scene, gid, RTC_INDEX_BUFFER); 91 | for (size_t i = 0; i < mesh.NumFaces(); i++) { 92 | *triangles++ = static_cast(mesh.face(i)[0]); 93 | *triangles++ = static_cast(mesh.face(i)[1]); 94 | *triangles++ = static_cast(mesh.face(i)[2]); 95 | } 96 | rtcUnmapBuffer(scene, gid, RTC_INDEX_BUFFER); 97 | rtcCommit(scene); 98 | 99 | // Load the triangle indices map and barycentric coordinates map 100 | const int tex_size = tex_size_in; 101 | const string albedo_index_map_filename("/home/phg/Data/Multilinear/albedo_index.png"); 102 | const string albedo_pixel_map_filename("/home/phg/Data/Multilinear/albedo_pixel.png"); 103 | const string valid_faces_indices_filename("/home/phg/Data/Multilinear/face_region_indices.txt"); 104 | 105 | QImage albedo_index_map = GetIndexMap(albedo_index_map_filename, mesh, true, tex_size); 106 | 107 | vector> albedo_pixel_map; 108 | QImage pixel_map_image; 109 | tie(pixel_map_image, albedo_pixel_map) = GetPixelCoordinatesMap(albedo_pixel_map_filename, 110 | albedo_index_map, 111 | mesh, 112 | false, 113 | tex_size); 114 | 115 | #if 0 116 | auto valid_faces_indices_quad = LoadIndices(valid_faces_indices_filename); 117 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 118 | vector valid_faces_indices; 119 | for(auto fidx : valid_faces_indices_quad) { 120 | valid_faces_indices.push_back(fidx*2); 121 | valid_faces_indices.push_back(fidx*2+1); 122 | } 123 | 124 | vector valid_faces_flag(mesh.NumFaces(), false); 125 | for(auto fidx : valid_faces_indices) valid_faces_flag[fidx] = true; 126 | #else 127 | vector valid_faces_flag(mesh.NumFaces(), true); 128 | #endif 129 | 130 | // Iterate over each pixel in the light map, row by row. 131 | printf("Rendering ambient occlusion (%d threads)...\n", 132 | omp_get_max_threads()); 133 | double begintime = omp_get_wtime(); 134 | vector results(tex_size*tex_size, 0); 135 | vector normals(tex_size*tex_size*3, 0); 136 | vector positions(tex_size*tex_size*3, 0); 137 | 138 | const uint32_t npixels = tex_size*tex_size; 139 | const float E = 0.00001f; 140 | 141 | srand(time(NULL)); 142 | 143 | vector> dirs(nsamples, vector(3, 0)); 144 | #ifdef EVEN_SAMPLING 145 | const int hstep = sqrt(nsamples*2); 146 | const int vstep = hstep / 2; 147 | for(int vi = 0, di = 0; vi < vstep; ++vi) { 148 | double phi = vi / static_cast(vstep - 1) * 3.1415926; 149 | for(int hi = 0; hi < hstep; ++hi, ++di) { 150 | double theta = hi / static_cast(hstep) * 3.1415926 * 2.0; 151 | dirs[di][0] = cos(theta)*cos(phi); 152 | dirs[di][1] = sin(theta)*cos(phi); 153 | dirs[di][2] = sin(phi); 154 | } 155 | } 156 | #else 157 | for(int i=0;i 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | using namespace std; 19 | 20 | #ifndef MKL_BLAS 21 | #define MKL_BLAS MKL_DOMAIN_BLAS 22 | #endif 23 | 24 | #define EIGEN_USE_MKL_ALL 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | using namespace Eigen; 32 | 33 | #include "glm/glm.hpp" 34 | #include "gli/gli.hpp" 35 | #include "glm/gtx/norm.hpp" 36 | 37 | #endif //FACESHAPEFROMSHADING_COMMON_H_H 38 | -------------------------------------------------------------------------------- /cost_functions.h: -------------------------------------------------------------------------------- 1 | #ifndef FACESHAPEFROMSHADING_COST_FUNCTIONS_H 2 | #define FACESHAPEFROMSHADING_COST_FUNCTIONS_H 3 | 4 | #include 5 | 6 | #include "ceres/ceres.h" 7 | #include "utils.h" 8 | 9 | struct DepthMapDataTerm { 10 | DepthMapDataTerm(double Ir, double Ig, double Ib, 11 | double ar, double ag, double ab, 12 | const VectorXd& lighting_coeffs, 13 | double dx, double dy, 14 | double weight = 1.0) 15 | : I(Vector3d(Ir, Ig, Ib)), a(Vector3d(ar, ag, ab)), 16 | lighting_coeffs(lighting_coeffs), dx(dx), dy(dy), weight(weight) {} 17 | 18 | ~DepthMapDataTerm() {} 19 | 20 | bool operator()(double const * const * parameters, double *residuals) const { 21 | double z = parameters[0][0], z_l = parameters[1][0], z_u = parameters[2][0]; 22 | 23 | double p = (z - z_l) / dx; 24 | double q = (z_u - z) / dy; 25 | 26 | double nx, ny, nz; 27 | 28 | double N = p * p + q * q + 1; 29 | nx = p / N; ny = q / N; nz = 1 / N; 30 | 31 | VectorXd Y = sphericalharmonics(nx, ny, nz); 32 | 33 | double LdotY = lighting_coeffs.transpose() * Y; 34 | 35 | Vector3d f = I - a * LdotY; 36 | 37 | residuals[0] = f[0] * weight; 38 | residuals[1] = f[1] * weight; 39 | residuals[2] = f[2] * weight; 40 | 41 | return true; 42 | } 43 | 44 | Vector3d I, a; 45 | VectorXd lighting_coeffs; 46 | double dx, dy; 47 | double weight; 48 | }; 49 | 50 | struct DepthMapIntegrabilityTerm { 51 | DepthMapIntegrabilityTerm(double dx, double dy, double weight = 1.0) 52 | : dx(dx), dy(dy), weight(weight) {} 53 | 54 | ~DepthMapIntegrabilityTerm() {} 55 | 56 | bool operator()(double const * const * parameters, double *residuals) const { 57 | double z = parameters[0][0], z_l = parameters[1][0]; 58 | double z_u = parameters[2][0], z_ul = parameters[3][0]; 59 | double z_uu = parameters[4][0], z_ll = parameters[5][0]; 60 | 61 | const double epsilon = 1e-16; 62 | 63 | auto get_normal = [&epsilon](double z, double z_u, double z_l, double dx, double dy) { 64 | double p = (z - z_l) / dx; 65 | double q = (z_u - z) / dy; 66 | 67 | double nx, ny, nz; 68 | 69 | double N = p * p + q * q + 1; 70 | 71 | nx = p / N; ny = q / N; nz = 1 / N; 72 | return make_tuple(nx, ny, nz); 73 | }; 74 | 75 | double nx, ny, nz; tie(nx, ny, nz) = get_normal(z, z_u, z_l, dx, dy); 76 | double nx_l, ny_l, nz_l; tie(nx_l, ny_l, nz_l) = get_normal(z_l, z_ul, z_ll, dx, dy); 77 | double nx_u, ny_u, nz_u; tie(nx_u, ny_u, nz_u) = get_normal(z_u, z_uu, z_ul, dx, dy); 78 | 79 | double denom = nz * nz_u * nz_l; 80 | double int_val = (nx_u * nz * nz_l - nx * nz_l * nz_u) - (ny * nz_l * nz_u - ny_l * nz * nz_u); 81 | 82 | if( fabs(denom) < epsilon) denom = denom>0?epsilon:-epsilon; 83 | int_val = int_val / epsilon; 84 | 85 | residuals[0] = int_val * weight; 86 | 87 | return true; 88 | } 89 | 90 | double dx, dy; 91 | double weight; 92 | }; 93 | 94 | struct DepthMapRegularizationTerm { 95 | DepthMapRegularizationTerm(const vector>& info, 96 | double z_ref_LoG, 97 | double weight) 98 | : info(info), z_ref_LoG(z_ref_LoG), weight(weight) {} 99 | 100 | bool operator()(double const * const * parameters, double *residuals) const { 101 | double z_LoG = 0; 102 | 103 | for(int i=0;i> info; 118 | double z_ref_LoG; 119 | double weight; 120 | }; 121 | 122 | struct NormalMapDataTerm { 123 | NormalMapDataTerm(double Ir, double Ig, double Ib, 124 | double ar, double ag, double ab, 125 | const VectorXd& lighting_coeffs, 126 | double weight = 1.0) 127 | : I(Vector3d(Ir, Ig, Ib)), a(Vector3d(ar, ag, ab)), 128 | lighting_coeffs(lighting_coeffs), weight(weight) {} 129 | 130 | ~NormalMapDataTerm() {} 131 | 132 | bool operator()(double const * const * parameters, double *residuals) const { 133 | double theta = parameters[0][0], phi = parameters[1][0]; 134 | 135 | double nx, ny, nz; 136 | tie(nx, ny, nz) = sphericalcoords2normal(theta, phi); 137 | 138 | VectorXd Y = sphericalharmonics(nx, ny, nz); 139 | 140 | double LdotY = lighting_coeffs.transpose() * Y; 141 | 142 | Vector3d f = I - a * LdotY; 143 | 144 | residuals[0] = f[0] * weight; 145 | residuals[1] = f[1] * weight; 146 | residuals[2] = f[2] * weight; 147 | 148 | return true; 149 | } 150 | 151 | Vector3d I, a; 152 | VectorXd lighting_coeffs; 153 | double weight; 154 | }; 155 | 156 | struct NormalMapDataTerm_analytic : public ceres::CostFunction { 157 | NormalMapDataTerm_analytic(double Ir, double Ig, double Ib, 158 | double ar, double ag, double ab, 159 | const VectorXd& lighting_coeffs, 160 | double weight = 1.0) 161 | : I(Vector3d(Ir, Ig, Ib)), a(Vector3d(ar, ag, ab)), lighting_coeffs(lighting_coeffs), weight(weight) { 162 | 163 | mutable_parameter_block_sizes()->clear(); 164 | mutable_parameter_block_sizes()->push_back(1); 165 | mutable_parameter_block_sizes()->push_back(1); 166 | set_num_residuals(3); 167 | } 168 | 169 | virtual bool Evaluate(double const *const *parameters, 170 | double *residuals, 171 | double **jacobians) const 172 | { 173 | const int num_dof = 9; 174 | 175 | double theta = parameters[0][0], phi = parameters[1][0]; 176 | 177 | double cosTheta = cos(theta), sinTheta = sin(theta); 178 | double cosPhi = cos(phi), sinPhi = sin(phi); 179 | 180 | double nx, ny, nz; 181 | tie(nx, ny, nz) = sphericalcoords2normal(theta, phi); 182 | 183 | VectorXd Y = sphericalharmonics(nx, ny, nz); 184 | 185 | double LdotY = lighting_coeffs.transpose() * Y; 186 | 187 | Vector3d f = I - a * LdotY; 188 | 189 | residuals[0] = f[0] * weight; 190 | residuals[1] = f[1] * weight; 191 | residuals[2] = f[2] * weight; 192 | 193 | if (jacobians != NULL) { 194 | assert(jacobians[0] != NULL); 195 | assert(jacobians[1] != NULL); 196 | 197 | MatrixXd dYdnormal = dY_dnormal(nx, ny, nz); 198 | 199 | VectorXd dYdtheta = dYdnormal * dnormal_dtheta(theta, phi); 200 | VectorXd dYdphi = dYdnormal * dnormal_dphi(theta, phi); 201 | 202 | double LdotdYdtheta = lighting_coeffs.transpose() * dYdtheta; 203 | double LdotdYdphi = lighting_coeffs.transpose() * dYdphi; 204 | 205 | // jacobians[0][i] = \frac{\partial E}{\partial \theta} 206 | jacobians[0][0] = -a[0] * LdotdYdtheta * weight; 207 | jacobians[0][1] = -a[1] * LdotdYdtheta * weight; 208 | jacobians[0][2] = -a[2] * LdotdYdtheta * weight; 209 | 210 | // jacobians[1][i] = \frac{\partial E}{\partial \phi} 211 | jacobians[1][0] = -a[0] * LdotdYdphi * weight; 212 | jacobians[1][1] = -a[1] * LdotdYdphi * weight; 213 | jacobians[1][2] = -a[2] * LdotdYdphi * weight; 214 | } 215 | return true; 216 | } 217 | 218 | Vector3d I, a; 219 | VectorXd lighting_coeffs; 220 | double weight; 221 | }; 222 | 223 | struct NormalMapIntegrabilityTerm { 224 | NormalMapIntegrabilityTerm(double dx, double dy, double weight) : dx(dx), dy(dy), weight(weight) {} 225 | 226 | double safe_division(double numer, double denom, double eps) const { 227 | if(fabs(denom) < eps) { 228 | denom = (denom<0)?-eps:eps; 229 | } 230 | return numer / denom; 231 | } 232 | 233 | double round_off(double val, double eps) const { 234 | if(fabs(val) < eps) { 235 | if(val < 0) return -eps; 236 | else return eps; 237 | } else return val; 238 | } 239 | 240 | bool operator()(double const * const * parameters, double *residuals) const { 241 | double theta = parameters[0][0], phi = parameters[1][0]; 242 | double theta_l = parameters[2][0], phi_l = parameters[3][0]; 243 | double theta_u = parameters[4][0], phi_u = parameters[5][0]; 244 | 245 | double tan_theta = tan(theta), sin_phi = sin(phi), cos_phi = cos(phi); 246 | double tan_theta_l = tan(theta_l), sin_phi_l = sin(phi_l), cos_phi_l = cos(phi_l); 247 | double tan_theta_u = tan(theta_u), sin_phi_u = sin(phi_u), cos_phi_u = cos(phi_u); 248 | 249 | if(weight == 0) { 250 | residuals[0] = 0; 251 | } else { 252 | if(fabs(theta) < 1e-3 && fabs(phi) < 1e-3) { 253 | // singular point 254 | residuals[0] = ((phi - phi_l) + (phi_u - phi) + (theta - theta_l) + (theta_u - theta)) * weight; 255 | } else { 256 | residuals[0] = ((tan_theta_u * sin_phi_u - tan_theta * sin_phi) - 257 | (tan_theta * cos_phi - tan_theta_l * cos_phi_l)) * weight; 258 | } 259 | } 260 | 261 | return true; 262 | } 263 | 264 | double dx, dy; 265 | double weight; 266 | }; 267 | 268 | struct NormalMapIntegrabilityTerm_analytic : public ceres::CostFunction { 269 | NormalMapIntegrabilityTerm_analytic(double dx, double dy, double weight) : dx(dx), dy(dy), weight(weight) { 270 | mutable_parameter_block_sizes()->clear(); 271 | for(int param_i=0;param_i<6;++param_i) 272 | mutable_parameter_block_sizes()->push_back(1); 273 | set_num_residuals(1); 274 | } 275 | 276 | double round_off(double val, double eps) const { 277 | if(fabs(val) < eps) { 278 | if(val < 0) return -eps; 279 | else return eps; 280 | } else return val; 281 | } 282 | 283 | virtual bool Evaluate(double const *const *parameters, 284 | double *residuals, 285 | double **jacobians) const 286 | { 287 | double theta = parameters[0][0], phi = parameters[1][0]; 288 | double theta_l = parameters[2][0], phi_l = parameters[3][0]; 289 | double theta_u = parameters[4][0], phi_u = parameters[5][0]; 290 | 291 | double tan_theta = tan(theta), sin_phi = sin(phi), cos_phi = cos(phi); 292 | double tan_theta_l = tan(theta_l), sin_phi_l = sin(phi_l), cos_phi_l = cos(phi_l); 293 | double tan_theta_u = tan(theta_u), sin_phi_u = sin(phi_u), cos_phi_u = cos(phi_u); 294 | 295 | if(weight == 0) { 296 | residuals[0] = 0; 297 | if(jacobians != NULL) { 298 | for(int param_i=0;param_i<6;++param_i) { 299 | if(jacobians[param_i] != NULL) jacobians[param_i][0] = 0; 300 | } 301 | } 302 | } else { 303 | if(fabs(theta) < 1e-3 && fabs(phi) < 1e-3) { 304 | // singular point 305 | residuals[0] = ((phi - phi_l) + (phi_u - phi) + (theta - theta_l) + (theta_u - theta)) * weight; 306 | if(jacobians != NULL) { 307 | for(int param_i=0;param_i<6;++param_i) assert(jacobians[param_i] != NULL); 308 | jacobians[0][0] = 0; 309 | jacobians[1][0] = 0; 310 | jacobians[2][0] = -1; 311 | jacobians[3][0] = -1; 312 | jacobians[4][0] = 1; 313 | jacobians[5][0] = 1; 314 | } 315 | } else { 316 | residuals[0] = ((tan_theta_u * sin_phi_u - tan_theta * sin_phi) - 317 | (tan_theta * cos_phi - tan_theta_l * cos_phi_l)) * weight; 318 | 319 | residuals[0] = ((tan_theta_u * sin_phi_u - tan_theta * sin_phi) - 320 | (tan_theta * cos_phi - tan_theta_l * cos_phi_l)) * weight; 321 | if(jacobians != NULL) { 322 | for(int param_i=0;param_i<6;++param_i) assert(jacobians[param_i] != NULL); 323 | 324 | { 325 | double dEdtheta = -(sin_phi + cos_phi) * 2.0 / max(cos(theta*2.0) + 1.0, 1e-16); 326 | // jacobians[0][0] = \frac{\partial E}{\partial \theta} 327 | jacobians[0][0] = dEdtheta * weight; 328 | 329 | double dEdphi = -tan_theta * (cos_phi - sin_phi); 330 | // jacobians[1][0] = \frac{\partial E}{\partial \phi} 331 | jacobians[1][0] = dEdphi * weight; 332 | } 333 | 334 | { 335 | double dEdtheta = cos_phi_l * 2.0 / max(cos(theta_l*2) + 1.0, 1e-16); 336 | // jacobians[2][0] = \frac{\partial E}{\partial \theta_l} 337 | jacobians[2][0] = dEdtheta * weight; 338 | 339 | double dEdphi = -tan_theta_l * sin_phi_l; 340 | // jacobians[3][0] = \frac{\partial E}{\partial \phi_l} 341 | jacobians[3][0] = dEdphi * weight; 342 | } 343 | 344 | { 345 | double dEdtheta = sin_phi_u * 2.0 / max(cos(theta_u*2.0) + 1.0, 1e-16); 346 | // jacobians[4][0] = \frac{\partial E}{\partial \theta_u} 347 | jacobians[4][0] = dEdtheta * weight; 348 | 349 | double dEdphi = cos_phi_u * tan_theta_u; 350 | // jacobians[5][0] = \frac{\partial E}{\partial \phi_u} 351 | jacobians[5][0] = dEdphi * weight; 352 | } 353 | } 354 | } 355 | } 356 | 357 | return true; 358 | } 359 | 360 | double dx, dy; 361 | double weight; 362 | }; 363 | 364 | struct NormalMapSmoothnessTerm { 365 | NormalMapSmoothnessTerm(double dx, double dy, double weight) : dx(dx), dy(dy), weight(weight) {} 366 | 367 | double safe_division(double numer, double denom, double eps) const { 368 | if(fabs(denom) < eps) { 369 | denom = (denom<0)?-eps:eps; 370 | } 371 | return numer / denom; 372 | } 373 | 374 | double round_off(double val, double eps) const { 375 | if(fabs(val) < eps) { 376 | if(val < 0) return -eps; 377 | else return eps; 378 | } else return val; 379 | } 380 | 381 | bool operator()(double const * const * parameters, double *residuals) const { 382 | double theta = parameters[0][0], phi = parameters[1][0]; 383 | double theta_l = parameters[2][0], phi_l = parameters[3][0]; 384 | double theta_u = parameters[4][0], phi_u = parameters[5][0]; 385 | 386 | double nx, ny, nz; 387 | tie(nx, ny, nz) = sphericalcoords2normal(theta, phi); 388 | 389 | double nx_l, ny_l, nz_l; 390 | tie(nx_l, ny_l, nz_l) = sphericalcoords2normal(theta_l, phi_l); 391 | 392 | double nx_u, ny_u, nz_u; 393 | tie(nx_u, ny_u, nz_u) = sphericalcoords2normal(theta_u, phi_u); 394 | 395 | if(weight == 0) { 396 | residuals[0] = 0; 397 | residuals[1] = 0; 398 | residuals[2] = 0; 399 | residuals[3] = 0; 400 | } else { 401 | nz = round_off(nz, 1e-16); 402 | nz_l = round_off(nz_l, 1e-16); 403 | nz_u = round_off(nz_u, 1e-16); 404 | 405 | residuals[0] = (nx_u * nz - nx * nz_u) * weight; 406 | residuals[1] = (ny_u * nz - ny * nz_u) * weight; 407 | residuals[2] = (nx_l * nz - nx * nz_l) * weight; 408 | residuals[3] = (ny_l * nz - ny * nz_l) * weight; 409 | } 410 | 411 | return true; 412 | } 413 | 414 | double dx, dy; 415 | double weight; 416 | }; 417 | 418 | struct NormalMapRegularizationTerm { 419 | NormalMapRegularizationTerm(const vector>& info, 420 | const Vector3d& normal_ref_LoG, 421 | double weight) 422 | : info(info), normal_ref_LoG(normal_ref_LoG), weight(weight) {} 423 | 424 | bool operator()(double const * const * parameters, double *residuals) const { 425 | Vector3d normal_LoG(0, 0, 0); 426 | 427 | for(int i=0;i(theta, phi); 435 | 436 | normal_LoG += Vector3d(nx, ny, nz) * kval; 437 | } 438 | 439 | residuals[0] = (normal_LoG(0) - normal_ref_LoG(0)) * weight; 440 | residuals[1] = (normal_LoG(1) - normal_ref_LoG(1)) * weight; 441 | residuals[2] = (normal_LoG(2) - normal_ref_LoG(2)) * weight; 442 | 443 | return true; 444 | } 445 | 446 | vector> info; 447 | Vector3d normal_ref_LoG; 448 | double weight; 449 | }; 450 | 451 | struct NormalMapRegularizationTerm_analytic : public ceres::CostFunction { 452 | NormalMapRegularizationTerm_analytic(const vector>& info, 453 | const Vector3d& normal_ref_LoG, 454 | double weight) 455 | : info(info), normal_ref_LoG(normal_ref_LoG), weight(weight) { 456 | mutable_parameter_block_sizes()->clear(); 457 | for(int i=0;ipush_back(1); 458 | set_num_residuals(3); 459 | } 460 | 461 | virtual bool Evaluate(double const *const *parameters, 462 | double *residuals, 463 | double **jacobians) const { 464 | Vector3d normal_LoG(0, 0, 0); 465 | 466 | for(int i=0;i(theta, phi); 474 | 475 | normal_LoG += Vector3d(nx, ny, nz) * kval; 476 | } 477 | 478 | residuals[0] = (normal_LoG(0) - normal_ref_LoG(0)) * weight; 479 | residuals[1] = (normal_LoG(1) - normal_ref_LoG(1)) * weight; 480 | residuals[2] = (normal_LoG(2) - normal_ref_LoG(2)) * weight; 481 | 482 | if (jacobians != NULL) { 483 | for(int i=0;i> info; 509 | Vector3d normal_ref_LoG; 510 | double weight; 511 | }; 512 | 513 | struct NormalMapAngleRegularizationTerm : public ceres::CostFunction { 514 | NormalMapAngleRegularizationTerm(double weight) : weight(weight) { 515 | mutable_parameter_block_sizes()->clear(); 516 | mutable_parameter_block_sizes()->push_back(1); 517 | set_num_residuals(1); 518 | } 519 | 520 | virtual bool Evaluate(double const *const *parameters, 521 | double *residuals, 522 | double **jacobians) const { 523 | residuals[0] = parameters[0][0] * weight; 524 | if (jacobians != NULL) { 525 | if( jacobians[0] != NULL ) { 526 | jacobians[0][0] = weight; 527 | } 528 | } 529 | return true; 530 | } 531 | 532 | double weight; 533 | }; 534 | 535 | #endif // FACESHAPEFROMSHADING_COST_FUNCTIONS_H 536 | -------------------------------------------------------------------------------- /defs.h: -------------------------------------------------------------------------------- 1 | #ifndef DEFS_H 2 | #define DEFS_H 3 | 4 | struct PixelInfo { 5 | PixelInfo() : fidx(-1) {} 6 | PixelInfo(int fidx, glm::vec3 bcoords) : fidx(fidx), bcoords(bcoords) {} 7 | 8 | int fidx; // trinagle index 9 | glm::vec3 bcoords; // bary centric coordinates 10 | }; 11 | 12 | struct ImageBundle { 13 | ImageBundle() {} 14 | ImageBundle(const string& filename, 15 | const QImage& image, 16 | const vector& points, 17 | const ReconstructionResult& params) 18 | : filename(filename), image(image), points(points), params(params) {} 19 | string filename; 20 | QImage image; 21 | vector points; 22 | ReconstructionResult params; 23 | }; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /explicit.cpp: -------------------------------------------------------------------------------- 1 | #include // smallpt, a Path Tracer by Kevin Beason, 2008 2 | #include // Make : g++ -O3 -fopenmp smallpt.cpp -o smallpt 3 | #include // Remove "-fopenmp" for g++ version < 4.2 4 | 5 | #include 6 | #include "Geometry/geometryutils.hpp" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | typedef CGAL::Simple_cartesian K; 14 | typedef K::FT FT; 15 | typedef K::Line_3 Line; 16 | typedef K::Point_3 Point; 17 | typedef K::Triangle_3 Triangle; 18 | typedef std::vector::iterator Iterator; 19 | typedef CGAL::AABB_triangle_primitive Primitive; 20 | typedef CGAL::AABB_traits AABB_triangle_traits; 21 | typedef CGAL::AABB_tree Tree; 22 | 23 | struct Vec { // Usage: time ./smallpt 5000 && xv image.ppm 24 | double x, y, z; // position, also color (r,g,b) 25 | Vec(double x_=0, double y_=0, double z_=0){ x=x_; y=y_; z=z_; } 26 | Vec operator+(const Vec &b) const { return Vec(x+b.x,y+b.y,z+b.z); } 27 | Vec operator-(const Vec &b) const { return Vec(x-b.x,y-b.y,z-b.z); } 28 | Vec operator*(double b) const { return Vec(x*b,y*b,z*b); } 29 | Vec mult(const Vec &b) const { return Vec(x*b.x,y*b.y,z*b.z); } 30 | Vec& norm(){ return *this = *this * (1/sqrt(x*x+y*y+z*z)); } 31 | double dot(const Vec &b) const { return x*b.x+y*b.y+z*b.z; } // cross: 32 | Vec operator%(Vec&b){return Vec(y*b.z-z*b.y,z*b.x-x*b.z,x*b.y-y*b.x);} 33 | friend ostream& operator<<(ostream& os, Vec v); 34 | }; 35 | inline ostream& operator<<(ostream& os, Vec v) { 36 | os << '(' << v.x << ", " << v.y << ", " << v.z << ')'; 37 | return os; 38 | } 39 | struct Ray { Vec o, d; Ray(Vec o_, Vec d_) : o(o_), d(d_) {} }; 40 | enum Refl_t { DIFF, SPEC, REFR }; // material types, used in radiance() 41 | 42 | struct Object { 43 | enum Type { 44 | SPHERE = 0, 45 | MESH, 46 | }; 47 | Object(Type t) : type(t) {} 48 | 49 | Type type; 50 | }; 51 | 52 | struct Sphere : Object{ 53 | double rad; // radius 54 | Vec p, e, c; // position, emission, color 55 | Refl_t refl; // reflection type (DIFFuse, SPECular, REFRactive) 56 | Sphere(double rad_, Vec p_, Vec e_, Vec c_, Refl_t refl_): 57 | Object(Object::SPHERE), 58 | rad(rad_), p(p_), e(e_), c(c_), refl(refl_) {} 59 | double intersect(const Ray &r) const { // returns distance, 0 if nohit 60 | Vec op = p-r.o; // Solve t^2*d.d + 2*t*(o-p).d + (o-p).(o-p)-R^2 = 0 61 | double t, eps=1e-4, b=op.dot(r.d), det=b*b-op.dot(op)+rad*rad; 62 | if (det<0) return 0; else det=sqrt(det); 63 | return (t=b-det)>eps ? t : ((t=b+det)>eps ? t : 0); 64 | } 65 | }; 66 | 67 | Vec compute_barycentric_coordinates(Vec p, Vec q1, Vec q2, Vec q3) { 68 | Vec e23 = q3 - q2, e21 = q1 - q2; 69 | Vec d1 = q1 - p, d2 = q2 - p, d3 = q3 - p; 70 | Vec oriN = e23 % e21; 71 | Vec n = oriN; 72 | n.norm(); 73 | 74 | double invBTN = 1.0 / n.dot(oriN); 75 | Vec bcoord; 76 | bcoord.x = n.dot(d2 % d3) * invBTN; 77 | bcoord.y = n.dot(d3 % d1) * invBTN; 78 | bcoord.z = 1 - bcoord.x - bcoord.y; 79 | 80 | return bcoord; 81 | } 82 | 83 | int good_count = 0; 84 | 85 | bool rayIntersectsTriangle(Ray r, Vec v0, Vec v1, Vec v2, 86 | double& t) { 87 | Vec e1 = v1 - v0; 88 | Vec e2 = v2 - v0; 89 | Vec N = (e1 % e2).norm(); 90 | 91 | double a2 = sqrt(N.dot(N)); 92 | 93 | double NdotD = N.dot(r.d); 94 | 95 | if(fabs(NdotD) < 1e-16) { 96 | //cout << "Failed at NdotD" << endl; 97 | return false; 98 | } 99 | 100 | //cout << "N = " << N << endl; 101 | //cout << "r.o = " << r.o << endl; 102 | //cout << "v0 = " << v0 << endl; 103 | t = -N.dot(r.o - v0) / NdotD; 104 | if(t < 0) { 105 | //cout << "Failed at t<0" << endl; 106 | //t = 1000.0; 107 | return false; 108 | } 109 | 110 | //t -= min(0.5 * t, 1e-3); 111 | 112 | // compute the intersection point using equation 1 113 | Vec P = r.o + r.d * t; 114 | 115 | // Step 2: inside-outside test 116 | Vec C; // vector perpendicular to triangle's plane 117 | 118 | // edge 0 119 | Vec vp0 = P - v0; 120 | C = e1 % vp0; 121 | if (N.dot(C) < 0) { 122 | //cout << "Failed at e0" << endl; 123 | return false; // P is on the right side 124 | } 125 | 126 | // edge 1 127 | Vec e3 = v2 - v1; 128 | Vec vp1 = P - v1; 129 | C = e3 % vp1; 130 | if (N.dot(C) < 0) { 131 | //cout << "Failed at e1" << endl; 132 | return false; // P is on the right side 133 | } 134 | 135 | // edge 2 136 | Vec vp2 = P - v2; 137 | C = e2 % vp2; 138 | if (N.dot(C) > 0) { 139 | //cout << "Failed at e2" << endl; 140 | return false; // P is on the right side; 141 | } 142 | 143 | t -= 1e-3; 144 | ++good_count; 145 | return true; // this ray hits the triangle 146 | } 147 | 148 | struct Mesh : Object { 149 | Mesh() : Object(Object::MESH) {} 150 | Mesh(const string& filename) : Object(Object::MESH), mesh(filename) { 151 | mesh.ComputeNormals(); 152 | } 153 | void buildTree(double scale, Point translation) { 154 | int nfaces = mesh.NumFaces(); 155 | triangles.reserve(nfaces); 156 | face_indices_map.resize(nfaces); 157 | for(int i=0,ioffset=0;iaccelerate_distance_queries(); 171 | } 172 | 173 | bool intersect(const Ray &r, double& t, Vec& n, Vec& nl, Vec& f) const { // returns distance, 0 if nohit 174 | K::Ray_3 ray(Point(r.o.x, r.o.y, r.o.z), K::Direction_3(K::Vector_3(r.d.x, r.d.y, r.d.z))); 175 | vector hits; 176 | tree->all_intersected_primitives(ray, back_inserter(hits)); 177 | if(hits.empty()) return false; 178 | 179 | Vector3d dir(r.d.x, r.d.y, r.d.z); 180 | Vector3d ori(r.o.x, r.o.y, r.o.z); 181 | 182 | bool has_intersection = false; 183 | //cout << hits.size() << endl; 184 | double inf = 1e10; 185 | t = inf; 186 | 187 | for(int i=0;i ti ) { 209 | t = min(t, ti); 210 | auto n0 = mesh.vertex_normal(v0idx); 211 | auto n1 = mesh.vertex_normal(v1idx); 212 | auto n2 = mesh.vertex_normal(v2idx); 213 | 214 | Vector3d nn = n0 * bc.x + n1 * bc.y + n2 * bc.z; 215 | nn.normalize(); 216 | n=Vec(nn[0], nn[1], nn[2]); 217 | nl=n.dot(r.d)<0?n:n*-1; 218 | } 219 | 220 | f = Vec(0.75, 0.5, 0.5); 221 | } 222 | } 223 | 224 | return has_intersection; 225 | } 226 | 227 | BasicMesh mesh; 228 | shared_ptr tree; 229 | std::vector triangles; 230 | vector face_indices_map; 231 | }; 232 | 233 | Sphere spheres[] = {//Scene: radius, position, emission, color, material 234 | //Sphere(1e5, Vec( 1e5+1,40.8,81.6), Vec(),Vec(.75,.25,.25),DIFF),//Left 235 | //Sphere(1e5, Vec(-1e5+99,40.8,81.6),Vec(),Vec(.25,.25,.75),DIFF),//Rght 236 | //Sphere(1e5, Vec(50,40.8, 1e5), Vec(),Vec(.75,.75,.75),DIFF),//Back 237 | //Sphere(1e5, Vec(50,40.8,-1e5+170), Vec(),Vec(), DIFF),//Frnt 238 | //Sphere(1e5, Vec(50, 1e5, 81.6), Vec(),Vec(.75,.75,.75),DIFF),//Botm 239 | //Sphere(1e5, Vec(50,-1e5+81.6,81.6),Vec(),Vec(.75,.75,.75),DIFF),//Top 240 | //Sphere(600, Vec(50,681.6-.27,81.6),Vec(12,12,12), Vec(), DIFF) //Lite 241 | //Sphere(16.5,Vec(73,16.5,78), Vec(),Vec(1,.5,.5)*.999, DIFF),//Glas 242 | //Sphere(10.0, Vec(27,43,47), Vec(),Vec(.25,.25, .95)*.999, DIFF),//Mirr 243 | //Sphere(16.5,Vec(27,16.5,47), Vec(),Vec(.75,.75, .75)*.999, DIFF),//Mirr 244 | //Sphere(1e5, Vec(0,-1e5,0), Vec(),Vec(.25,0.5,0.25)*.999, DIFF), 245 | //Sphere(1e5, Vec(0,0,-1e5), Vec(),Vec(0.5, 0.5,0.25)*.999, DIFF), 246 | }; 247 | 248 | Mesh mesh; 249 | 250 | inline double clamp(double x){ return x<0 ? 0 : x>1 ? 1 : x; } 251 | inline int toInt(double x){ return int(pow(clamp(x),1/2.2)*255+.5); } 252 | inline bool intersect_spheres(const Ray &r, double &t, int &id){ 253 | double n=sizeof(spheres)/sizeof(Sphere), d, inf=t=1e20; 254 | for(int i=int(n);i--;) if((d=spheres[i].intersect(r))&&d0)?L:-0.12345; 282 | 283 | return Vec(L, L, L); 284 | } 285 | 286 | struct path_info { 287 | path_info() : valid(false) {} 288 | path_info(int idx) : valid(false), pixel_id(idx), c(Vec(1, 1, 1)) {} 289 | 290 | bool valid; 291 | Vec light_dir; 292 | int pixel_id; 293 | Vec c; 294 | Vec cfinal; 295 | }; 296 | 297 | Vec radiance(const Ray &r, int depth, unsigned short *Xi, path_info& info){ 298 | switch(render_mode) { 299 | case GlobalIllumination: { 300 | double t; // distance to intersection 301 | int id=0; // id of intersected object 302 | double mesh_t=0; 303 | 304 | Vec n, nl, f; 305 | bool intersects_mesh = mesh.intersect(r, mesh_t, n, nl, f); 306 | //bool intersects_spheres = intersect_spheres(r, t, id); 307 | bool intersects_spheres = false; 308 | 309 | //if(intersects_mesh) return (n + Vec(1, 1, 1)).mult(Vec(.5, .5, .5)); 310 | 311 | /* 312 | if ((intersects_spheres && !intersects_mesh) || (intersects_spheres && intersects_mesh && t < mesh_t)) { 313 | const Sphere &obj = spheres[id]; // the hit object 314 | Vec x=r.o+r.d*t; 315 | n=(x-obj.p).norm(), nl=n.dot(r.d)<0?n:n*-1, f=obj.c; 316 | double p = f.x>f.y && f.x>f.z ? f.x : f.y>f.z ? f.y : f.z; // max refl 317 | if (++depth>5) if (erand48(Xi).1?Vec(0,1):Vec(1))%w).norm(), v=w%u; 321 | Vec d = (u*cos(r1)*r2s + v*sin(r1)*r2s + w*sqrt(1-r2)).norm(); 322 | return obj.e + f.mult(radiance(Ray(x,d),depth,Xi, info)); 323 | } 324 | }else */if(intersects_mesh) { 325 | info.valid = true; 326 | Vec x=r.o+r.d*(mesh_t - 1e-3); 327 | double p = f.x>f.y && f.x>f.z ? f.x : f.y>f.z ? f.y : f.z; // max refl 328 | if (++depth>5) { 329 | // maximum depth is 5 330 | if (erand48(Xi).1?Vec(0,1):Vec(1))%w).norm(), v=w%u; 339 | Vec d = (u*cos(r1)*r2s + v*sin(r1)*r2s + w*sqrt(1-r2)).norm(); 340 | info.c = info.c.mult(f); 341 | return f.mult(radiance(Ray(x,d),depth,Xi, info)); 342 | } 343 | if(depth == 0) { 344 | info.valid = false; 345 | return Vec(); 346 | } else { 347 | info.valid = true; 348 | info.light_dir = r.d; 349 | return sample_light(r); 350 | } 351 | 352 | #if 0 353 | else if (obj.refl == SPEC) // Ideal SPECULAR reflection 354 | return obj.e + f.mult(radiance(Ray(x,r.d-n*2*n.dot(r.d)),depth,Xi,info)); 355 | 356 | Ray reflRay(x, r.d-n*2*n.dot(r.d)); // Ideal dielectric REFRACTION 357 | bool into = n.dot(nl)>0; // Ray from outside going in? 358 | double nc=1, nt=1.5, nnt=into?nc/nt:nt/nc, ddn=r.d.dot(nl), cos2t; 359 | if ((cos2t=1-nnt*nnt*(1-ddn*ddn))<0) // Total internal reflection 360 | return obj.e + f.mult(radiance(reflRay,depth,Xi,info)); 361 | Vec tdir = (r.d*nnt - n*((into?1:-1)*(ddn*nnt+sqrt(cos2t)))).norm(); 362 | double a=nt-nc, b=nt+nc, R0=a*a/(b*b), c = 1-(into?-ddn:tdir.dot(n)); 363 | double Re=R0+(1-R0)*c*c*c*c*c,Tr=1-Re,P=.25+.5*Re,RP=Re/P,TP=Tr/(1-P); 364 | return obj.e + f.mult(depth>2 ? (erand48(Xi)

f.y && f.x>f.z ? f.x : f.y>f.z ? f.y : f.z; // max refl 386 | if (++depth>2) if (erand48(Xi).1?Vec(0,1):Vec(1))%w).norm(), v=w%u; 390 | Vec d = (u*cos(r1)*r2s + v*sin(r1)*r2s + w*sqrt(1-r2)).norm(); 391 | return obj.e + f.mult(radiance(Ray(x,d),depth,Xi,info)); 392 | } 393 | }else if(intersects_mesh) { 394 | Vec x=r.o+r.d*(mesh_t - 1e-3); 395 | double p = f.x>f.y && f.x>f.z ? f.x : f.y>f.z ? f.y : f.z; // max refl 396 | if (++depth>5) if (erand48(Xi).1?Vec(0,1):Vec(1))%w).norm(), v=w%u; 401 | Vec d = (u*cos(r1)*r2s + v*sin(r1)*r2s + w*sqrt(1-r2)).norm(); 402 | */ 403 | 404 | Vec d = r.d - nl * 2.0 * r.d.dot(nl); 405 | 406 | return f.mult(sample_light(Ray(x, d))); 407 | } 408 | if(depth == 0) return Vec(); 409 | else return sample_light(r); 410 | break; 411 | } 412 | case Normal: { 413 | double t; // distance to intersection 414 | int id=0; // id of intersected object 415 | double mesh_t=0; 416 | 417 | Vec n, nl, f; 418 | bool intersects_mesh = mesh.intersect(r, mesh_t, n, nl, f); 419 | bool intersects_spheres = intersect_spheres(r, t, id); 420 | 421 | if(intersects_mesh) return (n + Vec(1, 1, 1)).mult(Vec(.5, .5, .5)); 422 | else return Vec(); 423 | } 424 | } 425 | } 426 | 427 | int main(int argc, char *argv[]){ 428 | int w=640, h=480, samps = argc>=2 ? atoi(argv[1])/4 : 1; // # samples 429 | if(argc>2) { 430 | mesh = Mesh(argv[2]); 431 | mesh.buildTree(25.0, Point(45,40,55)); 432 | } 433 | 434 | if(argc>3) { 435 | render_mode = RenderingMode(atoi(argv[3])); 436 | } 437 | 438 | vector> traces; 439 | traces.resize(w*h, vector()); 440 | 441 | Ray cam(Vec(50,52,295.6), Vec(0,-0.042612,-1).norm()); // cam pos, dir 442 | Vec cx=Vec(w*.5135/h), cy=(cx%cam.d).norm()*.5135, r, *c=new Vec[w*h]; 443 | #pragma omp parallel for schedule(dynamic, 1) private(r) // OpenMP 444 | for (int y=0; y traces_id; 486 | vector traces_size; 487 | for(int i=0;i traces_data; 496 | for(int i=0;i // smallpt, a Path Tracer by Kevin Beason, 2008 2 | #include // Make : g++ -O3 -fopenmp smallpt.cpp -o smallpt 3 | #include // Remove "-fopenmp" for g++ version < 4.2 4 | struct Vec { // Usage: time ./smallpt 5000 && xv image.ppm 5 | double x, y, z; // position, also color (r,g,b) 6 | Vec(double x_=0, double y_=0, double z_=0){ x=x_; y=y_; z=z_; } 7 | Vec operator+(const Vec &b) const { return Vec(x+b.x,y+b.y,z+b.z); } 8 | Vec operator-(const Vec &b) const { return Vec(x-b.x,y-b.y,z-b.z); } 9 | Vec operator*(double b) const { return Vec(x*b,y*b,z*b); } 10 | Vec mult(const Vec &b) const { return Vec(x*b.x,y*b.y,z*b.z); } 11 | Vec& norm(){ return *this = *this * (1/sqrt(x*x+y*y+z*z)); } 12 | double dot(const Vec &b) const { return x*b.x+y*b.y+z*b.z; } // cross: 13 | Vec operator%(Vec&b){return Vec(y*b.z-z*b.y,z*b.x-x*b.z,x*b.y-y*b.x);} 14 | }; 15 | struct Ray { Vec o, d; Ray(Vec o_, Vec d_) : o(o_), d(d_) {} }; 16 | enum Refl_t { DIFF, SPEC, REFR }; // material types, used in radiance() 17 | struct Sphere { 18 | double rad; // radius 19 | Vec p, e, c; // position, emission, color 20 | Refl_t refl; // reflection type (DIFFuse, SPECular, REFRactive) 21 | Sphere(double rad_, Vec p_, Vec e_, Vec c_, Refl_t refl_): 22 | rad(rad_), p(p_), e(e_), c(c_), refl(refl_) {} 23 | double intersect(const Ray &r) const { // returns distance, 0 if nohit 24 | Vec op = p-r.o; // Solve t^2*d.d + 2*t*(o-p).d + (o-p).(o-p)-R^2 = 0 25 | double t, eps=1e-4, b=op.dot(r.d), det=b*b-op.dot(op)+rad*rad; 26 | if (det<0) return 0; else det=sqrt(det); 27 | return (t=b-det)>eps ? t : ((t=b+det)>eps ? t : 0); 28 | } 29 | }; 30 | Sphere spheres[] = {//Scene: radius, position, emission, color, material 31 | //Sphere(1e5, Vec( 1e5+1,40.8,81.6), Vec(),Vec(.75,.25,.25),DIFF),//Left 32 | //Sphere(1e5, Vec(-1e5+99,40.8,81.6),Vec(),Vec(.25,.25,.75),DIFF),//Rght 33 | //Sphere(1e5, Vec(50,40.8, 1e5), Vec(),Vec(.75,.75,.75),DIFF),//Back 34 | //Sphere(1e5, Vec(50,40.8,-1e5+170), Vec(),Vec(), DIFF),//Frnt 35 | Sphere(1e5, Vec(50, 1e5, 81.6), Vec(),Vec(.75,.75,.75),DIFF),//Botm 36 | //Sphere(1e5, Vec(50,-1e5+81.6,81.6),Vec(),Vec(.75,.75,.75),DIFF),//Top 37 | Sphere(16.5,Vec(27,16.5,47), Vec(),Vec(1,1,1)*.999, DIFF),//Mirr 38 | Sphere(16.5,Vec(73,16.5,78), Vec(),Vec(1,1,1)*.999, DIFF),//Glas 39 | Sphere(600, Vec(50,681.6-.27,81.6),Vec(12,12,12), Vec(), DIFF) //Lite 40 | }; 41 | inline double clamp(double x){ return x<0 ? 0 : x>1 ? 1 : x; } 42 | inline int toInt(double x){ return int(pow(clamp(x),1/2.2)*255+.5); } 43 | inline bool intersect(const Ray &r, double &t, int &id){ 44 | double n=sizeof(spheres)/sizeof(Sphere), d, inf=t=1e20; 45 | for(int i=int(n);i--;) if((d=spheres[i].intersect(r))&&df.y && f.x>f.z ? f.x : f.y>f.z ? f.y : f.z; // max refl 55 | if (++depth>5) if (erand48(Xi).1?Vec(0,1):Vec(1))%w).norm(), v=w%u; 59 | Vec d = (u*cos(r1)*r2s + v*sin(r1)*r2s + w*sqrt(1-r2)).norm(); 60 | return obj.e + f.mult(radiance(Ray(x,d),depth,Xi)); 61 | } 62 | #if 0 63 | else if (obj.refl == SPEC) // Ideal SPECULAR reflection 64 | return obj.e + f.mult(radiance(Ray(x,r.d-n*2*n.dot(r.d)),depth,Xi)); 65 | Ray reflRay(x, r.d-n*2*n.dot(r.d)); // Ideal dielectric REFRACTION 66 | bool into = n.dot(nl)>0; // Ray from outside going in? 67 | double nc=1, nt=1.5, nnt=into?nc/nt:nt/nc, ddn=r.d.dot(nl), cos2t; 68 | if ((cos2t=1-nnt*nnt*(1-ddn*ddn))<0) // Total internal reflection 69 | return obj.e + f.mult(radiance(reflRay,depth,Xi)); 70 | Vec tdir = (r.d*nnt - n*((into?1:-1)*(ddn*nnt+sqrt(cos2t)))).norm(); 71 | double a=nt-nc, b=nt+nc, R0=a*a/(b*b), c = 1-(into?-ddn:tdir.dot(n)); 72 | double Re=R0+(1-R0)*c*c*c*c*c,Tr=1-Re,P=.25+.5*Re,RP=Re/P,TP=Tr/(1-P); 73 | return obj.e + f.mult(depth>2 ? (erand48(Xi)

5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | #include "common.h" 16 | 17 | #include "ceres/ceres.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | 31 | namespace fs = boost::filesystem; 32 | 33 | #include "nlohmann/json.hpp" 34 | using json = nlohmann::json; 35 | 36 | #include "cost_functions.h" 37 | #include "defs.h" 38 | #include "utils.h" 39 | 40 | struct NormalConstraint { 41 | int fidx; 42 | Eigen::Vector3d bcoords; 43 | Eigen::Vector3d n; 44 | }; 45 | 46 | int main(int argc, char** argv) { 47 | QApplication a(argc, argv); 48 | glutInit(&argc, argv); 49 | 50 | //google::InitGoogleLogging(argv[0]); 51 | const string home_directory = QDir::homePath().toStdString(); 52 | cout << "Home dir: " << home_directory << endl; 53 | 54 | // load the settings file 55 | PhGUtils::message("Loading global settings ..."); 56 | json global_settings = json::parse(ifstream(home_directory + "/Codes/FaceShapeFromShading/settings.txt")); 57 | PhGUtils::message("done."); 58 | cout << setw(2) << global_settings << endl; 59 | 60 | const string model_filename(home_directory + "/Data/Multilinear/blendshape_core.tensor"); 61 | const string id_prior_filename(home_directory + "/Data/Multilinear/blendshape_u_0_aug.tensor"); 62 | const string exp_prior_filename(home_directory + "/Data/Multilinear/blendshape_u_1_aug.tensor"); 63 | const string template_mesh_filename(home_directory + "/Data/Multilinear/template.obj"); 64 | const string contour_points_filename(home_directory + "/Data/Multilinear/contourpoints.txt"); 65 | const string landmarks_filename(home_directory + "/Data/Multilinear/landmarks_73.txt"); 66 | const string albedo_index_map_filename(home_directory + "/Data/Multilinear/albedo_index.png"); 67 | const string albedo_pixel_map_filename(home_directory + "/Data/Multilinear/albedo_pixel.png"); 68 | const string mean_albedo_filename(home_directory + "/Data/Texture/mean_texture.png"); 69 | const string core_face_region_filename(home_directory + "/Data/Multilinear/albedos/core_face.png"); 70 | 71 | const string valid_faces_indices_filename(home_directory + "/Data/Multilinear/face_region_indices.txt"); 72 | const string face_boundary_indices_filename(home_directory + "/Data/Multilinear/face_boundary_indices.txt"); 73 | const string hair_region_filename(home_directory + "/Data/Multilinear/hair_region_indices.txt"); 74 | 75 | BasicMesh mesh(template_mesh_filename); 76 | auto landmarks = LoadIndices(landmarks_filename); 77 | auto contour_indices = LoadContourIndices(contour_points_filename); 78 | 79 | auto valid_faces_indices_quad = LoadIndices(valid_faces_indices_filename); 80 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 81 | vector valid_faces_indices; 82 | for(auto fidx : valid_faces_indices_quad) { 83 | valid_faces_indices.push_back(fidx*2); 84 | valid_faces_indices.push_back(fidx*2+1); 85 | } 86 | 87 | auto faces_boundary_indices_quad = LoadIndices(face_boundary_indices_filename); 88 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 89 | unordered_set face_boundary_indices; 90 | for(auto fidx : faces_boundary_indices_quad) { 91 | face_boundary_indices.insert(fidx*2); 92 | face_boundary_indices.insert(fidx*2+1); 93 | } 94 | 95 | auto hair_region_indices_quad = LoadIndices(hair_region_filename); 96 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 97 | unordered_set hair_region_indices; 98 | for(auto fidx : hair_region_indices_quad) { 99 | hair_region_indices.insert(fidx*2); 100 | hair_region_indices.insert(fidx*2+1); 101 | } 102 | 103 | MultilinearModel model(model_filename); 104 | 105 | // Start the main process 106 | { 107 | fs::path image_filename = argv[1]; 108 | fs::path pts_filename = argv[2]; 109 | fs::path res_filename = argv[3]; 110 | fs::path mask_filename = argv[4]; 111 | fs::path optimized_normal_filename = argv[5]; 112 | fs::path results_path = argv[6]; 113 | 114 | cout << "[" << image_filename << ", " << pts_filename << "]" << endl; 115 | 116 | auto image_points_pair = LoadImageAndPoints(image_filename.string(), pts_filename.string(), false); 117 | auto recon_results = LoadReconstructionResult(res_filename.string()); 118 | 119 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 120 | mesh.UpdateVertices(model.GetTM()); 121 | mesh.ComputeNormals(); 122 | 123 | { 124 | const int max_subdivisions = 1; 125 | for(int i=0;i triangles = triangles_indices_pair.first; 255 | vector triangle_indices_map = triangles_indices_pair.second; 256 | cerr << "Num triangles visible: " << triangles.size() << endl; 257 | 258 | // for each visible pixel, compute its bary-centric coordinates 259 | cv::Mat mask_image = cv::imread(mask_filename.string(), CV_LOAD_IMAGE_GRAYSCALE); 260 | QImage optimized_normal_image = QImage(optimized_normal_filename.string().c_str()); 261 | QImage bcoords_image = img; 262 | QImage normal_constraints_image = img; 263 | cv::Mat bcoords = cv::Mat(img.height(), img.width(), CV_64FC3); 264 | vector normal_constraints; 265 | for(int y=0,pidx=0;y(y, x); 268 | const int fidx = triangle_indices_map[pidx]; 269 | 270 | if(pix > 0 && fidx >= 0) { 271 | cout << pix << " " << fidx << endl; 272 | auto f = mesh.face(fidx); 273 | auto v0 = mesh.vertex(f[0]), v1 = mesh.vertex(f[1]), v2 = mesh.vertex(f[2]); 274 | 275 | auto p = depth_maps_no_rot.at(y, x); 276 | PhGUtils::Point3f bcoords; 277 | // Compute barycentric coordinates 278 | PhGUtils::computeBarycentricCoordinates(PhGUtils::Point3d(p[0], p[1], p[2]), 279 | PhGUtils::Point3d(v0[0], v0[1], v0[2]), 280 | PhGUtils::Point3d(v1[0], v1[1], v1[2]), 281 | PhGUtils::Point3d(v2[0], v2[1], v2[2]), 282 | bcoords); 283 | 284 | cout << bcoords.x << ", " << bcoords.y << ", " << bcoords.z << endl; 285 | bcoords.x = clamp(bcoords.x, 0, 1); 286 | bcoords.y = clamp(bcoords.y, 0, 1); 287 | bcoords.z = clamp(bcoords.z, 0, 1); 288 | 289 | Eigen::Vector3d bcoords_vec(bcoords.x, bcoords.y, bcoords.z); 290 | bcoords_vec = bcoords_vec / bcoords_vec.norm(); 291 | 292 | auto npix = optimized_normal_image.pixel(x, y); 293 | double nx, ny, nz; 294 | tie(nx, ny, nz) = std::make_tuple(qRed(npix)/255.0*2-1.0, 295 | qGreen(npix)/255.0*2-1.0, 296 | qBlue(npix)/255.0*2-1.0); 297 | Eigen::Vector3d normal_vec(nx, ny, nz); 298 | 299 | normal_constraints.push_back( NormalConstraint{fidx, bcoords_vec, normal_vec} ); 300 | 301 | bcoords_image.setPixel(x, y, qRgb(bcoords.x*255.0, bcoords.y*255.0, bcoords.z*255.0)); 302 | normal_constraints_image.setPixel(x, y, npix); 303 | } else { 304 | bcoords.at(y, x) = cv::Vec3d(0, 0, 0); 305 | } 306 | } 307 | } 308 | bcoords_image.save( (results_path / fs::path("bcoords.png")).string().c_str() ); 309 | normal_constraints_image.save( (results_path / fs::path("normal_constraints.png")).string().c_str() ); 310 | 311 | { 312 | ofstream fout( (results_path / fs::path("constraints.txt") ).string() ); 313 | for(auto c : normal_constraints) { 314 | fout << c.fidx << " "; 315 | fout << c.bcoords[0] << ' ' 316 | << c.bcoords[1] << ' ' 317 | << c.bcoords[2] << ' '; 318 | fout << c.n[0] << ' ' << c.n[1] << ' ' << c.n[2] << endl; 319 | } 320 | } 321 | } 322 | 323 | return 0; 324 | } 325 | -------------------------------------------------------------------------------- /refine_mesh_with_normal_exp.cpp: -------------------------------------------------------------------------------- 1 | #include "Geometry/geometryutils.hpp" 2 | #include "Utils/utility.hpp" 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | #include "common.h" 16 | 17 | #include "ceres/ceres.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | namespace fs = boost::filesystem; 33 | namespace po = boost::program_options; 34 | 35 | #include "nlohmann/json.hpp" 36 | using json = nlohmann::json; 37 | 38 | #include "cost_functions.h" 39 | #include "defs.h" 40 | #include "utils.h" 41 | 42 | struct NormalConstraint { 43 | int fidx; 44 | Eigen::Vector3d bcoords; 45 | Eigen::Vector3d n; 46 | }; 47 | 48 | po::variables_map ParseCommandlineOptions(int argc, char** argv) { 49 | po::options_description desc("Options"); 50 | desc.add_options() 51 | ("help", "Print help messages") 52 | ("image", po::value()->required(), "Input image file.") 53 | ("points", po::value()->required(), "Input points file.") 54 | ("init_recon", po::value()->required(), "Initial reconstruction path.") 55 | ("blendshapes_path", po::value()->required(), "Input blendshapes path.") 56 | ("mask", po::value()->required(), "Deformation mask file.") 57 | ("normals", po::value()->required(), "Optimized normals file.") 58 | ("output_dir", po::value()->required(), "Directory for output data.") 59 | ("subdivision", "Whether the input blendshapes are subdivided or not.") 60 | ("subdivision_depth", po::value(), "The depth of subdivision."); 61 | po::variables_map vm; 62 | 63 | try { 64 | po::store(po::parse_command_line(argc, argv, desc), vm); 65 | po::notify(vm); 66 | if(vm.count("help")) { 67 | cout << desc << endl; 68 | exit(1); 69 | } 70 | return vm; 71 | } catch(po::error& e) { 72 | cerr << "Error: " << e.what() << endl; 73 | cerr << desc << endl; 74 | exit(1); 75 | } 76 | } 77 | 78 | int main(int argc, char** argv) { 79 | po::variables_map vm = ParseCommandlineOptions(argc, argv); 80 | 81 | QApplication a(argc, argv); 82 | glutInit(&argc, argv); 83 | 84 | //google::InitGoogleLogging(argv[0]); 85 | const string home_directory = QDir::homePath().toStdString(); 86 | cout << "Home dir: " << home_directory << endl; 87 | 88 | // load the settings file 89 | PhGUtils::message("Loading global settings ..."); 90 | json global_settings = json::parse(ifstream(home_directory + "/Codes/FaceShapeFromShading/settings.txt")); 91 | PhGUtils::message("done."); 92 | cout << setw(2) << global_settings << endl; 93 | 94 | const string model_filename(home_directory + "/Data/Multilinear/blendshape_core.tensor"); 95 | const string id_prior_filename(home_directory + "/Data/Multilinear/blendshape_u_0_aug.tensor"); 96 | const string exp_prior_filename(home_directory + "/Data/Multilinear/blendshape_u_1_aug.tensor"); 97 | const string template_mesh_filename(home_directory + "/Data/Multilinear/template.obj"); 98 | const string contour_points_filename(home_directory + "/Data/Multilinear/contourpoints.txt"); 99 | const string landmarks_filename(home_directory + "/Data/Multilinear/landmarks_73.txt"); 100 | const string albedo_index_map_filename(home_directory + "/Data/Multilinear/albedo_index.png"); 101 | const string albedo_pixel_map_filename(home_directory + "/Data/Multilinear/albedo_pixel.png"); 102 | const string mean_albedo_filename(home_directory + "/Data/Texture/mean_texture.png"); 103 | const string core_face_region_filename(home_directory + "/Data/Multilinear/albedos/core_face.png"); 104 | 105 | const string valid_faces_indices_filename(home_directory + "/Data/Multilinear/face_region_indices.txt"); 106 | const string face_boundary_indices_filename(home_directory + "/Data/Multilinear/face_boundary_indices.txt"); 107 | const string hair_region_filename(home_directory + "/Data/Multilinear/hair_region_indices.txt"); 108 | 109 | BasicMesh mesh(template_mesh_filename); 110 | auto landmarks = LoadIndices(landmarks_filename); 111 | auto contour_indices = LoadContourIndices(contour_points_filename); 112 | 113 | auto valid_faces_indices_quad = LoadIndices(valid_faces_indices_filename); 114 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 115 | vector valid_faces_indices; 116 | for(auto fidx : valid_faces_indices_quad) { 117 | valid_faces_indices.push_back(fidx*2); 118 | valid_faces_indices.push_back(fidx*2+1); 119 | } 120 | 121 | auto faces_boundary_indices_quad = LoadIndices(face_boundary_indices_filename); 122 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 123 | unordered_set face_boundary_indices; 124 | for(auto fidx : faces_boundary_indices_quad) { 125 | face_boundary_indices.insert(fidx*2); 126 | face_boundary_indices.insert(fidx*2+1); 127 | } 128 | 129 | auto hair_region_indices_quad = LoadIndices(hair_region_filename); 130 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 131 | unordered_set hair_region_indices; 132 | for(auto fidx : hair_region_indices_quad) { 133 | hair_region_indices.insert(fidx*2); 134 | hair_region_indices.insert(fidx*2+1); 135 | } 136 | 137 | if(vm.count("subdivision")) { 138 | // HACK: subdivie the template mesh so it has the same topology as the input 139 | // blendshapes 140 | const int max_subdivisions = vm["subdivision_depth"].as(); 141 | for(int i=0;i(); 164 | fs::path pts_filename = vm["points"].as(); 165 | fs::path res_filename = vm["init_recon"].as(); 166 | fs::path mask_filename = vm["mask"].as(); 167 | fs::path optimized_normal_filename = vm["normals"].as(); 168 | fs::path results_path = vm["output_dir"].as(); 169 | string blendshapes_path = vm["blendshapes_path"].as(); 170 | 171 | // Load all the input blendshapes 172 | const int num_blendshapes = 46; 173 | vector blendshapes(num_blendshapes+1); 174 | for(int i=0;i<=num_blendshapes;++i) { 175 | blendshapes[i].LoadOBJMesh( blendshapes_path + "/" + "B_" + to_string(i) + ".obj" ); 176 | blendshapes[i].ComputeNormals(); 177 | } 178 | 179 | MultilinearModel model(model_filename); 180 | 181 | // Start the main process 182 | { 183 | cout << "[" << image_filename << ", " << pts_filename << "]" << endl; 184 | 185 | auto image_points_pair = LoadImageAndPoints(image_filename.string(), pts_filename.string(), false); 186 | auto recon_results = LoadReconstructionResult(res_filename.string()); 187 | 188 | ApplyWeights(mesh, blendshapes, recon_results.params_model.Wexp_FACS); 189 | 190 | { 191 | const int max_subdivisions = 1; 192 | for(int i=0;i triangles = triangles_indices_pair.first; 322 | vector triangle_indices_map = triangles_indices_pair.second; 323 | cerr << "Num triangles visible: " << triangles.size() << endl; 324 | 325 | // for each visible pixel, compute its bary-centric coordinates 326 | cv::Mat mask_image = cv::imread(mask_filename.string(), CV_LOAD_IMAGE_GRAYSCALE); 327 | QImage optimized_normal_image = QImage(optimized_normal_filename.string().c_str()); 328 | QImage bcoords_image = img; 329 | QImage normal_constraints_image = img; 330 | cv::Mat bcoords = cv::Mat(img.height(), img.width(), CV_64FC3); 331 | vector normal_constraints; 332 | for(int y=0,pidx=0;y(y, x); 335 | const int fidx = triangle_indices_map[pidx]; 336 | 337 | if(pix > 0 && fidx >= 0) { 338 | cout << pix << " " << fidx << endl; 339 | auto f = mesh.face(fidx); 340 | auto v0 = mesh.vertex(f[0]), v1 = mesh.vertex(f[1]), v2 = mesh.vertex(f[2]); 341 | 342 | auto p = depth_maps_no_rot.at(y, x); 343 | PhGUtils::Point3f bcoords; 344 | // Compute barycentric coordinates 345 | PhGUtils::computeBarycentricCoordinates(PhGUtils::Point3d(p[0], p[1], p[2]), 346 | PhGUtils::Point3d(v0[0], v0[1], v0[2]), 347 | PhGUtils::Point3d(v1[0], v1[1], v1[2]), 348 | PhGUtils::Point3d(v2[0], v2[1], v2[2]), 349 | bcoords); 350 | 351 | cout << bcoords.x << ", " << bcoords.y << ", " << bcoords.z << endl; 352 | bcoords.x = clamp(bcoords.x, 0, 1); 353 | bcoords.y = clamp(bcoords.y, 0, 1); 354 | bcoords.z = clamp(bcoords.z, 0, 1); 355 | 356 | Eigen::Vector3d bcoords_vec(bcoords.x, bcoords.y, bcoords.z); 357 | bcoords_vec = bcoords_vec / bcoords_vec.norm(); 358 | 359 | auto npix = optimized_normal_image.pixel(x, y); 360 | double nx, ny, nz; 361 | tie(nx, ny, nz) = std::make_tuple(qRed(npix)/255.0*2-1.0, 362 | qGreen(npix)/255.0*2-1.0, 363 | qBlue(npix)/255.0*2-1.0); 364 | Eigen::Vector3d normal_vec(nx, ny, nz); 365 | 366 | normal_constraints.push_back( NormalConstraint{fidx, bcoords_vec, normal_vec} ); 367 | 368 | bcoords_image.setPixel(x, y, qRgb(bcoords.x*255.0, bcoords.y*255.0, bcoords.z*255.0)); 369 | normal_constraints_image.setPixel(x, y, npix); 370 | } else { 371 | bcoords.at(y, x) = cv::Vec3d(0, 0, 0); 372 | } 373 | } 374 | } 375 | bcoords_image.save( (results_path / fs::path("bcoords.png")).string().c_str() ); 376 | normal_constraints_image.save( (results_path / fs::path("normal_constraints.png")).string().c_str() ); 377 | 378 | { 379 | ofstream fout( (results_path / fs::path("constraints.txt") ).string() ); 380 | for(auto c : normal_constraints) { 381 | fout << c.fidx << " "; 382 | fout << c.bcoords[0] << ' ' 383 | << c.bcoords[1] << ' ' 384 | << c.bcoords[2] << ' '; 385 | fout << c.n[0] << ' ' << c.n[1] << ' ' << c.n[2] << endl; 386 | } 387 | } 388 | } 389 | 390 | return 0; 391 | } 392 | -------------------------------------------------------------------------------- /settings.txt: -------------------------------------------------------------------------------- 1 | { 2 | "max_iters": 3, 3 | "preparation_only": true, 4 | "albedo": { 5 | "lambda": 256.0 6 | }, 7 | "lighting": { 8 | "saturated_pixels_threshold": 225, 9 | "dark_pixels_threshold": 25, 10 | "albedo_distance_bins": 1000, 11 | "lighting_pixels_ratio_lower": 1.0, 12 | "lighting_pixels_ratio_upper": 1.0, 13 | "num_dof": 9, 14 | "w_reg": 0.0001, 15 | "relaxation": 1.0 16 | }, 17 | "depth": { 18 | "num_iters": 3, 19 | "w_data": 1.0, 20 | "w_int": 0.0, 21 | "w_reg": 0.0, 22 | "w_smooth": 0.0, 23 | "integrability_threshold": 64.0, 24 | "optimization": { 25 | "max_iters": 10, 26 | "init_tr_radius": 0.01 27 | } 28 | }, 29 | "mean_texture_options": { 30 | "generate_mean_texture": true, 31 | "refine_method": "hsv", 32 | "hsv_threshold": 0.95, 33 | "mix_ratio": 0.95, 34 | "hsv_thresholds": { 35 | "FW": 0.95, 36 | "regular": 0.15, 37 | "comments": "Choose either 0.35 or 0.15 based on the experiment to run" 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | 3 | project(FaceShapeFromShading) 4 | 5 | # Find includes in corresponding build directories 6 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 7 | # Instruct CMake to run moc automatically when needed. 8 | set(CMAKE_AUTOMOC ON) 9 | 10 | set(CMAKE_AUTOUIC ON) 11 | 12 | include(CheckCXXCompilerFlag) 13 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 14 | CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) 15 | if(COMPILER_SUPPORTS_CXX11) 16 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 17 | elseif(COMPILER_SUPPORTS_CXX0X) 18 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") 19 | else() 20 | message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 21 | endif() 22 | 23 | # Boost 24 | find_package(Boost COMPONENTS filesystem timer program_options REQUIRED) 25 | include_directories(${Boost_INCLUDE_DIRS}) 26 | link_libraries(${Boost_LIBRARIES} -lboost_filesystem -lboost_system) 27 | 28 | # OpenMP 29 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") 30 | 31 | # OpenGL 32 | find_package(OpenGL REQUIRED) 33 | find_package(GLUT REQUIRED) 34 | include_directories( ${OPENGL_INCLUDE_DIRS} ${GLUT_INCLUDE_DIRS} ) 35 | link_libraries(${OPENGL_LIBRARIES} ${GLUT_LIBRARY}) 36 | 37 | find_package(GLEW REQUIRED) 38 | if (GLEW_FOUND) 39 | include_directories(${GLEW_INCLUDE_DIRS}) 40 | link_libraries(${GLEW_LIBRARIES}) 41 | endif() 42 | 43 | # Eigen 44 | find_package(Eigen) 45 | include_directories(${EIGEN_INCLUDE_DIR}) 46 | 47 | # MKL 48 | set(MKL_INCLUDE_DIRS "/home/phg/SDKs/intel/composer_xe/include") 49 | message([] ${MKL_INCLUDE_DIRS} ) 50 | include_directories(${MKL_INCLUDE_DIRS}) 51 | set(MKLROOT "/home/phg/SDKs/intel/composer_xe_2015/mkl") 52 | set(MKLLIBS_DIRS "${MKLROOT}/lib/intel64/") 53 | link_directories(${MKLLIBS_DIRS}) 54 | set(MKLLIBS "-Wl,--start-group -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -Wl,--end-group -liomp5 -ldl -lpthread -lm") 55 | 56 | # PhGLib 57 | include_directories("/home/phg/SDKs/PhGLib/include") 58 | link_directories("/home/phg/SDKs/PhGLib/lib") 59 | set(PhGLib "-lPhGLib") 60 | 61 | # Qt5 62 | find_package(Qt5Core) 63 | find_package(Qt5Widgets) 64 | find_package(Qt5OpenGL) 65 | 66 | # json 67 | message(STATUS "Found json cpp") 68 | message(STATUS "${CMAKE_CURRENT_LIST_DIR}/../json/include") 69 | include_directories("${CMAKE_CURRENT_LIST_DIR}/../json/include") 70 | 71 | # Ceres solver 72 | #find_package(Ceres REQUIRED) 73 | #include_directories(${CERES_INCLUDE_DIRS}) 74 | #link_libraries(${CERES_LIBRARIES}) 75 | 76 | # Face shape from shading program 77 | add_executable(test_fbo test_fbo.cpp) 78 | target_link_libraries(test_fbo 79 | Qt5::Core 80 | Qt5::Widgets 81 | Qt5::OpenGL) 82 | 83 | add_executable(test_transfer_color test_transfer_color.cpp) 84 | target_link_libraries(test_transfer_color 85 | Qt5::Core 86 | Qt5::Widgets 87 | Qt5::OpenGL 88 | ${MKLLIBS} 89 | ${PhGLib}) 90 | include_directories(..) 91 | -------------------------------------------------------------------------------- /tests/test_fbo.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void nativePainting() 6 | { 7 | static const float vertexPositions[] = { 8 | -0.8f, -0.8f, 0.0f, 9 | 0.8f, -0.8f, 0.0f, 10 | 0.0f, 0.8f, 0.0f 11 | }; 12 | 13 | static const float vertexColors[] = { 14 | 1.0f, 0.0f, 0.0f, 15 | 0.0f, 1.0f, 0.0f, 16 | 0.0f, 0.0f, 1.0f 17 | }; 18 | 19 | QOpenGLBuffer vertexPositionBuffer(QOpenGLBuffer::VertexBuffer); 20 | vertexPositionBuffer.create(); 21 | vertexPositionBuffer.setUsagePattern(QOpenGLBuffer::StaticDraw); 22 | vertexPositionBuffer.bind(); 23 | vertexPositionBuffer.allocate(vertexPositions, 9 * sizeof(float)); 24 | 25 | QOpenGLBuffer vertexColorBuffer(QOpenGLBuffer::VertexBuffer); 26 | vertexColorBuffer.create(); 27 | vertexColorBuffer.setUsagePattern(QOpenGLBuffer::StaticDraw); 28 | vertexColorBuffer.bind(); 29 | vertexColorBuffer.allocate(vertexColors, 9 * sizeof(float)); 30 | 31 | QOpenGLShaderProgram program; 32 | program.addShaderFromSourceCode(QOpenGLShader::Vertex, 33 | "#version 330\n" 34 | "in vec3 position;\n" 35 | "in vec3 color;\n" 36 | "out vec3 fragColor;\n" 37 | "void main() {\n" 38 | " fragColor = color;\n" 39 | " gl_Position = vec4(position, 1.0);\n" 40 | "}\n" 41 | ); 42 | program.addShaderFromSourceCode(QOpenGLShader::Fragment, 43 | "#version 330\n" 44 | "in vec3 fragColor;\n" 45 | "out vec4 color;\n" 46 | "void main() {\n" 47 | " color = vec4(fragColor, 1.0);\n" 48 | "}\n" 49 | ); 50 | program.link(); 51 | program.bind(); 52 | 53 | vertexPositionBuffer.bind(); 54 | program.enableAttributeArray("position"); 55 | program.setAttributeBuffer("position", GL_FLOAT, 0, 3); 56 | 57 | vertexColorBuffer.bind(); 58 | program.enableAttributeArray("color"); 59 | program.setAttributeBuffer("color", GL_FLOAT, 0, 3); 60 | 61 | glDrawArrays(GL_TRIANGLES, 0, 3); 62 | } 63 | 64 | QImage createImageWithFBO() 65 | { 66 | QSurfaceFormat format; 67 | format.setMajorVersion(3); 68 | format.setMinorVersion(3); 69 | 70 | QOffscreenSurface surface; 71 | surface.setFormat(format); 72 | surface.create(); 73 | 74 | QOpenGLContext context; 75 | context.setFormat(format); 76 | if (!context.create()) 77 | qFatal("Cannot create the requested OpenGL context!"); 78 | context.makeCurrent(&surface); 79 | 80 | const QRect drawRect(0, 0, 400, 400); 81 | const QSize drawRectSize = drawRect.size(); 82 | 83 | QOpenGLFramebufferObjectFormat fboFormat; 84 | fboFormat.setSamples(16); 85 | fboFormat.setAttachment(QOpenGLFramebufferObject::CombinedDepthStencil); 86 | 87 | QOpenGLFramebufferObject fbo(drawRectSize, fboFormat); 88 | fbo.bind(); 89 | 90 | QOpenGLPaintDevice device(drawRectSize); 91 | QPainter painter; 92 | painter.begin(&device); 93 | painter.setRenderHints(QPainter::Antialiasing | QPainter::HighQualityAntialiasing); 94 | 95 | painter.fillRect(drawRect, Qt::blue); 96 | 97 | painter.drawTiledPixmap(drawRect, QPixmap(":/qt-project.org/qmessagebox/images/qtlogo-64.png")); 98 | 99 | painter.setPen(QPen(Qt::green, 5)); 100 | painter.setBrush(Qt::red); 101 | painter.drawEllipse(0, 100, 400, 200); 102 | painter.drawEllipse(100, 0, 200, 400); 103 | 104 | painter.beginNativePainting(); 105 | nativePainting(); 106 | painter.endNativePainting(); 107 | 108 | painter.setPen(QPen(Qt::white, 0)); 109 | QFont font; 110 | font.setPointSize(24); 111 | painter.setFont(font); 112 | painter.drawText(drawRect, "Hello FBO", QTextOption(Qt::AlignCenter)); 113 | 114 | painter.end(); 115 | 116 | fbo.release(); 117 | return fbo.toImage(); 118 | } 119 | 120 | int main(int argc, char **argv) 121 | { 122 | QApplication app(argc, argv); 123 | 124 | QImage targetImage = createImageWithFBO(); 125 | targetImage.save("test_fbo.png"); 126 | 127 | return 0; 128 | } -------------------------------------------------------------------------------- /tests/test_transfer_color.cpp: -------------------------------------------------------------------------------- 1 | #include "../utils.h" 2 | 3 | int main(int argc, char** argv) { 4 | QImage source(argv[1]); 5 | QImage target(argv[2]); 6 | 7 | #if 0 8 | vector pix_s(source.width()*source.height()), pix_t(target.width()*target.height()); 9 | for(int i=0;i pix_s, pix_t; 13 | for(int i=0;i 0) { 16 | pix_t.push_back(i); 17 | } 18 | } 19 | pix_s = pix_t; 20 | #endif 21 | 22 | QImage transferred = TransferColor(source, target, pix_s, pix_t); 23 | 24 | source.save("source.png"); 25 | target.save("target.png"); 26 | transferred.save("transferred.png"); 27 | 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /utils.h: -------------------------------------------------------------------------------- 1 | #ifndef FACESHAPEFROMSHADING_UTILS_H 2 | #define FACESHAPEFROMSHADING_UTILS_H 3 | 4 | #include 5 | 6 | #include "Geometry/geometryutils.hpp" 7 | #include "Utils/utility.hpp" 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "defs.h" 23 | 24 | #include "boost/filesystem/operations.hpp" 25 | #include "boost/filesystem/path.hpp" 26 | #include 27 | 28 | namespace fs = boost::filesystem; 29 | 30 | #include "nlohmann/json.hpp" 31 | using json = nlohmann::json; 32 | 33 | template 34 | pair normal2sphericalcoords(T nx, T ny, T nz) { 35 | // nx = sin(theta) * sin(phi) 36 | // ny = sin(theta) * cos(phi) 37 | // nz = cos(theta) 38 | return make_pair(acos(nz), atan2(nx, ny)); 39 | } 40 | 41 | template 42 | tuple sphericalcoords2normal(double theta, double phi) { 43 | // nx = sin(theta) * sin(phi) 44 | // ny = sin(theta) * cos(phi) 45 | // nz = cos(theta) 46 | return make_tuple(sin(theta)*sin(phi), sin(theta)*cos(phi), cos(theta)); 47 | } 48 | 49 | inline Vector3d dnormal_dtheta(double theta, double phi) { 50 | // nx = sin(theta) * sin(phi) 51 | // ny = sin(theta) * cos(phi) 52 | // nz = cos(theta) 53 | 54 | // dnx_dtheta = cos(theta) * sin(phi) 55 | // dny_dtheta = cos(theta) * cos(phi) 56 | // dnz_dtheta = -sin(theta) 57 | return Vector3d(cos(theta)*sin(phi), cos(theta)*cos(phi), -sin(theta)); 58 | } 59 | 60 | inline Vector3d dnormal_dphi(double theta, double phi) { 61 | // nx = sin(theta) * sin(phi) 62 | // ny = sin(theta) * cos(phi) 63 | // nz = cos(theta) 64 | 65 | // dnx_dtheta = sin(theta) * cos(phi) 66 | // dny_dtheta = -sin(theta) * sin(phi) 67 | // dnz_dtheta = 0 68 | return Vector3d(sin(theta)*cos(phi), -sin(theta)*sin(phi), 0); 69 | } 70 | 71 | inline VectorXd sphericalharmonics(double nx, double ny, double nz) { 72 | VectorXd Y(9); 73 | Y(0) = 1.0; 74 | Y(1) = nx; Y(2) = ny; Y(3) = nz; 75 | Y(4) = nx * ny; Y(5) = nx * nz; Y(6) = ny * nz; 76 | Y(7) = nx * nx - ny * ny; Y(8) = 3 * nz * nz - 1; 77 | return Y; 78 | } 79 | 80 | inline MatrixXd dY_dnormal(double nx, double ny, double nz) { 81 | MatrixXd dYdnormal(9, 3); 82 | dYdnormal(0, 0) = 0; dYdnormal(0, 1) = 0; dYdnormal(0, 2) = 0; 83 | 84 | dYdnormal(1, 0) = 1; dYdnormal(1, 1) = 0; dYdnormal(1, 2) = 0; 85 | dYdnormal(2, 0) = 0; dYdnormal(2, 1) = 1; dYdnormal(2, 2) = 0; 86 | dYdnormal(3, 0) = 0; dYdnormal(3, 1) = 0; dYdnormal(3, 2) = 1; 87 | 88 | dYdnormal(4, 0) = ny; dYdnormal(4, 1) = nx; dYdnormal(4, 2) = 0; 89 | dYdnormal(5, 0) = nz; dYdnormal(5, 1) = 0; dYdnormal(5, 2) = nx; 90 | dYdnormal(6, 0) = 0; dYdnormal(6, 1) = nz; dYdnormal(6, 2) = ny; 91 | 92 | dYdnormal(7, 0) = 2*nx; dYdnormal(7, 1) = -2*ny; dYdnormal(7, 2) = 0; 93 | dYdnormal(8, 0) = 0; dYdnormal(8, 1) = 0; dYdnormal(8, 2) = 6 * nz; 94 | 95 | return dYdnormal; 96 | } 97 | 98 | template 99 | T clamp(T val, T lower, T upper) { 100 | return std::max(lower, std::min(upper, val)); 101 | } 102 | 103 | inline void encode_index(int idx, unsigned char& r, unsigned char& g, unsigned char& b) { 104 | r = static_cast(idx & 0xff); idx >>= 8; 105 | g = static_cast(idx & 0xff); idx >>= 8; 106 | b = static_cast(idx & 0xff); 107 | } 108 | 109 | inline int decode_index(unsigned char r, unsigned char g, unsigned char b, int& idx) { 110 | idx = b; idx <<= 8; idx |= g; idx <<= 8; idx |= r; 111 | return idx; 112 | } 113 | 114 | inline glm::dvec3 bilinear_sample(const QImage& img, double x, double y) { 115 | int x0 = floor(x), x1 = x0 + 1; 116 | int y0 = floor(y), y1 = y0 + 1; 117 | 118 | if(x0 < 0 || y0 < 0) return glm::dvec3(-1, -1, -1); 119 | if(x1 >= img.width() || y1 >= img.height()) return glm::dvec3(-1, -1, -1); 120 | 121 | double c0 = x - x0, c0c = 1 - c0; 122 | double c1 = y - y0, c1c = 1 - c1; 123 | 124 | QRgb pix = img.pixel(x0, y0); 125 | return glm::dvec3(qRed(pix), qGreen(pix), qBlue(pix)); 126 | 127 | /* 128 | 129 | p00 ------ p01 130 | |___| c1c | 131 | |c0c | 132 | | | 133 | p10 ------ p11 134 | 135 | */ 136 | 137 | QRgb p00 = img.pixel(x0, y0); double f00 = c0c * c1c; 138 | QRgb p01 = img.pixel(x1, y0); double f01 = c0c * c1; 139 | QRgb p10 = img.pixel(x0, y1); double f10 = c0 * c1c; 140 | QRgb p11 = img.pixel(x1, y1); double f11 = c0 * c1; 141 | 142 | auto interpolate_channel = [&](QRgb tl, QRgb tr, QRgb bl, QRgb br, std::function extractor) -> double { 143 | return f00 * extractor(tl) + f01 * extractor(tr) + f10 * extractor(bl) + f11 * extractor(br); 144 | }; 145 | 146 | double r = interpolate_channel(p00, p01, p10, p11, [](QRgb pix) { return qRed(pix); }); 147 | double g = interpolate_channel(p00, p01, p10, p11, [](QRgb pix) { return qGreen(pix); }); 148 | double b = interpolate_channel(p00, p01, p10, p11, [](QRgb pix) { return qBlue(pix); }); 149 | 150 | return glm::dvec3(r, g, b); 151 | } 152 | 153 | inline QRgb jet_color_QRgb(double ratio) { 154 | double r = max(0.0, min(1.0, (ratio - 0.5) / 0.25)); 155 | double g = 0; 156 | double b = 1.0 - max(0.0, min(1.0, (ratio - 0.25) / 0.25 )); 157 | if(ratio < 0.5) { 158 | g = min(1.0, ratio / 0.25); 159 | } else { 160 | g = 1.0 - max(0.0, (ratio - 0.75) / 0.25); 161 | } 162 | return qRgb(r*255, g*255, b*255); 163 | } 164 | 165 | inline glm::dvec3 jet_color(double ratio) { 166 | double r = max(0.0, min(1.0, (ratio - 0.5) / 0.25)); 167 | double g = 0; 168 | double b = 1.0 - max(0.0, min(1.0, (ratio - 0.25) / 0.25 )); 169 | if(ratio < 0.5) { 170 | g = min(1.0, ratio / 0.25); 171 | } else { 172 | g = 1.0 - max(0.0, (ratio - 0.75) / 0.25); 173 | } 174 | return glm::dvec3(r*255, g*255, b*255); 175 | } 176 | 177 | inline int get_image_index(const string& filename) { 178 | return std::stoi(filename.substr(0, filename.size()-4)); 179 | } 180 | 181 | inline pair, vector> FindTrianglesIndices(const QImage& img) { 182 | int w = img.width(), h = img.height(); 183 | set S; 184 | vector indices_map(w*h); 185 | for(int i=0, pidx = 0;i(qRed(pix)); 189 | unsigned char g = static_cast(qGreen(pix)); 190 | unsigned char b = static_cast(qBlue(pix)); 191 | 192 | if(r == 0 && g == 0 && b == 0) { 193 | indices_map[pidx] = -1; 194 | continue; 195 | } 196 | else { 197 | int idx; 198 | decode_index(r, g, b, idx); 199 | S.insert(idx); 200 | indices_map[pidx] = idx; 201 | } 202 | } 203 | } 204 | return make_pair(S, indices_map); 205 | } 206 | 207 | inline MatrixXd ComputeLoGKernel(int k, double sigma) { 208 | MatrixXd kernel(2*k+1, 2*k+1); 209 | const double sigma2 = sigma * sigma; 210 | const double sigma4 = sigma2 * sigma2; 211 | const double sigma6 = sigma2 * sigma4; 212 | 213 | double S = 0.0; 214 | for(int y=-k, i=0;y<=k;++y, ++i) { 215 | double y2 = y * y; 216 | for(int x=-k, j=0;x<=k;++x, ++j) { 217 | double x2 = x * x; 218 | double val = exp(-(x2 + y2) / (2 * sigma2)); 219 | kernel(i, j) = val; 220 | S += val; 221 | } 222 | } 223 | 224 | const double PI = 3.1415926535897; 225 | double S2 = 0.0; 226 | for(int y=-k, i=0;y<=k;++y, ++i) { 227 | double y2 = y * y; 228 | for(int x=-k, j=0;x<=k;++x, ++j) { 229 | double x2 = x * x; 230 | kernel(i, j) *= (x2 + y2 - 2 * sigma2); 231 | kernel(i, j) /= S; 232 | S2 += kernel(i, j); 233 | } 234 | } 235 | 236 | S2 /= ((2*k+1) * (2*k+1)); 237 | 238 | for(int i=0;i<2*k+1;++i) { 239 | for(int j=0;j<2*k+1;++j) { 240 | kernel(i, j) -= S2; 241 | } 242 | } 243 | 244 | return kernel; 245 | } 246 | 247 | inline Vector3d rgb2lab(double r, double g, double b) { 248 | Vector3d rgb(r, g, b); 249 | Matrix3d RGB2LMS; 250 | RGB2LMS << 0.3811, 0.5783, 0.0402, 251 | 0.1967, 0.7244, 0.0782, 252 | 0.0241, 0.1288, 0.8444; 253 | Matrix3d mb, mc; 254 | mb << 1.0/sqrt(3.0), 0, 0, 255 | 0, 1.0/sqrt(6.0), 0, 256 | 0, 0, 1.0/sqrt(2.0); 257 | mc << 1, 1, 1, 258 | 1, 1, -2, 259 | 1, -1, 0; 260 | Matrix3d LMS2lab = mb * mc; 261 | Vector3d Lab = LMS2lab * RGB2LMS * rgb; 262 | return Lab; 263 | } 264 | 265 | static QImage TransferColor(const QImage& source, const QImage& target, 266 | const vector& valid_pixels_s, 267 | const vector& valid_pixels_t) { 268 | // Make a copy 269 | QImage result = source; 270 | 271 | const int num_rows_s = source.height(), num_cols_s = source.width(); 272 | const int num_rows_t = target.height(), num_cols_t = target.width(); 273 | const size_t num_pixels_s = valid_pixels_s.size(); 274 | const size_t num_pixels_t = valid_pixels_t.size(); 275 | 276 | Matrix3d RGB2LMS, LMS2RGB; 277 | RGB2LMS << 0.3811, 0.5783, 0.0402, 278 | 0.1967, 0.7244, 0.0782, 279 | 0.0241, 0.1288, 0.8444; 280 | LMS2RGB << 4.4679, -3.5873, 0.1193, 281 | -1.2186, 2.3809, -0.1624, 282 | 0.0497, -0.2439, 1.2045; 283 | 284 | Matrix3d b, c, b2, c2; 285 | b << 1.0/sqrt(3.0), 0, 0, 286 | 0, 1.0/sqrt(6.0), 0, 287 | 0, 0, 1.0/sqrt(2.0); 288 | c << 1, 1, 1, 289 | 1, 1, -2, 290 | 1, -1, 0; 291 | b2 << sqrt(3.0)/3.0, 0, 0, 292 | 0, sqrt(6.0)/6.0, 0, 293 | 0, 0, sqrt(2.0)/2.0; 294 | c2 << 1, 1, 1, 295 | 1, 1, -1, 296 | 1, -2, 0; 297 | Matrix3d LMS2lab = b * c; 298 | Matrix3d lab2LMS = c2 * b2; 299 | 300 | auto unpack_pixel = [](QRgb pix) { 301 | int r = max(1, qRed(pix)), g = max(1, qGreen(pix)), b = max(1, qBlue(pix)); 302 | return make_tuple(r, g, b); 303 | }; 304 | 305 | auto compute_image_stats = [&](const QImage& img, const vector& valid_pixels) { 306 | const size_t num_pixels = valid_pixels.size(); 307 | const int num_cols = img.width(), num_rows = img.height(); 308 | MatrixXd pixels(3, num_pixels); 309 | 310 | cout << num_cols << 'x' << num_rows << endl; 311 | 312 | for(size_t i=0;i(est_im(0, i) * 255.0, 0, 255), 376 | clamp(est_im(1, i) * 255.0, 0, 255), 377 | clamp(est_im(2, i) * 255.0, 0, 255))); 378 | } 379 | return result; 380 | } 381 | 382 | inline QImage GetIndexMap(const string& albedo_index_map_filename, 383 | const BasicMesh& mesh, 384 | bool generate_index_map = true, 385 | int tex_size = 2048) { 386 | QImage albedo_index_map; 387 | if(QFile::exists(albedo_index_map_filename.c_str()) && (!generate_index_map)) { 388 | PhGUtils::message("loading index map for albedo."); 389 | albedo_index_map = QImage(albedo_index_map_filename.c_str()); 390 | albedo_index_map.save("albedo_index.png"); 391 | } else { 392 | OffscreenMeshVisualizer visualizer(tex_size, tex_size); 393 | visualizer.BindMesh(mesh); 394 | visualizer.SetRenderMode(OffscreenMeshVisualizer::Texture); 395 | visualizer.SetMVPMode(OffscreenMeshVisualizer::OrthoNormal); 396 | QImage img = visualizer.Render(); 397 | img.save("albedo_index.png"); 398 | albedo_index_map = img; 399 | } 400 | return albedo_index_map; 401 | } 402 | 403 | inline pair>> GetPixelCoordinatesMap( 404 | const string& albedo_pixel_map_filename, 405 | const QImage& albedo_index_map, 406 | const BasicMesh& mesh, 407 | bool gen_pixel_map = true, 408 | int tex_size = 2048) { 409 | 410 | vector> albedo_pixel_map(tex_size, vector(tex_size)); 411 | 412 | // Generate pixel map for albedo 413 | QImage pixel_map_image; 414 | if(QFile::exists(albedo_pixel_map_filename.c_str()) && (!gen_pixel_map)) { 415 | pixel_map_image = QImage(albedo_pixel_map_filename.c_str()); 416 | 417 | PhGUtils::message("generating pixel map for albedo ..."); 418 | boost::timer::auto_cpu_timer t("pixel map for albedo generation time = %w seconds.\n"); 419 | 420 | for(int i=0;i(qRed(pix)); 424 | unsigned char g = static_cast(qGreen(pix)); 425 | unsigned char b = static_cast(qBlue(pix)); 426 | if(r == 0 && g == 0 && b == 0) continue; 427 | int fidx; 428 | decode_index(r, g, b, fidx); 429 | 430 | QRgb bcoords_pix = pixel_map_image.pixel(j, i); 431 | 432 | float x = static_cast(qRed(bcoords_pix)) / 255.0f; 433 | float y = static_cast(qGreen(bcoords_pix)) / 255.0f; 434 | float z = static_cast(qBlue(bcoords_pix)) / 255.0f; 435 | albedo_pixel_map[i][j] = PixelInfo(fidx, glm::vec3(x, y, z)); 436 | } 437 | } 438 | //pixel_map_image.save("albedo_pixel.png"); 439 | PhGUtils::message("done."); 440 | } else { 441 | /// @FIXME antialiasing issue because of round-off error 442 | pixel_map_image = QImage(tex_size, tex_size, QImage::Format_ARGB32); 443 | pixel_map_image.fill(0); 444 | PhGUtils::message("generating pixel map for albedo ..."); 445 | boost::timer::auto_cpu_timer t("pixel map for albedo generation time = %w seconds.\n"); 446 | 447 | for(int i=0;i(tex_size); 450 | double x = (j + 0.5) / static_cast(tex_size); 451 | 452 | QRgb pix = albedo_index_map.pixel(j, i); 453 | unsigned char r = static_cast(qRed(pix)); 454 | unsigned char g = static_cast(qGreen(pix)); 455 | unsigned char b = static_cast(qBlue(pix)); 456 | if(r == 0 && g == 0 && b == 0) continue; 457 | int fidx; 458 | decode_index(r, g, b, fidx); 459 | 460 | auto f = mesh.face_texture(fidx); 461 | auto t0 = mesh.texture_coords(f[0]), t1 = mesh.texture_coords(f[1]), t2 = mesh.texture_coords(f[2]); 462 | 463 | using PhGUtils::Point3f; 464 | using PhGUtils::Point2d; 465 | Point3f bcoords; 466 | // Compute barycentric coordinates 467 | PhGUtils::computeBarycentricCoordinates(Point2d(x, y), 468 | Point2d(t0[0], t0[1]), Point2d(t1[0], t1[1]), Point2d(t2[0], t2[1]), 469 | bcoords); 470 | //cerr << bcoords << endl; 471 | albedo_pixel_map[i][j] = PixelInfo(fidx, glm::vec3(bcoords.x, bcoords.y, bcoords.z)); 472 | 473 | pixel_map_image.setPixel(j, i, qRgb(bcoords.x*255, bcoords.y*255, bcoords.z*255)); 474 | } 475 | } 476 | pixel_map_image.save("albedo_pixel.jpg"); 477 | PhGUtils::message("done."); 478 | } 479 | 480 | return make_pair(pixel_map_image, albedo_pixel_map); 481 | } 482 | 483 | inline void ApplyWeights( 484 | BasicMesh& mesh, 485 | const vector& blendshapes, 486 | const VectorXd& weights 487 | ) { 488 | const int num_blendshapes = 46; 489 | MatrixX3d verts0 = blendshapes[0].vertices(); 490 | MatrixX3d verts = verts0; 491 | for(int j=1;j<=num_blendshapes;++j) { 492 | verts += (blendshapes[j].vertices() - verts0) * weights(j); 493 | } 494 | mesh.vertices() = verts; 495 | mesh.ComputeNormals(); 496 | } 497 | 498 | inline tuple>> GenerateMeanTexture( 499 | const vector image_bundles, 500 | MultilinearModel& model, 501 | const vector& blendshapes, 502 | BasicMesh& mesh, 503 | int tex_size, 504 | vector>& albedo_pixel_map, 505 | vector>& mean_texture, 506 | vector>& mean_texture_weight, 507 | cv::Mat& mean_texture_mat, 508 | const string& mean_albedo_filename, 509 | const fs::path& results_path, 510 | const string& options) { 511 | QImage mean_texture_image; 512 | vector> face_indices_maps; 513 | { 514 | json settings = json::parse(options); 515 | 516 | cout << settings << endl; 517 | 518 | bool generate_mean_texture = settings["generate_mean_texture"]; 519 | bool use_blendshapes = settings["use_blendshapes"]; 520 | 521 | // use a larger scale when generating mean texture with blendshapes 522 | // since blendshapes are subdivided meshes and each triangle is much smaller 523 | double scale_factor = 6.0; 524 | if(use_blendshapes) scale_factor = 8.0; 525 | 526 | for(auto& bundle : image_bundles) { 527 | // get the geometry of the mesh, update normal 528 | if(use_blendshapes) { 529 | ApplyWeights(mesh, blendshapes, bundle.params.params_model.Wexp_FACS); 530 | } else { 531 | model.ApplyWeights(bundle.params.params_model.Wid, bundle.params.params_model.Wexp); 532 | mesh.UpdateVertices(model.GetTM()); 533 | mesh.ComputeNormals(); 534 | } 535 | 536 | // for each image bundle, render the mesh to FBO with culling to get the visible triangles 537 | OffscreenMeshVisualizer visualizer(bundle.image.width() * scale_factor, bundle.image.height() * scale_factor); 538 | visualizer.SetMVPMode(OffscreenMeshVisualizer::CamPerspective); 539 | visualizer.SetRenderMode(OffscreenMeshVisualizer::Mesh); 540 | visualizer.BindMesh(mesh); 541 | visualizer.SetCameraParameters(bundle.params.params_cam); 542 | visualizer.SetMeshRotationTranslation(bundle.params.params_model.R, bundle.params.params_model.T); 543 | visualizer.SetIndexEncoded(true); 544 | visualizer.SetEnableLighting(false); 545 | QImage img = visualizer.Render(); 546 | img.save("mesh.png"); 547 | 548 | // find the visible triangles from the index map 549 | auto triangles_indices_pair = FindTrianglesIndices(img); 550 | set triangles = triangles_indices_pair.first; 551 | face_indices_maps.push_back(triangles_indices_pair.second); 552 | cerr << triangles.size() << endl; 553 | 554 | // get the projection parameters 555 | glm::dmat4 Rmat = glm::eulerAngleYXZ(bundle.params.params_model.R[0], bundle.params.params_model.R[1], 556 | bundle.params.params_model.R[2]); 557 | glm::dmat4 Tmat = glm::translate(glm::dmat4(1.0), 558 | glm::dvec3(bundle.params.params_model.T[0], 559 | bundle.params.params_model.T[1], 560 | bundle.params.params_model.T[2])); 561 | glm::dmat4 Mview = Tmat * Rmat; 562 | 563 | // for each visible triangle, compute the coordinates of its 3 corners 564 | QImage img_vertices = img; 565 | map> triangles_projected; 566 | vector transforms(mesh.NumFaces()); 567 | for(auto tidx : triangles) { 568 | auto face_i = mesh.face(tidx); 569 | auto v0_mesh = mesh.vertex(face_i[0]); 570 | auto v1_mesh = mesh.vertex(face_i[1]); 571 | auto v2_mesh = mesh.vertex(face_i[2]); 572 | glm::dvec3 v0_tri = ProjectPoint(glm::dvec3(v0_mesh[0], v0_mesh[1], v0_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 573 | glm::dvec3 v1_tri = ProjectPoint(glm::dvec3(v1_mesh[0], v1_mesh[1], v1_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 574 | glm::dvec3 v2_tri = ProjectPoint(glm::dvec3(v2_mesh[0], v2_mesh[1], v2_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 575 | triangles_projected[tidx] = vector{v0_tri, v1_tri, v2_tri}; 576 | 577 | img_vertices.setPixel(v0_tri.x, img.height()-1-v0_tri.y, qRgb(255, 255, 255)); 578 | img_vertices.setPixel(v1_tri.x, img.height()-1-v1_tri.y, qRgb(255, 255, 255)); 579 | img_vertices.setPixel(v2_tri.x, img.height()-1-v2_tri.y, qRgb(255, 255, 255)); 580 | 581 | // TODO Warp this triangle into the destination 582 | // auto tex_coords_i = mesh.face_texture(tidx); 583 | // cv::Point2f dst_points[3] = { 584 | // cv::Point2f(v0_tri.x, img.height()-1-v0_tri.y), 585 | // cv::Point2f(v1_tri.x, img.height()-1-v1_tri.y), 586 | // cv::Point2f(v2_tri.x, img.height()-1-v2_tri.y), 587 | // }; 588 | // cv::Point2f src_points[3] = { 589 | // cv::Point2f(mesh.texture_coords(tex_coords_i[0])[0], 1 - mesh.texture_coords(tex_coords_i[0])[1]) * tex_size, 590 | // cv::Point2f(mesh.texture_coords(tex_coords_i[1])[0], 1 - mesh.texture_coords(tex_coords_i[1])[1]) * tex_size, 591 | // cv::Point2f(mesh.texture_coords(tex_coords_i[2])[0], 1 - mesh.texture_coords(tex_coords_i[2])[1]) * tex_size 592 | // }; 593 | // cv::Mat transform_i = cv::getAffineTransform(src_points, dst_points); 594 | // transforms[tidx] = transform_i; 595 | } 596 | img_vertices.save("mesh_with_vertices.png"); 597 | 598 | // QImage tex_img_warped(tex_size, tex_size, QImage::Format_ARGB32); 599 | // tex_img_warped.fill(0); 600 | // for(int i=0;i= mesh.NumFaces()) continue; 622 | 623 | // auto face_i = mesh.face(fidx); 624 | // auto v0_mesh = mesh.vertex(face_i[0]); 625 | // auto v1_mesh = mesh.vertex(face_i[1]); 626 | // auto v2_mesh = mesh.vertex(face_i[2]); 627 | // glm::dvec3 v0_tri = ProjectPoint(glm::dvec3(v0_mesh[0], v0_mesh[1], v0_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 628 | // glm::dvec3 v1_tri = ProjectPoint(glm::dvec3(v1_mesh[0], v1_mesh[1], v1_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 629 | // glm::dvec3 v2_tri = ProjectPoint(glm::dvec3(v2_mesh[0], v2_mesh[1], v2_mesh[2]), Mview, bundle.params.params_cam) * scale_factor; 630 | // triangles_projected.push_back(vector{v0_tri, v1_tri, v2_tri}); 631 | 632 | glm::dvec3 v0_tri, v1_tri, v2_tri; 633 | auto tri_verts_2d = triangles_projected.at(fidx); 634 | v0_tri = tri_verts_2d[0]; 635 | v1_tri = tri_verts_2d[1]; 636 | v2_tri = tri_verts_2d[2]; 637 | 638 | using PhGUtils::Point3f; 639 | using PhGUtils::Point2d; 640 | Point3f bcoords; 641 | // Compute barycentric coordinates 642 | PhGUtils::computeBarycentricCoordinates(Point2d(j+0.5, i+0.5), 643 | Point2d(v0_tri.x, img.height()-1-v0_tri.y), 644 | Point2d(v1_tri.x, img.height()-1-v1_tri.y), 645 | Point2d(v2_tri.x, img.height()-1-v2_tri.y), 646 | bcoords); 647 | // Color the pixel 648 | img_bcoords.setPixel(j, i, qRgb(bcoords.x*255, bcoords.y*255, bcoords.z*255)); 649 | } 650 | } 651 | img_bcoords.save("mesh_with_bcoords.png"); 652 | 653 | if(generate_mean_texture) { 654 | // populate the current texture map 655 | QImage tex_img_i(tex_size, tex_size, QImage::Format_ARGB32); 656 | tex_img_i.fill(0); 657 | 658 | // for each pixel in the texture map, use backward projection to obtain pixel value in the input image 659 | // accumulate the texels in average texel map 660 | for(int i=0;i(i, j) = cv::Vec3d(0, 0, 0); 716 | continue; 717 | } else { 718 | glm::dvec3 texel = (mean_texture[i][j] + mean_texture[i][tex_size-1-j]) / (weight_ij + weight_ij_s); 719 | mean_texture[i][j] = texel; 720 | mean_texture[i][tex_size-1-j] = texel; 721 | mean_texture_image.setPixel(j, i, qRgb(texel.r, texel.g, texel.b)); 722 | mean_texture_image.setPixel(tex_size-1-j, i, qRgb(texel.r, texel.g, texel.b)); 723 | 724 | mean_texture_mat.at(i, j) = cv::Vec3d(texel.x, texel.y, texel.z); 725 | mean_texture_mat.at(i, tex_size-1-j) = cv::Vec3d(texel.x, texel.y, texel.z); 726 | } 727 | } 728 | } 729 | } else { 730 | for(int i=0;i(i, j) = cv::Vec3d(0, 0, 0); 736 | continue; 737 | } else { 738 | glm::dvec3 texel = mean_texture[i][j] / weight_ij; 739 | mean_texture[i][j] = texel; 740 | mean_texture_image.setPixel(j, i, qRgb(texel.r, texel.g, texel.b)); 741 | mean_texture_mat.at(i, j) = cv::Vec3d(texel.x, texel.y, texel.z); 742 | } 743 | } 744 | } 745 | } 746 | 747 | string refine_method = settings["refine_method"]; 748 | 749 | cv::Mat mean_texture_refined_mat = mean_texture_mat; 750 | if(refine_method == "mean_shift") { 751 | cv::resize(mean_texture_mat, mean_texture_mat, cv::Size(), 0.25, 0.25); 752 | mean_texture_refined_mat = StatsUtils::MeanShiftSegmentation(mean_texture_refined_mat, 5.0, 30.0, 0.5); 753 | mean_texture_refined_mat = 0.25 * mean_texture_mat + 0.75 * mean_texture_refined_mat; 754 | mean_texture_refined_mat = StatsUtils::MeanShiftSegmentation(mean_texture_refined_mat, 10.0, 30.0, 0.5); 755 | mean_texture_refined_mat = 0.25 * mean_texture_mat + 0.75 * mean_texture_refined_mat; 756 | mean_texture_refined_mat = StatsUtils::MeanShiftSegmentation(mean_texture_refined_mat, 20.0, 30.0, 0.5); 757 | mean_texture_refined_mat = 0.25 * mean_texture_mat + 0.75 * mean_texture_refined_mat; 758 | cv::resize(mean_texture_refined_mat, mean_texture_refined_mat, cv::Size(), 4.0, 4.0); 759 | } else if (refine_method == "hsv") { 760 | cout << "Refine using hsv method ..." << endl; 761 | // refine using core face region and clustering in hsv space 762 | const string core_face_region_filename = settings["core_face_region_filename"]; 763 | auto core_face_region = cv::imread(core_face_region_filename.c_str(), CV_LOAD_IMAGE_GRAYSCALE); 764 | 765 | vector valid_pixels; 766 | for(int i=0;i(i, j); 769 | if( c > 0 ) valid_pixels.push_back(glm::ivec2(i, j)); 770 | } 771 | } 772 | cout << "valid pixels = " << valid_pixels.size() << endl; 773 | 774 | glm::dvec3 mean_color(0, 0, 0); 775 | for(auto p : valid_pixels) { 776 | cv::Vec3d pix = mean_texture_refined_mat.at(p.x, p.y); 777 | mean_color.r += pix[0] / 255.0; 778 | mean_color.g += pix[1] / 255.0; 779 | mean_color.b += pix[2] / 255.0; 780 | } 781 | mean_color /= valid_pixels.size(); 782 | cv::Vec3d mean_color_vec(mean_color.r*255.0, mean_color.g*255.0, mean_color.b*255.0); 783 | 784 | QColor mean_color_qt = QColor::fromRgbF(mean_color.r, mean_color.g, mean_color.b); 785 | glm::dvec3 mean_hsv; 786 | mean_color_qt.getHsvF(&mean_hsv.r, &mean_hsv.g, &mean_hsv.b); 787 | 788 | const double distance_threshold = settings["hsv_threshold"]; 789 | // convert the entire image to hsv 790 | for(int i=0;i(i, j); 793 | QColor pix_color = QColor::fromRgb(pix[0], pix[1], pix[2]); 794 | glm::dvec3 pix_hsv; 795 | pix_color.getHsvF(&pix_hsv.r, &pix_hsv.g, &pix_hsv.b); 796 | //double d_ij = fabs(pix_hsv.r - mean_hsv.r); 797 | double d_ij = glm::distance2(pix_hsv, mean_hsv); 798 | if(d_ij < distance_threshold) { 799 | // Change this ratio to control how much details to include in the albedo 800 | const double mix_ratio = settings["mix_ratio"]; 801 | mean_texture_refined_mat.at(i, j) = mean_color_vec * mix_ratio + mean_texture_refined_mat.at(i, j) * (1-mix_ratio); 802 | } 803 | } 804 | } 805 | } else { 806 | // no refinement 807 | } 808 | 809 | QImage mean_texture_image_refined(tex_size, tex_size, QImage::Format_ARGB32); 810 | for(int i=0;i(i, j); 813 | mean_texture_image_refined.setPixel(j, i, qRgb(pix[0], pix[1], pix[2])); 814 | } 815 | } 816 | 817 | mean_texture_image.save( (results_path / fs::path("mean_texture.png")).string().c_str() ); 818 | mean_texture_image_refined.save( (results_path / fs::path("mean_texture_refined.png")).string().c_str() ); 819 | mean_texture_image = mean_texture_image_refined; 820 | } else { 821 | mean_texture_image = QImage(mean_albedo_filename.c_str()); 822 | mean_texture_image.save( (results_path / fs::path("mean_texture.png")).string().c_str() ); 823 | } 824 | } 825 | 826 | return make_tuple(mean_texture_image, face_indices_maps); 827 | } 828 | 829 | #endif //FACESHAPEFROMSHADING_UTILS_H 830 | --------------------------------------------------------------------------------