├── .gitignore ├── .gitmodules ├── BlendshapeGeneration ├── .gitignore ├── CMakeLists.txt ├── blendshapegeneration.cpp ├── blendshapegeneration.h ├── blendshapegeneration.pro ├── blendshapegeneration.qrc ├── blendshapegeneration.ui ├── blendshaperefiner.cpp ├── blendshaperefiner.h ├── blendshaperefiner_old.cpp ├── blendshaperefiner_old.h ├── blendshapes_driver.cpp ├── blendshapesweightswidget.cpp ├── blendshapesweightswidget.h ├── blendshapesweightswidget.ui ├── cereswrapper.h ├── common.cpp ├── common.h ├── compute_details.cpp ├── deformationtransfer.cpp ├── expressiontransfer.cpp ├── interactiverigging.cpp ├── interactiverigging.h ├── landmarks_74_new.txt ├── laplaciandeformation.cpp ├── main.cpp ├── meshdeformer.cpp ├── meshdeformer.h ├── meshfitter.cpp ├── meshtransferer.cpp ├── meshtransferer.h ├── ndarray.hpp ├── pointcloud.cpp ├── pointcloud.h ├── testcases.cpp ├── testcases.h └── triangle_gradient.h └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | build-blendshapegeneration-Desktop-Debug/* 2 | build-blendshapegeneration-Desktop-Release/* 3 | BlendshapeGeneration-release/* 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "BlendshapeGeneration/MultilinearReconstruction"] 2 | path = BlendshapeGeneration/MultilinearReconstruction 3 | url = ../MultilinearReconstruction.git 4 | [submodule "BlendshapeGeneration/json"] 5 | path = BlendshapeGeneration/json 6 | url = https://github.com/nlohmann/json 7 | -------------------------------------------------------------------------------- /BlendshapeGeneration/.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | /blendshapegeneration.pro.user 3 | .idea/* 4 | cmake-build-release/* 5 | -------------------------------------------------------------------------------- /BlendshapeGeneration/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.11) 2 | 3 | project(BlendshapeGeneration) 4 | 5 | # Find includes in corresponding build directories 6 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 7 | # Instruct CMake to run moc automatically when needed. 8 | set(CMAKE_AUTOMOC ON) 9 | set(CMAKE_AUTOUIC ON) 10 | 11 | include(CheckCXXCompilerFlag) 12 | CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) 13 | CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) 14 | if(COMPILER_SUPPORTS_CXX11) 15 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 16 | elseif(COMPILER_SUPPORTS_CXX0X) 17 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") 18 | else() 19 | message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") 20 | endif() 21 | 22 | # Boost 23 | find_package(Boost COMPONENTS filesystem timer program_options REQUIRED) 24 | include_directories(${Boost_INCLUDE_DIRS}) 25 | link_libraries(${Boost_LIBRARIES} -lboost_filesystem -lboost_system) 26 | 27 | # OpenMP 28 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") 29 | 30 | # OpenCV 31 | find_package( OpenCV REQUIRED ) 32 | message("-- ${OPENCV_LIBS}") 33 | link_libraries(${OpenCV_LIBS}) 34 | 35 | # OpenGL 36 | find_package(OpenGL REQUIRED) 37 | find_package(GLUT REQUIRED) 38 | include_directories( ${OPENGL_INCLUDE_DIRS} ${GLUT_INCLUDE_DIRS} ) 39 | link_libraries(${OPENGL_LIBRARIES} ${GLUT_LIBRARY}) 40 | 41 | find_package(GLEW REQUIRED) 42 | if (GLEW_FOUND) 43 | include_directories(${GLEW_INCLUDE_DIRS}) 44 | link_libraries(${GLEW_LIBRARIES}) 45 | endif() 46 | 47 | # GLI 48 | #find_package(GLI REQUIRED) 49 | #if (GLI_FOUND) 50 | # message([] ${GLI_INCLUDE_DIRS}) 51 | # include_directories(${GLI_INCLUDE_DIRS}) 52 | #endif() 53 | 54 | # Eigen 55 | find_package(Eigen) 56 | include_directories(${EIGEN3_INCLUDE_DIRS}) 57 | message("-- Eigen include path is ${EIGEN3_INCLUDE_DIRS}") 58 | 59 | # SuiteSparse 60 | set(SUITESPARSE_INCLUDE_DIR "/usr/include/suitesparse") 61 | include_directories(${SUITESPARSE_INCLUDE_DIR}) 62 | 63 | # MKL 64 | set(MKL_INCLUDE_DIRS "/opt/intel/mkl/include") 65 | message([] ${MKL_INCLUDE_DIRS} ) 66 | include_directories(${MKL_INCLUDE_DIRS}) 67 | set(MKLROOT "/opt/intel/mkl") 68 | set(MKLLIBS_DIRS "${MKLROOT}/lib/intel64/" "/opt/intel/lib/intel64_lin") 69 | link_directories(${MKLLIBS_DIRS}) 70 | set(MKLLIBS "-Wl,--start-group -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -Wl,--end-group -liomp5 -ldl -lpthread -lm") 71 | 72 | # PhGLib 73 | include_directories("$ENV{HOME}/SDKs/PhGLib/include") 74 | link_directories("$ENV{HOME}/SDKs/PhGLib/lib") 75 | set(PhGLib "-lPhGLib") 76 | 77 | # Qt5 78 | find_package(Qt5Core) 79 | find_package(Qt5Widgets) 80 | find_package(Qt5OpenGL) 81 | 82 | # Ceres solver 83 | find_package(Ceres REQUIRED) 84 | include_directories(${CERES_INCLUDE_DIRS}) 85 | link_libraries(${CERES_LIBRARIES}) 86 | 87 | include_directories("${CMAKE_CURRENT_LIST_DIR}/MultilinearReconstruction/third_party/json/include") 88 | 89 | 90 | add_library(meshdeformer meshdeformer.cpp) 91 | target_link_libraries(meshdeformer 92 | Qt5::Core 93 | Qt5::Widgets 94 | Qt5::OpenGL 95 | CGAL 96 | ${MKLLIBS} 97 | ${OPENCV_LIBS} 98 | ${PhGLib} 99 | basicmesh 100 | ioutilities 101 | multilinearmodel 102 | reporter 103 | offscreenmeshvisualizer 104 | ) 105 | 106 | add_library(meshtransferer meshtransferer.cpp) 107 | target_link_libraries(meshtransferer 108 | Qt5::Core 109 | Qt5::Widgets 110 | Qt5::OpenGL 111 | CGAL 112 | ${MKLLIBS} 113 | ${OPENCV_LIBS} 114 | ${PhGLib} 115 | basicmesh 116 | ioutilities 117 | multilinearmodel 118 | reporter 119 | offscreenmeshvisualizer 120 | ) 121 | 122 | # Face shape from shading program 123 | add_executable(BlendshapeGeneration main.cpp 124 | blendshapegeneration.cpp 125 | interactiverigging.cpp 126 | blendshapesweightswidget.cpp 127 | blendshapesweightswidget.h 128 | pointcloud.cpp blendshaperefiner.cpp blendshaperefiner.h blendshaperefiner_old.cpp blendshaperefiner_old.h) 129 | 130 | target_link_libraries(BlendshapeGeneration 131 | Qt5::Core 132 | Qt5::Widgets 133 | Qt5::OpenGL 134 | CGAL 135 | ${MKLLIBS} 136 | ${OPENCV_LIBS} 137 | ${PhGLib} 138 | basicmesh 139 | ioutilities 140 | multilinearmodel 141 | reporter 142 | meshdeformer 143 | meshtransferer) 144 | 145 | add_executable(compute_details compute_details.cpp) 146 | target_link_libraries(compute_details 147 | Qt5::Core 148 | Qt5::Widgets 149 | Qt5::OpenGL 150 | CGAL 151 | ${MKLLIBS} 152 | ${PhGLib} 153 | basicmesh 154 | ioutilities 155 | multilinearmodel 156 | meshdeformer) 157 | 158 | add_executable(laplaciandeformation laplaciandeformation.cpp) 159 | target_link_libraries(laplaciandeformation 160 | Qt5::Core 161 | Qt5::Widgets 162 | Qt5::OpenGL 163 | CGAL 164 | ${MKLLIBS} 165 | ${PhGLib} 166 | basicmesh 167 | ioutilities 168 | multilinearmodel 169 | meshdeformer) 170 | 171 | add_executable(deformationtransfer deformationtransfer.cpp) 172 | target_link_libraries(deformationtransfer 173 | Qt5::Core 174 | Qt5::Widgets 175 | Qt5::OpenGL 176 | CGAL 177 | ${MKLLIBS} 178 | ${PhGLib} 179 | basicmesh 180 | ioutilities 181 | multilinearmodel 182 | meshtransferer) 183 | 184 | add_executable(expressiontransfer expressiontransfer.cpp) 185 | target_link_libraries(expressiontransfer 186 | Qt5::Core 187 | Qt5::Widgets 188 | Qt5::OpenGL 189 | CGAL 190 | ${MKLLIBS} 191 | ${PhGLib} 192 | basicmesh 193 | ioutilities 194 | multilinearmodel 195 | meshdeformer 196 | meshtransferer 197 | offscreenmeshvisualizer 198 | meshvisualizer) 199 | 200 | add_executable(meshfitter meshfitter.cpp) 201 | target_link_libraries(meshfitter 202 | Qt5::Core 203 | Qt5::Widgets 204 | Qt5::OpenGL 205 | CGAL 206 | ${MKLLIBS} 207 | ${PhGLib} 208 | basicmesh 209 | ioutilities 210 | multilinearmodel 211 | meshdeformer 212 | meshtransferer 213 | offscreenmeshvisualizer 214 | meshvisualizer) 215 | 216 | add_executable(blendshapes_driver blendshapes_driver.cpp) 217 | target_link_libraries(blendshapes_driver 218 | Qt5::Core 219 | Qt5::Widgets 220 | Qt5::OpenGL 221 | CGAL 222 | ${MKLLIBS} 223 | ${PhGLib} 224 | basicmesh 225 | ioutilities 226 | multilinearmodel 227 | meshdeformer 228 | meshtransferer 229 | offscreenmeshvisualizer 230 | meshvisualizer) 231 | 232 | link_directories(MultilinearReconstruction) 233 | 234 | add_subdirectory(MultilinearReconstruction) 235 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapegeneration.cpp: -------------------------------------------------------------------------------- 1 | #include "blendshapegeneration.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | typedef CGAL::Simple_cartesian K; 15 | typedef K::FT FT; 16 | typedef K::Line_3 Line; 17 | typedef K::Point_3 Point; 18 | typedef K::Triangle_3 Triangle; 19 | typedef std::vector::iterator Iterator; 20 | typedef CGAL::AABB_triangle_primitive Primitive; 21 | typedef CGAL::AABB_traits AABB_triangle_traits; 22 | typedef CGAL::AABB_tree Tree; 23 | 24 | OffscreenBlendshapeVisualizer::OffscreenBlendshapeVisualizer(int w, int h) 25 | : use_side_view(false), canvasW(w), canvasH(h), has_texture(false) { 26 | // Load and Parse rendering settings 27 | { 28 | const string home_directory = QDir::homePath().toStdString(); 29 | cout << "Home dir: " << home_directory << endl; 30 | std::ifstream fin(home_directory + "/Data/Settings/blendshape_vis.json"); 31 | fin >> rendering_settings; 32 | //cout << rendering_settings << endl; 33 | } 34 | 35 | SetBasicRenderingParams(); 36 | } 37 | OffscreenBlendshapeVisualizer::~OffscreenBlendshapeVisualizer() {} 38 | 39 | void OffscreenBlendshapeVisualizer::SetBasicRenderingParams() { 40 | // Basic setup 41 | sceneScale = rendering_settings["scene_scale"]; 42 | cameraPos = QVector3D( 43 | rendering_settings["campos"][0], 44 | rendering_settings["campos"][1], 45 | rendering_settings["campos"][2]); 46 | 47 | string render_mode = rendering_settings["mode"]; 48 | if(render_mode == "perspective") { 49 | projectionMode = PERSPECTIVE; 50 | } else { 51 | projectionMode = ORTHONGONAL; 52 | } 53 | } 54 | 55 | void OffscreenBlendshapeVisualizer::loadMesh(const string& filename) { 56 | mesh.LoadOBJMesh(filename); 57 | mesh.ComputeNormals(); 58 | computeDistance(); 59 | } 60 | 61 | void OffscreenBlendshapeVisualizer::loadReferenceMesh(const string& filename) { 62 | refmesh.LoadOBJMesh(filename); 63 | refmesh.ComputeNormals(); 64 | computeDistance(); 65 | } 66 | 67 | namespace { 68 | BasicMesh AlignMesh(const BasicMesh& source, const BasicMesh& target) { 69 | BasicMesh aligned = source; 70 | 71 | bool use_face_region = true; 72 | vector valid_faces; 73 | if(!use_face_region) { 74 | valid_faces.resize(target.NumFaces()); 75 | for(int i=0;i triangles; 90 | triangles.reserve(nfaces); 91 | 92 | { 93 | boost::timer::auto_cpu_timer t("[mesh alignment] data preparation time = %w seconds.\n"); 94 | for(int i=0,ioffset=0;i> correspondence; 118 | for(int i=0;i 0.1) { 141 | continue; 142 | } else { 143 | correspondence.push_back(make_pair(point_i, Vector3d(qx, qy, qz))); 144 | } 145 | } 146 | } 147 | 148 | // Solve for R and t using the correspondence 149 | const int nconstraints = correspondence.size(); 150 | cout << iters << ": " << nconstraints << endl; 151 | MatrixX3d P, Q; 152 | P.resize(nconstraints, 3); Q.resize(nconstraints, 3); 153 | for(int i=0;i svd(S, ComputeThinU | ComputeThinV); 166 | auto sigma = svd.singularValues(); 167 | MatrixXd U = svd.matrixU(); 168 | MatrixXd V = svd.matrixV(); 169 | MatrixXd VdUt = V * U.transpose(); 170 | double detVU = VdUt.determinant(); 171 | Matrix3d eye3 = Matrix3d::Identity(); 172 | eye3(2, 2) = detVU; 173 | Matrix3d R = V * eye3 * U.transpose(); 174 | Vector3d t = qbar - R * pbar; 175 | 176 | // Apply R and t to the vertices of aligned 177 | aligned.vertices() = 178 | ((R * aligned.vertices().transpose() + t.replicate(1, aligned.NumVertices())).transpose()).eval(); 179 | } 180 | 181 | return aligned; 182 | } 183 | 184 | BasicMesh AlignMeshTranslation(const BasicMesh& source, const BasicMesh& target) { 185 | BasicMesh aligned = source; 186 | 187 | bool use_face_region = true; 188 | vector valid_faces; 189 | if(!use_face_region) { 190 | valid_faces.resize(target.NumFaces()); 191 | for(int i=0;i triangles; 206 | triangles.reserve(nfaces); 207 | 208 | { 209 | boost::timer::auto_cpu_timer t("[mesh alignment] data preparation time = %w seconds.\n"); 210 | for(int i=0,ioffset=0;i> correspondence; 234 | for(int i=0;i 0.1) { 257 | continue; 258 | } else { 259 | correspondence.push_back(make_pair(point_i, Vector3d(qx, qy, qz))); 260 | } 261 | } 262 | } 263 | 264 | // Solve for R and t using the correspondence 265 | const int nconstraints = correspondence.size(); 266 | cout << iters << ": " << nconstraints << endl; 267 | MatrixX3d P, Q; 268 | P.resize(nconstraints, 3); Q.resize(nconstraints, 3); 269 | for(int i=0;i svd(S, ComputeThinU | ComputeThinV); 282 | auto sigma = svd.singularValues(); 283 | MatrixXd U = svd.matrixU(); 284 | MatrixXd V = svd.matrixV(); 285 | MatrixXd VdUt = V * U.transpose(); 286 | double detVU = VdUt.determinant(); 287 | Matrix3d eye3 = Matrix3d::Identity(); 288 | eye3(2, 2) = detVU; 289 | Matrix3d R = V * eye3 * U.transpose(); 290 | Vector3d t = qbar - R * pbar; 291 | 292 | // Apply translation part to the vertices of aligned 293 | aligned.vertices() = (aligned.vertices() + t.replicate(1, aligned.NumVertices()).transpose()).eval(); 294 | } 295 | 296 | return aligned; 297 | } 298 | } 299 | 300 | void OffscreenBlendshapeVisualizer::computeDistance() { 301 | boost::timer::auto_cpu_timer t("Distance computation time = %w seconds.\n"); 302 | if( refmesh.NumFaces() <= 0 ) return; 303 | 304 | int nfaces = refmesh.NumFaces(); 305 | 306 | std::vector triangles; 307 | triangles.reserve(nfaces); 308 | 309 | { 310 | boost::timer::auto_cpu_timer t("[compute distance] data preparation time = %w seconds.\n"); 311 | for(int i=0,ioffset=0;i valid_faces; 336 | if(!use_face_region) { 337 | valid_faces.resize(refmesh.NumFaces()); 338 | for(int i=0;irefmesh.NumVertices()?1:0; 351 | for(int i=0;i face_region_indices_new; 353 | for(auto fidx : valid_faces) { 354 | int fidx_base = fidx*4; 355 | face_region_indices_new.push_back(fidx_base); 356 | face_region_indices_new.push_back(fidx_base+1); 357 | face_region_indices_new.push_back(fidx_base+2); 358 | face_region_indices_new.push_back(fidx_base+3); 359 | } 360 | valid_faces = face_region_indices_new; 361 | } 362 | } 363 | set valid_vertices; 364 | if(use_face_region) { 365 | for(auto fidx : valid_faces) { 366 | auto f_i = mesh_cpy.face(fidx); 367 | valid_vertices.insert(f_i[0]); 368 | valid_vertices.insert(f_i[1]); 369 | valid_vertices.insert(f_i[2]); 370 | } 371 | } else { 372 | for(int i=0;i 0.1) { 405 | dists[i] = 0.1; 406 | } 407 | } 408 | } 409 | } 410 | 411 | void OffscreenBlendshapeVisualizer::setupViewing() { 412 | glViewport(0, 0, canvasW, canvasH); 413 | glMatrixMode(GL_PROJECTION); 414 | glLoadIdentity(); 415 | 416 | switch(projectionMode) 417 | { 418 | case ORTHONGONAL: 419 | { 420 | glOrtho (-1.0 * sceneScale, 1.0 * sceneScale, 421 | -1.0 * sceneScale, 1.0 * sceneScale, 422 | -1.0 * sceneScale, 1.0 * sceneScale); 423 | break; 424 | } 425 | case FRUSTUM: 426 | { 427 | glFrustum(-1.0, 1.0, -1.0, 1.0, 1.0, 2.0); 428 | break; 429 | } 430 | case PERSPECTIVE: 431 | { 432 | gluPerspective(25.0, (float)width()/(float)height(), 0.01f, sceneScale * 10.0); 433 | break; 434 | } 435 | default: 436 | break; 437 | } 438 | 439 | glMatrixMode (GL_MODELVIEW); 440 | 441 | glLoadIdentity (); 442 | 443 | if(projectionMode==PERSPECTIVE) 444 | { 445 | //QVector3D scaledCameraPos = cameraPos / trackBall.getScale(); 446 | QVector3D scaledCameraPos = cameraPos; 447 | gluLookAt( 448 | scaledCameraPos.x(), scaledCameraPos.y(), scaledCameraPos.z(), 449 | 0.0, 0.0, 0.0, 450 | 0.0, 1.0, 0.0); 451 | } 452 | } 453 | 454 | void OffscreenBlendshapeVisualizer::paint() 455 | { 456 | boost::timer::auto_cpu_timer t("render time = %w seconds.\n"); 457 | 458 | // Create a offscreen surface for drawing 459 | QSurfaceFormat format; 460 | format.setMajorVersion(3); 461 | format.setMinorVersion(3); 462 | 463 | QOffscreenSurface surface; 464 | surface.setFormat(format); 465 | surface.create(); 466 | 467 | QOpenGLContext context; 468 | context.setFormat(format); 469 | if (!context.create()) { 470 | qFatal("Cannot create the requested OpenGL context!"); 471 | } 472 | context.makeCurrent(&surface); 473 | 474 | const QRect drawRect(0, 0, canvasW, canvasH); 475 | const QSize drawRectSize = drawRect.size(); 476 | 477 | QOpenGLFramebufferObjectFormat fboFormat; 478 | fboFormat.setSamples(16); 479 | fboFormat.setAttachment(QOpenGLFramebufferObject::Depth); 480 | 481 | QOpenGLFramebufferObject fbo(drawRectSize, fboFormat); 482 | fbo.bind(); 483 | 484 | setupViewing(); 485 | 486 | glEnable(GL_DEPTH_TEST); 487 | 488 | enableLighting(); 489 | if(use_side_view) { 490 | glPushMatrix(); 491 | glRotatef(-90, 0, 1, 0); 492 | } 493 | 494 | if( mesh.NumFaces() > 0 ) { 495 | if( refmesh.NumFaces() > 0 ) drawMeshWithColor(mesh); 496 | else drawMesh(mesh); 497 | } else { 498 | cout << "Drawing mesh ..." << endl; 499 | glDepthFunc(GL_LEQUAL); 500 | drawMesh(refmesh); 501 | 502 | cout << "Drawing vertices mesh ..." << endl; 503 | glDepthFunc(GL_ALWAYS); 504 | drawMeshVerticesWithColor(mesh); 505 | } 506 | 507 | if(use_side_view) { 508 | glPopMatrix(); 509 | } 510 | 511 | disableLighting(); 512 | 513 | if( !dists.empty() ) { 514 | #if 0 515 | double minVal = (*std::min_element(dists.begin(), dists.end())); 516 | double maxVal = (*std::max_element(dists.begin(), dists.end())); 517 | #else 518 | double minVal = 0.0; 519 | double maxVal = error_range; 520 | #endif 521 | string minStr = "min: " + to_string(minVal); 522 | string maxStr = "max: " + to_string(maxVal); 523 | 524 | drawColorBar(minVal, maxVal); 525 | 526 | QPainter painter; 527 | QOpenGLPaintDevice device(drawRectSize); 528 | painter.begin(&device); 529 | painter.setRenderHints(QPainter::Antialiasing | QPainter::HighQualityAntialiasing); 530 | 531 | painter.setPen(Qt::blue); 532 | painter.setBrush(Qt::blue); 533 | painter.drawText(25, 25, QString(minStr.c_str())); 534 | 535 | painter.setPen(Qt::red); 536 | painter.setBrush(Qt::red); 537 | painter.drawText(25, 45, QString(maxStr.c_str())); 538 | 539 | 540 | // draw texts for the color bar 541 | painter.setPen(Qt::black); 542 | painter.setBrush(Qt::black); 543 | { 544 | double left = 0.85 * width(); 545 | double w = 0.025 * width(); 546 | double top = 0.25 * height(); 547 | //double h = 0.5 * height() / (nsegments-1); 548 | int ntexts = 6; 549 | for(int i=0;i(left + w), 554 | static_cast(top+hpos + 5.0), QString(str.c_str())); 555 | } 556 | } 557 | 558 | painter.end(); 559 | } 560 | 561 | rendered_img = fbo.toImage(); 562 | fbo.release(); 563 | } 564 | 565 | void OffscreenBlendshapeVisualizer::enableLighting() 566 | { 567 | enabled_lights.clear(); 568 | 569 | // Setup material 570 | auto& mat_specular_json = rendering_settings["material"]["specular"]; 571 | GLfloat mat_specular[] = { 572 | mat_specular_json[0], 573 | mat_specular_json[1], 574 | mat_specular_json[2], 575 | mat_specular_json[3] 576 | }; 577 | 578 | auto& mat_diffuse_json = rendering_settings["material"]["diffuse"]; 579 | GLfloat mat_diffuse[] = { 580 | mat_diffuse_json[0], 581 | mat_diffuse_json[1], 582 | mat_diffuse_json[2], 583 | mat_diffuse_json[3] 584 | }; 585 | 586 | auto& mat_shininess_json = rendering_settings["material"]["shininess"]; 587 | GLfloat mat_shininess[] = {mat_shininess_json}; 588 | 589 | glMaterialfv(GL_FRONT_AND_BACK, GL_SPECULAR, mat_specular); 590 | glMaterialfv(GL_FRONT_AND_BACK, GL_SHININESS, mat_shininess); 591 | glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, mat_diffuse); 592 | 593 | // Setup Lights 594 | auto setup_light = [&](json light_json) { 595 | auto light_i = GL_LIGHT0 + enabled_lights.size(); 596 | 597 | GLfloat light_position[] = { 598 | light_json["pos"][0], 599 | light_json["pos"][1], 600 | light_json["pos"][2], 601 | light_json["pos"][3] 602 | }; 603 | GLfloat light_ambient[] = { 604 | light_json["ambient"][0], 605 | light_json["ambient"][1], 606 | light_json["ambient"][2], 607 | light_json["ambient"][3] 608 | }; 609 | GLfloat light_diffuse[] = { 610 | light_json["diffuse"][0], 611 | light_json["diffuse"][1], 612 | light_json["diffuse"][2], 613 | light_json["diffuse"][3] 614 | }; 615 | GLfloat light_specular[] = { 616 | light_json["specular"][0], 617 | light_json["specular"][1], 618 | light_json["specular"][2], 619 | light_json["specular"][3] 620 | }; 621 | 622 | glLightfv(light_i, GL_POSITION, light_position); 623 | glLightfv(light_i, GL_DIFFUSE, light_diffuse); 624 | glLightfv(light_i, GL_SPECULAR, light_specular); 625 | glLightfv(light_i, GL_AMBIENT, light_ambient); 626 | 627 | enabled_lights.push_back(light_i); 628 | }; 629 | 630 | for(json::iterator it = rendering_settings["lights"].begin(); 631 | it != rendering_settings["lights"].end(); 632 | ++it) 633 | { 634 | setup_light(*it); 635 | } 636 | 637 | glEnable(GL_LIGHTING); 638 | for(auto light_i : enabled_lights) glEnable(light_i); 639 | } 640 | 641 | void OffscreenBlendshapeVisualizer::disableLighting() 642 | { 643 | for(auto light_i : enabled_lights) glDisable(light_i); 644 | enabled_lights.clear(); 645 | 646 | glDisable(GL_LIGHTING); 647 | } 648 | 649 | void OffscreenBlendshapeVisualizer::drawColorBar(double minVal, double maxVal) { 650 | 651 | // Store current state 652 | glPushAttrib(GL_ALL_ATTRIB_BITS); 653 | 654 | // setup 2D view 655 | glViewport(0, 0, width(), height()); 656 | glMatrixMode(GL_MODELVIEW); 657 | glPushMatrix(); 658 | glLoadIdentity(); 659 | 660 | glMatrixMode(GL_PROJECTION); 661 | glPushMatrix(); 662 | glLoadIdentity(); 663 | glOrtho(0, width(), height(), 0, -1, 1); 664 | 665 | int nsegments = 128; 666 | 667 | double left = 0.85 * width(); 668 | double w = 0.025 * width(); 669 | double top = 0.25 * height(); 670 | double h = 0.5 * height() / (nsegments-1); 671 | 672 | vector colors(nsegments); 673 | for(int i=0;i(rendering_settings["ao_power"])), 740 | float(mat_diffuse_json[1]) * pow(ao_value, static_cast(rendering_settings["ao_power"])), 741 | float(mat_diffuse_json[2]) * pow(ao_value, static_cast(rendering_settings["ao_power"])), 742 | float(mat_diffuse_json[3]) 743 | }; 744 | glMaterialfv(GL_FRONT_AND_BACK, GL_DIFFUSE, mat_diffuse); 745 | }; 746 | 747 | bool use_customized_normals = !normals.empty(); 748 | 749 | glBegin(GL_TRIANGLES); 750 | for(int i=0;i 0 ) { 975 | if( refmesh.NumFaces() > 0 ) drawMeshWithColor(mesh); 976 | else drawMesh(mesh); 977 | } else { 978 | cout << "Drawing mesh ..." << endl; 979 | glDepthFunc(GL_LEQUAL); 980 | drawMesh(refmesh); 981 | cout << "Drawing vertices mesh ..." << endl; 982 | glDepthFunc(GL_ALWAYS); 983 | drawMeshVerticesWithColor(mesh); 984 | } 985 | disableLighting(); 986 | 987 | if( !dists.empty() ) { 988 | #if 0 989 | double minVal = (*std::min_element(dists.begin(), dists.end())); 990 | double maxVal = (*std::max_element(dists.begin(), dists.end())); 991 | #else 992 | double minVal = 0.0; 993 | double maxVal = 0.05; 994 | #endif 995 | string minStr = "min: " + to_string(minVal); 996 | string maxStr = "max: " + to_string(maxVal); 997 | glColor4f(0, 0, 1, 1); 998 | renderText(25, 25, QString(minStr.c_str())); 999 | glColor4f(1, 0, 0, 1); 1000 | renderText(25, 45, QString(maxStr.c_str())); 1001 | 1002 | drawColorBar(minVal, maxVal); 1003 | } 1004 | } 1005 | 1006 | void BlendshapeVisualizer::drawColorBar(double minVal, double maxVal) { 1007 | // setup 2D view 1008 | glViewport(0, 0, width(), height()); 1009 | glMatrixMode(GL_MODELVIEW); 1010 | glLoadIdentity(); 1011 | glMatrixMode(GL_PROJECTION); 1012 | glLoadIdentity(); 1013 | glOrtho(0, width(), height(), 0, -1, 1); 1014 | 1015 | int nsegments = 128; 1016 | 1017 | double left = 0.85 * width(); 1018 | double w = 0.025 * width(); 1019 | double top = 0.25 * height(); 1020 | double h = 0.5 * height() / (nsegments-1); 1021 | 1022 | vector colors(nsegments); 1023 | for(int i=0;i triangles; 1100 | triangles.reserve(nfaces); 1101 | for(int i=0,ioffset=0;i 0.1) dists[i] = -1; 1139 | } 1140 | } 1141 | } 1142 | 1143 | void BlendshapeVisualizer::drawMesh(const BasicMesh &m) 1144 | { 1145 | glColor4f(0.75, 0.75, 0.75, 1.0); 1146 | glBegin(GL_TRIANGLES); 1147 | for(int i=0;iloadMesh(filename.toStdString()); 1310 | } 1311 | 1312 | void BlendshapeGeneration::slot_loadReferenceMesh() 1313 | { 1314 | QString filename = QFileDialog::getOpenFileName(); 1315 | canvas->loadReferenceMesh(filename.toStdString()); 1316 | } 1317 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapegeneration.h: -------------------------------------------------------------------------------- 1 | #ifndef BLENDSHAPEGENERATION_H 2 | #define BLENDSHAPEGENERATION_H 3 | 4 | #include 5 | #include "ui_blendshapegeneration.h" 6 | 7 | #include "OpenGL/gl3dcanvas.h" 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include "nlohmann/json.hpp" 14 | using json = nlohmann::json; 15 | 16 | class OffscreenBlendshapeVisualizer { 17 | public: 18 | OffscreenBlendshapeVisualizer(int w, int h); 19 | ~OffscreenBlendshapeVisualizer(); 20 | 21 | void setRenderingSettings(json settings) { 22 | rendering_settings = settings; 23 | SetBasicRenderingParams(); 24 | } 25 | 26 | void SetSideView(bool val) { 27 | use_side_view = val; 28 | } 29 | 30 | void SetAmbientOcclusion(const vector& AO) { 31 | ao = AO; 32 | } 33 | void SetNormals(const vector& ns) { 34 | normals = ns; 35 | } 36 | void SetSkipFaces(const vector& skip_faces_in, bool subdivided=false) { 37 | vector faces_to_skip; 38 | for(auto i : skip_faces_in) { 39 | faces_to_skip.push_back(2*i); 40 | faces_to_skip.push_back(2*i+1); 41 | } 42 | if(subdivided) { 43 | //cout << "subdivided" << endl; 44 | for(auto i : faces_to_skip) { 45 | skip_faces.insert(4*i); 46 | skip_faces.insert(4*i+1); 47 | skip_faces.insert(4*i+2); 48 | skip_faces.insert(4*i+3); 49 | } 50 | } else { 51 | skip_faces = set(faces_to_skip.begin(), faces_to_skip.end()); 52 | } 53 | } 54 | 55 | void SetErrorRange(float v) { 56 | error_range = v; 57 | } 58 | 59 | void SetAlignMesh(bool v) { 60 | align_mesh = v; 61 | } 62 | 63 | void SetAlignMeshTranslation(bool v) { 64 | align_mesh_translation = v; 65 | } 66 | 67 | void SetFaceOnlyMode(bool v) { 68 | face_only_mode = v; 69 | } 70 | 71 | void loadMesh(const string& filename); 72 | void loadReferenceMesh(const string& filename); 73 | 74 | void setTexture(const QImage& img) { 75 | texture = img; 76 | has_texture = true; 77 | } 78 | 79 | double getError() const { 80 | boost::timer::auto_cpu_timer t("[compute error] error computation time = %w seconds.\n"); 81 | double avg_error = 0; 82 | int cnt = 0; 83 | for(auto x : dists) { 84 | if(x >= 0) { 85 | ++cnt; 86 | avg_error += x; 87 | } 88 | } 89 | return avg_error / cnt; 90 | } 91 | 92 | int width() const { return canvasW; } 93 | int height() const { return canvasH; } 94 | 95 | QImage render() { 96 | paint(); 97 | return rendered_img; 98 | } 99 | 100 | QImage getImage() { 101 | return rendered_img; 102 | } 103 | 104 | void repaint() { 105 | 106 | } 107 | 108 | protected: 109 | void SetBasicRenderingParams(); 110 | void setupViewing(); 111 | void paint(); 112 | void enableLighting(); 113 | void disableLighting(); 114 | void computeDistance(); 115 | void drawMesh(const BasicMesh& m); 116 | void drawMeshWithColor(const BasicMesh& m); 117 | void drawMeshVerticesWithColor(const BasicMesh& m); 118 | void drawColorBar(double, double); 119 | 120 | private: 121 | bool use_side_view; 122 | vector dists; 123 | vector ao, normals; 124 | BasicMesh mesh; 125 | BasicMesh refmesh; 126 | float error_range; 127 | bool align_mesh; 128 | bool align_mesh_translation; 129 | bool face_only_mode; 130 | 131 | set skip_faces; 132 | 133 | double sceneScale; 134 | QVector3D cameraPos; 135 | 136 | int canvasW, canvasH; 137 | enum ProjectionMode { 138 | ORTHONGONAL, 139 | FRUSTUM, 140 | PERSPECTIVE 141 | } projectionMode; 142 | QImage rendered_img; 143 | 144 | bool has_texture; 145 | QImage texture; 146 | 147 | json rendering_settings; 148 | vector enabled_lights; 149 | }; 150 | 151 | class BlendshapeVisualizer : public GL3DCanvas { 152 | Q_OBJECT 153 | public: 154 | BlendshapeVisualizer(QWidget *parent = 0); 155 | ~BlendshapeVisualizer(); 156 | 157 | virtual QSize sizeHint() const { 158 | return QSize(600, 600); 159 | } 160 | 161 | virtual QSize minimumSizeHint() const { 162 | return QSize(600, 600); 163 | } 164 | 165 | void SetAmbientOcclusion(const vector& AO) { 166 | ao = AO; 167 | } 168 | void SetNormals(const vector& ns) { 169 | normals = ns; 170 | } 171 | void SetSideView(bool val) { 172 | use_side_view = val; 173 | } 174 | void setMesh(const BasicMesh& m); 175 | void loadMesh(const string &filename); 176 | void loadReferenceMesh(const string &filename); 177 | 178 | QImage getImage() { 179 | return this->grabFrameBuffer(); 180 | } 181 | 182 | QImage render() { 183 | repaint(); 184 | return this->grabFrameBuffer(); 185 | } 186 | 187 | protected: 188 | virtual void paintGL(); 189 | 190 | void enableLighting(); 191 | void disableLighting(); 192 | void computeDistance(); 193 | void drawMesh(const BasicMesh &m); 194 | void drawMeshWithColor(const BasicMesh &m); 195 | void drawMeshVerticesWithColor(const BasicMesh& m); 196 | void drawColorBar(double, double); 197 | 198 | private: 199 | bool use_side_view; 200 | vector dists; 201 | vector ao, normals; 202 | BasicMesh mesh; 203 | BasicMesh refmesh; 204 | }; 205 | 206 | class BlendshapeGeneration : public QMainWindow 207 | { 208 | Q_OBJECT 209 | 210 | public: 211 | BlendshapeGeneration(bool silent, QWidget *parent = 0); 212 | ~BlendshapeGeneration(); 213 | 214 | virtual QSize sizeHint() const { 215 | return QSize(600, 600); 216 | } 217 | 218 | void SetSideView(bool val) { 219 | if(silent) { 220 | ocanvas->SetSideView(val); 221 | } else { 222 | canvas->SetSideView(val); 223 | } 224 | } 225 | 226 | void SetTexture(const string& filename) { 227 | texture = QImage(filename.c_str()); 228 | if(silent) { 229 | ocanvas->setTexture(texture); 230 | } 231 | } 232 | 233 | void SetAmbientOcclusion(const string& filename) { 234 | vector ao = LoadFloats(filename); 235 | if(silent) { 236 | ocanvas->SetAmbientOcclusion(ao); 237 | } else { 238 | canvas->SetAmbientOcclusion(ao); 239 | } 240 | } 241 | 242 | void SetNormals(const string& filename) { 243 | vector normals = LoadFloats(filename); 244 | if (silent) { 245 | ocanvas->SetNormals(normals); 246 | } else { 247 | canvas->SetNormals(normals); 248 | } 249 | } 250 | 251 | void LoadRenderingSettings(const string& filename) { 252 | json settings; 253 | ifstream fin(filename); 254 | fin >> settings; 255 | 256 | if(silent) { 257 | ocanvas->setRenderingSettings(settings); 258 | } 259 | } 260 | 261 | void LoadMeshes(const string& mesh, const string& refmesh) { 262 | if(silent) { 263 | ocanvas->loadMesh(mesh); 264 | ocanvas->loadReferenceMesh(refmesh); 265 | avg_error = ocanvas->getError(); 266 | } else { 267 | canvas->loadMesh(mesh); 268 | canvas->loadReferenceMesh(refmesh); 269 | canvas->repaint(); 270 | //avg_error = canvas->getError(); 271 | } 272 | } 273 | 274 | void LoadMesh(const string& mesh) { 275 | if(silent) { 276 | ocanvas->loadMesh(mesh); 277 | } else { 278 | canvas->loadMesh(mesh); 279 | canvas->repaint(); 280 | } 281 | } 282 | 283 | void LoadSkipFaces(const string& filename, bool subdivided) { 284 | auto indices = LoadIndices(filename); 285 | if(silent) { 286 | ocanvas->SetSkipFaces(indices, subdivided); 287 | } 288 | } 289 | 290 | void SetErrorRange(float v) { 291 | if(silent) { 292 | ocanvas->SetErrorRange(v); 293 | } 294 | } 295 | 296 | void SetAlignMesh(bool v) { 297 | if(silent) { 298 | ocanvas->SetAlignMesh(v); 299 | } 300 | } 301 | 302 | void SetAlignMeshTranslation(bool v) { 303 | if(silent) { 304 | ocanvas->SetAlignMeshTranslation(v); 305 | } 306 | } 307 | 308 | void SetFaceOnlyMode(bool v) { 309 | if(silent) { 310 | ocanvas->SetFaceOnlyMode(v); 311 | } 312 | } 313 | 314 | void Save(const string& filename) { 315 | boost::timer::auto_cpu_timer t("[save image] image save time = %w seconds.\n"); 316 | if(silent) { 317 | QImage pixmap = ocanvas->render(); 318 | pixmap.save(filename.c_str()); 319 | } else { 320 | QImage pixmap = canvas->render(); 321 | pixmap.save(filename.c_str()); 322 | } 323 | } 324 | 325 | void SaveError(const string& filename) { 326 | boost::timer::auto_cpu_timer t("[save error] error save time = %w seconds.\n"); 327 | ofstream fout(filename); 328 | fout << avg_error << endl; 329 | fout.close(); 330 | } 331 | 332 | private slots: 333 | void slot_loadMesh(); 334 | void slot_loadReferenceMesh(); 335 | 336 | private: 337 | Ui::BlendshapeGenerationClass ui; 338 | 339 | bool silent; 340 | double avg_error; 341 | BlendshapeVisualizer *canvas; 342 | OffscreenBlendshapeVisualizer *ocanvas; 343 | 344 | QImage texture; 345 | }; 346 | 347 | #endif // BLENDSHAPEGENERATION_H 348 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapegeneration.pro: -------------------------------------------------------------------------------- 1 | QT += core widgets opengl 2 | HEADERS += blendshapegeneration.h \ 3 | common.h \ 4 | sparsematrix.h \ 5 | densematrix.h \ 6 | densevector.h \ 7 | ndarray.hpp \ 8 | sparsematrix.h \ 9 | cereswrapper.h \ 10 | meshtransferer.h \ 11 | meshdeformer.h \ 12 | pointcloud.h \ 13 | basicmesh.h \ 14 | testcases.h \ 15 | utils.h 16 | SOURCES += blendshapegeneration.cpp \ 17 | main.cpp \ 18 | common.cpp \ 19 | sparsematrix.cpp \ 20 | densematrix.cpp \ 21 | densevector.cpp \ 22 | meshtransferer.cpp \ 23 | meshdeformer.cpp \ 24 | pointcloud.cpp \ 25 | basicmesh.cpp \ 26 | testcases.cpp 27 | RESOURCES += blendshapegeneration.qrc 28 | 29 | FORMS += \ 30 | blendshapegeneration.ui 31 | 32 | CONFIG += c++11 33 | CONFIG -= app_bundle 34 | 35 | QMAKE_CXXFLAGS += -m64 -frounding-math 36 | 37 | macx { 38 | message(building for mac os) 39 | INCLUDEPATH += /usr/local/include /opt/intel/mkl/include 40 | LIBS += -L/opt/intel/lib -L/opt/intel/mkl/lib -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -liomp5 -ldl -lpthread -lm 41 | 42 | INCLUDEPATH += /Users/phg/SDKs/PhGLib/include 43 | LIBS += -L/Users/phg/SDKs/PhGLib/lib -lPhGLib 44 | 45 | INCLUDEPATH += /Users/phg/SDKs/SuiteSparse/SuiteSparse_config /Users/phg/SDKs/SuiteSparse/CHOLMOD/Include 46 | LIBS += -L/Users/phg/SDKs/SuiteSparse/SuiteSparse_config -L/Users/phg/SDKs/SuiteSparse/AMD/Lib -L/Users/phg/SDKs/SuiteSparse/COLAMD/Lib -L/Users/phg/SDKs/SuiteSparse/CHOLMOD/Lib -lcholmod -lamd -lcolamd -lsuitesparseconfig 47 | 48 | LIBS += -framework Accelerate 49 | } 50 | 51 | linux-g++ { 52 | message(building for linux) 53 | 54 | QMAKE_CXXFLAGS += -fopenmp 55 | 56 | # mkl 57 | INCLUDEPATH += /usr/local/include /home/phg/SDKs/intel/mkl/include 58 | LIBS += -L/home/phg/SDKs/intel/lib/intel64/ -L/home/phg/SDKs/intel/mkl/lib/intel64/ -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -liomp5 -ldl -lpthread -lm 59 | 60 | # PhGLib 61 | INCLUDEPATH += /home/phg/SDKs/PhGLib/include 62 | LIBS += -L/home/phg/SDKs/PhGLib/lib -lPhGLib 63 | 64 | # ceres 65 | INCLUDEPATH += /home/phg/SDKs/ceres-solver-1.10.0/include /usr/include/eigen3 66 | LIBS += -L/home/phg/SDKs/ceres-solver-1.10.0/lib -lceres -lglog -gflags 67 | 68 | # CHOLMOD 69 | #INCLUDEPATH += /home/phg/SDKs/SuiteSparse/SuiteSparse_config /home/phg/SDKs/SuiteSparse/CHOLMOD/Include 70 | #LIBS += -L/home/phg/SDKs/SuiteSparse/SuiteSparse_config -L/home/phg/SDKs/SuiteSparse/AMD/Lib -L/home/phg/SDKs/SuiteSparse/COLAMD/Lib -L/home/phg/SDKs/SuiteSparse/CHOLMOD/Lib -lcholmod -lamd -lcolamd -lsuitesparseconfig 71 | LIBS += -lcholmod -lcamd -lamd -lccolamd -lcolamd -lsuitesparseconfig -lcxsparse 72 | 73 | # CGAL 74 | LIBS += -lCGAL -lboost_system 75 | 76 | # GLEW 77 | INCLUDEPATH += /home/phg/SDKs/glew-1.12.0/include 78 | LIBS += -L/home/phg/SDKs/glew-1.12.0/lib -lGLEW 79 | 80 | # GLUT 81 | INCLUDEPATH += /home/phg/SDKs/freeglut-2.8.1/include 82 | LIBS += -lGLU -lglut 83 | } 84 | 85 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapegeneration.qrc: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapegeneration.ui: -------------------------------------------------------------------------------- 1 | 2 | 3 | BlendshapeGenerationClass 4 | 5 | 6 | 7 | 0 8 | 0 9 | 600 10 | 400 11 | 12 | 13 | 14 | BlendshapeGeneration 15 | 16 | 17 | 18 | 19 | 20 | 0 21 | 0 22 | 600 23 | 27 24 | 25 | 26 | 27 | 28 | File 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | TopToolBarArea 38 | 39 | 40 | false 41 | 42 | 43 | 44 | 45 | 46 | Load Mesh 47 | 48 | 49 | 50 | 51 | Load Reference Mesh 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshaperefiner.h: -------------------------------------------------------------------------------- 1 | #ifndef FACESHAPEFROMSHADING_BLENDSHAPEREFINER_H 2 | #define FACESHAPEFROMSHADING_BLENDSHAPEREFINER_H 3 | 4 | #include "common.h" 5 | 6 | #include "ndarray.hpp" 7 | #include "triangle_gradient.h" 8 | #include "pointcloud.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "boost/filesystem/operations.hpp" 17 | #include "boost/filesystem/path.hpp" 18 | #include 19 | 20 | namespace fs = boost::filesystem; 21 | 22 | #include "nlohmann/json.hpp" 23 | using json = nlohmann::json; 24 | 25 | struct ImageBundle { 26 | ImageBundle() {} 27 | ImageBundle(const string& filename, const QImage& image, const vector& points, const ReconstructionResult& params) 28 | : filename(filename), image(image), points(points), params(params) {} 29 | string filename; 30 | QImage image; 31 | vector points; 32 | ReconstructionResult params; 33 | }; 34 | 35 | class BlendshapeRefiner { 36 | public: 37 | BlendshapeRefiner(json settings = json{}); 38 | ~BlendshapeRefiner() {} 39 | 40 | void SetBlendshapeCount(int count) { num_shapes = count; } 41 | void SetResourcesPath(const string& path); 42 | void SetReconstructionsPath(const string& path); 43 | void SetPointCloudsPath(const string& path); 44 | void SetExampleMeshesPath(const string& path); 45 | void SetInputBlendshapesPath(const string& path); 46 | void SetBlendshapesPath(const string& path); 47 | 48 | void LoadTemplateMeshes(const string& path, const string& basename); 49 | void LoadSelectionFile(const string& selection_filename); 50 | void LoadInputReconstructionResults(const string& settings_filename); 51 | void LoadInputPointClouds(); 52 | void LoadInputExampleMeshes(); 53 | 54 | void Refine(bool initialize_only=false, 55 | bool disable_neutral_opt=false); 56 | void Refine_EBFR(); 57 | 58 | private: 59 | void LoadInitialBlendshapes(); 60 | void CreateTrainingShapes(); 61 | void InitializeBlendshapes(); 62 | 63 | MatrixXd LoadPointCloud(const string& filename, const glm::dmat4& R); 64 | 65 | vector RefineBlendshapes(const vector &S, 66 | const vector > &Sgrad, 67 | const vector &A, 68 | const vector &B, const BasicMesh &B00, 69 | const vector &alpha, 70 | double beta, double gamma, 71 | const vector > &prior, 72 | const MatrixXd& w_prior, 73 | const vector &stationary_vertices, 74 | const vector &stationary_faces_set, 75 | bool refine_neutral_only = true); 76 | 77 | vector RefineBlendshapes_EBFR(const vector &S, 78 | const vector > &Sgrad, 79 | const vector &A, 80 | const vector &B, const BasicMesh &B00, 81 | const vector &alpha, 82 | double beta, double gamma, 83 | const vector > &prior, 84 | const MatrixXd& w_prior, 85 | const vector stationary_indices); 86 | 87 | 88 | VectorXd EstimateWeights(const BasicMesh &S, 89 | const BasicMesh &B0, 90 | const vector &dB, 91 | const VectorXd &w0, // init value 92 | const VectorXd &wp, // prior 93 | double w_prior, 94 | int itmax, 95 | const vector& valid_vertices = vector()); 96 | 97 | protected: 98 | string FullFile(const fs::path& path, const string& filename) { 99 | return (path / fs::path(filename)).string(); 100 | } 101 | string InBlendshapesDirectory(const string& filename) { 102 | return FullFile(blendshapes_path, filename); 103 | } 104 | 105 | private: 106 | MultilinearModel model; 107 | MultilinearModelPrior model_prior; 108 | BasicMesh template_mesh; 109 | 110 | int num_shapes; 111 | vector A; // template blendshapes 112 | 113 | vector Binit; // transferred/initial blendshapes 114 | vector B; // refined blendshapes 115 | 116 | int num_poses; 117 | bool use_init_blendshapes; 118 | bool do_subdivision; 119 | bool blendshapes_subdivided; 120 | bool mask_nose_and_fore_head; 121 | 122 | vector image_bundles; 123 | vector point_clouds; 124 | vector S0; // initial training shapes 125 | vector S; // point cloud deformed training shapes 126 | vector Slandmarks; 127 | 128 | fs::path resources_path, reconstructions_path, input_blendshapes_path; 129 | fs::path point_clouds_path, example_meshes_path; 130 | fs::path blendshapes_path; 131 | vector selection_indices; 132 | 133 | unordered_set hair_region_indices, 134 | nose_forehead_indices, 135 | extended_hair_region_indices; 136 | 137 | Reporter reporter; 138 | }; 139 | 140 | #endif //FACESHAPEFROMSHADING_BLENDSHAPEREFINER_H 141 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshaperefiner_old.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by phg on 9/24/17. 3 | // 4 | 5 | #include "blendshaperefiner_old.h" 6 | 7 | struct PointResidual { 8 | PointResidual(double x, double y, double z, int idx, const vector &dB) 9 | : mx(x), my(y), mz(z), vidx(idx), dB(dB) {} 10 | 11 | template 12 | bool operator()(const T* const alpha, T* residual) const { 13 | T p[3]; 14 | p[0] = T(0); p[1] = T(0); p[2] = T(0); 15 | int nshapes = dB.size(); 16 | // compute 17 | for(int i=0;i &dB; 31 | }; 32 | 33 | struct PriorResidue { 34 | PriorResidue(double *prior):mprior(prior){} 35 | 36 | template 37 | bool operator()(const T* const alpha, T* residual) const { 38 | int nshapes = 46; 39 | for(int i=0;i estimateWeights(const BasicMesh &S, 47 | const BasicMesh &B0, 48 | const vector &dB, 49 | const Array1D &w0, // init value 50 | const Array1D &wp, // prior 51 | double w_prior, 52 | int itmax) { 53 | Array1D w = w0; 54 | 55 | Problem problem; 56 | 57 | // add all constraints 58 | int nverts = S.NumVertices(); 59 | for(int i=0;i( 67 | new PointResidual(dx, dy, dz, i, dB)); 68 | problem.AddResidualBlock(costfun, NULL, w.data.get()); 69 | } 70 | 71 | // add prior if necessary 72 | if( fabs(w_prior) > 1e-6 ) { 73 | problem.AddResidualBlock( 74 | new AutoDiffCostFunction(new PriorResidue(wp.data.get())), 75 | NULL, w.data.get()); 76 | } 77 | 78 | cout << "w0 = " << endl; 79 | cout << w << endl; 80 | // set the solver options 81 | Solver::Options options; 82 | options.linear_solver_type = ceres::DENSE_QR; 83 | options.minimizer_progress_to_stdout = true; 84 | 85 | options.num_threads = 8; 86 | options.num_linear_solver_threads = 8; 87 | 88 | Solver::Summary summary; 89 | ceres::Solve(options, &problem, &summary); 90 | 91 | cout << summary.BriefReport() << endl; 92 | 93 | return w; 94 | } 95 | 96 | vector refineBlendShapes(const vector &S, 97 | const vector> &Sgrad, 98 | const vector &B, 99 | const vector> &alpha, 100 | double beta, double gamma, 101 | const Array2D prior, 102 | const Array2D w_prior, 103 | const vector stationary_indices) { 104 | const BasicMesh &B0 = B[0]; 105 | 106 | int nfaces = B0.NumFaces(); 107 | int nverts = B0.NumVertices(); 108 | 109 | int nposes = S.size(); 110 | int nshapes = B.size() - 1; 111 | 112 | // compute the deformation gradients for B0 113 | Array2D B0grad(nfaces, 9); 114 | Array1D DB(nfaces); 115 | for(int i=0;i; 125 | using SparseMatrixd = Eigen::SparseMatrix; 126 | vector Adata_coeffs; 127 | 128 | // fill the upper part of A 129 | for(int i=0;i M(nfaces); 143 | 144 | #pragma omp parallel for 145 | for(int j=0;j A_coeffs = Adata_coeffs; 162 | for(int i=0;i A(nrows, ncols); 178 | A.setFromTriplets(A_coeffs.begin(), A_coeffs.end()); 179 | A.makeCompressed(); 180 | 181 | Eigen::SparseMatrix AtA = (A.transpose() * A).pruned(); 182 | 183 | const double epsilon = 0.0;//1e-9; 184 | Eigen::SparseMatrix eye(ncols, ncols); 185 | for(int j=0;j> solver; 189 | solver.compute(AtA); 190 | if(solver.info()!=Success) { 191 | cerr << "Failed to decompose matrix A." << endl; 192 | exit(-1); 193 | } 194 | 195 | VectorXd Atb = A.transpose() * b; 196 | VectorXd Mv = solver.solve(Atb); 197 | if(solver.info()!=Success) { 198 | cerr << "Failed to solve A\\b." << endl; 199 | exit(-1); 200 | } 201 | 202 | // convert Mv to nshapes x 9 matrix 203 | M[j] = MatrixXd(nshapes, 9); 204 | for(int i=0;i M(nfaces); 221 | 222 | #pragma omp parallel for 223 | for(int j=0;j B_new(nshapes); 262 | for(int i=0;i Bgrad_i(nfaces); 265 | for(int j=0;j estimateNeutralFace(const BasicMesh& target) { 280 | MultilinearModel model("/home/phg/Data/Multilinear/blendshape_core.tensor"); 281 | MultilinearModelPrior model_prior; 282 | model_prior.load("/home/phg/Data/Multilinear/blendshape_u_0_aug.tensor", 283 | "/home/phg/Data/Multilinear/blendshape_u_1_aug.tensor"); 284 | 285 | // Just the neutral expression 286 | VectorXd w_exp_FACS = VectorXd::Zero(47); 287 | w_exp_FACS(0) = 1.0; 288 | 289 | VectorXd w_exp = (w_exp_FACS.transpose() * model_prior.Uexp).eval(); 290 | 291 | model.UpdateTM1(w_exp); 292 | 293 | Tensor2 tm1 = model.GetTM1(); 294 | // tm1_mat^T * w_exp -> verts 295 | MatrixXd tm1_mat = tm1.GetData(); 296 | cout << tm1_mat.rows() << 'x' << tm1_mat.cols() << endl; 297 | 298 | int nverts = tm1.cols() / 3; 299 | VectorXd target_verts(tm1_mat.cols()); 300 | for(int i=0;i A(nshapes+1); 339 | vector B(nshapes+1); 340 | vector B_ref(nshapes+1); 341 | vector S0(nposes); // ground truth training meshes 342 | 343 | // load the landmarks 344 | vector landmarks = LoadIndices("/home/phg/Data/Multilinear/landmarks_73.txt"); 345 | 346 | // load the template blendshapes and ground truth blendshapes 347 | for(int i=0;i<=nshapes;++i) { 348 | A[i].LoadOBJMesh(A_path + "shape_" + to_string(i) + ".obj"); 349 | B_ref[i].LoadOBJMesh(B_path + "shape_" + to_string(i) + ".obj"); 350 | } 351 | 352 | // Estimate a neutral face using the multi-linear model 353 | VectorXd w_id; 354 | tie(w_id, B[0]) = estimateNeutralFace(B_ref[0]); 355 | 356 | // reference shapes for convenience 357 | auto& A0 = A[0]; 358 | auto& B0 = B[0]; 359 | 360 | vector valid_faces = B0.filterFaces([&B0](Vector3i fi) { 361 | Vector3d c = (B0.vertex(fi[0]) + B0.vertex(fi[1]) + B0.vertex(fi[2]))/ 3.0; 362 | return c[2] > -0.75; 363 | }); 364 | 365 | // load the training poses 366 | for(int i=0;i stationary_indices = B0.filterVertices([=](const Vector3d& v) { 372 | return v[2] <= -0.45; 373 | }); 374 | cout << "stationary vertices = " << stationary_indices.size() << endl; 375 | 376 | ColorStream(ColorOutput::Blue) << "creating initial set of blendshapes using deformation transfer."; 377 | // use deformation transfer to create an initial set of blendshapes 378 | MeshTransferer transferer; 379 | transferer.setSource(A0); // set source and compute deformation gradient 380 | transferer.setTarget(B0); // set target and compute deformation gradient 381 | transferer.setStationaryVertices(stationary_indices); 382 | 383 | for(int i=1;i<=nshapes;++i) { 384 | ColorStream(ColorOutput::Green)<< "creating shape " << i; 385 | B[i] = transferer.transfer(A[i]); 386 | B[i].Write("Binit_" + to_string(i) + ".obj"); 387 | } 388 | auto B_init = B; 389 | ColorStream(ColorOutput::Blue)<< "initial set of blendshapes created."; 390 | 391 | const bool synthesizeTrainingPoses = true; 392 | vector> alpha_ref(nposes); 393 | if( synthesizeTrainingPoses ) { 394 | // estimate the blendshape weights from the input training poses, then use 395 | // the estimated weights to generate a new set of training poses 396 | 397 | // compute the delta shapes of the renference shapes 398 | vector dB_ref(nshapes); 399 | for(int i=0;i::ones(nshapes) * 0.25, 408 | Array1D::zeros(nshapes), 409 | 0.0, 5); 410 | cout << alpha_ref[i] << endl; 411 | } 412 | 413 | // use the reference weights to build an initial fitted set of training shapes 414 | vector Sgen(nposes); 415 | for(int i=0;i::random(nshapes); 435 | } 436 | 437 | // create point clouds from Sgen 438 | vector P(nposes); 439 | for(int i=0;i S(nposes); // meshes reconstructed from point clouds 446 | // use Laplacian deformation to reconstruct a set of meshes from the sampled 447 | // point clouds 448 | MeshDeformer deformer; 449 | deformer.setSource(B0); 450 | deformer.setLandmarks(landmarks); 451 | deformer.setValidFaces(valid_faces); 452 | 453 | for(int i=0;i S = S0; 471 | // Use the generated shapes as initial reconstructions 472 | auto S_init = Sgen; 473 | #endif 474 | 475 | // XXX Need to refer to the blendshape refiner and change the code below to a 476 | // full optimization of the entire set of blendshapes 477 | 478 | // compute deformation gradient prior from the template mesh 479 | ColorStream(ColorOutput::Blue)<< "computing priors."; 480 | int nfaces = A0.NumFaces(); 481 | 482 | Array2D MB0 = Array2D::zeros(nfaces, 9); 483 | Array2D MA0 = Array2D::zeros(nfaces, 9); 484 | 485 | for(int j=0;j prior = Array2D::zeros(nshapes, 9*nfaces); 496 | Array2D w_prior = Array2D::zeros(nshapes, nfaces); 497 | for(int i=0;i Pi = Array2D::zeros(nfaces, 9); 501 | auto w_prior_i = w_prior.rowptr(i); 502 | for(int j=0;j dB(nshapes); 534 | for(int i=0;i> alpha(nposes); 540 | for(int i=0;i::random(nshapes), 543 | Array1D::ones(nshapes)*0.25, 544 | Array1D::zeros(nshapes), 545 | 0.0, 5); 546 | } 547 | auto alpha_init = alpha; 548 | 549 | ColorStream(ColorOutput::Red)<< "initialization done."; 550 | 551 | // reset the parameters 552 | B = B_init; 553 | S = S_init; 554 | alpha = alpha_init; 555 | 556 | // main blendshape refinement loop 557 | 558 | // compute deformation gradients for S 559 | // Note: this deformation gradient cannot be used to call 560 | // MeshTransferer::transfer directly. See this function for 561 | // details 562 | vector> Sgrad(nposes); 563 | for(int i=0;i::zeros(nfaces, 9); 565 | for(int j=0;j grads; 584 | for(int j=0;j 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "meshdeformer.h" 18 | #include "meshtransferer.h" 19 | #include "ndarray.hpp" 20 | #include "cereswrapper.h" 21 | #include "triangle_gradient.h" 22 | 23 | Array1D estimateWeights(const BasicMesh &S, 24 | const BasicMesh &B0, 25 | const vector &dB, 26 | const Array1D &w0, // init value 27 | const Array1D &wp, // prior 28 | double w_prior, 29 | int itmax); 30 | 31 | vector refineBlendShapes(const vector &S, 32 | const vector> &Sgrad, 33 | const vector &B, 34 | const vector> &alpha, 35 | double beta, double gamma, 36 | const Array2D prior, 37 | const Array2D w_prior, 38 | const vector stationary_indices); 39 | 40 | pair estimateNeutralFace(const BasicMesh& target); 41 | 42 | void blendShapeGeneration(); 43 | 44 | #endif //BLENDSHAPEGENERATION_BLENDSHAPEREFINER_OLD_H 45 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapes_driver.cpp: -------------------------------------------------------------------------------- 1 | #ifndef MKL_BLAS 2 | #define MKL_BLAS MKL_DOMAIN_BLAS 3 | #endif 4 | 5 | #define EIGEN_USE_MKL_ALL 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "MultilinearReconstruction/common.h" 14 | #include "MultilinearReconstruction/basicmesh.h" 15 | #include "MultilinearReconstruction/constraints.h" 16 | #include "MultilinearReconstruction/costfunctions.h" 17 | #include "MultilinearReconstruction/ioutilities.h" 18 | #include "MultilinearReconstruction/multilinearmodel.h" 19 | #include "MultilinearReconstruction/parameters.h" 20 | #include "MultilinearReconstruction/statsutils.h" 21 | #include "MultilinearReconstruction/utils.hpp" 22 | 23 | #include "MultilinearReconstruction/OffscreenMeshVisualizer.h" 24 | 25 | #include "boost/filesystem/operations.hpp" 26 | #include "boost/filesystem/path.hpp" 27 | 28 | namespace fs = boost::filesystem; 29 | 30 | using namespace Eigen; 31 | 32 | vector LoadBlendshapes(const string& path) { 33 | const string prefix = "B_"; 34 | const int num_blendshapes = 47; 35 | vector blendshapes(num_blendshapes); 36 | #pragma omp parallel for 37 | for(int i=0;i& blendshapes, 49 | const string& rendering_settings_filename, 50 | const string& output_image_filename, 51 | bool use_head_pose, 52 | bool scale_output=true) { 53 | 54 | int imgw = img.width(); 55 | int imgh = img.height(); 56 | if(scale_output) { 57 | const int target_size = 640; 58 | double scale = static_cast(target_size) / imgw; 59 | imgw *= scale; 60 | imgh *= scale; 61 | } 62 | 63 | BasicMesh mesh = blendshapes.front(); 64 | auto recon_results = LoadReconstructionResult(res_filename); 65 | 66 | // Interpolate the blendshapes 67 | { 68 | MatrixX3d v = mesh.vertices(); 69 | for(int i=1;i<47;++i) { 70 | const double weight_i = recon_results.params_model.Wexp_FACS[i]; 71 | if(weight_i > 0) v += (blendshapes[i].vertices() - mesh.vertices()) * weight_i; 72 | } 73 | mesh.vertices() = v; 74 | } 75 | mesh.ComputeNormals(); 76 | 77 | OffscreenMeshVisualizer visualizer(imgw, imgh); 78 | 79 | visualizer.LoadRenderingSettings(rendering_settings_filename); 80 | visualizer.SetMVPMode(OffscreenMeshVisualizer::CamPerspective); 81 | visualizer.SetRenderMode(OffscreenMeshVisualizer::TexturedMesh); 82 | visualizer.BindMesh(mesh); 83 | visualizer.BindTexture(img); 84 | 85 | visualizer.SetCameraParameters(recon_results.params_cam); 86 | if(use_head_pose) 87 | visualizer.SetMeshRotationTranslation(recon_results.params_model.R, recon_results.params_model.T); 88 | else 89 | visualizer.SetMeshRotationTranslation(Vector3d(0, 0, 0), Vector3d(0, 0, -12)); 90 | visualizer.SetIndexEncoded(false); 91 | visualizer.SetEnableLighting(true); 92 | 93 | QImage output_img = visualizer.Render(true); 94 | output_img.save(output_image_filename.c_str()); 95 | } 96 | 97 | int main(int argc, char** argv) { 98 | QApplication app(argc, argv); 99 | if(argc<9) { 100 | cout << "Usage: " << argv[0] << " texture_img res_path prefix start_idx end_idx blendshapes_path output_path rendering_settings" << endl; 101 | return 1; 102 | } 103 | 104 | const string texture_filename = argv[1]; 105 | const string res_path = argv[2]; 106 | const string prefix = argv[3]; 107 | const int start_idx = std::stoi(string(argv[4])); 108 | const int end_idx = std::stoi(string(argv[5])); 109 | const string blendshapes_path = argv[6]; 110 | const string output_path = argv[7]; 111 | const string rendering_settings_filename = argv[8]; 112 | 113 | bool use_head_pose = true; 114 | if(argc > 9 && string(argv[9]) == "no_pose") use_head_pose = false; 115 | 116 | vector blendshapes = LoadBlendshapes(blendshapes_path); 117 | QImage texture(texture_filename.c_str()); 118 | 119 | #pragma omp parallel for 120 | for(int idx=start_idx;idx<=end_idx;++idx) { 121 | char buff[100]; 122 | snprintf(buff, sizeof(buff), "%05d", idx); 123 | string idx_str = buff; 124 | string res_filename = res_path + "/" + prefix + idx_str + ".jpg.res"; 125 | string output_filename = output_path + "/" + idx_str + ".png"; 126 | cout << "Rendering to " << output_filename << endl; 127 | VisualizeReconstructionResult(texture, res_filename, blendshapes, 128 | rendering_settings_filename, output_filename, use_head_pose); 129 | } 130 | return 0; 131 | } 132 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapesweightswidget.cpp: -------------------------------------------------------------------------------- 1 | #include "blendshapesweightswidget.h" 2 | #include "ui_blendshapesweightswidget.h" 3 | 4 | #include 5 | #include 6 | 7 | BlendshapesWeightsWidget::BlendshapesWeightsWidget(QWidget *parent) : 8 | QWidget(parent), 9 | ui(new Ui::BlendshapesWeightsWidget) 10 | { 11 | ui->setupUi(this); 12 | 13 | setLayout(ui->gridLayout); 14 | 15 | for(int i=0;i<46;++i) { 16 | auto* slider = new QSlider(Qt::Horizontal); 17 | sliders.push_back(slider); 18 | int r = i % 23; 19 | int c = (i / 23) * 2 + 1; 20 | ui->gridLayout->addWidget(new QLabel(QString::fromStdString(to_string(i+1))), r, c-1); 21 | ui->gridLayout->addWidget(slider, r, c); 22 | 23 | connect(slider, SIGNAL(valueChanged(int)), this, SLOT(slot_sliderChanged(int))); 24 | } 25 | } 26 | 27 | BlendshapesWeightsWidget::~BlendshapesWeightsWidget() 28 | { 29 | delete ui; 30 | } 31 | 32 | void BlendshapesWeightsWidget::slot_sliderChanged(int v) 33 | { 34 | auto s = dynamic_cast(sender()); 35 | for(int i=0;i 7 | #include 8 | #include 9 | using namespace std; 10 | 11 | namespace Ui { 12 | class BlendshapesWeightsWidget; 13 | } 14 | 15 | class BlendshapesWeightsWidget : public QWidget 16 | { 17 | Q_OBJECT 18 | 19 | public: 20 | explicit BlendshapesWeightsWidget(QWidget *parent = 0); 21 | ~BlendshapesWeightsWidget(); 22 | 23 | signals: 24 | void sig_sliderChanged(int, int); 25 | 26 | private slots: 27 | void slot_sliderChanged(int); 28 | 29 | private: 30 | Ui::BlendshapesWeightsWidget *ui; 31 | 32 | vector sliders; 33 | }; 34 | 35 | #endif // BLENDSHAPESWEIGHTSWIDGET_H 36 | -------------------------------------------------------------------------------- /BlendshapeGeneration/blendshapesweightswidget.ui: -------------------------------------------------------------------------------- 1 | 2 | 3 | BlendshapesWeightsWidget 4 | 5 | 6 | 7 | 0 8 | 0 9 | 801 10 | 889 11 | 12 | 13 | 14 | Form 15 | 16 | 17 | 18 | 19 | 10 20 | 10 21 | 781 22 | 871 23 | 24 | 25 | 26 | 27 | QLayout::SetMaximumSize 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /BlendshapeGeneration/cereswrapper.h: -------------------------------------------------------------------------------- 1 | #ifndef CERESWRAPPER_H 2 | #define CERESWRAPPER_H 3 | 4 | #include "common.h" 5 | 6 | #include "ceres/ceres.h" 7 | 8 | using ceres::AutoDiffCostFunction; 9 | using ceres::NumericDiffCostFunction; 10 | using ceres::CostFunction; 11 | using ceres::Problem; 12 | using ceres::Solver; 13 | using ceres::Solve; 14 | 15 | struct CostFunctor { 16 | template 17 | bool operator()(const T* const x, T* residual) const { 18 | residual[0] = T(10.0) - x[0]; 19 | return true; 20 | } 21 | }; 22 | 23 | #endif // CERESWRAPPER_H 24 | -------------------------------------------------------------------------------- /BlendshapeGeneration/common.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | -------------------------------------------------------------------------------- /BlendshapeGeneration/common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define USE_BOOST 0 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #if USE_BOOST 15 | #include 16 | #else 17 | #include 18 | #endif 19 | #include 20 | #if USE_BOOST 21 | #include 22 | #else 23 | #include 24 | #endif 25 | #include 26 | #include 27 | #include 28 | #if USE_BOOST 29 | #include 30 | using boost::shared_ptr; 31 | #endif 32 | #include 33 | #include 34 | 35 | using namespace std; 36 | 37 | #include "Utils/Timer.h" 38 | 39 | #ifndef MKL_BLAS 40 | #define MKL_BLAS MKL_DOMAIN_BLAS 41 | #endif 42 | 43 | #define EIGEN_USE_MKL_ALL 44 | 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | //using namespace Eigen; 51 | -------------------------------------------------------------------------------- /BlendshapeGeneration/compute_details.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include 5 | #include 6 | 7 | #include "testcases.h" 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "meshdeformer.h" 17 | #include "meshtransferer.h" 18 | #include "cereswrapper.h" 19 | 20 | #include "Geometry/matrix.hpp" 21 | #include "triangle_gradient.h" 22 | 23 | #include "boost/filesystem/operations.hpp" 24 | #include "boost/filesystem/path.hpp" 25 | #include 26 | 27 | namespace { 28 | Eigen::MatrixXd load_matrix(const string& filename) { 29 | vector lines = ReadFileByLine(filename); 30 | Eigen::MatrixXd mat; 31 | vector> elems; 32 | std::transform(lines.begin(), lines.end(), std::back_inserter(elems), 33 | [](const string& line) { 34 | vector parts; 35 | boost::algorithm::split(parts, line, 36 | boost::algorithm::is_any_of(" "), 37 | boost::algorithm::token_compress_on); 38 | auto parts_end = std::remove_if(parts.begin(), parts.end(), 39 | [](const string &s) { 40 | return s.empty(); 41 | }); 42 | vector row(std::distance(parts.begin(), parts_end)); 43 | std::transform(parts.begin(), parts_end, row.begin(), 44 | [](const string &s) { 45 | return std::stod(s); 46 | }); 47 | return row; 48 | }); 49 | const int nrows = elems.size(), ncols = elems.front().size(); 50 | mat.resize(nrows, ncols); 51 | for(int i=0;i& weights) : wexp(wexp), weights(weights) {} 73 | virtual void operator()(const BasicMesh& corase_mesh, const BasicMesh& detailed_mesh, const string& output_filename) = 0; 74 | virtual void write_diff(ostream& os) = 0; 75 | 76 | virtual void write(const string& filename) { 77 | cout << "Writing diff to " << filename << endl; 78 | ofstream fout(filename); 79 | write_wexp(fout); 80 | write_diff(fout); 81 | } 82 | 83 | void write_wexp(ostream& os) { 84 | cout << "Writing wexp ..." << endl; 85 | os << wexp.size() << endl; 86 | for(int i=0;i weights; 94 | }; 95 | 96 | class VertexDiffer : public MeshDifferBase { 97 | public: 98 | VertexDiffer(const Eigen::VectorXd& wexp, const vector& weights) : MeshDifferBase(wexp, weights) {} 99 | void operator()(const BasicMesh& coarse_mesh, const BasicMesh& detailed_mesh, const string& output_filename) override { 100 | const int num_verts = coarse_mesh.NumVertices(); 101 | assert(detailed_mesh.NumVertices() == num_verts); 102 | assert(weights.size() == num_verts); 103 | 104 | diffs = detailed_mesh.vertices() - coarse_mesh.vertices(); 105 | 106 | #if 0 107 | Eigen::Vector3d mean_diff = (diffs.colwise().sum() / num_verts).eval(); 108 | 109 | cout << diffs.rows() << " vs " << weights.size() << endl; 110 | 111 | for(int i=0;i& weights) : MeshDifferBase(wexp, weights) {} 131 | void operator()(const BasicMesh& coarse_mesh, const BasicMesh& detailed_mesh, const string& output_filename) override { 132 | const int num_verts = coarse_mesh.NumVertices(); 133 | assert(detailed_mesh.NumVertices() == num_verts); 134 | assert(weights.size() == num_verts); 135 | 136 | diffs.resize(num_verts, 3); 137 | for(int i=0;i 151 | void compute_details_exp( 152 | const string& res_filename, 153 | const string& init_bs_path, 154 | const string& detailed_mesh_filename, 155 | const string& output_fitted_mesh_filename, 156 | const string& output_diff_filename 157 | ) { 158 | const string home_directory = QDir::homePath().toStdString(); 159 | cout << "Home dir: " << home_directory << endl; 160 | const string hair_mask_path(home_directory + "/Data/Multilinear/hair_region_indices.txt"); 161 | auto hair_region_indices_quad = LoadIndices(hair_mask_path); 162 | vector hair_region_indices; 163 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 164 | for(auto fidx : hair_region_indices_quad) { 165 | hair_region_indices.push_back(fidx*2); 166 | hair_region_indices.push_back(fidx*2+1); 167 | } 168 | // HACK: each valid face i becomes [4i, 4i+1, 4i+2, 4i+3] after the each 169 | // subdivision. See BasicMesh::Subdivide for details 170 | const int max_subdivisions = 1; 171 | for(int i=0;i hair_region_indices_new; 173 | for(auto fidx : hair_region_indices) { 174 | int fidx_base = fidx*4; 175 | hair_region_indices_new.push_back(fidx_base); 176 | hair_region_indices_new.push_back(fidx_base+1); 177 | hair_region_indices_new.push_back(fidx_base+2); 178 | hair_region_indices_new.push_back(fidx_base+3); 179 | } 180 | hair_region_indices = hair_region_indices_new; 181 | } 182 | unordered_set hair_region_indices_set(hair_region_indices.begin(), hair_region_indices.end()); 183 | 184 | const string datapath(home_directory + "/Data/FaceWarehouse_Data_0/"); 185 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 186 | 187 | const int num_blendshapes = 46; 188 | vector blendshapes(num_blendshapes+1); 189 | #pragma omp parallel for 190 | for(int i=0;i<=num_blendshapes;++i) { 191 | blendshapes[i].LoadOBJMesh( init_bs_path + "/" + "B_" + to_string(i) + ".obj" ); 192 | blendshapes[i].ComputeNormals(); 193 | } 194 | 195 | BasicMesh m = blendshapes[0]; 196 | 197 | auto recon_results = LoadReconstructionResult(res_filename); 198 | cout << "Recon results loaded." << endl; 199 | { 200 | MatrixX3d verts0 = blendshapes[0].vertices(); 201 | MatrixX3d verts = verts0; 202 | for(int j=1;j<=num_blendshapes;++j) { 203 | verts += (blendshapes[j].vertices() - verts0) * recon_results.params_model.Wexp_FACS(j); 204 | } 205 | m.vertices() = verts; 206 | m.ComputeNormals(); 207 | } 208 | 209 | m.Write(output_fitted_mesh_filename); 210 | 211 | BasicMesh detailed_mesh; 212 | detailed_mesh.LoadOBJMesh(detailed_mesh_filename); 213 | 214 | // Compute per-vertex weights 215 | unordered_set hair_region_vertices; 216 | for(int i=0;i vert_weights(m.NumVertices(), 1.0); 233 | for(auto vi : hair_region_vertices) { 234 | double y = m.vertex(vi)[1]; 235 | vert_weights[vi] = (y - min_y) / (max_y - min_y); 236 | } 237 | 238 | // Compute the difference map 239 | MeshDiffer differ(recon_results.params_model.Wexp_FACS, vert_weights); 240 | differ(m, detailed_mesh, output_diff_filename); 241 | } 242 | 243 | template 244 | void compute_details_exp( 245 | const string& res_filename, 246 | const string& detailed_mesh_filename, 247 | const string& coarse_mesh_file, 248 | const string& output_diff_filename 249 | ) { 250 | const string home_directory = QDir::homePath().toStdString(); 251 | cout << "Home dir: " << home_directory << endl; 252 | const string hair_mask_path(home_directory + "/Data/Multilinear/hair_region_indices.txt"); 253 | auto hair_region_indices_quad = LoadIndices(hair_mask_path); 254 | vector hair_region_indices; 255 | // @HACK each quad face is triangulated, so the indices change from i to [2*i, 2*i+1] 256 | for(auto fidx : hair_region_indices_quad) { 257 | hair_region_indices.push_back(fidx*2); 258 | hair_region_indices.push_back(fidx*2+1); 259 | } 260 | // HACK: each valid face i becomes [4i, 4i+1, 4i+2, 4i+3] after the each 261 | // subdivision. See BasicMesh::Subdivide for details 262 | const int max_subdivisions = 1; 263 | for(int i=0;i hair_region_indices_new; 265 | for(auto fidx : hair_region_indices) { 266 | int fidx_base = fidx*4; 267 | hair_region_indices_new.push_back(fidx_base); 268 | hair_region_indices_new.push_back(fidx_base+1); 269 | hair_region_indices_new.push_back(fidx_base+2); 270 | hair_region_indices_new.push_back(fidx_base+3); 271 | } 272 | hair_region_indices = hair_region_indices_new; 273 | } 274 | unordered_set hair_region_indices_set(hair_region_indices.begin(), hair_region_indices.end()); 275 | 276 | const string datapath(home_directory + "/Data/FaceWarehouse_Data_0/"); 277 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 278 | 279 | BasicMesh m; 280 | m.LoadOBJMesh(coarse_mesh_file); 281 | m.ComputeNormals(); 282 | 283 | auto recon_results = LoadReconstructionResult(res_filename); 284 | cout << "Recon results loaded." << endl; 285 | 286 | BasicMesh detailed_mesh; 287 | detailed_mesh.LoadOBJMesh(detailed_mesh_filename); 288 | detailed_mesh.ComputeNormals(); 289 | 290 | // Compute per-vertex weights 291 | unordered_set hair_region_vertices; 292 | for(int i=0;i vert_weights(m.NumVertices(), 1.0); 309 | for(auto vi : hair_region_vertices) { 310 | double y = m.vertex(vi)[1]; 311 | vert_weights[vi] = (y - min_y) / (max_y - min_y); 312 | } 313 | 314 | // Compute the difference map 315 | cout << "Computing diffs ..." << endl; 316 | MeshDiffer differ(recon_results.params_model.Wexp_FACS, vert_weights); 317 | differ(m, detailed_mesh, output_diff_filename); 318 | } 319 | 320 | void visualize_details( 321 | const string& res_filename, 322 | const string& coarse_mesh_filename, 323 | const string& pca_comp_filename, 324 | const string& mapping_matrix_filename, 325 | const string& output_detailed_mesh_file 326 | ) { 327 | const string home_directory = QDir::homePath().toStdString(); 328 | cout << "Home dir: " << home_directory << endl; 329 | const string datapath(home_directory + "/Data/FaceWarehouse_Data_0/"); 330 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 331 | 332 | BasicMesh m; 333 | m.LoadOBJMesh(coarse_mesh_filename); 334 | 335 | auto recon_results = LoadReconstructionResult(res_filename); 336 | cout << "Recon results loaded." << endl; 337 | 338 | // Load pca components 339 | Eigen::MatrixXd pca_components = load_matrix(pca_comp_filename); 340 | Eigen::MatrixXd mapping_matrix = load_matrix(mapping_matrix_filename); 341 | 342 | Eigen::VectorXd pca_coeffs = (mapping_matrix * recon_results.params_model.Wexp_FACS).eval(); 343 | cout << pca_coeffs << endl; 344 | 345 | Eigen::VectorXd diff = (pca_components.transpose() * pca_coeffs).eval(); 346 | m.vertices() += Eigen::Map>(diff.data(), m.NumVertices(), 3); 347 | m.Write(output_detailed_mesh_file); 348 | } 349 | 350 | void visualize_details_normal(const string& res_filename, 351 | const string& coarse_mesh_filename, 352 | const string& pca_comp_filename, 353 | const string& mapping_matrix_filename, 354 | const string& output_normals_file) { 355 | const string home_directory = QDir::homePath().toStdString(); 356 | cout << "Home dir: " << home_directory << endl; 357 | const string datapath(home_directory + "/Data/FaceWarehouse_Data_0/"); 358 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 359 | 360 | auto recon_results = LoadReconstructionResult(res_filename); 361 | cout << "Recon results loaded." << endl; 362 | 363 | BasicMesh m; 364 | m.LoadOBJMesh(coarse_mesh_filename); 365 | m.ComputeNormals(); 366 | 367 | // Load pca components 368 | Eigen::MatrixXd pca_components = load_matrix(pca_comp_filename); 369 | Eigen::MatrixXd mapping_matrix = load_matrix(mapping_matrix_filename); 370 | 371 | Eigen::VectorXd pca_coeffs = (mapping_matrix * recon_results.params_model.Wexp_FACS).eval(); 372 | cout << pca_coeffs << endl; 373 | 374 | Eigen::VectorXd diff = (pca_components.transpose() * pca_coeffs).eval(); 375 | for(int i=0;i blendshapes(num_blendshapes+1); 401 | #pragma omp parallel for 402 | for(int i=0;i<=num_blendshapes;++i) { 403 | blendshapes[i].LoadOBJMesh( init_bs_path + "/" + "B_" + to_string(i) + ".obj" ); 404 | blendshapes[i].ComputeNormals(); 405 | } 406 | 407 | BasicMesh m = blendshapes[0]; 408 | 409 | auto recon_results = LoadReconstructionResult(res_filename); 410 | cout << "Recon results loaded." << endl; 411 | { 412 | MatrixX3d verts0 = blendshapes[0].vertices(); 413 | MatrixX3d verts = verts0; 414 | for(int j=1;j<=num_blendshapes;++j) { 415 | verts += (blendshapes[j].vertices() - verts0) * recon_results.params_model.Wexp_FACS(j); 416 | } 417 | m.vertices() = verts; 418 | m.ComputeNormals(); 419 | } 420 | m.Write(output_mesh_filename); 421 | 422 | // Load pca components 423 | Eigen::MatrixXd pca_components = load_matrix(pca_comp_filename); 424 | Eigen::MatrixXd mapping_matrix = load_matrix(mapping_matrix_filename); 425 | 426 | const float max_elem_val = 5.0; 427 | Eigen::VectorXd pca_coeffs = (mapping_matrix * recon_results.params_model.Wexp_FACS).eval(); 428 | pca_coeffs = pca_coeffs.cwiseMax(-max_elem_val).cwiseMin(max_elem_val).eval(); 429 | 430 | const float max_norm = 18.0; 431 | const float clamp_norm = 15.0; 432 | float coeffs_norm = pca_coeffs.norm(); 433 | if(coeffs_norm > max_norm) { 434 | pca_coeffs *= sqrt(clamp_norm / coeffs_norm); 435 | } 436 | cout << pca_coeffs << endl; 437 | 438 | Eigen::VectorXd diff = (pca_components.transpose() * pca_coeffs).eval(); 439 | for(int i=0;i(res_file, 482 | init_bs_path, 483 | detailed_mesh_file, 484 | output_fitted_mesh_file, 485 | output_difference_map); 486 | } else if (option == "-v" ) { 487 | string res_file = argv[argidx++]; 488 | string coarse_mesh_file = argv[argidx++]; 489 | string output_detailed_mesh_file = argv[argidx++]; 490 | string pca_comp_file = argv[argidx++]; 491 | string mapping_matrix_file = argv[argidx++]; 492 | visualize_details(res_file, 493 | coarse_mesh_file, 494 | pca_comp_file, 495 | mapping_matrix_file, 496 | output_detailed_mesh_file); 497 | } else if (option == "-cn") { 498 | string res_file = argv[argidx++]; 499 | string detailed_mesh_file = argv[argidx++]; 500 | string coarse_mesh_file = argv[argidx++]; 501 | string output_normal_map = argv[argidx++]; 502 | compute_details_exp(res_file, 503 | detailed_mesh_file, 504 | coarse_mesh_file, 505 | output_normal_map); 506 | } else if (option == "-vn") { 507 | string res_file = argv[argidx++]; 508 | string coarse_mesh_file = argv[argidx++]; 509 | string output_normals_file = argv[argidx++]; 510 | string pca_comp_file = argv[argidx++]; 511 | string mapping_matrix_file = argv[argidx++]; 512 | visualize_details_normal(res_file, 513 | coarse_mesh_file, 514 | pca_comp_file, 515 | mapping_matrix_file, 516 | output_normals_file); 517 | } else if (option == "-vne") { 518 | string res_file = argv[argidx++]; 519 | string init_bs_path = argv[argidx++]; 520 | string output_mesh_file = argv[argidx++]; 521 | string output_normals_file = argv[argidx++]; 522 | string pca_comp_file = argv[argidx++]; 523 | string mapping_matrix_file = argv[argidx++]; 524 | visualize_details_normal_exp(res_file, 525 | init_bs_path, 526 | pca_comp_file, 527 | mapping_matrix_file, 528 | output_normals_file, 529 | output_mesh_file); 530 | } 531 | 532 | return 0; 533 | #endif 534 | } 535 | -------------------------------------------------------------------------------- /BlendshapeGeneration/deformationtransfer.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include 5 | 6 | #include "testcases.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "meshdeformer.h" 16 | #include "meshtransferer.h" 17 | #include "cereswrapper.h" 18 | 19 | #include "Geometry/matrix.hpp" 20 | #include "triangle_gradient.h" 21 | 22 | #include "boost/filesystem/operations.hpp" 23 | #include "boost/filesystem/path.hpp" 24 | #include 25 | 26 | void deformationTransfer(const string& s0_name = string(), const string& t0_name = string(), 27 | const string& s_name = string(), const string& t_name = string()) { 28 | const string datapath = "/home/phg/Data/FaceWarehouse_Data_0/"; 29 | 30 | BasicMesh S0; 31 | if(s0_name.empty()) 32 | S0.LoadOBJMesh(datapath + "Tester_1/Blendshape/shape_0.obj"); 33 | else 34 | S0.LoadOBJMesh(s0_name); 35 | 36 | BasicMesh T0; 37 | if(t0_name.empty()) 38 | T0.LoadOBJMesh(datapath + "Tester_106/Blendshape/shape_0.obj"); 39 | else 40 | T0.LoadOBJMesh(t0_name); 41 | 42 | // use deformation transfer to create an initial set of blendshapes 43 | MeshTransferer transferer; 44 | 45 | transferer.setSource(S0); // set source and compute deformation gradient 46 | transferer.setTarget(T0); // set target and compute deformation gradient 47 | 48 | // find the stationary set of verteces 49 | vector stationary_indices = T0.filterVertices([=](const Vector3d& v) { 50 | return v[2] <= -0.45; 51 | }); 52 | transferer.setStationaryVertices(stationary_indices); 53 | 54 | BasicMesh S; 55 | if(s_name.empty()) 56 | S.LoadOBJMesh(datapath + "Tester_1/Blendshape/shape_22.obj"); 57 | else 58 | S.LoadOBJMesh(s_name); 59 | 60 | BasicMesh T = transferer.transfer(S); 61 | if(t_name.empty()) 62 | T.Write("transferred.obj"); 63 | else 64 | T.Write(t_name); 65 | } 66 | 67 | void printUsage() { 68 | cout << "Deformation transfer: [program] -d" << endl; 69 | } 70 | 71 | int main(int argc, char *argv[]) 72 | { 73 | google::InitGoogleLogging(argv[0]); 74 | 75 | #if RUN_TESTS 76 | TestCases::testCeres(); 77 | return 0; 78 | #else 79 | 80 | if( argc < 2 ) { 81 | printUsage(); 82 | } 83 | 84 | if(argc > 4) { 85 | for(int i=1;i<4;++i) 86 | cout << argv[i] << endl; 87 | deformationTransfer(argv[1], argv[2], argv[3], argv[4]); 88 | } else { 89 | string option = argv[1]; 90 | 91 | if( option == "-d") { 92 | deformationTransfer(); 93 | } 94 | } 95 | 96 | return 0; 97 | #endif 98 | } -------------------------------------------------------------------------------- /BlendshapeGeneration/expressiontransfer.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include 5 | 6 | #include "testcases.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "meshdeformer.h" 18 | #include "meshtransferer.h" 19 | #include "cereswrapper.h" 20 | 21 | #include "Geometry/matrix.hpp" 22 | #include "triangle_gradient.h" 23 | 24 | #include "boost/filesystem/operations.hpp" 25 | #include "boost/filesystem/path.hpp" 26 | #include 27 | 28 | void expressionTransfer(const string& res_filename, const string& blendshapes_path) { 29 | // Load the reconstruction results 30 | auto recon_results = LoadReconstructionResult(res_filename); 31 | 32 | // Read in all blendshapes 33 | const int nshapes = 46; 34 | vector B(nshapes+1); 35 | #pragma omp parallel for 36 | for(int i=0;i<=nshapes;++i) { 37 | B[i] = BasicMesh(blendshapes_path + "/B_" + std::to_string(i) + ".obj"); 38 | } 39 | 40 | // Create the transferred shape 41 | BasicMesh mesh = B[0]; 42 | for(int i=1;i<=nshapes;++i) { 43 | auto dBi = B[i].vertices() - B[0].vertices(); 44 | mesh.vertices() += dBi * recon_results.params_model.Wexp_FACS[i]; 45 | } 46 | 47 | mesh.Write("exp_transferred.obj"); 48 | 49 | MultilinearModel model("/home/phg/Data/Multilinear/blendshape_core.tensor"); 50 | MultilinearModelPrior model_prior; 51 | model_prior.load("/home/phg/Data/Multilinear/blendshape_u_0_aug.tensor", 52 | "/home/phg/Data/Multilinear/blendshape_u_1_aug.tensor"); 53 | 54 | 55 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 56 | BasicMesh mesh0("/home/phg/Data/Multilinear/template.obj"); 57 | mesh0.UpdateVertices(model.GetTM()); 58 | mesh0.ComputeNormals(); 59 | 60 | // Render the transferred mesh 61 | // for each image bundle, render the mesh to FBO with culling to get the visible triangles 62 | #if 0 63 | OffscreenMeshVisualizer visualizer(recon_results.params_cam.image_size.x, recon_results.params_cam.image_size.y); 64 | visualizer.SetMVPMode(OffscreenMeshVisualizer::CamPerspective); 65 | visualizer.SetRenderMode(OffscreenMeshVisualizer::Mesh); 66 | visualizer.BindMesh(mesh); 67 | visualizer.SetCameraParameters(recon_results.params_cam); 68 | visualizer.SetMeshRotationTranslation(recon_results.params_model.R, recon_results.params_model.T); 69 | visualizer.SetIndexEncoded(false); 70 | 71 | QImage img = visualizer.Render(); 72 | img.save("transferred.png"); 73 | #else 74 | { 75 | MeshVisualizer* w = new MeshVisualizer("Reconstruction result", mesh0); 76 | w->SetMeshRotationTranslation(recon_results.params_model.R, recon_results.params_model.T); 77 | w->SetCameraParameters(recon_results.params_cam); 78 | 79 | float scale = 640.0 / recon_results.params_cam.image_size.y; 80 | w->resize(recon_results.params_cam.image_size.x * scale, recon_results.params_cam.image_size.y * scale); 81 | w->show(); 82 | } 83 | 84 | { 85 | MeshVisualizer* w = new MeshVisualizer("Transferred result", mesh); 86 | w->SetMeshRotationTranslation(recon_results.params_model.R, recon_results.params_model.T); 87 | w->SetCameraParameters(recon_results.params_cam); 88 | 89 | float scale = 640.0 / recon_results.params_cam.image_size.y; 90 | w->resize(recon_results.params_cam.image_size.x * scale, recon_results.params_cam.image_size.y * scale); 91 | w->show(); 92 | } 93 | #endif 94 | } 95 | 96 | void printUsage(const string& program_name) { 97 | cout << "Usage: " << program_name << " from_exp" << endl; 98 | } 99 | 100 | void create_mesh(const string& filename) { 101 | auto recon_results = LoadReconstructionResult(filename); 102 | MultilinearModel model("/home/phg/Data/Multilinear/blendshape_core.tensor"); 103 | MultilinearModelPrior model_prior; 104 | model_prior.load("/home/phg/Data/Multilinear/blendshape_u_0_aug.tensor", 105 | "/home/phg/Data/Multilinear/blendshape_u_1_aug.tensor"); 106 | 107 | 108 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 109 | BasicMesh mesh0("/home/phg/Data/Multilinear/template.obj"); 110 | mesh0.UpdateVertices(model.GetTM()); 111 | mesh0.ComputeNormals(); 112 | 113 | cout << recon_results.params_model.R << endl; 114 | 115 | glm::dmat4 Rmat = glm::eulerAngleYXZ(-recon_results.params_model.R[0], 116 | -recon_results.params_model.R[1], 117 | -recon_results.params_model.R[2]); 118 | Rmat = glm::transpose(Rmat); 119 | for(int i=0;i 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | InteractiveRigging::InteractiveRigging(QWidget* parent) 11 | : QMainWindow(parent) { 12 | canvas = new BlendshapeVisualizer(this); 13 | 14 | cur_idx = 0; 15 | 16 | setCentralWidget(canvas); 17 | 18 | weights_widget.show(); 19 | 20 | connect(&weights_widget, SIGNAL(sig_sliderChanged(int, int)), this, SLOT(slot_weightsChanged(int,int))); 21 | } 22 | 23 | InteractiveRigging::~InteractiveRigging() {} 24 | 25 | void InteractiveRigging::slot_weightsChanged(int widx, int tick) { 26 | weights[widx] = tick * 0.01; 27 | UpdateShape(); 28 | canvas->setMesh(mesh); 29 | } 30 | 31 | void InteractiveRigging::UpdateShape() { 32 | MatrixX3d v = vertices[0]; 33 | for(int i=1;i<47;++i) { 34 | if(weights[i] > 0) v += vertices[i] * weights[i]; 35 | } 36 | mesh.vertices() = v; 37 | //mesh.ComputeNormals(); 38 | } 39 | 40 | void InteractiveRigging::LoadBlendshapes(const string& path) { 41 | const string prefix = "B_"; 42 | const int num_blendshapes = 47; 43 | blendshapes.resize(num_blendshapes); 44 | for(int i=0;isetMesh(mesh); 61 | 62 | grabKeyboard(); 63 | } 64 | 65 | void InteractiveRigging::keyPressEvent(QKeyEvent* e) { 66 | switch (e->key()) { 67 | case Qt::Key_Left: { 68 | cur_idx = (cur_idx + 1) % blendshapes.size(); 69 | mesh = blendshapes[cur_idx]; 70 | canvas->setMesh(mesh); 71 | break; 72 | } 73 | case Qt::Key_Right: { 74 | cur_idx = (cur_idx - 1); 75 | cur_idx = (cur_idx < 0)?cur_idx+blendshapes.size():cur_idx; 76 | mesh = blendshapes[cur_idx]; 77 | canvas->setMesh(mesh); 78 | break; 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /BlendshapeGeneration/interactiverigging.h: -------------------------------------------------------------------------------- 1 | #ifndef INTERACTIVE_RIGGING_H 2 | #define INTERACTIVE_RIGGING_H 3 | 4 | #include 5 | 6 | #include "OpenGL/gl3dcanvas.h" 7 | #include 8 | 9 | #include "blendshapegeneration.h" 10 | #include "blendshapesweightswidget.h" 11 | 12 | class InteractiveRigging : public QMainWindow { 13 | Q_OBJECT 14 | public: 15 | InteractiveRigging(QWidget *parent=0); 16 | ~InteractiveRigging(); 17 | 18 | virtual QSize sizeHint() const { 19 | return QSize(720, 720); 20 | } 21 | 22 | virtual QSize minimumSizeHint() const { 23 | return QSize(720, 720); 24 | } 25 | 26 | void LoadBlendshapes(const string& path); 27 | 28 | virtual void keyPressEvent(QKeyEvent* e); 29 | 30 | private slots: 31 | void slot_weightsChanged(int, int); 32 | 33 | private: 34 | void UpdateShape(); 35 | 36 | 37 | private: 38 | vector blendshapes; 39 | 40 | BlendshapeVisualizer *canvas; 41 | BlendshapesWeightsWidget weights_widget; 42 | 43 | BasicMesh mesh; 44 | vector vertices; 45 | vector weights; 46 | 47 | int cur_idx; 48 | }; 49 | 50 | #endif // INTERACTIVE_RIGGING_H 51 | -------------------------------------------------------------------------------- /BlendshapeGeneration/landmarks_74_new.txt: -------------------------------------------------------------------------------- 1 | 11284 2 | 5500 3 | 10652 4 | 3842 5 | 3854 6 | 3865 7 | 10573 8 | 3611 9 | 6541 10 | 6770 11 | 6740 12 | 2748 13 | 2573 14 | 9759 15 | 2580 16 | 9251 17 | 2136 18 | 7168 19 | 2134 20 | 2123 21 | 2124 22 | 10751 23 | 4247 24 | 712 25 | 709 26 | 4245 27 | 4248 28 | 576 29 | 4349 30 | 10820 31 | 4392 32 | 1965 33 | 2171 34 | 7100 35 | 9445 36 | 10733 37 | 3439 38 | 364 39 | 10459 40 | 6274 41 | 6388 42 | 1789 43 | 6335 44 | 9236 45 | 10453 46 | 8994 47 | 174 48 | 3237 49 | 182 50 | 8814 51 | 6090 52 | 1620 53 | 6150 54 | 8864 55 | 6117 56 | 3190 57 | 3231 58 | 10306 59 | 3275 60 | 3272 61 | 6164 62 | 6144 63 | 3280 64 | 10334 65 | 8940 66 | 10687 67 | 4356 68 | 10880 69 | 4339 70 | 2170 71 | 9443 72 | 9447 73 | 9310 -------------------------------------------------------------------------------- /BlendshapeGeneration/laplaciandeformation.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include 5 | 6 | #include "testcases.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "meshdeformer.h" 16 | #include "meshtransferer.h" 17 | #include "cereswrapper.h" 18 | 19 | #include "Geometry/matrix.hpp" 20 | #include "triangle_gradient.h" 21 | 22 | #include "boost/filesystem/operations.hpp" 23 | #include "boost/filesystem/path.hpp" 24 | #include 25 | 26 | void laplacianDeformation() { 27 | const string datapath = "/home/phg/Data/FaceWarehouse_Data_0/"; 28 | 29 | BasicMesh m; 30 | m.LoadOBJMesh(datapath + "Tester_1/Blendshape/shape_0.obj"); 31 | // m.BuildHalfEdgeMesh(); 32 | // cout << "subdivide..." << endl; 33 | // m.Subdivide(); 34 | // m.BuildHalfEdgeMesh(); 35 | // cout << "subdivide..." << endl; 36 | // m.Subdivide(); 37 | 38 | MeshDeformer deformer; 39 | deformer.setSource(m); 40 | 41 | vector landmarks = LoadIndices(datapath+"landmarks_74_new.txt"); 42 | deformer.setLandmarks(landmarks); 43 | 44 | vector valid_faces = m.filterFaces([&m](Vector3i fi) { 45 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2]))/ 3.0; 46 | return c[2] > -0.75; 47 | }); 48 | deformer.setValidFaces(valid_faces); 49 | 50 | int objidx = 1; 51 | BasicMesh T; 52 | T.LoadOBJMesh(datapath + "Tester_1/TrainingPose/pose_" + to_string(objidx) + ".obj"); 53 | // T.BuildHalfEdgeMesh(); 54 | // T.Subdivide(); 55 | // T.BuildHalfEdgeMesh(); 56 | // T.Subdivide(); 57 | 58 | MatrixX3d lm_points(landmarks.size(), 3); 59 | for(int i=0;i landmarks = LoadIndices(datapath+"landmarks_74_new.txt"); 84 | deformer.setLandmarks(landmarks); 85 | 86 | vector valid_faces = m.filterFaces([&m](Vector3i fi) { 87 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2]))/ 3.0; 88 | return c[2] > -0.75; 89 | }); 90 | deformer.setValidFaces(valid_faces); 91 | 92 | int objidx = 1; 93 | BasicMesh T; 94 | T.LoadOBJMesh(target_mesh_file); 95 | 96 | BasicMesh D = deformer.deformWithMesh(T, MatrixX3d(), itmax, 3); 97 | 98 | D.Write("deformed" + to_string(objidx) + ".obj"); 99 | } 100 | 101 | void laplacianDeformation_pointcloud( 102 | const string& res_filename, 103 | const string& pointcloud_filename, 104 | const string& output_mesh_filename 105 | ) { 106 | const string datapath("/home/phg/Data/FaceWarehouse_Data_0/"); 107 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 108 | const string model_filename("/home/phg/Data/Multilinear/blendshape_core.tensor"); 109 | 110 | // Update the mesh with the blendshape weights 111 | MultilinearModel model(model_filename); 112 | auto recon_results = LoadReconstructionResult(res_filename); 113 | cout << "Recon results loaded." << endl; 114 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 115 | 116 | glm::dmat4 R = glm::eulerAngleZ(-recon_results.params_model.R[2]) 117 | * glm::eulerAngleX(-recon_results.params_model.R[1]) 118 | * glm::eulerAngleY(-recon_results.params_model.R[0]); 119 | 120 | ifstream fin(pointcloud_filename); 121 | vector points; 122 | points.reserve(100000); 123 | while(fin) { 124 | double x, y, z; 125 | fin >> x >> y >> z; 126 | 127 | // rotate the input point cloud to regular view 128 | glm::dvec4 pt0 = R * glm::dvec4(x, y, z, 1.0); 129 | 130 | points.push_back(Vector3d(pt0.x, pt0.y, pt0.z)); 131 | } 132 | cout << "Points loaded: " << points.size() << " points." << endl; 133 | 134 | MatrixXd P(points.size(), 3); 135 | for(int i=0;i valid_faces = m.filterFaces([&m](Vector3i fi) { 154 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2]))/ 3.0; 155 | return c[2] > -0.5; 156 | }); 157 | deformer.setValidFaces(valid_faces); 158 | // No landmarks 159 | deformer.setLandmarks(vector()); 160 | 161 | BasicMesh D = deformer.deformWithPoints(P, lm_points, 20); 162 | D.Write("deformed" + to_string(i) + ".obj"); 163 | 164 | // replace it 165 | m = D; 166 | 167 | if(max_iters > 1 && i != max_iters - 1) { 168 | m.BuildHalfEdgeMesh(); 169 | m.Subdivide(); 170 | } 171 | } 172 | 173 | m.Write(output_mesh_filename); 174 | } 175 | 176 | void laplacianDeformation_normals( 177 | const string& res_filename, 178 | const string& normals_filename, 179 | const string& output_mesh_filename, 180 | int itmax 181 | ) { 182 | const string datapath("/home/phg/Data/FaceWarehouse_Data_0/"); 183 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 184 | const string model_filename("/home/phg/Data/Multilinear/blendshape_core.tensor"); 185 | 186 | // Update the mesh with the blendshape weights 187 | MultilinearModel model(model_filename); 188 | auto recon_results = LoadReconstructionResult(res_filename); 189 | cout << "Recon results loaded." << endl; 190 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 191 | 192 | glm::dmat4 R = glm::eulerAngleZ(-recon_results.params_model.R[2]) 193 | * glm::eulerAngleX(-recon_results.params_model.R[1]) 194 | * glm::eulerAngleY(-recon_results.params_model.R[0]); 195 | 196 | ifstream fin(normals_filename); 197 | vector normals; 198 | normals.reserve(100000); 199 | while(fin) { 200 | int fidx; 201 | double bx, by, bz, nx, ny, nz; 202 | fin >> fidx >> bx >> by >> bz >> nx >> ny >> nz; 203 | 204 | // rotate the input normal to regular view 205 | glm::dvec4 n0 = R * glm::dvec4(nx, ny, nz, 1.0); 206 | 207 | normals.push_back(NormalConstraint{fidx, Eigen::Vector3d(bx, by, bz), Eigen::Vector3d(n0.x, n0.y, n0.z)}); 208 | } 209 | cout << "Normals loaded: " << normals.size() << " normals." << endl; 210 | 211 | BasicMesh m; 212 | m.LoadOBJMesh(mesh_filename); 213 | m.UpdateVertices(model.GetTM()); 214 | 215 | const int num_subdivision = 1; 216 | for(int i=0;i valid_faces = m.filterFaces([&m](Vector3i fi) { 228 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2])) / 3.0; 229 | return c[2] > -0.5; 230 | }); 231 | deformer.setValidFaces(valid_faces); 232 | // No landmarks 233 | deformer.setLandmarks(vector()); 234 | 235 | BasicMesh D = deformer.deformWithNormals(normals, MatrixX3d(), itmax); 236 | 237 | D.Write(output_mesh_filename); 238 | } 239 | 240 | void laplacianDeformation_normals_exp( 241 | const string& init_bs_path, 242 | const string& res_filename, 243 | const string& normals_filename, 244 | const string& output_mesh_filename, 245 | int itmax, 246 | bool subdivided 247 | ) { 248 | cout << subdivided << endl; 249 | const string datapath("/home/phg/Data/FaceWarehouse_Data_0/"); 250 | 251 | const int num_blendshapes = 46; 252 | vector blendshapes(num_blendshapes+1); 253 | #pragma omp parallel for 254 | for(int i=0;i<=num_blendshapes;++i) { 255 | blendshapes[i].LoadOBJMesh( init_bs_path + "/" + "B_" + to_string(i) + ".obj" ); 256 | blendshapes[i].ComputeNormals(); 257 | } 258 | 259 | BasicMesh m = blendshapes[0]; 260 | 261 | auto recon_results = LoadReconstructionResult(res_filename); 262 | cout << "Recon results loaded." << endl; 263 | { 264 | MatrixX3d verts0 = blendshapes[0].vertices(); 265 | MatrixX3d verts = verts0; 266 | for(int j=1;j<=num_blendshapes;++j) { 267 | verts += (blendshapes[j].vertices() - verts0) * recon_results.params_model.Wexp_FACS(j); 268 | } 269 | m.vertices() = verts; 270 | m.ComputeNormals(); 271 | } 272 | 273 | const int num_subdivision = subdivided?0:1; 274 | cout << num_subdivision << endl; 275 | for(int i=0;i normals; 288 | normals.reserve(100000); 289 | while(fin) { 290 | int fidx; 291 | double bx, by, bz, nx, ny, nz; 292 | fin >> fidx >> bx >> by >> bz >> nx >> ny >> nz; 293 | 294 | // rotate the input normal to regular view 295 | glm::dvec4 n0 = R * glm::dvec4(nx, ny, nz, 1.0); 296 | 297 | normals.push_back(NormalConstraint{fidx, Eigen::Vector3d(bx, by, bz), Eigen::Vector3d(n0.x, n0.y, n0.z)}); 298 | } 299 | cout << "Normals loaded: " << normals.size() << " normals." << endl; 300 | 301 | MeshDeformer deformer; 302 | deformer.setSource(m); 303 | 304 | // Filter the faces to reduce the search range 305 | vector valid_faces = m.filterFaces([&m](Vector3i fi) { 306 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2])) / 3.0; 307 | return c[2] > -0.5; 308 | }); 309 | deformer.setValidFaces(valid_faces); 310 | // No landmarks 311 | deformer.setLandmarks(vector()); 312 | 313 | BasicMesh D = deformer.deformWithNormals(normals, MatrixX3d(), itmax); 314 | 315 | D.Write(output_mesh_filename); 316 | } 317 | 318 | void laplacianDeformation_mesh( 319 | const string& res_filename, 320 | const string& target_mesh_filename, 321 | const string& output_mesh_filename, 322 | int itmax 323 | ) { 324 | const string datapath("/home/phg/Data/FaceWarehouse_Data_0/"); 325 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 326 | const string model_filename("/home/phg/Data/Multilinear/blendshape_core.tensor"); 327 | 328 | // Update the mesh with the blendshape weights 329 | MultilinearModel model(model_filename); 330 | auto recon_results = LoadReconstructionResult(res_filename); 331 | cout << "Recon results loaded." << endl; 332 | model.ApplyWeights(recon_results.params_model.Wid, recon_results.params_model.Wexp); 333 | 334 | 335 | BasicMesh target; 336 | target.LoadOBJMesh(target_mesh_filename); 337 | 338 | 339 | BasicMesh m; 340 | m.LoadOBJMesh(mesh_filename); 341 | m.UpdateVertices(model.GetTM()); 342 | 343 | const int num_subdivision = 1; 344 | for(int i=0;i valid_faces = m.filterFaces([&m](Vector3i fi) { 356 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2])) / 3.0; 357 | return c[2] > -0.5; 358 | }); 359 | deformer.setValidFaces(valid_faces); 360 | // No landmarks 361 | deformer.setLandmarks(vector()); 362 | 363 | BasicMesh D = deformer.deformWithMesh(target, MatrixX3d(), itmax); 364 | 365 | D.Write(output_mesh_filename); 366 | } 367 | 368 | void laplacianDeformation_mesh_exp( 369 | const string& res_filename, 370 | const string& init_bs_path, 371 | const string& target_mesh_filename, 372 | const string& output_mesh_filename, 373 | int itmax, 374 | bool subdivided 375 | ) { 376 | const string datapath("/home/phg/Data/FaceWarehouse_Data_0/"); 377 | const string mesh_filename(datapath + "Tester_1/Blendshape/shape_0.obj"); 378 | 379 | const int num_blendshapes = 46; 380 | vector blendshapes(num_blendshapes+1); 381 | #pragma omp parallel for 382 | for(int i=0;i<=num_blendshapes;++i) { 383 | blendshapes[i].LoadOBJMesh( init_bs_path + "/" + "B_" + to_string(i) + ".obj" ); 384 | blendshapes[i].ComputeNormals(); 385 | } 386 | 387 | BasicMesh m = blendshapes[0]; 388 | 389 | auto recon_results = LoadReconstructionResult(res_filename); 390 | cout << "Recon results loaded." << endl; 391 | { 392 | MatrixX3d verts0 = blendshapes[0].vertices(); 393 | MatrixX3d verts = verts0; 394 | for(int j=1;j<=num_blendshapes;++j) { 395 | verts += (blendshapes[j].vertices() - verts0) * recon_results.params_model.Wexp_FACS(j); 396 | } 397 | m.vertices() = verts; 398 | m.ComputeNormals(); 399 | } 400 | 401 | const int num_subdivision = subdivided?0:1; 402 | cout << num_subdivision << endl; 403 | for(int i=0;i valid_faces = m.filterFaces([&m](Vector3i fi) { 418 | Vector3d c = (m.vertex(fi[0]) + m.vertex(fi[1]) + m.vertex(fi[2])) / 3.0; 419 | return c[2] > -0.5; 420 | }); 421 | deformer.setValidFaces(valid_faces); 422 | // No landmarks 423 | deformer.setLandmarks(vector()); 424 | 425 | BasicMesh D = deformer.deformWithMesh(target, MatrixX3d(), itmax); 426 | 427 | D.Write(output_mesh_filename); 428 | } 429 | 430 | void printUsage() { 431 | cout << "Laplacian deformation: [program] -l" << endl; 432 | cout << "Laplacian deformation with point cloud: [program] -lp" << endl; 433 | cout << "Laplacian deformation with normals: [program] -ln" << endl; 434 | } 435 | 436 | int main(int argc, char *argv[]) 437 | { 438 | google::InitGoogleLogging(argv[0]); 439 | QApplication a(argc, argv); 440 | glutInit(&argc, argv); 441 | 442 | #if RUN_TESTS 443 | TestCases::testCeres(); 444 | return 0; 445 | #else 446 | 447 | if( argc < 2 ) { 448 | printUsage(); 449 | return 0; 450 | } 451 | 452 | string option = argv[1]; 453 | 454 | if( option == "-l" ) { 455 | laplacianDeformation(); 456 | } else if( option == "-lp" ) { 457 | string res_file = argc>2?argv[2]:"/home/phg/Data/InternetRecon0/yaoming/4.jpg.res"; 458 | string pointcloud_file = argc>3?argv[3]:"/home/phg/Data/InternetRecon0/yaoming/SFS/masked_optimized_point_cloud_7.txt"; 459 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 460 | laplacianDeformation_pointcloud(res_file, pointcloud_file, output_mesh_file); 461 | } else if( option == "-ln" ) { 462 | string res_file = argc>2?argv[2]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/1.jpg.res"; 463 | string normals_file = argc>3?argv[3]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/constraints.txt"; 464 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 465 | int itmax = argc>5?stoi(argv[5]):2; 466 | laplacianDeformation_normals(res_file, normals_file, output_mesh_file, itmax); 467 | } else if( option == "-ln_exp" ) { 468 | for(int i=0;i2?argv[2]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/1.jpg.res"; 472 | string normals_file = argc>3?argv[3]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/constraints.txt"; 473 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 474 | int itmax = argc>5?stoi(argv[5]):2; 475 | string init_bs_path = argc>6?argv[6]:""; 476 | bool subdivided = argc>7?string(argv[7])=="1":false; 477 | laplacianDeformation_normals_exp(init_bs_path, res_file, normals_file, output_mesh_file, itmax, subdivided); 478 | } else if( option == "-lm" ) { 479 | string res_file = argc>2?argv[2]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/1.jpg.res"; 480 | string target_mesh_file = argc>3?argv[3]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/deformed.obj"; 481 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 482 | int itmax = argc>5?stoi(argv[5]):20; 483 | laplacianDeformation_mesh(res_file, target_mesh_file, output_mesh_file, itmax); 484 | } else if( option == "-lm_exp" ) { 485 | string res_file = argc>2?argv[2]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/1.jpg.res"; 486 | string target_mesh_file = argc>3?argv[3]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/deformed.obj"; 487 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 488 | int itmax = argc>5?stoi(argv[5]):20; 489 | string init_bs_path = argc>6?argv[6]:""; 490 | bool subdivided = argc>7?string(argv[7])=="1":false; 491 | laplacianDeformation_mesh_exp(res_file, init_bs_path, target_mesh_file, output_mesh_file, itmax, subdivided); 492 | } else if( option == "-lms" ) { 493 | string source_mesh_file = argc>2?argv[2]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/deformed.obj"; 494 | string target_mesh_file = argc>3?argv[3]:"/home/phg/Data/SFS_test/Andy_Lau_400x400/outputs/deformed.obj"; 495 | string output_mesh_file = argc>4?argv[4]:"deformed.obj"; 496 | int itmax = argc>5?stoi(argv[5]):20; 497 | laplacianDeformation_meshes(source_mesh_file, target_mesh_file, output_mesh_file, itmax); 498 | } 499 | 500 | return 0; 501 | #endif 502 | } 503 | -------------------------------------------------------------------------------- /BlendshapeGeneration/main.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include "interactiverigging.h" 5 | #include 6 | #include 7 | 8 | #include "testcases.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "blendshaperefiner.h" 19 | #include "blendshaperefiner_old.h" 20 | #include "meshdeformer.h" 21 | #include "meshtransferer.h" 22 | #include "cereswrapper.h" 23 | 24 | #include "Geometry/matrix.hpp" 25 | #include "triangle_gradient.h" 26 | 27 | #include "boost/filesystem/operations.hpp" 28 | #include "boost/filesystem/path.hpp" 29 | #include "boost/program_options.hpp" 30 | #include "boost/timer/timer.hpp" 31 | 32 | namespace fs = boost::filesystem; 33 | 34 | #include "nlohmann/json.hpp" 35 | using json = nlohmann::json; 36 | 37 | void blendShapeGeneration_mesh( 38 | const string& source_path, 39 | bool subdivision, 40 | bool disable_neutral_opt, 41 | bool mask_nose_and_fore_head 42 | ) { 43 | BlendshapeRefiner refiner( 44 | json{ 45 | {"use_init_blendshapes", false}, 46 | {"subdivision", subdivision}, 47 | {"blendshapes_subdivided", false}, 48 | {"mask_nose_and_fore_head", mask_nose_and_fore_head} 49 | }); 50 | 51 | const string home_directory = QDir::homePath().toStdString(); 52 | cout << "Home dir: " << home_directory << endl; 53 | 54 | refiner.SetBlendshapeCount(46); 55 | refiner.LoadTemplateMeshes(home_directory + "/Data/FaceWarehouse_Data_0/Tester_1/Blendshape/", "shape_"); 56 | 57 | refiner.SetResourcesPath(source_path); 58 | refiner.SetReconstructionsPath(source_path); 59 | refiner.SetExampleMeshesPath(source_path + "/normal_constraints"); 60 | refiner.SetInputBlendshapesPath(home_directory + "/Data/FaceWarehouse_Data_0/Tester_1/Blendshape/"); 61 | refiner.SetBlendshapesPath(source_path + "/blendshapes"); 62 | 63 | refiner.LoadSelectionFile("selection_sfs.txt"); 64 | refiner.LoadInputReconstructionResults("settings.txt"); 65 | refiner.LoadInputExampleMeshes(); 66 | 67 | refiner.Refine(false, disable_neutral_opt); 68 | } 69 | 70 | void blendShapeGeneration_mesh_blendshapes( 71 | const string& source_path, 72 | const string& recon_path, 73 | const string& meshes_path, 74 | const string& input_blendshapes_path, 75 | const string& blendshapes_path, 76 | bool subdivision, 77 | bool blendshapes_subdivided, 78 | bool initialize_only, 79 | bool disable_neutral_opt, 80 | bool mask_nose_and_fore_head 81 | ) { 82 | BlendshapeRefiner refiner( 83 | json{ 84 | {"use_init_blendshapes", true}, 85 | {"subdivision", subdivision}, 86 | {"blendshapes_subdivided", blendshapes_subdivided}, 87 | {"mask_nose_and_fore_head", mask_nose_and_fore_head} 88 | } 89 | ); 90 | 91 | const string home_directory = QDir::homePath().toStdString(); 92 | cout << "Home dir: " << home_directory << endl; 93 | 94 | refiner.SetBlendshapeCount(46); 95 | refiner.LoadTemplateMeshes(home_directory + "/Data/FaceWarehouse_Data_0/Tester_1/Blendshape/", "shape_"); 96 | 97 | refiner.SetResourcesPath(source_path); 98 | refiner.SetReconstructionsPath(recon_path); 99 | refiner.SetExampleMeshesPath(meshes_path); 100 | refiner.SetInputBlendshapesPath(input_blendshapes_path); 101 | refiner.SetBlendshapesPath(blendshapes_path); 102 | 103 | refiner.LoadSelectionFile("selection_sfs.txt"); 104 | refiner.LoadInputReconstructionResults("settings.txt"); 105 | refiner.LoadInputExampleMeshes(); 106 | 107 | refiner.Refine(initialize_only, disable_neutral_opt); 108 | } 109 | 110 | void blendShapeGeneration_pointcloud( 111 | const string& source_path, 112 | int template_id, 113 | bool subdivision, 114 | bool disable_neutral_opt, 115 | bool mask_nose_and_fore_head) { 116 | BlendshapeRefiner refiner( 117 | json{ 118 | {"use_init_blendshapes", false}, 119 | {"subdivision", subdivision}, 120 | {"blendshapes_subdivided", false}, 121 | {"mask_nose_and_fore_head", mask_nose_and_fore_head} 122 | }); 123 | 124 | const string home_directory = QDir::homePath().toStdString(); 125 | cout << "Home dir: " << home_directory << endl; 126 | refiner.SetBlendshapeCount(46); 127 | refiner.LoadTemplateMeshes(home_directory + "/Data/FaceWarehouse_Data_0/Tester_" + to_string(template_id) + "/Blendshape/", "shape_"); 128 | 129 | refiner.SetResourcesPath(source_path); 130 | refiner.SetReconstructionsPath(source_path); 131 | refiner.SetPointCloudsPath(source_path + "/SFS"); 132 | refiner.SetInputBlendshapesPath(home_directory + "/Data/FaceWarehouse_Data_0/Tester_" + to_string(template_id) + "/Blendshape/"); 133 | refiner.SetBlendshapesPath(source_path + "/blendshapes"); 134 | 135 | refiner.LoadSelectionFile("selection_sfs.txt"); 136 | refiner.LoadInputReconstructionResults("settings.txt"); 137 | refiner.LoadInputPointClouds(); 138 | 139 | refiner.Refine(false, disable_neutral_opt); 140 | } 141 | 142 | void blendShapeGeneration_pointcloud_blendshapes( 143 | const string& source_path, 144 | const string& recon_path, 145 | const string& point_clouds_path, 146 | const string& input_blendshapes_path, 147 | const string& blendshapes_path, 148 | int template_id, 149 | bool subdivision, 150 | bool blendshapes_subdivided, 151 | bool initialize_only, 152 | bool disable_neutral_opt, 153 | bool mask_nose_and_fore_head 154 | ) { 155 | BlendshapeRefiner refiner( 156 | json{ 157 | {"use_init_blendshapes", true}, 158 | {"subdivision", subdivision}, 159 | {"blendshapes_subdivided", blendshapes_subdivided}, 160 | {"mask_nose_and_fore_head", mask_nose_and_fore_head} 161 | } 162 | ); 163 | const string home_directory = QDir::homePath().toStdString(); 164 | cout << "Home dir: " << home_directory << endl; 165 | refiner.SetBlendshapeCount(46); 166 | refiner.LoadTemplateMeshes(home_directory + "/Data/FaceWarehouse_Data_0/Tester_" + to_string(template_id) + "/Blendshape/", "shape_"); 167 | 168 | refiner.SetResourcesPath(source_path); 169 | refiner.SetReconstructionsPath(recon_path); 170 | refiner.SetPointCloudsPath(point_clouds_path); 171 | refiner.SetInputBlendshapesPath(input_blendshapes_path); 172 | refiner.SetBlendshapesPath(blendshapes_path); 173 | 174 | refiner.LoadSelectionFile("selection_sfs.txt"); 175 | refiner.LoadInputReconstructionResults("settings.txt"); 176 | refiner.LoadInputPointClouds(); 177 | 178 | refiner.Refine(initialize_only, disable_neutral_opt); 179 | } 180 | 181 | void blendShapeGeneration_pointcloud_EBFR() { 182 | BlendshapeRefiner refiner; 183 | const string home_directory = QDir::homePath().toStdString(); 184 | cout << "Home dir: " << home_directory << endl; 185 | refiner.SetBlendshapeCount(46); 186 | refiner.LoadTemplateMeshes(home_directory + "/Storage/Data/FaceWarehouse_Data_0/Tester_1/Blendshape/", "shape_"); 187 | 188 | // yaoming 189 | refiner.SetResourcesPath(home_directory + "/Storage/Data/InternetRecon0/yaoming/crop/"); 190 | refiner.LoadInputReconstructionResults("settings.txt"); 191 | refiner.LoadInputPointClouds(); 192 | 193 | // // Turing 194 | // refiner.SetResourcesPath(home_directory + "/Storage/Data/InternetRecon2/Allen_Turing/"); 195 | // refiner.LoadInputReconstructionResults("setting.txt"); 196 | // refiner.LoadInputPointClouds(); 197 | 198 | refiner.Refine_EBFR(); 199 | } 200 | 201 | int main(int argc, char *argv[]) 202 | { 203 | QApplication a(argc, argv); 204 | google::InitGoogleLogging(argv[0]); 205 | glutInit(&argc, argv); 206 | 207 | namespace po = boost::program_options; 208 | po::options_description desc("Options"); 209 | desc.add_options() 210 | ("help", "Print help messages") 211 | ("oldfasion", "Generate blendshapes the old way.") 212 | ("pointclouds", "Generate blendshapes from point clouds") 213 | ("pointclouds_with_init_shapes", "Generate blendshapes from point clouds and a set of initial blendshapes") 214 | ("pointclouds_from_meshes", "Generate blendshapes from point clouds sampled from meshes") 215 | ("pointclouds_from_meshes_with_init_shapes", "Generate blendshapes from point clouds sampled from meshes and a set of initial blendshapes") 216 | ("ebfr", "Generate blendshapes using example based facial rigging method") 217 | ("rigging", "Interactive rigging mode with specified blendshapes") 218 | ("template_id", po::value()->default_value(1), "Index for the template blendshapes") 219 | ("repo_path", po::value(), "Path to images repo.") 220 | ("recon_path", po::value(), "Path to large scale reconstruction results") 221 | ("pointclouds_path", po::value(), "Path to input point clouds") 222 | ("meshes_path", po::value(), "Path to input example meshes") 223 | ("init_blendshapes_path", po::value(), "Path to initial blendshapes") 224 | ("blendshapes_path", po::value(), "Path to output blendshapes") 225 | ("initialize_only", "Only perform initialization") 226 | ("no_neutral", "Disable neutral shape optimzation") 227 | ("subdivided", "Indicate the input blendshapes are subdivided") 228 | ("subdivision", "Enable subdivision") 229 | ("mask_nose_and_fore_head", "Use fore head and nose mask") 230 | ("ref_mesh", po::value(), "Reference mesh for distance computation") 231 | ("mesh", po::value(), "Mesh to visualize") 232 | ("vis", "Visualize blendshape mesh") 233 | ("align", "Align mesh for error computation") 234 | ("align_translation", "Align mesh for error computation, translation only") 235 | ("face_only", "Visualize the difference in face region only") 236 | ("range", po::value()->default_value(0.05), "Range for error visualization") 237 | ("rendering_settings", po::value(), "Rendering settings to use") 238 | ("skip_faces", po::value(), "Faces to skip rendering") 239 | ("texture", po::value(), "Texture for the mesh") 240 | ("ambient_occlusion", po::value(), "Ambient occlusion file") 241 | ("normals", po::value(), "Normals file") 242 | ("sideview", "Visualize the blendshape mesh in side view") 243 | ("silent", "Silent visualization using offscreen drawing") 244 | ("save,s", po::value(), "Save the result to a file"); 245 | po::variables_map vm; 246 | 247 | try { 248 | po::store(po::parse_command_line(argc, argv, desc), vm); 249 | po::notify(vm); 250 | 251 | if (vm.count("help")) { 252 | cout << desc << endl; 253 | return 1; 254 | } 255 | 256 | if (vm.count("oldfasion")) { 257 | blendShapeGeneration(); 258 | } else if (vm.count("ebfr")) { 259 | throw runtime_error("This is no longer supported."); 260 | blendShapeGeneration_pointcloud_EBFR(); 261 | } else if (vm.count("rigging")) { 262 | QApplication a(argc, argv); 263 | InteractiveRigging rigging; 264 | rigging.LoadBlendshapes(vm["blendshapes_path"].as()); 265 | rigging.show(); 266 | return a.exec(); 267 | } else if (vm.count("pointclouds")) { 268 | if (vm.count("repo_path")) { 269 | blendShapeGeneration_pointcloud(vm["repo_path"].as(), 270 | vm["template_id"].as(), 271 | vm.count("subdivision"), 272 | vm.count("no_neutral"), 273 | vm.count("mask_nose_and_fore_head")); 274 | } else { 275 | throw po::error("Need to specify repo_path"); 276 | } 277 | } else if (vm.count("pointclouds_from_meshes")) { 278 | if (vm.count("repo_path")) { 279 | blendShapeGeneration_mesh(vm["repo_path"].as(), 280 | vm.count("subdivision"), 281 | vm.count("no_neutral"), 282 | vm.count("mask_nose_and_fore_head")); 283 | } else { 284 | throw po::error("Need to specify repo_path"); 285 | } 286 | } else if (vm.count("pointclouds_with_init_shapes")) { 287 | if(vm.count("repo_path") 288 | && vm.count("recon_path") 289 | && vm.count("pointclouds_path") 290 | && vm.count("init_blendshapes_path") 291 | && vm.count("blendshapes_path")) { 292 | blendShapeGeneration_pointcloud_blendshapes( 293 | vm["repo_path"].as(), 294 | vm["recon_path"].as(), 295 | vm["pointclouds_path"].as(), 296 | vm["init_blendshapes_path"].as(), 297 | vm["blendshapes_path"].as(), 298 | vm["template_id"].as(), 299 | vm.count("subdivision"), 300 | vm.count("subdivided"), 301 | vm.count("initialize_only"), 302 | vm.count("no_neutral"), 303 | vm.count("mask_nose_and_fore_head") 304 | ); 305 | } else { 306 | throw po::error("Need to specify repo_path, recon_path, pointclouds_path, init_blendshapes_path, blendshapes_path"); 307 | } 308 | } else if (vm.count("pointclouds_from_meshes_with_init_shapes")) { 309 | if(vm.count("repo_path") 310 | && vm.count("recon_path") 311 | && vm.count("meshes_path") 312 | && vm.count("init_blendshapes_path") 313 | && vm.count("blendshapes_path")) { 314 | blendShapeGeneration_mesh_blendshapes( 315 | vm["repo_path"].as(), 316 | vm["recon_path"].as(), 317 | vm["meshes_path"].as(), 318 | vm["init_blendshapes_path"].as(), 319 | vm["blendshapes_path"].as(), 320 | vm.count("subdivision"), 321 | vm.count("subdivided"), 322 | vm.count("initialize_only"), 323 | vm.count("no_neutral"), 324 | vm.count("mask_nose_and_fore_head") 325 | ); 326 | } else { 327 | throw po::error("Need to specify repo_path, recon_path, pointclouds_path, init_blendshapes_path, blendshapes_path"); 328 | } 329 | } else if(vm.count("vis")) { 330 | bool save_result = vm.count("save"); 331 | bool compare_mode = vm.count("ref_mesh"); 332 | bool sideview = vm.count("sideview"); 333 | 334 | string input_mesh_file; 335 | if(vm.count("mesh")) { 336 | input_mesh_file = vm["mesh"].as(); 337 | } else { 338 | throw po::error("Need to specify mesh"); 339 | } 340 | 341 | BlendshapeGeneration w(vm.count("silent")); 342 | if(!vm.count("silent")) w.show(); 343 | w.SetSideView(sideview); 344 | 345 | if(vm.count("texture")) w.SetTexture(vm["texture"].as()); 346 | if(vm.count("ambient_occlusion")) w.SetAmbientOcclusion(vm["ambient_occlusion"].as()); 347 | if(vm.count("normals")) w.SetNormals(vm["normals"].as()); 348 | if(vm.count("rendering_settings")) w.LoadRenderingSettings(vm["rendering_settings"].as()); 349 | if(vm.count("skip_faces")) w.LoadSkipFaces(vm["skip_faces"].as(), vm.count("subdivided")); 350 | w.SetErrorRange(vm["range"].as()); 351 | w.SetAlignMesh(vm.count("align")); 352 | w.SetAlignMeshTranslation(vm.count("align_translation")); 353 | w.SetFaceOnlyMode(vm.count("face_only")); 354 | 355 | if(compare_mode) { 356 | string ref_mesh_file = vm["ref_mesh"].as(); 357 | 358 | w.LoadMeshes(input_mesh_file, ref_mesh_file); 359 | w.setWindowTitle(input_mesh_file.c_str()); 360 | } else { 361 | w.LoadMesh(input_mesh_file); 362 | w.setWindowTitle(input_mesh_file.c_str()); 363 | } 364 | 365 | if(save_result) { 366 | cout << "Saving results ..." << endl; 367 | w.repaint(); 368 | for(int i=0;i<10;++i) 369 | qApp->processEvents(); 370 | 371 | w.Save(vm["save"].as()); 372 | qApp->processEvents(); 373 | 374 | if(compare_mode) w.SaveError(vm["save"].as() + ".error"); 375 | return 0; 376 | } else { 377 | return a.exec(); 378 | } 379 | } 380 | 381 | } catch(po::error& e) { 382 | cerr << "Error: " << e.what() << endl; 383 | cerr << desc << endl; 384 | return 1; 385 | } 386 | 387 | return 0; 388 | } 389 | -------------------------------------------------------------------------------- /BlendshapeGeneration/meshdeformer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "common.h" 3 | #include 4 | //#include "pointcloud.h" 5 | 6 | struct ICPCorrespondence { 7 | int tidx; // triangle index 8 | double bcoords[3]; // bary-centric coordinates 9 | double hit[3]; // point on triangle 10 | double d; 11 | double weight; // weight of this point 12 | }; 13 | 14 | struct NormalConstraint { 15 | int fidx; 16 | Eigen::Vector3d bcoords, n; 17 | }; 18 | 19 | class MeshDeformer 20 | { 21 | public: 22 | MeshDeformer(); 23 | ~MeshDeformer(); 24 | 25 | void setSource(const BasicMesh &src) { 26 | S = src; 27 | 28 | // find the neighbor information of every vertex in the source mesh 29 | const int nfaces = S.NumFaces(); 30 | const int nverts = S.NumVertices(); 31 | 32 | N.resize(nverts); 33 | for (int i = 0; i < nfaces; ++i) { 34 | auto Fi = S.face(i); 35 | int v1 = Fi[0], v2 = Fi[1], v3 = Fi[2]; 36 | 37 | N[v1].insert(v2); N[v1].insert(v3); 38 | N[v2].insert(v1); N[v2].insert(v3); 39 | N[v3].insert(v1); N[v3].insert(v2); 40 | } 41 | } 42 | void setLandmarks(const vector &lms) { landmarks = lms; } 43 | void setLandmarks(const VectorXi& lms) { 44 | landmarks.resize(lms.size()); 45 | for(int i=0;i &fidx) { 48 | valid_faces = fidx; 49 | 50 | #if 0 51 | { 52 | ofstream fout("valid_faces.txt"); 53 | for(int i : fidx) { 54 | fout << i << endl; 55 | } 56 | fout.close(); 57 | } 58 | #endif 59 | 60 | valid_vertices.clear(); 61 | unordered_set valid_vertices_set; 62 | for(int i : valid_faces) { 63 | auto fi = S.face(i); 64 | valid_vertices_set.insert(fi[0]); 65 | valid_vertices_set.insert(fi[1]); 66 | valid_vertices_set.insert(fi[2]); 67 | } 68 | valid_vertices.assign(valid_vertices_set.begin(), valid_vertices_set.end()); 69 | 70 | // Set the fixed faces 71 | fixed_faces.clear(); 72 | unordered_set valid_faces_set(valid_faces.begin(), valid_faces.end()); 73 | for(int i=0;i fixed_vertices_set; 80 | 81 | struct edge_t { 82 | edge_t() {} 83 | edge_t(int s, int t) : s(s), t(t) {} 84 | bool operator<(const edge_t& other) const { 85 | if(s < other.s) return true; 86 | else if( s > other.s ) return false; 87 | else return t < other.t; 88 | } 89 | int s, t; 90 | }; 91 | map counter; 92 | for(int i : fixed_faces) { 93 | auto fi = S.face(i); 94 | fixed_vertices_set.insert(fi[0]); 95 | fixed_vertices_set.insert(fi[1]); 96 | fixed_vertices_set.insert(fi[2]); 97 | 98 | auto add_edge = [&counter](int s, int t) { 99 | edge_t e(min(s, t), max(s, t)); 100 | if(counter.count(e)) ++counter[e]; 101 | else counter[e] = 1; 102 | }; 103 | 104 | add_edge(fi[0], fi[1]); 105 | add_edge(fi[1], fi[2]); 106 | add_edge(fi[2], fi[0]); 107 | } 108 | fixed_vertices.assign(fixed_vertices_set.begin(), fixed_vertices_set.end()); 109 | 110 | // Find out the boundary vertices for the fixed region 111 | unordered_set boundary_vertices_set; 112 | fixed_faces_boundary_vertices.clear(); 113 | for(auto p : counter) { 114 | if(p.second == 1) { 115 | boundary_vertices_set.insert(p.first.s); 116 | boundary_vertices_set.insert(p.first.t); 117 | } 118 | } 119 | 120 | // Also add neighboring vertices of the valid vertices, if needed 121 | for(int i : valid_vertices) { 122 | for(int j : N[i]) { 123 | if(valid_vertices_set.count(j) == 0) 124 | boundary_vertices_set.insert(j); 125 | } 126 | } 127 | 128 | fixed_faces_boundary_vertices.assign(boundary_vertices_set.begin(), boundary_vertices_set.end()); 129 | 130 | std::sort(fixed_faces_boundary_vertices.begin(), fixed_faces_boundary_vertices.end()); 131 | std::sort(valid_vertices.begin(), valid_vertices.end()); 132 | 133 | #if 0 134 | { 135 | ofstream fout("boundary_vertices.txt"); 136 | for(int i : fixed_faces_boundary_vertices) { 137 | fout << i << endl; 138 | } 139 | fout.close(); 140 | } 141 | #endif 142 | } 143 | 144 | BasicMesh deformWithMesh(const BasicMesh &T, const MatrixX3d &lm_points, int itmax = 10, int points_per_face = 32); 145 | BasicMesh deformWithPoints(const MatrixX3d &P, const MatrixX3d &lm_points, int itmax = 10); 146 | BasicMesh deformWithNormals(const vector &normals, const MatrixX3d &lm_points, int itmax = 10); 147 | 148 | protected: 149 | vector findClosestPoints_projection(const MatrixX3d &P, const BasicMesh &mesh); 150 | vector findClosestPoints_tree(const MatrixX3d &P, const BasicMesh &mesh); 151 | vector findClosestPoints_bruteforce(const MatrixX3d &P, const BasicMesh &mesh); 152 | ICPCorrespondence findClosestPoint_triangle(double px, double py, double pz, 153 | const Vector3d& v0, const Vector3d& v1, const Vector3d& v2); 154 | 155 | private: 156 | BasicMesh S; 157 | vector landmarks; 158 | 159 | // The set of vertices/faces to deform 160 | vector valid_vertices; 161 | vector valid_faces; 162 | vector vertex_index_map; 163 | 164 | // The set of fixed vertices/faces 165 | vector fixed_faces; 166 | vector fixed_vertices; 167 | vector fixed_faces_boundary_vertices; 168 | 169 | vector> N; 170 | }; 171 | -------------------------------------------------------------------------------- /BlendshapeGeneration/meshfitter.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include "blendshapegeneration.h" 4 | #include 5 | 6 | #include "testcases.h" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "meshdeformer.h" 18 | #include "meshtransferer.h" 19 | #include "cereswrapper.h" 20 | 21 | #include "Geometry/matrix.hpp" 22 | #include "triangle_gradient.h" 23 | 24 | #include "boost/filesystem/operations.hpp" 25 | #include "boost/filesystem/path.hpp" 26 | #include 27 | 28 | struct CombinedIdentityExpressionCostFunction { 29 | CombinedIdentityExpressionCostFunction( 30 | const MultilinearModel& model, 31 | const BasicMesh& target 32 | ) : model(model), target(target) { 33 | } 34 | 35 | bool operator()(const double *const *params, double* residual) const { 36 | VectorXd wid = Map(params[0], 50).eval(); 37 | VectorXd wexp = Map(params[1], 25).eval(); 38 | 39 | // Apply the weights to MM 40 | model.ApplyWeights(wid, wexp); 41 | 42 | auto tm = model.GetTM(); 43 | const int num_residues = tm.size() / 3; 44 | 45 | // Compute the residues 46 | #pragma omp parallel for 47 | for(int i=0;i 86 | *cost_function = new ceres::DynamicNumericDiffCostFunction( 87 | new CombinedIdentityExpressionCostFunction( 88 | model, 89 | mesh 90 | ) 91 | ); 92 | cout << mesh.NumVertices() << endl; 93 | cost_function->AddParameterBlock(wid.size()); 94 | cost_function->AddParameterBlock(wexp.size()); 95 | cost_function->SetNumResiduals(mesh.NumVertices()); 96 | problem.AddResidualBlock(cost_function, NULL, vector{wid.data(), wexp.data()}); 97 | 98 | // Add prior terms 99 | 100 | // Solve it 101 | { 102 | boost::timer::auto_cpu_timer timer_solve( 103 | "Problem solve time = %w seconds.\n"); 104 | ceres::Solver::Options options; 105 | options.max_num_iterations = 30; 106 | options.num_threads = 8; 107 | options.num_linear_solver_threads = 8; 108 | //options.minimizer_type = ceres::LINE_SEARCH; 109 | //options.line_search_direction_type = ceres::STEEPEST_DESCENT; 110 | options.minimizer_progress_to_stdout = true; 111 | ceres::Solver::Summary summary; 112 | ceres::Solve(options, &problem, &summary); 113 | summary.BriefReport(); 114 | } 115 | 116 | // Output the final mesh 117 | model.ApplyWeights(wid, wexp); 118 | BasicMesh output_mesh = mesh; 119 | output_mesh.UpdateVertices(model.GetTM()); 120 | output_mesh.Write(output_mesh_filename); 121 | 122 | // Output the fitted identity and expression 123 | { 124 | ofstream fout(output_mesh_filename + ".weights"); 125 | auto write_vector = [](const VectorXd& v, ofstream& os) { 126 | os << v.rows() << ' '; 127 | for(int i=0;i S1grad(nfaces); 35 | 36 | for(int j=0;j &S1grad) 47 | { 48 | if( !(S0set && T0set) ) { 49 | throw "S0 or T0 not set."; 50 | } 51 | 52 | auto &S = S1grad; 53 | auto &T = T0grad; 54 | 55 | int nfaces = S0.NumFaces(); 56 | int nverts = S0.NumVertices(); 57 | 58 | // assemble sparse matrix A 59 | int nrowsA = nfaces * 3; 60 | int nsv = stationary_vertices.size(); 61 | int nrowsC = nsv; 62 | int nrows = nrowsA + nrowsC; 63 | int ncols = nverts; 64 | int ntermsA = nfaces*9; 65 | int ntermsC = stationary_vertices.size(); 66 | int nterms = ntermsA + ntermsC; 67 | 68 | using Tripletd = Eigen::Triplet; 69 | using SparseMatrixd = Eigen::SparseMatrix; 70 | 71 | PhGUtils::Timer tmatrix; 72 | tmatrix.tic(); 73 | 74 | vector A_coeffs; 75 | A_coeffs.reserve(nterms); 76 | 77 | SparseMatrixd A(nrows, ncols); 78 | 79 | // fill in the deformation gradient part 80 | for(int i=0, ioffset=0;i D_coeffs; 142 | D_coeffs.reserve(nrowsA); 143 | for(int i=0;i> solver; 171 | solver.compute(GtDG); 172 | if(solver.info()!=Eigen::Success) { 173 | cerr << "Failed to decompose matrix A." << endl; 174 | exit(-1); 175 | } 176 | 177 | MatrixXd x = solver.solve(GtDc); 178 | 179 | if(solver.info()!=Eigen::Success) { 180 | cerr << "Failed to solve A\\b." << endl; 181 | exit(-1); 182 | } 183 | tsolve.toc("solving linear equations"); 184 | 185 | // make a copy of T0 186 | BasicMesh Td = T0; 187 | 188 | // change the vertices with x 189 | for(int i=0;i 6 | #include "Geometry/matrix.hpp" 7 | 8 | class MeshTransferer 9 | { 10 | public: 11 | MeshTransferer(); 12 | ~MeshTransferer(); 13 | 14 | void setSource(const BasicMesh &src); 15 | void setTarget(const BasicMesh &tgt); 16 | void setStationaryVertices(const vector &sv) { stationary_vertices = sv; } 17 | 18 | // transfer using a target shape 19 | BasicMesh transfer(const BasicMesh &S1); 20 | // transfer using a per-face deformation gradient 21 | BasicMesh transfer(const vector &S1grad); 22 | 23 | protected: 24 | void computeS0grad(); 25 | void computeT0grad(); 26 | 27 | private: 28 | bool S0set, T0set; 29 | BasicMesh S0, T0; 30 | vector S0grad, T0grad; 31 | vector Ds; 32 | vector stationary_vertices; 33 | }; 34 | 35 | #endif // MESHTRANSFERER_H 36 | -------------------------------------------------------------------------------- /BlendshapeGeneration/ndarray.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NDARRAY_HPP 2 | #define NDARRAY_HPP 3 | 4 | #include "common.h" 5 | 6 | template 7 | struct Array1D { 8 | Array1D():nrow(0), data(shared_ptr(nullptr)){} 9 | Array1D(int n):nrow(n), data(shared_ptr(new T[n])){} 10 | Array1D(const Array1D &other):nrow(other.nrow) { 11 | data = shared_ptr(new T[nrow]); 12 | memcpy(data.get(), other.data.get(), sizeof(T)*nrow); 13 | } 14 | 15 | // Array1D(Array1D &&other):nrow(other.nrow) { 16 | // data = other.data; 17 | // other.data.reset(); 18 | // } 19 | Array1D clone() const { 20 | Array1D ret(nrow); 21 | memcpy(ret.data.get(), data.get(), sizeof(T)*nrow); 22 | return ret; 23 | } 24 | Array1D &operator=(const Array1D &rhs) { 25 | if( this != &rhs) { 26 | nrow = rhs.nrow; 27 | data = shared_ptr(new T[nrow]); 28 | memcpy(data.get(), rhs.data.get(), sizeof(T)*nrow); 29 | } 30 | return (*this); 31 | } 32 | // Array1D &operator=(Array1D &&rhs) { 33 | // if( this != &rhs) { 34 | // nrow = rhs.nrow; 35 | // data = rhs.data; 36 | // rhs.data.reset(); 37 | // } 38 | // return (*this); 39 | // } 40 | 41 | static Array1D ones(int n) { 42 | Array1D ret(n); 43 | for(int i=0;i 80 | friend ostream& operator<<(ostream &os, const Array1D &A); 81 | 82 | int nrow; 83 | shared_ptr data; 84 | }; 85 | 86 | template 87 | ostream &operator<<(ostream &os, const Array1D &A) 88 | { 89 | for(int i=0;i 95 | struct Array2D { 96 | Array2D():nrow(0), ncol(0), data(shared_ptr(nullptr)){} 97 | Array2D(int m, int n):nrow(m), ncol(n), data(shared_ptr(new T[m*n])){} 98 | Array2D(const Array2D &other):nrow(other.nrow), ncol(other.ncol) { 99 | data = shared_ptr(new T[other.nrow*other.ncol]); 100 | memcpy(data.get(), other.data.get(), sizeof(T)*nrow*ncol); 101 | } 102 | // Array2D(Array2D &&other):nrow(other.nrow), ncol(other.ncol) { 103 | // data = other.data; 104 | // other.data.reset(); 105 | // } 106 | 107 | Array2D clone() const { 108 | Array2D ret(nrow, ncol); 109 | memcpy(ret.data.get(), data.get(), sizeof(T)*nrow*ncol); 110 | return ret; 111 | } 112 | 113 | Array2D& operator=(const Array2D& rhs) { 114 | if( this != &rhs ) { 115 | nrow = rhs.nrow; 116 | ncol = rhs.ncol; 117 | data = shared_ptr(new T[nrow*ncol]); 118 | memcpy(data.get(), rhs.data.get(), sizeof(T)*nrow*ncol); 119 | } 120 | return (*this); 121 | } 122 | 123 | // Array2D& operator=(Array2D&& rhs) { 124 | // if( this != &rhs ) { 125 | // nrow = rhs.nrow; 126 | // ncol = rhs.ncol; 127 | // data = rhs.data; 128 | // rhs.data.reset(); 129 | // } 130 | // return (*this); 131 | // } 132 | 133 | static Array2D zeros(int m, int n) { 134 | Array2D ret(m, n); 135 | memset(ret.data.get(), 0, sizeof(T)*m*n); 136 | return ret; 137 | } 138 | 139 | T& operator()(int r, int c) { return data.get()[r*ncol+c]; } 140 | const T& operator()(int r, int c) const { return data.get()[r*ncol+c]; } 141 | 142 | T& operator()(int idx) { return data.get()[idx]; } 143 | const T& operator()(int idx) const { return data.get()[idx]; } 144 | 145 | Array2D row(const vector &rowIndices) { 146 | Array2D ret(rowIndices.size(), ncol); 147 | for(int i=0;i 181 | friend Array2D operator*(AT lhs, const Array2D &rhs); 182 | 183 | void resize(int m, int n) { 184 | nrow = m; ncol = n; 185 | data.reset(new T[m*n]); 186 | } 187 | 188 | T* rowptr(int ridx) const{ 189 | return data.get() + ridx*ncol; 190 | } 191 | 192 | void save(const string &filename) { 193 | ofstream fout(filename); 194 | fout << (*this); 195 | fout.close(); 196 | } 197 | 198 | template 199 | friend ostream& operator<<(ostream &os, const Array2D &A); 200 | 201 | int nrow, ncol; 202 | shared_ptr data; 203 | }; 204 | 205 | template 206 | ostream &operator<<(ostream &os, const Array2D &A) { 207 | for(int i=0;i 217 | Array2D operator*(T lhs, const Array2D &rhs) { 218 | Array2D ret = rhs; 219 | int n = rhs.nrow * rhs.ncol; 220 | for(int i=0;i points; 19 | }; 20 | 21 | 22 | inline ostream& operator<<(ostream &os, const PointCloud &P) 23 | { 24 | os << P.points; 25 | return os; 26 | } 27 | 28 | -------------------------------------------------------------------------------- /BlendshapeGeneration/testcases.cpp: -------------------------------------------------------------------------------- 1 | #include "testcases.h" 2 | 3 | #if 0 4 | void TestCases::testMatrix() 5 | { 6 | DenseMatrix A = DenseMatrix::random(5, 5); 7 | cout << "A = \n" << A << endl; 8 | auto B = A.inv(); 9 | cout << "B = \n" << B << endl; 10 | cout << "Bt = \n" << B.transposed() << endl; 11 | cout << "A*B = \n" << A * B << endl; 12 | cout << "B*A = \n" << B * A << endl; 13 | } 14 | 15 | void TestCases::testSaprseMatrix() 16 | { 17 | /* 18 | 2 -1 0 0 0 19 | -1 2 -1 0 0 20 | 0 -1 2 -1 0 21 | 0 0 -1 2 -1 22 | 0 0 0 -1 2 23 | */ 24 | SparseMatrix M(5, 5, 13); 25 | M.append(0, 0, 2); M.append(0, 1, -1); 26 | M.append(1, 0, -1); M.append(1, 1, 2); M.append(1, 2, -1); 27 | M.append(2, 1, -1); M.append(2, 2, 2); M.append(2, 3, -1); 28 | M.append(3, 2, -1); M.append(3, 3, 2); M.append(3, 4, -1); 29 | M.append(4, 3, -1); M.append(4, 4, 2); 30 | M.append(2, 1, 2); 31 | 32 | auto MtM = M.selfProduct(); 33 | cholmod_print_sparse(MtM, "MtM", global::cm); 34 | 35 | DenseVector b(5); 36 | for(int i=0;i<5;++i) b(i) = 1.0; 37 | 38 | auto x = M.solve(b, true); 39 | for (int i = 0; i < x.length(); ++i) cout << x(i) << ' '; 40 | cout << endl; 41 | DenseVector b1 = M * x; 42 | for (int i = 0; i < b1.length(); ++i) cout << b1(i) << ' '; 43 | cout << endl; 44 | } 45 | #endif 46 | 47 | void TestCases::testCeres() { 48 | 49 | // The variable to solve for with its initial value. 50 | double initial_x = 5.0; 51 | double x = initial_x; 52 | 53 | // Build the problem. 54 | Problem problem; 55 | 56 | // Set up the only cost function (also known as residual). This uses 57 | // auto-differentiation to obtain the derivative (jacobian). 58 | CostFunction* cost_function = 59 | new AutoDiffCostFunction(new CostFunctor); 60 | problem.AddResidualBlock(cost_function, NULL, &x); 61 | 62 | // Run the solver! 63 | Solver::Options options; 64 | options.linear_solver_type = ceres::DENSE_QR; 65 | options.minimizer_progress_to_stdout = true; 66 | Solver::Summary summary; 67 | Solve(options, &problem, &summary); 68 | 69 | std::cout << summary.BriefReport() << "\n"; 70 | std::cout << "x : " << initial_x 71 | << " -> " << x << "\n"; 72 | } 73 | -------------------------------------------------------------------------------- /BlendshapeGeneration/testcases.h: -------------------------------------------------------------------------------- 1 | #ifndef TESTCASES_H 2 | #define TESTCASES_H 3 | 4 | //#include "densematrix.h" 5 | //#include "densevector.h" 6 | //#include "sparsematrix.h" 7 | #include "cereswrapper.h" 8 | 9 | class TestCases 10 | { 11 | public: 12 | //static void testMatrix(); 13 | //static void testSaprseMatrix(); 14 | static void testCeres(); 15 | }; 16 | 17 | #endif // TESTCASES_H 18 | -------------------------------------------------------------------------------- /BlendshapeGeneration/triangle_gradient.h: -------------------------------------------------------------------------------- 1 | #ifndef TRIANGLE_GRADIENT_H 2 | #define TRIANGLE_GRADIENT_H 3 | 4 | #include "Geometry/matrix.hpp" 5 | #include 6 | 7 | inline PhGUtils::Matrix3x3d triangleGradient(const BasicMesh &m, int fidx) { 8 | auto f = m.face(fidx); 9 | 10 | Vector3d v0 = m.vertex(f[0]); 11 | Vector3d v1 = m.vertex(f[1]); 12 | Vector3d v2 = m.vertex(f[2]); 13 | 14 | Vector3d v1mv0 = v1 - v0; 15 | Vector3d v2mv0 = v2 - v0; 16 | 17 | PhGUtils::Vector3d v0v1(v1mv0[0], v1mv0[1], v1mv0[2]); 18 | PhGUtils::Vector3d v0v2(v2mv0[0], v2mv0[1], v2mv0[2]); 19 | 20 | PhGUtils::Vector3d n(v0v1, v0v2); 21 | PhGUtils::Vector3d nn = n.normalized(); 22 | 23 | PhGUtils::Matrix3x3d G(v0v1, v0v2, nn); 24 | 25 | double d = 0.5 * dot(n, nn); 26 | 27 | return G; 28 | } 29 | 30 | inline pair triangleGradient2(const BasicMesh &m, int fidx) { 31 | auto f = m.face(fidx); 32 | 33 | Vector3d v0 = m.vertex(f[0]); 34 | Vector3d v1 = m.vertex(f[1]); 35 | Vector3d v2 = m.vertex(f[2]); 36 | 37 | Vector3d v1mv0 = v1 - v0; 38 | Vector3d v2mv0 = v2 - v0; 39 | 40 | PhGUtils::Vector3d v0v1(v1mv0[0], v1mv0[1], v1mv0[2]); 41 | PhGUtils::Vector3d v0v2(v2mv0[0], v2mv0[1], v2mv0[2]); 42 | 43 | PhGUtils::Vector3d n(v0v1, v0v2); 44 | PhGUtils::Vector3d nn = n.normalized(); 45 | 46 | PhGUtils::Matrix3x3d G(v0v1, v0v2, nn); 47 | 48 | double d = 0.5 * n.dot(nn); 49 | 50 | return make_pair(G, d); 51 | } 52 | #endif // TRIANGLE_GRADIENT_H 53 | 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BlendshapeGenerationCPP 2 | 3 | ## Dependencies 4 | * Boost 1.63 5 | * ceres solver 1.12.0 6 | * OpenCV 3.4 7 | * freeglut 8 | * GLEW 9 | * Eigen 3.3.3 10 | * SuiteSparse 4.5.3 11 | * Intel MKL 12 | * Qt5 13 | * PhGLib 14 | 15 | ## Compile 16 | ```bash 17 | git clone --recursive https://github.com/phg1024/BlendshapeGenerationCPP.git 18 | cd BlendshapeGenerationCPP/BlendshapeGeneration 19 | mkdir build 20 | cd build 21 | cmake .. -DCMAKE_BUILT_TYPE=Release -DCMAKE_C_COMPILER=icc -DCMAKE_CXX_COMPILER=icpc 22 | make -j8 23 | ``` 24 | --------------------------------------------------------------------------------