├── .gitignore ├── CMakeLists.txt ├── LICENSE.md ├── README.md ├── _config.yml ├── cmake ├── FindEigen3.cmake └── FindOpenMesh.cmake ├── examples └── weights.csv └── src ├── Args.h ├── ebfr ├── BlendshapeSolver.cpp ├── BlendshapeSolver.h ├── GradientSolver.cpp ├── GradientSolver.h ├── Gradients.cpp ├── Gradients.h ├── Parameter.h ├── Rig.cpp ├── Rig.h ├── SolverBase.cpp ├── SolverBase.h ├── VertexSolver.cpp ├── VertexSolver.h ├── WeightsSolver.cpp └── WeightsSolver.h ├── main.cpp ├── shared ├── CSV.cpp ├── CSV.h ├── FS.cpp ├── FS.h ├── Matrix.h ├── Mesh.cpp ├── Mesh.h ├── SolverUtil.cpp ├── SolverUtil.h ├── Timing.h ├── Util.cpp └── Util.h └── test ├── gradient.cpp ├── posegen.cpp ├── vertex.cpp └── weights.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | cmake-build-debug/ 3 | 4 | cmake-build-release/ 5 | 6 | results/ 7 | 8 | .DS_Store 9 | 10 | output/ 11 | 12 | .idea/ 13 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | project(examplebasedfacialrigging) 3 | 4 | set(CMAKE_CXX_STANDARD 17) 5 | 6 | # additional CMake modules 7 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") 8 | 9 | # OpenMesh 10 | set(OPENMESH_LIBRARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../OpenMesh/lib) # <<< 11 | find_package(OpenMesh REQUIRED) 12 | include_directories(${OPENMESH_INCLUDE_DIRS}) 13 | 14 | # Eigen 15 | set(ENV{EIGEN3_ROOT} ${CMAKE_CURRENT_SOURCE_DIR}/../eigen) # <<< 16 | find_package(Eigen3 REQUIRED) 17 | include_directories(${EIGEN3_INCLUDE_DIR}) 18 | 19 | # CXXOpts 20 | set(CXXOPTS_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cxxopts/include) # <<< 21 | include_directories(${CXXOPTS_INCLUDE_DIR}) 22 | 23 | set_property( 24 | DIRECTORY 25 | APPEND PROPERTY COMPILE_DEFINITIONS _USE_MATH_DEFINES 26 | ) 27 | 28 | set(EBFR_SOURCE src/ebfr/GradientSolver.cpp src/ebfr/GradientSolver.h src/ebfr/Gradients.cpp src/ebfr/Gradients.h src/ebfr/Parameter.h src/ebfr/BlendshapeSolver.cpp src/ebfr/BlendshapeSolver.h src/ebfr/Rig.cpp src/ebfr/Rig.h src/ebfr/SolverBase.cpp src/ebfr/SolverBase.h src/ebfr/VertexSolver.cpp src/ebfr/VertexSolver.h src/ebfr/WeightsSolver.cpp src/ebfr/WeightsSolver.h) 29 | set(SHARED_SOURCE src/shared/CSV.cpp src/shared/CSV.h src/shared/FS.cpp src/shared/FS.h src/shared/Matrix.h src/shared/Mesh.cpp src/shared/Mesh.h src/shared/SolverUtil.cpp src/shared/SolverUtil.h src/shared/Timing.h src/shared/Util.cpp src/shared/Util.h) 30 | 31 | set(EBFR_LIBRARIES ${OPENMESH_LIBRARIES} Eigen3::Eigen) 32 | if(APPLE) 33 | set(EBFR_LIBRARIES ${EBFR_LIBRARIES} "-framework Accelerate" ${CMAKE_DL_LIBS}) 34 | endif() 35 | 36 | add_executable(ebfr ${SHARED_SOURCE} ${EBFR_SOURCE} src/main.cpp src/Args.h) 37 | TARGET_LINK_LIBRARIES(ebfr ${EBFR_LIBRARIES}) 38 | 39 | add_executable(test-gradient ${SHARED_SOURCE} ${EBFR_SOURCE} src/test/gradient.cpp src/Args.h) 40 | TARGET_LINK_LIBRARIES(test-gradient ${EBFR_LIBRARIES}) 41 | 42 | add_executable(test-vertex ${SHARED_SOURCE} ${EBFR_SOURCE} src/test/vertex.cpp src/Args.h) 43 | TARGET_LINK_LIBRARIES(test-vertex ${EBFR_LIBRARIES}) 44 | 45 | add_executable(test-weights ${SHARED_SOURCE} ${EBFR_SOURCE} src/test/weights.cpp src/Args.h) 46 | TARGET_LINK_LIBRARIES(test-weights ${EBFR_LIBRARIES}) 47 | 48 | add_executable(pose-gen src/shared/CSV.cpp src/shared/CSV.h src/shared/FS.cpp src/shared/FS.h src/shared/Matrix.h src/shared/Mesh.cpp src/shared/Mesh.h src/shared/Timing.h src/shared/Util.cpp src/shared/Util.h src/ebfr/Rig.cpp src/ebfr/Rig.h src/test/posegen.cpp) 49 | TARGET_LINK_LIBRARIES(pose-gen ${OPENMESH_LIBRARIES}) -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Copyright (C) 2020 Kyle 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Implementation of [Example-Based Facial Rigging](https://www.hao-li.com/Hao_Li/Hao_Li_-_publications_%5BExample-Based_Facial_Rigging%5D.html) 2 | 3 | A method for generating a facial blendshape rig from a source rig and a set of corresponding example poses. 4 | The method alternates between optimizing for the blendshapes, utilizing an approach derived from [Deformation Transfer](#useful-references), and for the blending weights. 5 | The resulting blendshapes carry the source's controller semantics and maintain the identity of the target. 6 | 7 | Implemented in C++17 8 | 9 | Using: 10 | * [Eigen 3](https://eigen.tuxfamily.org/index.php?title=Main_Page) 11 | * [OpenMesh](https://www.graphics.rwth-aachen.de/software/openmesh/) 12 | * [CXXOpts](https://github.com/jarro2783/cxxopts) 13 | 14 | ## Build 15 | ### Dependencies 16 | Download and install Eigen, OpenMesh, and CXXOpts according to their documentation. 17 | 18 | ### CMakeLists.txt 19 | Adjustments to CMakeLists.txt may be necessary depending on your environment. 20 | Lines marked with "<<<" are likely candidates. 21 | 22 | ### Build 23 | To prepare the build environment, create a build directory and run cmake inside of it. 24 | ```commandline 25 | mkdir build 26 | cd build 27 | cmake .. 28 | ``` 29 | To build the Example-Based Facial Rigging, make the 'ebfr' target. 30 | ```commandline 31 | make ebfr 32 | ``` 33 | 34 | ## Execution 35 | ```commandline 36 | ebfr --source-blendshapes "../data/source/blendshapes" --source-poses "../data/source/poses/" --source-weights "../data/source/poses/weights.csv" --target-neutral "../data/target/blendshapes/neutral.obj" --target-poses "../data/target/poses/" --target-weights "../data/target/poses/weights.csv" --output "output" 37 | ``` 38 | * --source-blendshapes: Path to the directory containing the source blendshape mesh files 39 | * Blendshapes are expected to be named sequentially from '0' and in OBJ format 40 | * --source-poses: Path to the directory containing the source pose mesh files 41 | * Pose mesh names are expected to match the names found in the weights file 42 | * --source-weights: Path to the source pose-weights file 43 | * See Weights CSV below 44 | * --target-neutral: Path to the target neutral mesh file 45 | * Neutral mesh is expected to be an OBJ file 46 | * --target-poses: Path to the directory containing the target pose mesh files 47 | * Pose mesh names are expected to match the names found in the weights file 48 | * --target-weights: Path to the target (estimated) pose-weights file 49 | * See Weights CSV below 50 | * --output Path to a directory to write the final target blendshapes 51 | 52 | ### Weights CSV 53 | #### Format 54 | **Header Row** 55 | 56 | Pose, 0, 1, ..., # of blendshapes 57 | 58 | **Row** 59 | 60 | PoseName, w0, w1, ..., wN 61 | 62 | PoseName should match the name of a mesh in the corresponding poses directory 63 | 64 | #####Example 65 | Pose | 0 | 1 | 2 | 3 | 4 66 | ---- | --- | --- | --- | --- | --- 67 | Smile | 0.1 | 0.2 | 0.3 | 0.4 | 0 68 | 69 | 70 | ## Notes 71 | * There must be a one-to-one correspondence between source and target meshes. 72 | * The source and target poses must also correspond to one another. 73 | * All meshes are, at this time, expected to be in OBJ format 74 | * Runtime on a reasonable PC is roughly 50s with 13k vertices, 28 blendshapes, and 10 example poses. 75 | 76 | ## Useful References 77 | [Deformation Transfer for Triangle Meshes by Sumner, Popovic. 2004](https://people.csail.mit.edu/sumner/research/deftransfer/) ([Project](https://github.com/kyle-gh/DeformationTransfer)) 78 | 79 | [Deformation Transfer for Detail-Preserving Surface Editing by Botsch, Sumner, Pauly, and Gross. 2006](https://lgg.epfl.ch/publications/2006/botsch_2006_DTD.pdf) -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-modernist -------------------------------------------------------------------------------- /cmake/FindEigen3.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find Eigen3 lib 2 | # 3 | # This module supports requiring a minimum version, e.g. you can do 4 | # find_package(Eigen3 3.1.2) 5 | # to require version 3.1.2 or newer of Eigen3. 6 | # 7 | # Once done this will define 8 | # 9 | # EIGEN3_FOUND - system has eigen lib with correct version 10 | # EIGEN3_INCLUDE_DIR - the eigen include directory 11 | # EIGEN3_VERSION - eigen version 12 | # 13 | # and the following imported target: 14 | # 15 | # Eigen3::Eigen - The header-only Eigen library 16 | # 17 | # This module reads hints about search locations from 18 | # the following environment variables: 19 | # 20 | # EIGEN3_ROOT 21 | # EIGEN3_ROOT_DIR 22 | 23 | # Copyright (c) 2006, 2007 Montel Laurent, 24 | # Copyright (c) 2008, 2009 Gael Guennebaud, 25 | # Copyright (c) 2009 Benoit Jacob 26 | # Redistribution and use is allowed according to the terms of the 2-clause BSD license. 27 | 28 | if(NOT Eigen3_FIND_VERSION) 29 | if(NOT Eigen3_FIND_VERSION_MAJOR) 30 | set(Eigen3_FIND_VERSION_MAJOR 2) 31 | endif() 32 | if(NOT Eigen3_FIND_VERSION_MINOR) 33 | set(Eigen3_FIND_VERSION_MINOR 91) 34 | endif() 35 | if(NOT Eigen3_FIND_VERSION_PATCH) 36 | set(Eigen3_FIND_VERSION_PATCH 0) 37 | endif() 38 | 39 | set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}") 40 | endif() 41 | 42 | macro(_eigen3_check_version) 43 | file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) 44 | 45 | string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}") 46 | set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}") 47 | string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}") 48 | set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}") 49 | string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}") 50 | set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}") 51 | 52 | set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION}) 53 | if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION}) 54 | set(EIGEN3_VERSION_OK FALSE) 55 | else() 56 | set(EIGEN3_VERSION_OK TRUE) 57 | endif() 58 | 59 | if(NOT EIGEN3_VERSION_OK) 60 | 61 | message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, " 62 | "but at least version ${Eigen3_FIND_VERSION} is required") 63 | endif() 64 | endmacro() 65 | 66 | if (EIGEN3_INCLUDE_DIR) 67 | 68 | # in cache already 69 | _eigen3_check_version() 70 | set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) 71 | set(Eigen3_FOUND ${EIGEN3_VERSION_OK}) 72 | 73 | else () 74 | 75 | # search first if an Eigen3Config.cmake is available in the system, 76 | # if successful this would set EIGEN3_INCLUDE_DIR and the rest of 77 | # the script will work as usual 78 | find_package(Eigen3 ${Eigen3_FIND_VERSION} NO_MODULE QUIET) 79 | 80 | if(NOT EIGEN3_INCLUDE_DIR) 81 | find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library 82 | HINTS 83 | ENV EIGEN3_ROOT 84 | ENV EIGEN3_ROOT_DIR 85 | PATHS 86 | ${CMAKE_INSTALL_PREFIX}/include 87 | ${KDE4_INCLUDE_DIR} 88 | PATH_SUFFIXES eigen3 eigen 89 | ) 90 | endif() 91 | 92 | if(EIGEN3_INCLUDE_DIR) 93 | _eigen3_check_version() 94 | endif() 95 | 96 | include(FindPackageHandleStandardArgs) 97 | find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK) 98 | 99 | mark_as_advanced(EIGEN3_INCLUDE_DIR) 100 | 101 | endif() 102 | 103 | if(EIGEN3_FOUND AND NOT TARGET Eigen3::Eigen) 104 | add_library(Eigen3::Eigen INTERFACE IMPORTED) 105 | set_target_properties(Eigen3::Eigen PROPERTIES 106 | INTERFACE_INCLUDE_DIRECTORIES "${EIGEN3_INCLUDE_DIR}") 107 | endif() 108 | -------------------------------------------------------------------------------- /cmake/FindOpenMesh.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Try to find OPENMESH 3 | # Once done this will define 4 | # 5 | # OPENMESH_FOUND - system has OPENMESH 6 | # OPENMESH_INCLUDE_DIRS - the OPENMESH include directories 7 | # OPENMESH_LIBRARIES - Link these to use OPENMESH 8 | # OPENMESH_LIBRARY_DIR - directory where the libraries are included 9 | # 10 | # Copyright 2015 Computer Graphics Group, RWTH Aachen University 11 | # Authors: Jan Möbius 12 | # Hans-Christian Ebke 13 | # 14 | # 15 | #=========================================================================== 16 | # 17 | # OpenMesh 18 | # Copyright (c) 2001-2015, RWTH-Aachen University 19 | # Department of Computer Graphics and Multimedia 20 | # All rights reserved. 21 | # www.openmesh.org 22 | # 23 | #--------------------------------------------------------------------------- 24 | # This file is part of OpenMesh. 25 | #--------------------------------------------------------------------------- 26 | # 27 | # Redistribution and use in source and binary forms, with or without 28 | # modification, are permitted provided that the following conditions 29 | # are met: 30 | # 31 | # 1. Redistributions of source code must retain the above copyright notice, 32 | # this list of conditions and the following disclaimer. 33 | # 34 | # 2. Redistributions in binary form must reproduce the above copyright 35 | # notice, this list of conditions and the following disclaimer in the 36 | # documentation and/or other materials provided with the distribution. 37 | # 38 | # 3. Neither the name of the copyright holder nor the names of its 39 | # contributors may be used to endorse or promote products derived from 40 | # this software without specific prior written permission. 41 | # 42 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 43 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 44 | # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 45 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER 46 | # OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 47 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 48 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 49 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 50 | # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 51 | # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 52 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 53 | # 54 | #=========================================================================== 55 | 56 | cmake_minimum_required(VERSION 2.8.9) 57 | 58 | #if already found via finder or simulated finder in openmesh CMakeLists.txt, skip the search 59 | IF (NOT OPENMESH_FOUND) 60 | SET (SEARCH_PATHS 61 | /usr/local/ 62 | /usr/ 63 | "${CMAKE_SOURCE_DIR}/OpenMesh/src/OpenMesh" 64 | "${CMAKE_SOURCE_DIR}/libs_required/OpenMesh/src/OpenMesh" 65 | "${CMAKE_SOURCE_DIR}/../OpenMesh/src" 66 | "C:/Program Files/OpenMesh 8.1" 67 | "C:/Program Files/OpenMesh 8.0" 68 | "C:/Program Files/OpenMesh 7.2" 69 | "C:/Program Files/OpenMesh 7.1" 70 | "C:/Program Files/OpenMesh 7.0" 71 | "C:/Program Files/OpenMesh 6.3" 72 | "C:/Program Files/OpenMesh 6.2" 73 | "C:/Program Files/OpenMesh 6.1" 74 | "C:/Program Files/OpenMesh 6.0" 75 | "C:/Program Files/OpenMesh 5.2" 76 | "C:/Program Files/OpenMesh 5.1" 77 | "C:/Program Files/OpenMesh 5.0" 78 | "C:/Program Files/OpenMesh 4.2" 79 | "C:/Program Files/OpenMesh 4.1" 80 | "C:/Program Files/OpenMesh 4.0" 81 | "C:/Program Files/OpenMesh 3.4" 82 | "C:/Program Files/OpenMesh 3.3" 83 | "C:/Program Files/OpenMesh 3.2" 84 | "C:/Program Files/OpenMesh 3.1" 85 | "C:/Program Files/OpenMesh 3.0" 86 | "C:/Program Files/OpenMesh 2.4.1" 87 | "C:/Program Files/OpenMesh 2.4" 88 | "C:/Program Files/OpenMesh 2.0/include" 89 | "C:/libs/OpenMesh 8.1" 90 | "C:/libs/OpenMesh 8.0" 91 | "C:/libs/OpenMesh 7.1" 92 | "C:/libs/OpenMesh 7.0" 93 | "C:/libs/OpenMesh 6.3" 94 | "C:/libs/OpenMesh 6.2" 95 | "C:/libs/OpenMesh 6.1" 96 | "C:/libs/OpenMesh 6.0" 97 | "C:/libs/OpenMesh 5.2" 98 | "C:/libs/OpenMesh 5.1" 99 | "C:/libs/OpenMesh 5.0" 100 | "C:/libs/OpenMesh 4.2" 101 | "C:/libs/OpenMesh 4.1" 102 | "C:/libs/OpenMesh 4.0" 103 | "C:/libs/OpenMesh 3.4" 104 | "C:/libs/OpenMesh 3.3" 105 | "C:/libs/OpenMesh 3.2" 106 | "C:/libs/OpenMesh 3.1" 107 | "C:/libs/OpenMesh 3.0" 108 | "C:/libs/OpenMesh 2.4.1" 109 | "C:/libs/OpenMesh 2.4" 110 | "${OPENMESH_LIBRARY_DIR}" 111 | ) 112 | 113 | FIND_PATH (OPENMESH_INCLUDE_DIR OpenMesh/Core/Mesh/PolyMeshT.hh 114 | PATHS ${SEARCH_PATHS} 115 | PATH_SUFFIXES include) 116 | 117 | FIND_LIBRARY(OPENMESH_CORE_LIBRARY_RELEASE NAMES OpenMeshCore 118 | PATHS ${SEARCH_PATHS} 119 | PATH_SUFFIXES lib lib64) 120 | 121 | FIND_LIBRARY(OPENMESH_CORE_LIBRARY_DEBUG NAMES OpenMeshCore 122 | PATHS ${SEARCH_PATHS} 123 | PATH_SUFFIXES lib lib64) 124 | 125 | FIND_LIBRARY(OPENMESH_TOOLS_LIBRARY_RELEASE NAMES OpenMeshTools 126 | PATHS ${SEARCH_PATHS} 127 | PATH_SUFFIXES lib lib64) 128 | 129 | FIND_LIBRARY(OPENMESH_TOOLS_LIBRARY_DEBUG NAMES OpenMeshTools 130 | PATHS ${SEARCH_PATHS} 131 | PATH_SUFFIXES lib lib64) 132 | 133 | #select configuration depending on platform (optimized... on windows) 134 | include(SelectLibraryConfigurations) 135 | select_library_configurations( OPENMESH_TOOLS ) 136 | select_library_configurations( OPENMESH_CORE ) 137 | 138 | set(OPENMESH_LIBRARIES ${OPENMESH_CORE_LIBRARY} ${OPENMESH_TOOLS_LIBRARY} ) 139 | set(OPENMESH_INCLUDE_DIRS ${OPENMESH_INCLUDE_DIR} ) 140 | 141 | #checks, if OPENMESH was found and sets OPENMESH_FOUND if so 142 | include(FindPackageHandleStandardArgs) 143 | find_package_handle_standard_args(OpenMesh DEFAULT_MSG 144 | OPENMESH_CORE_LIBRARY OPENMESH_TOOLS_LIBRARY OPENMESH_INCLUDE_DIR) 145 | 146 | #sets the library dir 147 | if ( OPENMESH_CORE_LIBRARY_RELEASE ) 148 | get_filename_component(_OPENMESH_LIBRARY_DIR ${OPENMESH_CORE_LIBRARY_RELEASE} PATH) 149 | else( OPENMESH_CORE_LIBRARY_RELEASE ) 150 | get_filename_component(_OPENMESH_LIBRARY_DIR ${OPENMESH_CORE_LIBRARY_DEBUG} PATH) 151 | endif( OPENMESH_CORE_LIBRARY_RELEASE ) 152 | set (OPENMESH_LIBRARY_DIR "${_OPENMESH_LIBRARY_DIR}" CACHE PATH "The directory where the OpenMesh libraries can be found.") 153 | 154 | 155 | mark_as_advanced(OPENMESH_INCLUDE_DIR OPENMESH_CORE_LIBRARY_RELEASE OPENMESH_CORE_LIBRARY_DEBUG OPENMESH_TOOLS_LIBRARY_RELEASE OPENMESH_TOOLS_LIBRARY_DEBUG OPENMESH_LIBRARY_DIR) 156 | endif() 157 | -------------------------------------------------------------------------------- /examples/weights.csv: -------------------------------------------------------------------------------- 1 | Pose,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27, 2 | Pose-0,0.345353,1,0.581241,0.488861,0,0,0.818279,0,0,0.673687,0,0,0,0,0,0,0,0,0,0,0,0,0.602341,0,0.520808,0.378253,0,0, 3 | Pose-1,0,0,0,0,0,0,0,0,0,0.25,0,0,0,0.563655,0,0,0,0.532522,0,0,0,0,0,0,0.647146,0.967019,0,1, 4 | Pose-2,1,0.511465,0,0,0,0,0,0.800021,0.346119,0,0,0.25,0,0,0,0,0.927894,0,0,0,0,0,0,0,0.25,0.449971,0,0, 5 | Pose-3,0,0,0,0,0,0,0,0,0,0.948885,0,0,0,0,1,0.875907,0,0,0,0,0,0,0,0.999832,0,0,0,1, 6 | Pose-4,0,0,0,0,0,0,0,0,0,0,0,0,0.322126,0,0,0,0.603599,0,0,0,0,0,0,0,0,0,0,0.591676, 7 | Pose-5,0.25,0.828291,0,0,0.465128,0.938324,0.445339,0,0,0,0,0,0,0,0,0,0.872942,0,0,0,0,0,0,0,0,0,0,0, 8 | -------------------------------------------------------------------------------- /src/Args.h: -------------------------------------------------------------------------------- 1 | // 2 | // Args.h 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef EXAMPLEBASEDFACIALRIGGING_ARGS_H 10 | #define EXAMPLEBASEDFACIALRIGGING_ARGS_H 11 | 12 | #include 13 | 14 | struct Args { 15 | std::string srcBlendshapeDir; 16 | std::string srcPoseDir; 17 | std::string srcWeightsPath; 18 | std::string tgtNeutralPath; 19 | std::string tgtPoseDir; 20 | std::string tgtWeightsPath; 21 | std::string vertexMaskPath; 22 | std::string outputPath; 23 | std::string debugPath; 24 | 25 | bool read(int argc, char *argv[]) { 26 | cxxopts::Options options("ebfr", "Generate a facial blendshape rig from example poses"); 27 | 28 | options.add_options() 29 | ("source-blendshapes", "Path to the directory containing the source blendshape", cxxopts::value()) 30 | ("source-poses", "Path to the directory containing the source pose mesh files", cxxopts::value()) 31 | ("source-weights", "Path to the source pose-weights file", cxxopts::value()) 32 | 33 | ("target-neutral", "Path to the target neutral mesh file", cxxopts::value()) 34 | ("target-poses", "Path to the directory containing the target pose mesh files", cxxopts::value()) 35 | ("target-weights", "Path to the target (estimated) pose-weights file", cxxopts::value()) 36 | 37 | ("vertex-mask", "Path to the vertex mask file", cxxopts::value()) 38 | 39 | ("output", "Path to a directory to write the final target blendshapes", cxxopts::value()) 40 | ("debug", "Path to a directory to save in-progress data (meshes, weights, etc.", cxxopts::value()); 41 | 42 | try { 43 | auto result = options.parse(argc, argv); 44 | 45 | if (!result.count("source-blendshapes") || !result.count("source-poses") || !result.count("source-weights") || 46 | !result.count("target-neutral") || !result.count("target-poses") || !result.count("target-weights") || 47 | !result.count("output") 48 | ) { 49 | std::cout << options.help() << std::endl; 50 | exit(1); 51 | } 52 | 53 | srcBlendshapeDir = result["source-blendshapes"].as(); 54 | srcPoseDir = result["source-poses"].as(); 55 | srcWeightsPath = result["source-weights"].as(); 56 | tgtNeutralPath = result["target-neutral"].as(); 57 | tgtPoseDir = result["target-poses"].as(); 58 | tgtWeightsPath = result["target-weights"].as(); 59 | vertexMaskPath = result["vertex-mask"].as(); 60 | outputPath = result["output"].as(); 61 | 62 | if (result.count("debug")) { 63 | debugPath = result["debug"].as(); 64 | } 65 | } 66 | catch (const cxxopts::OptionException &e) { 67 | std::cout << "error parsing options: " << e.what() << std::endl; 68 | exit(1); 69 | } 70 | 71 | return true; 72 | } 73 | }; 74 | 75 | #endif //EXAMPLEBASEDFACIALRIGGING_ARGS_H 76 | -------------------------------------------------------------------------------- /src/ebfr/BlendshapeSolver.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // BlendshapeSolver.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "BlendshapeSolver.h" 10 | 11 | #include "../shared/SolverUtil.h" 12 | #include "../shared/Timing.h" 13 | 14 | BlendshapeSolver::BlendshapeSolver() 15 | : _numIterations(10) { 16 | setBlendshapeSolveConsts(ParameterD(ParameterD::Continuous, {{0, 0.5}, 17 | {_numIterations, 0.1}})); 18 | setRegularizationConsts(ParameterD(0.1), ParameterD(2.0)); 19 | setWeightSolveConsts(ParameterD(ParameterD::Continuous, {{0, 1000}, 20 | {_numIterations, 100}})); 21 | } 22 | 23 | void BlendshapeSolver::setNumIterations(int num) { 24 | _numIterations = num; 25 | } 26 | 27 | void BlendshapeSolver::setRegularizationConsts(const ParameterD &k, const ParameterD &theta) { 28 | _gradientSolver.setRegularizationConsts(k, theta); 29 | } 30 | 31 | void BlendshapeSolver::setBlendshapeSolveConsts(const ParameterD &beta) { 32 | _gradientSolver.setBlendshapeSolveConsts(beta); 33 | } 34 | 35 | void BlendshapeSolver::setWeightSolveConsts(const ParameterD &lambda) { 36 | _weightsSolver.setLambda(lambda); 37 | } 38 | 39 | void BlendshapeSolver::setVertexStepCallback(VertexSolver::StepCallback callback) { 40 | _vertexSolver.setStepCallback(callback); 41 | } 42 | 43 | void BlendshapeSolver::setWeightsStepCallback(WeightsSolver::StepCallback callback) { 44 | _weightsSolver.setStepCallback(callback); 45 | } 46 | 47 | void BlendshapeSolver::setDebugPath(const std::string &path) { 48 | _gradientSolver.setDebugPath(path); 49 | _vertexSolver.setDebugPath(path); 50 | _weightsSolver.setDebugPath(path); 51 | } 52 | 53 | void BlendshapeSolver::setMultithreaded(bool enable) { 54 | _gradientSolver.setMultithreaded(enable); 55 | _vertexSolver.setMultithreaded(enable); 56 | _weightsSolver.setMultithreaded(enable); 57 | } 58 | 59 | bool BlendshapeSolver::setSource(RigPtr rig) { 60 | _source = rig; 61 | 62 | _sourceGradients = std::make_shared(); 63 | _sourceGradients->calculate(rig, false); 64 | 65 | return _source != nullptr; 66 | } 67 | 68 | bool BlendshapeSolver::setTarget(RigPtr rig) { 69 | _target = rig; 70 | 71 | _targetGradients = std::make_shared(); 72 | _targetGradients->calculate(rig, true); 73 | 74 | return _target != nullptr; 75 | } 76 | 77 | RigPtr BlendshapeSolver::getSource() const { 78 | return _source; 79 | } 80 | 81 | GradientsPtr BlendshapeSolver::getSourceGradients() const { 82 | return _sourceGradients; 83 | } 84 | 85 | RigPtr BlendshapeSolver::getTarget() const { 86 | return _target; 87 | } 88 | 89 | GradientsPtr BlendshapeSolver::getTargetGradients() const { 90 | return _targetGradients; 91 | } 92 | 93 | bool BlendshapeSolver::solve() { 94 | TIMER_START(Solve) 95 | 96 | init(); 97 | 98 | initGradient(); 99 | 100 | initVertex(); 101 | 102 | initWeights(); 103 | 104 | for (auto i = 0; i < _numIterations; i++) { 105 | TIMER_START(Iteration) 106 | 107 | // Optimize Blendshapes 108 | 109 | TIMER_START(GradientSolve); 110 | 111 | // Estimates blendshapes gradients 112 | if (!_gradientSolver.solve(i)) 113 | return false; 114 | 115 | TIMER_END(GradientSolve); 116 | 117 | TIMER_START(VertexSolve) 118 | 119 | // Calculates blendshapes vertices from gradients 120 | // See Deform. Transfer 121 | if (!_vertexSolver.solve(i)) 122 | return false; 123 | 124 | TIMER_END(VertexSolve) 125 | 126 | // TIMER_START(RebuildGradients) 127 | // 128 | // rebuildGradients(); 129 | // 130 | // TIMER_END(RebuildGradients); 131 | 132 | // rebuildBlendshapes(); 133 | 134 | TIMER_START(WeightsSolver) 135 | 136 | // Estimates blendshape weights per pose 137 | if (!_weightsSolver.solve(i)) 138 | return false; 139 | 140 | TIMER_END(WeightsSolver) 141 | 142 | // TIMER_START(RebuildPoses); 143 | // 144 | // rebuildPoses(); 145 | // 146 | // TIMER_END(RebuildPoses); 147 | 148 | TIMER_END(Iteration) 149 | } 150 | 151 | TIMER_END(Solve) 152 | 153 | return true; 154 | } 155 | 156 | bool BlendshapeSolver::testGradient(int iter) { 157 | if (iter == 0) { 158 | init(); 159 | 160 | initGradient(); 161 | } 162 | 163 | TIMER_START(GradientSolve); 164 | 165 | if (!_gradientSolver.solve(iter)) 166 | return false; 167 | 168 | TIMER_END(GradientSolve); 169 | 170 | return true; 171 | } 172 | 173 | bool BlendshapeSolver::testVertex(int iter, int bs) { 174 | if (iter == 0) { 175 | init(); 176 | 177 | initVertex(); 178 | } else { 179 | 180 | TIMER_START(VertexSolve); 181 | 182 | if (!_vertexSolver.solve(iter, bs)) 183 | return false; 184 | 185 | TIMER_END(VertexSolve); 186 | } 187 | 188 | return true; 189 | } 190 | 191 | bool BlendshapeSolver::testWeights(int iter) { 192 | if (iter == 0) { 193 | init(); 194 | 195 | initWeights(); 196 | } else { 197 | TIMER_START(WeightsSolver) 198 | 199 | // Estimates blendshape weights per pose 200 | if (!_weightsSolver.solve(iter)) 201 | return false; 202 | 203 | TIMER_END(WeightsSolver) 204 | } 205 | 206 | return true; 207 | } 208 | 209 | void BlendshapeSolver::init() { 210 | } 211 | 212 | void BlendshapeSolver::initGradient() { 213 | TIMER_START(GradientInit); 214 | 215 | _gradientSolver.setSource(_source, _sourceGradients); 216 | _gradientSolver.setTarget(_target, _targetGradients); 217 | _gradientSolver.init(); 218 | 219 | TIMER_END(GradientInit); 220 | } 221 | 222 | void BlendshapeSolver::initVertex() { 223 | TIMER_START(VertexInit); 224 | 225 | _vertexSolver.setSource(_source, _sourceGradients); 226 | _vertexSolver.setTarget(_target, _targetGradients); 227 | _vertexSolver.init(); 228 | 229 | TIMER_END(VertexInit); 230 | } 231 | 232 | void BlendshapeSolver::initWeights() { 233 | TIMER_START(WeightsInit); 234 | 235 | _weightsSolver.setSource(_source, _sourceGradients); 236 | _weightsSolver.setTarget(_target, _targetGradients); 237 | _weightsSolver.init(); 238 | 239 | TIMER_END(WeightsInit); 240 | } 241 | 242 | void BlendshapeSolver::rebuildBlendshapes() { 243 | const auto neutral = _target->neutral(); 244 | 245 | for (auto i = 0; i < _target->numBlendshapes(); i++) { 246 | auto blendshape = _target->blendshape(i).mesh(); 247 | 248 | AddVertices(blendshape, neutral, -1, blendshape); 249 | } 250 | } 251 | 252 | void BlendshapeSolver::rebuildGradients() { 253 | for (auto i = 0; i < _target->numBlendshapes(); i++) { 254 | //AddVertices(_target->neutral(), _target->blendshape(i).mesh(), 1, temp); 255 | 256 | // M_b 257 | CalculateFrames(_target->blendshape(i).mesh(), _targetGradients->blendshapeM[i]); 258 | } 259 | } 260 | 261 | void BlendshapeSolver::rebuildPoses() { 262 | for (auto pose = 0; pose < _target->numPoses(); pose++) { 263 | const auto weights = _target->pose(pose).weights(); 264 | 265 | auto poseMesh = _target->pose(pose).mesh(); 266 | 267 | CopyVertices(poseMesh, _target->neutral()); 268 | 269 | for (auto bs = 0; bs < _target->numBlendshapes(); bs++) { 270 | AddVertices(poseMesh, _target->blendshape(bs).mesh(), weights[bs]); 271 | } 272 | 273 | CalculateFrames(poseMesh, _targetGradients->poseM[pose]); 274 | } 275 | } 276 | 277 | -------------------------------------------------------------------------------- /src/ebfr/BlendshapeSolver.h: -------------------------------------------------------------------------------- 1 | // 2 | // BlendshapeSolver.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Resolver_hpp 10 | #define Resolver_hpp 11 | 12 | #include 13 | #include 14 | 15 | #include "../shared/Matrix.h" 16 | 17 | #include "Rig.h" 18 | 19 | #include "GradientSolver.h" 20 | #include "VertexSolver.h" 21 | #include "WeightsSolver.h" 22 | 23 | class BlendshapeSolver { 24 | public: 25 | BlendshapeSolver(); 26 | 27 | void setRegularizationConsts(const ParameterD &k, const ParameterD &theta); 28 | 29 | void setBlendshapeSolveConsts(const ParameterD &beta); 30 | 31 | void setWeightSolveConsts(const ParameterD &lambda); 32 | 33 | void setNumIterations(int num); 34 | 35 | void setVertexStepCallback(VertexSolver::StepCallback callback); 36 | 37 | void setWeightsStepCallback(WeightsSolver::StepCallback callback); 38 | 39 | void setDebugPath(const std::string &path); 40 | 41 | void setMultithreaded(bool enable); 42 | 43 | bool setSource(RigPtr rig); 44 | 45 | bool setTarget(RigPtr rig); 46 | 47 | RigPtr getSource() const; 48 | 49 | GradientsPtr getSourceGradients() const; 50 | 51 | RigPtr getTarget() const; 52 | 53 | GradientsPtr getTargetGradients() const; 54 | 55 | bool solve(); 56 | 57 | bool testGradient(int iter); 58 | 59 | bool testVertex(int iter, int bs); 60 | 61 | bool testWeights(int iter); 62 | 63 | private: 64 | RigPtr _source; 65 | 66 | GradientsPtr _sourceGradients; 67 | 68 | RigPtr _target; 69 | 70 | GradientsPtr _targetGradients; 71 | 72 | int _numIterations; 73 | 74 | // Stage A - Solve for Blendshape Gradients 75 | GradientSolver _gradientSolver; 76 | 77 | // Stage T - Solve for Vertices, from Gradients 78 | VertexSolver _vertexSolver; 79 | 80 | // Stage B - Solve for Weights 81 | WeightsSolver _weightsSolver; 82 | 83 | void init(); 84 | 85 | void initGradient(); 86 | 87 | void initVertex(); 88 | 89 | void initWeights(); 90 | 91 | void rebuildBlendshapes(); 92 | 93 | void rebuildGradients(); 94 | 95 | void rebuildPoses(); 96 | }; 97 | 98 | #endif /* Resolver_hpp */ 99 | -------------------------------------------------------------------------------- /src/ebfr/GradientSolver.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // GradientSolver.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "GradientSolver.h" 10 | 11 | #include 12 | #include 13 | 14 | GradientSolver::GradientSolver() 15 | : SolverBase() 16 | , _mSize(_Matrix::SizeAtCompileTime) 17 | , _mCols(_Matrix::ColsAtCompileTime) 18 | , _mRows(_Matrix::RowsAtCompileTime) 19 | , _regK(0.1) 20 | , _regTheta(2) 21 | , _beta(0.5) // -> 0.1 22 | { 23 | setMultithreaded(true); 24 | } 25 | 26 | void GradientSolver::setRegularizationConsts(const ParameterD &k, const ParameterD &theta) { 27 | _regK = k; 28 | _regTheta = theta; 29 | } 30 | 31 | void GradientSolver::setBlendshapeSolveConsts(const ParameterD &beta) { 32 | _beta = beta; 33 | } 34 | 35 | void GradientSolver::init() { 36 | calculateMStars(); 37 | calculateWs(); 38 | } 39 | 40 | bool GradientSolver::solve(int iter) { 41 | std::cout 42 | << std::endl 43 | << "Gradient Solver [" << iter << "]" << std::endl 44 | << "\tK: " << _regK(iter) << std::endl 45 | << "\tTheta: " << _regTheta(iter) << std::endl 46 | << "\tBeta: " << _beta(iter) << std::endl 47 | << std::endl; 48 | 49 | SolverBase::solve(iter); 50 | 51 | _betaIter = _beta(_iteration); 52 | 53 | std::mutex logMutex; 54 | 55 | auto solverOp = 56 | [this, &logMutex] 57 | (int threadId, size_t faceStart, size_t faceEnd) { 58 | if (_debug && threadId >= 0) { 59 | logMutex.lock(); 60 | std::cout << "Thread [" << threadId << "-GS] Start: Faces " << faceStart << " -> " << faceEnd << " (" << (faceEnd - faceStart) << ")" << std::endl; 61 | logMutex.unlock(); 62 | } 63 | 64 | const auto rows = (_source->numPoses() * _mSize) + 65 | (_source->numPoses() * (_source->numBlendshapes() - 1) * _mSize); 66 | const auto cols = _target->numBlendshapes() * _mSize; 67 | 68 | TripletList a; 69 | SparseMatrix A(rows, cols); 70 | 71 | MatrixX c(rows, 1); 72 | MatrixX x; 73 | 74 | a.clear(); 75 | c.setZero(); 76 | 77 | Index faceIndex = faceStart; 78 | Index numFaces = 0; 79 | 80 | Eigen::LLT solver; 81 | 82 | while (true) { 83 | if (faceIndex >= faceEnd) 84 | break; 85 | 86 | Index face = _source->face(faceIndex); 87 | 88 | a.clear(); 89 | c.setZero(); 90 | 91 | appendGradientFit(face, a, c); 92 | appendGradientRegularization(face, a, c); 93 | 94 | A.setFromTriplets(a.begin(), a.end()); 95 | A.makeCompressed(); 96 | 97 | const auto At = A.transpose(); 98 | 99 | solver.compute(At * A); 100 | if (!checkSolverError(solver.info())) 101 | break; 102 | 103 | x = solver.solve(At * c); 104 | 105 | copyBlendshapeMTo(x, _targetGradients->blendshapeM, face); 106 | 107 | faceIndex++; 108 | numFaces++; 109 | } 110 | 111 | if (_debug && threadId >= 0) { 112 | logMutex.lock(); 113 | std::cout << "Thread [" << threadId << "-GS] Complete - " << numFaces << " Faces" << std::endl; 114 | logMutex.unlock(); 115 | } 116 | }; 117 | 118 | if (_useMultithreaded) { 119 | std::vector pool; 120 | 121 | const auto numThreads = std::thread::hardware_concurrency(); 122 | const auto size = _source->numFaces(); 123 | const auto segmentSize = (size / numThreads) + 1; 124 | auto segmentStart = 0; 125 | 126 | for (auto i = 0; i < numThreads; i++) { 127 | pool.push_back( 128 | std::thread( 129 | solverOp, i, 130 | segmentStart, 131 | std::min(segmentStart + segmentSize, size) 132 | ) 133 | ); 134 | 135 | segmentStart += segmentSize; 136 | } 137 | 138 | for (auto &thread : pool) { 139 | thread.join(); 140 | } 141 | } else { 142 | solverOp(-1, 0, _target->numFaces()); 143 | } 144 | 145 | return true; 146 | } 147 | 148 | void GradientSolver::calculateMStars() { 149 | const auto numFaces = _source->numFaces(true); 150 | 151 | _mStar.resize(_source->numBlendshapes()); 152 | 153 | const auto &s0 = _sourceGradients->blendshapeM[0]; 154 | const auto &s0Inv = _sourceGradients->blendshapeMInv[0]; 155 | 156 | const auto &t0 = _targetGradients->blendshapeM[0]; 157 | 158 | for (auto bs = 1; bs < _source->numBlendshapes(); bs++) { 159 | auto &mStar = _mStar[bs]; 160 | 161 | mStar.resize(numFaces); 162 | 163 | const auto &si = _sourceGradients->blendshapeM[bs]; 164 | 165 | for (auto face = 0; face < numFaces; face++) { 166 | const auto m = (((s0[face] + si[face]) * s0Inv[face]) * t0[face]) - t0[face]; 167 | mStar[face] = m; 168 | } 169 | } 170 | } 171 | 172 | void GradientSolver::calculateWs() { 173 | const auto numFaces = _source->numFaces(true); 174 | 175 | _w.resize(_source->numBlendshapes()); 176 | 177 | const auto k = _regK(_iteration); 178 | const auto t = _regTheta(_iteration); 179 | 180 | for (auto bs = 0; bs < _source->numBlendshapes(); bs++) { 181 | const auto &blendshapeM = _sourceGradients->blendshapeM[bs]; 182 | 183 | auto &blendshapeW = _w[bs]; 184 | blendshapeW.resize(numFaces); 185 | 186 | for (auto face = 0; face < numFaces; face++) { 187 | const auto &mA = blendshapeM[face]; 188 | const auto mAf = mA.norm(); 189 | 190 | auto &w = blendshapeW[face]; 191 | 192 | w = std::pow((1 + mAf) / (k + mAf), t); 193 | } 194 | } 195 | } 196 | 197 | void GradientSolver::appendGradientFit(Index face, TripletList &a, MatrixX &c) const { 198 | appendGradientFitWeights(face, a); 199 | appendGradientFit(face, c); 200 | } 201 | 202 | void GradientSolver::appendGradientFitWeights(Index face, TripletList &a) const { 203 | for (auto pose = 0; pose < _source->numPoses(); pose++) { 204 | const auto row = rowIndex(pose, false); 205 | 206 | for (auto bs = 0; bs < _target->numBlendshapes(); bs++) { 207 | const auto weight = _target->weight(pose, bs); 208 | 209 | if (weight == 0.0) 210 | continue; 211 | 212 | const auto col = colIndex(bs, false); 213 | 214 | for (auto i = 0; i < _mSize; i++) { 215 | a.emplace_back(row + i, col + i, weight); 216 | } 217 | } 218 | } 219 | } 220 | 221 | void GradientSolver::appendGradientFit(Index face, MatrixX &c) const { 222 | for (auto pose = 0; pose < _source->numPoses(); pose++) { 223 | appendGradientFit(face, pose, c); 224 | } 225 | } 226 | 227 | void GradientSolver::appendGradientFit(Index face, Index pose, MatrixX &c) const { 228 | const auto &s = _targetGradients->poseM[pose][face]; 229 | const auto &n = _targetGradients->blendshapeM[0][face]; 230 | 231 | const auto row = rowIndex(pose, false); 232 | 233 | for (auto i = 0; i < _mCols; i++) { 234 | for (auto j = 0; j < _mRows; j++) { 235 | const auto offset = (i * _mRows) + j; 236 | 237 | c(row + offset, 0) = s(j, i) - n(j, i); 238 | } 239 | } 240 | } 241 | 242 | void GradientSolver::appendGradientRegularization(Index face, TripletList &a, MatrixX &c) const { 243 | for (auto pose = 0; pose < _target->numPoses(); pose++) { 244 | appendGradientRegularization(face, pose, a, c); 245 | } 246 | } 247 | 248 | void GradientSolver::appendGradientRegularization(Index face, Index pose, TripletList &a, MatrixX &c) const { 249 | for (Index bs = 1; bs < _target->numBlendshapes(); bs++) { 250 | appendGradientRegularization(face, pose, bs, a, c); 251 | } 252 | } 253 | 254 | void GradientSolver::appendGradientRegularization(Index face, Index pose, Index bs, TripletList &a, MatrixX &c) const { 255 | //const Index row = rowIndex(pose, true); 256 | const Index col = colIndex(bs, true); 257 | 258 | const Index row = 259 | (_source->numPoses() * _mSize) + 260 | (((_source->numBlendshapes() - 1) * _mSize * pose) + (_mSize * (bs - 1))); 261 | 262 | const auto wbeta = _w[bs][face] * _betaIter; 263 | 264 | if (wbeta == 0.0) 265 | return; 266 | 267 | const auto &mStar = _mStar[bs][face]; 268 | 269 | // M^B_i * wbeta 270 | for (auto i = 0; i < _mSize; i++) { 271 | a.emplace_back(row + i, col + i, wbeta); 272 | } 273 | 274 | for (auto i = 0; i < _mCols; i++) { 275 | for (auto j = 0; j < _mRows; j++) { 276 | const auto offset = (i * _mRows) + j; //index(j, i); 277 | 278 | c(row + offset, 0) = wbeta * mStar(j, i); 279 | } 280 | } 281 | } 282 | 283 | Index GradientSolver::rowIndex(Index pose, bool isReg) const { 284 | if (isReg) 285 | return (Index) (_source->numPoses() + pose) * _mSize; 286 | 287 | return (Index) (pose * _mSize); 288 | } 289 | 290 | Index GradientSolver::colIndex(Index bs, bool isReg) const { 291 | return (Index) (bs * _mSize); 292 | } 293 | 294 | Index GradientSolver::index(Index row, Index col) const { 295 | return (row * _mCols) + col; 296 | } 297 | 298 | void GradientSolver::copyBlendshapeMTo(MatrixX &x, std::vector> &ms, Index face) const { 299 | // Don't overwrite BS0/Neutral M 300 | auto row = _mSize; 301 | 302 | for (auto bs = 1; bs < _target->numBlendshapes(); bs++) { 303 | auto &m = ms[bs][face]; 304 | 305 | for (auto i = 0; i < _mCols; i++) { 306 | for (auto j = 0; j < _mRows; j++) { 307 | m(j, i) = x(row, 0); 308 | row++; 309 | } 310 | } 311 | } 312 | } 313 | -------------------------------------------------------------------------------- /src/ebfr/GradientSolver.h: -------------------------------------------------------------------------------- 1 | // 2 | // GradientSolver.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef GradientSolver_hpp 10 | #define GradientSolver_hpp 11 | 12 | #include "SolverBase.h" 13 | 14 | class GradientSolver : public SolverBase { 15 | public: 16 | GradientSolver(); 17 | 18 | void setRegularizationConsts(const ParameterD &k, const ParameterD &theta); 19 | 20 | void setBlendshapeSolveConsts(const ParameterD &beta); 21 | 22 | virtual void init(); 23 | 24 | virtual bool solve(int iter); 25 | 26 | private: 27 | typedef Matrix3x3 _Matrix; 28 | 29 | const Index _mSize; 30 | const Index _mCols; 31 | const Index _mRows; 32 | 33 | ParameterD _regK; 34 | ParameterD _regTheta; 35 | ParameterD _beta; 36 | 37 | double _betaIter; 38 | 39 | std::vector> _w; 40 | 41 | std::vector> _mStar; 42 | 43 | void calculateMStars(); 44 | 45 | void calculateWs(); 46 | 47 | void appendGradientFit(Index face, TripletList &a, MatrixX &c) const; 48 | 49 | void appendGradientFitWeights(Index face, TripletList &a) const; 50 | 51 | void appendGradientFit(Index face, MatrixX &c) const; 52 | 53 | void appendGradientFit(Index face, Index pose, MatrixX &c) const; 54 | 55 | void appendGradientFit(Index face, Index pose, Index bs, MatrixX &c) const; 56 | 57 | void appendGradientRegularization(Index face, TripletList &a, MatrixX &c) const; 58 | 59 | void appendGradientRegularization(Index face, Index pose, TripletList &a, MatrixX &c) const; 60 | 61 | void appendGradientRegularization(Index face, Index pose, Index bs, TripletList &a, MatrixX &c) const; 62 | 63 | Index rowIndex(Index pose, bool isReg) const; 64 | 65 | Index colIndex(Index bs, bool isReg) const; 66 | 67 | Index index(Index row, Index col) const; 68 | 69 | void copyBlendshapeMTo(MatrixX &x, std::vector> &ms, Index face) const; 70 | }; 71 | 72 | #endif /* GradientSolver_hpp */ 73 | -------------------------------------------------------------------------------- /src/ebfr/Gradients.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Gradients.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "Gradients.h" 10 | 11 | #include "../shared/SolverUtil.h" 12 | 13 | void Gradients::calculate(RigPtr rig, bool isTarget) { 14 | const auto &poses = rig->poses(); 15 | const auto &blendshapes = rig->blendshapes(); 16 | 17 | //// Calculate Frames - M_(A_i) 18 | poseM.resize(poses.size()); 19 | 20 | for (auto i = 0; i < poses.size(); i++) 21 | CalculateFrames(poses[i].mesh(), poseM[i]); 22 | 23 | blendshapeM.resize(blendshapes.size()); 24 | 25 | auto neutral = rig->neutral(); 26 | 27 | for (auto bs = 0; bs < rig->numBlendshapes(); bs++) { 28 | const auto mesh = rig->blendshape(bs).mesh(); 29 | auto &m = blendshapeM[bs]; 30 | 31 | if (bs == 0) { 32 | CalculateFrames(mesh, m); 33 | } else { 34 | if (isTarget) { 35 | GenerateEmptyFrames(neutral, m); 36 | } else { 37 | CalculateFrames(mesh, m); 38 | } 39 | } 40 | } 41 | 42 | blendshapeMInv.resize(1); 43 | blendshapeMInv[0].resize(blendshapeM[0].size()); 44 | 45 | for (auto i = 0; i < blendshapeM[0].size(); i++) { 46 | blendshapeMInv[0][i] = blendshapeM[0][i].inverse(); 47 | } 48 | } -------------------------------------------------------------------------------- /src/ebfr/Gradients.h: -------------------------------------------------------------------------------- 1 | // 2 | // Gradients.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Gradients_hpp 10 | #define Gradients_hpp 11 | 12 | #include "../shared/Matrix.h" 13 | #include "Rig.h" 14 | 15 | #include 16 | 17 | class Gradients { 18 | public: 19 | std::vector> blendshapeM; 20 | std::vector> blendshapeMInv; 21 | 22 | std::vector> blendshapeG; 23 | 24 | std::vector> poseM; 25 | 26 | void calculate(RigPtr rig, bool isTarget); 27 | }; 28 | 29 | typedef std::shared_ptr GradientsPtr; 30 | 31 | #endif /* Gradients_hpp */ 32 | -------------------------------------------------------------------------------- /src/ebfr/Parameter.h: -------------------------------------------------------------------------------- 1 | // 2 | // Parameter.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Parameter_hpp 10 | #define Parameter_hpp 11 | 12 | #include 13 | #include 14 | 15 | template 16 | class Parameter { 17 | public: 18 | enum Type { 19 | Step, 20 | Continuous, 21 | }; 22 | 23 | struct Entry { 24 | int time; 25 | T value; 26 | }; 27 | 28 | Parameter(T v) { 29 | setConstant(v); 30 | } 31 | 32 | Parameter(Type type, const std::vector& entries) 33 | : _type(type) 34 | , _schedule(entries) 35 | , _isModified(true) 36 | { 37 | } 38 | 39 | void setConstant(T v) { 40 | _schedule.push_back(Entry{-1, v}); 41 | _isModified = true; 42 | } 43 | 44 | void setType(Type type) { 45 | _type = type; 46 | } 47 | 48 | void addEntry(int t, T v) { 49 | _schedule.push_back(Entry{t, v}); 50 | _isModified = true; 51 | } 52 | 53 | T operator()(int t) { 54 | if (_schedule.empty()) 55 | return (T) 0; 56 | 57 | if (_schedule[0].time == -1) 58 | return _schedule[0].value; 59 | 60 | if (_isModified) { 61 | std::sort(_schedule.begin(), _schedule.end(), 62 | [](const Entry &l, const Entry &r) { return l.time < r.time; }); 63 | _isModified = false; 64 | } 65 | 66 | auto index = _schedule.size(); 67 | 68 | for (auto i = 0; i < _schedule.size(); i++) { 69 | if (t < _schedule[i].time) { 70 | index = i; 71 | } 72 | } 73 | 74 | if (index == 0) { 75 | return _schedule[0].value; 76 | } else if (index == _schedule.size()) { 77 | return _schedule[index - 1].value; 78 | } else { 79 | if (_type == Step) { 80 | return _schedule[index - 1].value; 81 | } else if (_type == Continuous) { 82 | const auto &from = _schedule[index - 1]; 83 | const auto &to = _schedule[index]; 84 | 85 | auto i = (double) (t - from.time) / (double) (to.time - from.time); 86 | 87 | return ((to.value - from.value) * i) + from.value; 88 | } 89 | } 90 | 91 | return (T) 0; 92 | } 93 | 94 | private: 95 | bool _isModified; 96 | 97 | Type _type; 98 | 99 | std::vector _schedule; 100 | }; 101 | 102 | typedef Parameter ParameterD; 103 | 104 | #endif /* Parameter_hpp */ 105 | -------------------------------------------------------------------------------- /src/ebfr/Rig.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Rig.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "Rig.h" 10 | 11 | #include "../shared/FS.h" 12 | #include "../shared/CSV.h" 13 | #include "../shared/Mesh.h" 14 | 15 | #include 16 | #include 17 | 18 | void Rig::load(const std::string& dirPath, const std::string& posePath, const std::string& weightsPath, const std::string& vertexMaskPath, bool isTarget) 19 | { 20 | std::cout 21 | << "Reading " << (isTarget ? "Target" : "Source") << " Rig" << std::endl 22 | << "\t" << dirPath << std::endl; 23 | 24 | if (isTarget) { 25 | loadNeutral(dirPath); 26 | } else { 27 | loadBlendshapes(dirPath); 28 | } 29 | 30 | PoseCSV csv; 31 | if (!csv.open(weightsPath)) { 32 | std::cerr << "Failed to open Pose CSV " << weightsPath << std::endl; 33 | return; 34 | } 35 | 36 | std::cout 37 | << "\tPoses" << std::endl 38 | << "\t\t" << posePath << std::endl; 39 | 40 | std::vector posePaths; 41 | std::vector> poseWeights; 42 | 43 | std::string poseName; 44 | std::vector poseWeight; 45 | 46 | while (csv.next()) { 47 | csv.values(poseName, poseWeight); 48 | 49 | if (!std::any_of(poseWeight.begin(), poseWeight.end(), [](double w) { return w > 0.0; })) 50 | continue; 51 | 52 | // Neutral/BS0 53 | poseWeight.insert(poseWeight.begin(), 1); 54 | 55 | posePaths.push_back(JoinPath(posePath, poseName + ".obj")); 56 | poseWeights.push_back(poseWeight); 57 | 58 | std::cout << "\t\t" << poseName << " - "; 59 | for (auto w : poseWeight) 60 | std::cout << w << " "; 61 | std::cout << std::endl; 62 | } 63 | 64 | loadPoses(posePaths, poseWeights); 65 | 66 | if (!vertexMaskPath.empty()) { 67 | loadVertexMask(vertexMaskPath); 68 | 69 | std::cout << "Modified Verticies: " << vertices().size() << " / " << numVertices(true) << std::endl; 70 | std::cout << "Modified Faces: " << faces().size() << " / " << numFaces(true) << std::endl; 71 | } 72 | 73 | return; 74 | } 75 | 76 | void Rig::loadBlendshapes(const std::string &dirPath) { 77 | std::vector blendshapePaths; 78 | ListFiles(JoinPath(dirPath, "(\\d*).obj"), blendshapePaths); 79 | blendshapePaths.insert(blendshapePaths.begin(), JoinPath(dirPath, "neutral.obj")); 80 | 81 | loadBlendshapes(blendshapePaths); 82 | } 83 | 84 | void Rig::loadBlendshapes(const std::vector &paths) { 85 | _blendshapes.resize(paths.size()); 86 | 87 | for (auto i = 0; i < paths.size(); i++) { 88 | _blendshapes[i].setMesh(ReadMesh(paths[i]), i > 0); 89 | } 90 | } 91 | 92 | void Rig::loadNeutral(const std::string &path) { 93 | if (_blendshapes.empty()) { 94 | _blendshapes.resize(1); 95 | } 96 | 97 | _blendshapes[0].setMesh(ReadMesh(path), false); 98 | } 99 | 100 | void Rig::generateEmptyBlendshapes(size_t num) { 101 | _blendshapes.resize(num); 102 | 103 | for (auto i = 0; i < num; i++) { 104 | if (_blendshapes[i].mesh() != nullptr) { 105 | continue; 106 | } 107 | 108 | auto mesh = MakeMesh(_poses[0].mesh()); 109 | 110 | SetVertices(mesh, Mesh::Point(0, 0, 0)); 111 | 112 | _blendshapes[i].setMesh(mesh, false); 113 | } 114 | } 115 | 116 | void Rig::randomizeWeights() { 117 | std::mt19937 g(0); 118 | 119 | std::normal_distribution<> a{0, 0.5}; 120 | std::normal_distribution<> b{0, 0.1}; 121 | 122 | for (auto i = 0; i < numPoses(); i++) { 123 | auto &ws = weights(i); 124 | 125 | for (auto &w : ws) { 126 | w = std::clamp(w * (1.0 - a(g)) + std::abs(b(g)), 0.0, 1.0); 127 | } 128 | } 129 | } 130 | 131 | MeshPtr Rig::generatePose(int pose) const { 132 | return generatePose(weights(pose)); 133 | } 134 | 135 | MeshPtr Rig::generatePose(const std::vector &weights) const { 136 | auto target = MakeMesh(neutral()); 137 | 138 | for (auto bs = 1; bs < numBlendshapes(); bs++) { 139 | AddVertices(target, blendshape(bs).mesh(), weights[bs], target); 140 | } 141 | 142 | return target; 143 | } 144 | 145 | void Rig::loadPoses(const std::vector &paths, const std::vector &weights) { 146 | _poses.resize(paths.size()); 147 | 148 | for (auto i = 0; i < paths.size(); i++) { 149 | auto mesh = ReadMesh(paths[i]); 150 | 151 | _poses[i].setMesh(mesh); 152 | _poses[i].setWeights(weights[i]); 153 | } 154 | } 155 | 156 | void Rig::loadVertexMask(const std::string &path) { 157 | std::ifstream file; 158 | file.open(path); 159 | 160 | int vid; 161 | while (!file.eof()) { 162 | file >> vid; 163 | _vertices.emplace_back(vid); 164 | } 165 | 166 | file.close(); 167 | 168 | auto mesh = _poses[0].mesh(); 169 | 170 | for (auto v : _vertices) { 171 | const auto vh = mesh->vertex_handle(v); 172 | 173 | for (auto iter = mesh->cvf_begin(vh), end = mesh->cvf_end(vh); iter != end; iter++) { 174 | _faces.emplace_back(iter->idx()); 175 | } 176 | } 177 | 178 | std::sort(_faces.begin(), _faces.end()); 179 | auto lastF = std::unique(_faces.begin(), _faces.end()); 180 | if (lastF != _faces.end()) { 181 | _faces.erase(lastF, _faces.end()); 182 | } 183 | 184 | _vertices.clear(); 185 | _vertices.reserve(_faces.size() * 3); 186 | 187 | for (auto f : _faces) { 188 | const auto fh = mesh->face_handle(f); 189 | 190 | for (auto iter = mesh->cfv_begin(fh), end = mesh->cfv_end(fh); iter != end; iter++) { 191 | _vertices.emplace_back(iter->idx()); 192 | } 193 | } 194 | 195 | std::sort(_vertices.begin(), _vertices.end()); 196 | 197 | auto lastV = std::unique(_vertices.begin(), _vertices.end()); 198 | if (lastV != _vertices.end()) { 199 | _vertices.erase(lastV, _vertices.end()); 200 | } 201 | } 202 | 203 | void Rig::findModified() { 204 | const auto numV = numVertices(); 205 | 206 | // for (auto bs = 1; bs < _blendshapes.size(); bs++) 207 | // { 208 | // const auto& isFixed = _blendshapes[bs].isFixed(); 209 | // 210 | // for (auto i = 0; i < numV; i++) 211 | // { 212 | // if (!isFixed[i]) { 213 | // if (i == 0) { 214 | // std::cout << _blendshapes[bs].mesh()->point(_blendshapes[bs].mesh()->vertex_handle(i)) << std::endl; 215 | // std::cout <points(); 223 | for (auto pose : _poses) { 224 | const auto points = pose.mesh()->points(); 225 | 226 | for (auto i = 0; i < numV; i++) { 227 | const auto diff = points[i] - neutral[i]; 228 | if (!isNearZero(diff, 0.5)) { 229 | _vertices.emplace_back(i); 230 | } 231 | } 232 | } 233 | 234 | std::sort(_vertices.begin(), _vertices.end()); 235 | auto lastV = std::unique(_vertices.begin(), _vertices.end()); 236 | if (lastV != _vertices.end()) { 237 | _vertices.erase(lastV, _vertices.end()); 238 | } 239 | 240 | 241 | auto mesh = _poses[0].mesh(); 242 | 243 | for (auto v : _vertices) { 244 | const auto vh = mesh->vertex_handle(v); 245 | 246 | for (auto iter = mesh->cvf_begin(vh), end = mesh->cvf_end(vh); iter != end; iter++) { 247 | _faces.emplace_back(iter->idx()); 248 | } 249 | } 250 | 251 | std::sort(_faces.begin(), _faces.end()); 252 | auto lastF = std::unique(_faces.begin(), _faces.end()); 253 | if (lastF != _faces.end()) { 254 | _faces.erase(lastF, _faces.end()); 255 | } 256 | } 257 | 258 | size_t Rig::faceIndex(size_t index) const { 259 | if (_faces.empty()) { 260 | return index; 261 | } 262 | 263 | auto iter = std::lower_bound(_faces.begin(), _faces.end(), index); 264 | if (iter == _faces.end()) { 265 | return 0; 266 | } 267 | 268 | return iter - _faces.begin(); 269 | } 270 | 271 | size_t Rig::vertexIndex(size_t index) const { 272 | if (_vertices.empty()) { 273 | return index; 274 | } 275 | 276 | auto iter = std::lower_bound(_vertices.begin(), _vertices.end(), index); 277 | if (iter == _vertices.end()) { 278 | return 0; 279 | } 280 | 281 | return iter - _vertices.begin(); 282 | } 283 | -------------------------------------------------------------------------------- /src/ebfr/Rig.h: -------------------------------------------------------------------------------- 1 | // 2 | // Rig.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Rig_hpp 10 | #define Rig_hpp 11 | 12 | #include "../shared/Mesh.h" 13 | 14 | #include 15 | #include 16 | 17 | //class Weights : public std::vector 18 | //{}; 19 | typedef std::vector Weights; 20 | 21 | class Pose { 22 | public: 23 | Pose() 24 | : _mesh(nullptr) { 25 | } 26 | 27 | Pose(MeshPtr pose) 28 | : _mesh(pose) { 29 | } 30 | 31 | Pose(MeshPtr mesh, const Weights &w) 32 | : _mesh(mesh), _weights(w) { 33 | } 34 | 35 | MeshPtr mesh() const { return _mesh; } 36 | 37 | const Weights &weights() const { return _weights; } 38 | 39 | Weights &weights() { return _weights; } 40 | 41 | void setMesh(MeshPtr pose) { 42 | _mesh = pose; 43 | } 44 | 45 | void setWeights(const Weights &w) { 46 | _weights = w; 47 | } 48 | 49 | private: 50 | MeshPtr _mesh; 51 | 52 | Weights _weights; 53 | }; 54 | 55 | class Blendshape { 56 | public: 57 | Blendshape() 58 | : _mesh(nullptr) { 59 | } 60 | 61 | Blendshape(MeshPtr blendshape) 62 | : _mesh(nullptr) { 63 | setMesh(blendshape); 64 | } 65 | 66 | MeshPtr mesh() const { return _mesh; } 67 | 68 | void setMesh(MeshPtr blendshape, bool checkFixed = true) { 69 | _mesh = blendshape; 70 | 71 | if (checkFixed) { 72 | for (auto vertIter = _mesh->vertices_begin(), vertEnd = _mesh->vertices_end(); 73 | vertIter != vertEnd; vertIter++) { 74 | const auto vert = *vertIter; 75 | 76 | if (isNearZero(_mesh->point(vert), 0.5)) { 77 | _fixed.emplace_back(vert.idx()); 78 | } 79 | } 80 | } 81 | } 82 | 83 | size_t numFixed() const { 84 | return _fixed.size(); 85 | } 86 | 87 | bool isFixed(int v) const { 88 | return std::binary_search(_fixed.begin(), _fixed.end(), v); 89 | } 90 | 91 | const std::vector &fixed() const { 92 | return _fixed; 93 | } 94 | 95 | private: 96 | MeshPtr _mesh; 97 | 98 | std::vector _fixed; 99 | }; 100 | 101 | class Rig { 102 | public: 103 | void load(const std::string &dirPath, const std::string &posePath, const std::string &weightsPath, 104 | const std::string &vertexMaskPath, bool isTarget); 105 | 106 | void loadBlendshapes(const std::string &dirPath); 107 | 108 | void loadBlendshapes(const std::vector &paths); 109 | 110 | void loadNeutral(const std::string &path); 111 | 112 | void loadPoses(const std::vector &paths, const std::vector &weights); 113 | 114 | void loadVertexMask(const std::string &path); 115 | 116 | void findModified(); 117 | 118 | void generateEmptyBlendshapes(size_t num); 119 | 120 | void randomizeWeights(); 121 | 122 | MeshPtr generatePose(int pose) const; 123 | 124 | MeshPtr generatePose(const std::vector &weights) const; 125 | 126 | MeshPtr neutral() const { return _blendshapes[0].mesh(); } 127 | 128 | size_t numVertices(bool all = false) const { 129 | return _vertices.empty() || all ? neutral()->n_vertices() : _vertices.size(); 130 | } 131 | 132 | size_t numFaces(bool all = false) const { return _faces.empty() || all ? neutral()->n_faces() : _faces.size(); } 133 | 134 | const std::vector &vertices() const { return _vertices; } 135 | 136 | const std::vector &faces() const { return _faces; } 137 | 138 | size_t face(size_t index) const { return _faces.empty() ? index : _faces[index]; } 139 | 140 | size_t faceIndex(size_t index) const; 141 | 142 | size_t vertex(size_t index) const { return _vertices.empty() ? index : _vertices[index]; } 143 | 144 | size_t vertexIndex(size_t index) const; 145 | 146 | std::vector &blendshapes() { return _blendshapes; } 147 | 148 | const std::vector &blendshapes() const { return _blendshapes; } 149 | 150 | Blendshape &blendshape(int bs) { return _blendshapes[bs]; } 151 | 152 | const Blendshape &blendshape(int bs) const { return _blendshapes[bs]; } 153 | 154 | size_t numBlendshapes() const { return _blendshapes.size(); } 155 | 156 | std::vector &poses() { return _poses; } 157 | 158 | const std::vector &poses() const { return _poses; } 159 | 160 | Pose &pose(int pose) { return _poses[pose]; } 161 | 162 | const Pose &pose(int pose) const { return _poses[pose]; } 163 | 164 | size_t numPoses() const { return _poses.size(); } 165 | 166 | std::vector weights() const { 167 | std::vector weights; 168 | for (const auto &pose : _poses) { weights.push_back(pose.weights()); } 169 | return weights; 170 | } 171 | 172 | Weights &weights(int pose) { return _poses[pose].weights(); } 173 | 174 | const Weights &weights(int pose) const { return _poses[pose].weights(); } 175 | 176 | double weight(int pose, int bs) const { return weights(pose)[bs]; } 177 | 178 | size_t numActiveWeights(int pose) const { 179 | return std::count_if(weights(pose).begin(), weights(pose).end(), [](double v) { return v > 0.0; }); 180 | } 181 | 182 | private: 183 | std::vector _blendshapes; 184 | 185 | std::vector _poses; 186 | 187 | std::vector _vertices; 188 | std::vector _faces; 189 | }; 190 | 191 | typedef std::shared_ptr RigPtr; 192 | 193 | inline RigPtr MakeRig() { 194 | return std::make_shared(); 195 | } 196 | 197 | #endif /* Rig_hpp */ 198 | -------------------------------------------------------------------------------- /src/ebfr/SolverBase.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // SolverBase.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "SolverBase.h" 10 | 11 | SolverBase::SolverBase() 12 | : _iteration(0) 13 | , _source(nullptr) 14 | , _sourceGradients(nullptr) 15 | , _target(nullptr) 16 | , _targetGradients(nullptr) 17 | , _useMultithreaded(false) 18 | , _debugPath() 19 | , _debug(false) 20 | { 21 | } 22 | 23 | bool SolverBase::setSource(RigPtr rig, GradientsPtr gradients) { 24 | _source = rig; 25 | _sourceGradients = gradients; 26 | 27 | return _source != nullptr && _sourceGradients != nullptr; 28 | } 29 | 30 | bool SolverBase::setTarget(RigPtr rig, GradientsPtr gradients) { 31 | _target = rig; 32 | _targetGradients = gradients; 33 | 34 | return _target != nullptr && _targetGradients != nullptr; 35 | } 36 | 37 | void SolverBase::setMultithreaded(bool enable) { 38 | _useMultithreaded = enable; 39 | } 40 | 41 | bool SolverBase::checkSolverError(Eigen::ComputationInfo info) const { 42 | if (info == Eigen::Success) 43 | return true; 44 | 45 | std::cerr << "**Eigen Solver Error: " << Error(info) << std::endl; 46 | return false; 47 | } 48 | 49 | -------------------------------------------------------------------------------- /src/ebfr/SolverBase.h: -------------------------------------------------------------------------------- 1 | // 2 | // SolverBase.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef SolverBase_hpp 10 | #define SolverBase_hpp 11 | 12 | #include 13 | #include 14 | 15 | #include "../shared/Matrix.h" 16 | #include "../shared/Mesh.h" 17 | 18 | #include "Rig.h" 19 | #include "Gradients.h" 20 | #include "Parameter.h" 21 | 22 | typedef unsigned int Index; 23 | 24 | class SolverBase { 25 | public: 26 | typedef std::function StepCallback; 27 | 28 | SolverBase(); 29 | 30 | bool setSource(RigPtr rig, GradientsPtr gradients); 31 | 32 | bool setTarget(RigPtr rig, GradientsPtr gradients); 33 | 34 | void setMultithreaded(bool enable); 35 | 36 | void setDebugPath(const std::string &path) { _debugPath = path; } 37 | 38 | void setStepCallback(StepCallback callback) { _callback = callback; } 39 | 40 | virtual void init() {} 41 | 42 | virtual bool solve(int iter) { 43 | _iteration = iter; 44 | return true; 45 | } 46 | 47 | protected: 48 | bool _debug; 49 | 50 | int _iteration; 51 | 52 | bool _useMultithreaded; 53 | 54 | RigPtr _source; 55 | GradientsPtr _sourceGradients; 56 | 57 | RigPtr _target; 58 | GradientsPtr _targetGradients; 59 | 60 | StepCallback _callback; 61 | 62 | std::string _debugPath; 63 | 64 | bool checkSolverError(Eigen::ComputationInfo info) const; 65 | }; 66 | 67 | #endif /* SolverBase_hpp */ 68 | -------------------------------------------------------------------------------- /src/ebfr/VertexSolver.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // VertexSolver.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "VertexSolver.h" 10 | 11 | #include "../shared/Timing.h" 12 | 13 | #include 14 | 15 | VertexSolver::VertexSolver() 16 | : SolverBase() 17 | , _mSize(_Matrix::SizeAtCompileTime) 18 | , _mCols(_Matrix::ColsAtCompileTime) 19 | , _mRows(_Matrix::RowsAtCompileTime) 20 | , _vSize(_Vector::SizeAtCompileTime) 21 | , _usePhantom(false) 22 | , _fixedWeight(0.5) 23 | , _maxFixed(100) 24 | , _randomFixed(true) 25 | { 26 | 27 | } 28 | 29 | void VertexSolver::init() { 30 | _fixedVertices.resize(_source->numBlendshapes()); 31 | 32 | for (auto bs = 1; bs < _source->numBlendshapes(); bs++) { 33 | const auto blendshape = _source->blendshape(bs); 34 | 35 | auto &fixedVertices = _fixedVertices[bs]; 36 | 37 | fixedVertices.resize(blendshape.numFixed()); 38 | std::copy(blendshape.fixed().begin(), blendshape.fixed().end(), fixedVertices.begin()); 39 | 40 | if (_maxFixed != -1 && _maxFixed < fixedVertices.size()) { 41 | if (_randomFixed) { 42 | std::mt19937 g(0); 43 | 44 | std::shuffle(fixedVertices.begin(), fixedVertices.end(), g); 45 | 46 | fixedVertices.resize(_maxFixed); 47 | 48 | std::sort(fixedVertices.begin(), fixedVertices.end()); 49 | } else { 50 | fixedVertices.resize(_maxFixed); 51 | } 52 | } 53 | } 54 | 55 | _solvers.resize(_target->numBlendshapes()); 56 | for (auto &solver : _solvers) { 57 | solver.initialized = false; 58 | } 59 | } 60 | 61 | bool VertexSolver::solve(int iter) { 62 | return solve(iter, -1); 63 | } 64 | 65 | bool VertexSolver::solve(int iter, int bs) { 66 | std::cout 67 | << std::endl 68 | << "Vertex Solver [" << iter << "]" 69 | << std::endl; 70 | 71 | SolverBase::solve(iter); 72 | 73 | if (bs != -1) { 74 | transfer(bs); 75 | } else { 76 | for (bs = 1; bs < _target->numBlendshapes(); bs++) { 77 | transfer(bs); 78 | } 79 | } 80 | 81 | if (_callback != nullptr) 82 | _callback(iter, _target, _debugPath); 83 | 84 | return true; 85 | } 86 | 87 | bool VertexSolver::transfer(Index bs) { 88 | if (_debug) { 89 | std::cout << "\tBlendshape [" << bs << "]" << std::endl; 90 | } 91 | 92 | auto &solver = _solvers[bs]; 93 | 94 | if (!solver.initialized) { 95 | SparseMatrix a; 96 | 97 | constructA(bs, a); 98 | 99 | TIMER_START(Compute); 100 | 101 | solver.at = a.transpose(); 102 | 103 | solver.solver.compute(solver.at * a); 104 | 105 | TIMER_END(Compute); 106 | 107 | if (!checkSolverError(solver.solver)) { 108 | std::cerr << "Vertex Solver failed to init" << std::endl; 109 | return false; 110 | } 111 | 112 | solver.initialized = true; 113 | } 114 | 115 | constructC(bs, solver.c); 116 | 117 | solver.x = solver.solver.solve(solver.at * solver.c); 118 | 119 | if (!checkSolverError(solver.solver)) 120 | return false; 121 | 122 | copyTo(bs, solver.x); 123 | 124 | return true; 125 | } 126 | 127 | void VertexSolver::constructA(int bs, SparseMatrix &a) { 128 | auto rows = _target->numFaces(true) * _mSize; 129 | 130 | rows += _fixedVertices[bs].size() * _vSize; 131 | 132 | auto cols = _target->numVertices(true) * _vSize; 133 | 134 | if (_usePhantom) { 135 | cols += _target->numFaces(true) * _vSize; 136 | } 137 | 138 | if (_debug) { 139 | std::cout 140 | << "Constructing A" << std::endl 141 | << "\tSize: " << rows << " x " << cols << std::endl; 142 | } 143 | 144 | TripletList m; 145 | m.reserve(rows * 10); 146 | 147 | MatrixE e; 148 | 149 | for (auto face = 0; face < _target->numFaces(true); face++) { 150 | constructE(face, e); 151 | 152 | constructA(face, e, m); 153 | } 154 | 155 | constructFixedA(bs, m); 156 | 157 | a.resize(rows, cols); 158 | a.setZero(); 159 | 160 | a.setFromTriplets(m.begin(), m.end()); 161 | 162 | a.makeCompressed(); 163 | } 164 | 165 | void VertexSolver::constructE(const Index face, MatrixE &e) const { 166 | const auto &mInv = _targetGradients->blendshapeMInv[0][face]; 167 | 168 | e << -mInv(0, 0) - mInv(1, 0) - (_usePhantom ? mInv(2, 0) : 0), 169 | mInv(0, 0), 170 | mInv(1, 0), 171 | mInv(2, 0), 172 | 173 | -mInv(0, 1) - mInv(1, 1) - (_usePhantom ? mInv(2, 1) : 0), 174 | mInv(0, 1), 175 | mInv(1, 1), 176 | mInv(2, 1), 177 | 178 | -mInv(0, 2) - mInv(1, 2) - (_usePhantom ? mInv(2, 2) : 0), 179 | mInv(0, 2), 180 | mInv(1, 2), 181 | mInv(2, 2); 182 | } 183 | 184 | void VertexSolver::constructA(const Index face, const MatrixE &e, TripletList &m) { 185 | const auto numVerts = _usePhantom ? 4 : 3; 186 | 187 | Index vertexIdx[4]; 188 | vertexIndices(face, vertexIdx); 189 | 190 | auto row = face * _mSize; 191 | 192 | for (int coord = 0; coord < 3; coord++) { 193 | for (int eqn = 0; eqn < 3; eqn++, row++) { 194 | for (int vert = 0; vert < numVerts; vert++) { 195 | const int vertIdx = (int) vertexIdx[vert]; 196 | 197 | m.emplace_back(row, vertIdx + coord, e(eqn, vert)); 198 | } 199 | } 200 | } 201 | } 202 | 203 | void VertexSolver::constructFixedA(Index bs, TripletList &m) { 204 | const auto &fixed = _fixedVertices[bs]; 205 | 206 | auto row = (int) (_target->numFaces() * _mSize); 207 | 208 | for (auto i : fixed) { 209 | auto idx = vertexIndex(i); 210 | 211 | for (auto j = 0; j < 3; j++) { 212 | m.emplace_back(row, idx, _fixedWeight); 213 | 214 | row++; 215 | idx++; 216 | } 217 | } 218 | } 219 | 220 | void VertexSolver::constructC(Index bs, MatrixX &c) { 221 | auto rows = _target->numFaces(true) * 9; 222 | 223 | rows += _fixedVertices[bs].size() * _vSize; 224 | 225 | if (_debug) { 226 | std::cout 227 | << "Constructing C" << std::endl 228 | << "\tSize: " << rows << " x " << 1 << std::endl; 229 | } 230 | 231 | c.resize(rows, 1); 232 | c.setZero(); 233 | 234 | const auto &neutralM = _targetGradients->blendshapeM[0]; 235 | const auto &neutralMInv = _targetGradients->blendshapeMInv[0]; 236 | const auto &blendshapeM = _targetGradients->blendshapeM[bs]; 237 | 238 | const auto I = Matrix3x3::Identity(); 239 | 240 | for (auto face = 0; face < _target->numFaces(true); face++) { 241 | const auto &m = blendshapeM[face]; 242 | 243 | if (m.isZero()) { 244 | constructC(face, I, c); 245 | } else { 246 | constructC(face, (neutralM[face] + m) * neutralMInv[face], c); 247 | } 248 | } 249 | 250 | constructFixedC(bs, c); 251 | } 252 | 253 | void VertexSolver::constructC(const Index face, const Matrix3x3 &q, MatrixX &c) { 254 | auto row = face * MatrixQ::RowsAtCompileTime; 255 | 256 | for (auto i = 0; i < 3; i++) { 257 | for (auto j = 0; j < 3; j++) { 258 | c(row, 0) = q(i, j); 259 | row++; 260 | } 261 | } 262 | } 263 | 264 | void VertexSolver::constructFixedC(Index bs, MatrixX &c) { 265 | const auto &fixed = _fixedVertices[bs]; 266 | 267 | auto mesh = _target->neutral(); 268 | 269 | auto row = (int) (_target->numFaces() * _mSize); 270 | 271 | for (auto i : fixed) { 272 | const auto vert = mesh->vertex_handle(i); 273 | const auto &p = mesh->point(vert); 274 | 275 | for (auto j = 0; j < 3; j++) { 276 | c(row, 0) = _fixedWeight * p[j]; 277 | 278 | row++; 279 | } 280 | } 281 | } 282 | 283 | void VertexSolver::copyTo(Index bs, MatrixX &x) const { 284 | auto target = _target->blendshape(bs).mesh(); 285 | 286 | for (auto v = 0; v < _target->numVertices(true); v++) { 287 | const auto vh = target->vertex_handle(v); 288 | const auto idx = vertexIndex(v); 289 | 290 | target->point(vh) = OpenMesh::Vec3d(x(idx, 0), x(idx + 1, 0), x(idx + 2, 0));; 291 | } 292 | 293 | if (_debug) { 294 | auto neutral = _target->neutral(); 295 | const auto &fixedVertices = _fixedVertices[bs]; 296 | 297 | auto printVert = [&target, &neutral](auto vert, bool flagged = false) { 298 | const auto p = target->point(target->vertex_handle(vert)); 299 | const auto np = neutral->point(neutral->vertex_handle(vert)); 300 | 301 | std::cout << "\tVert " << vert << ": " << p << " :: " << p - np << (flagged ? " *" : "") << std::endl; 302 | }; 303 | 304 | std::cout << "Vertices: " << std::endl; 305 | printVert(target->n_vertices() / 3); 306 | printVert(target->n_vertices() / 2); 307 | printVert(2 * (target->n_vertices() / 3)); 308 | 309 | if (!fixedVertices.empty()) { 310 | std::cout << "Fixed: " << std::endl; 311 | printVert(fixedVertices[fixedVertices.size() / 3]); 312 | printVert(fixedVertices[fixedVertices.size() / 2]); 313 | printVert(2 * (fixedVertices[fixedVertices.size() / 3])); 314 | } 315 | } 316 | } 317 | 318 | Index VertexSolver::vertexIndex(Index idx) const { 319 | return (Index) (idx * _vSize); 320 | } 321 | 322 | void VertexSolver::vertexIndices(const Index face, Index vertices[]) const { 323 | const auto mesh = _target->blendshape(0).mesh(); 324 | const auto fh = mesh->face_handle(face); 325 | 326 | size_t i = 0; 327 | for (auto vertIter = mesh->cfv_begin(fh), vertEnd = mesh->cfv_end(fh); 328 | i < 3 && vertIter != vertEnd; vertIter++, i++) { 329 | vertices[i] = vertexIndex(vertIter->idx()); 330 | } 331 | 332 | vertices[3] = (Index) ((_target->numVertices(true) + fh.idx()) * _vSize); 333 | } 334 | 335 | bool VertexSolver::checkSolverError(const Solver &solver) const { 336 | return SolverBase::checkSolverError(solver.info()); 337 | } 338 | -------------------------------------------------------------------------------- /src/ebfr/VertexSolver.h: -------------------------------------------------------------------------------- 1 | // 2 | // VertexSolver.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef VertexSolver_hpp 10 | #define VertexSolver_hpp 11 | 12 | #include "SolverBase.h" 13 | 14 | class VertexSolver : public SolverBase { 15 | public: 16 | VertexSolver(); 17 | 18 | virtual void init(); 19 | 20 | virtual bool solve(int iter); 21 | 22 | bool solve(int iter, int bs); 23 | 24 | private: 25 | typedef Matrix3x3 _Matrix; 26 | typedef Vector3 _Vector; 27 | 28 | typedef Eigen::Matrix MatrixE; 29 | typedef Eigen::Matrix MatrixQ; 30 | typedef Eigen::SimplicialLDLT Solver; 31 | 32 | const size_t _mSize; 33 | const size_t _mCols; 34 | const size_t _mRows; 35 | 36 | const size_t _vSize; 37 | 38 | const bool _usePhantom; 39 | const double _fixedWeight; 40 | const int _maxFixed; 41 | const bool _randomFixed; 42 | 43 | std::vector> _fixedVertices; 44 | 45 | class SolverData { 46 | public: 47 | SolverData() 48 | : initialized(false) 49 | , at() 50 | , c() 51 | , x() 52 | , solver() 53 | {} 54 | 55 | SolverData(const SolverData &other) 56 | : SolverData() 57 | {} 58 | 59 | bool initialized; 60 | 61 | SparseMatrix at; 62 | 63 | MatrixX c; 64 | 65 | MatrixX x; 66 | 67 | Solver solver; 68 | }; 69 | 70 | std::vector _solvers; 71 | 72 | StepCallback _callback; 73 | 74 | bool transfer(Index bs); 75 | 76 | void constructA(int bs, SparseMatrix &a); 77 | 78 | void constructA(Index face, const MatrixE &e, TripletList &m); 79 | 80 | void constructFixedA(Index bs, TripletList &m); 81 | 82 | void constructE(Index face, MatrixE &e) const; 83 | 84 | void constructC(Index bs, MatrixX &c); 85 | 86 | void constructC(Index face, const Matrix3x3 &q, MatrixX &c); 87 | 88 | void constructFixedC(Index bs, MatrixX &c); 89 | 90 | void copyTo(Index bs, MatrixX &x) const; 91 | 92 | Index vertexIndex(Index idx) const; 93 | 94 | void vertexIndices(Index face, Index vertices[]) const; 95 | 96 | bool checkSolverError(const Solver &solver) const; 97 | }; 98 | 99 | #endif /* VertexSolver_hpp */ 100 | -------------------------------------------------------------------------------- /src/ebfr/WeightsSolver.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // WeightsSolver.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "WeightsSolver.h" 10 | 11 | #include 12 | #include 13 | 14 | int WeightsSolver::WeightsFunctor::operator()(const Eigen::VectorXd &x, Eigen::VectorXd &fvec) const { 15 | const auto numVertices = a->rows() / 3; 16 | const auto numWeights = estimateW.size(); 17 | 18 | // ||Ax - c||^2 19 | // where A are the blendshapes, x are the weights, and c is (pose - neutral) 20 | MatrixX vDiff = (*a * x).rowwise().sum() - *c; 21 | vDiff = vDiff.cwiseProduct(vDiff); 22 | 23 | for (auto i = 0; i < numVertices; i++) { 24 | const auto sum = vDiff.block<3, 1>(i * 3, 0).sum(); 25 | fvec(i) = sum;//std::sqrt(sum); 26 | } 27 | 28 | // Regularization 29 | // ||x - w*||^2 30 | // where x are the weights and w* are the user-estimated weights 31 | VectorX wDiff = estimateW - x; 32 | wDiff = wDiff.cwiseProduct(wDiff); 33 | wDiff *= lambda; 34 | 35 | for (auto i = 0; i < numWeights; i++) 36 | fvec(i + numVertices) = wDiff(i); 37 | 38 | return 0; 39 | } 40 | 41 | int WeightsSolver::WeightsFunctor::inputs() const { 42 | return estimateW.size(); 43 | } 44 | 45 | int WeightsSolver::WeightsFunctor::values() const { 46 | // # of vertices + # of weights 47 | return (a->rows() / 3) + estimateW.size(); 48 | } 49 | 50 | WeightsSolver::WeightsSolver() 51 | : SolverBase() 52 | , _lambda(1000) // -> 100 53 | , _maxIterations(10) 54 | , _minWeight(0.0) 55 | , _maxWeight(1.0) 56 | { 57 | 58 | } 59 | 60 | void WeightsSolver::setLambda(const ParameterD &lambda) { 61 | _lambda = lambda; 62 | } 63 | 64 | void WeightsSolver::init() { 65 | const auto rows = _target->numVertices() * Vector3::SizeAtCompileTime; 66 | const auto cols = _target->numBlendshapes() - 1; 67 | 68 | _a.resize(rows, cols); 69 | 70 | // Save the user-provided weight estimates and the precalculate the 71 | // "c" matrix (pose - neutral). 72 | _estimateWs.resize(_target->numPoses()); 73 | _Cs.resize(_target->numPoses()); 74 | 75 | for (auto pose = 0; pose < _target->numPoses(); pose++) { 76 | copyWeightsTo(_target->weights(pose), _estimateWs[pose]); 77 | 78 | _Cs[pose].resize(_target->numVertices() * Vector3::SizeAtCompileTime); 79 | 80 | appendWeightFit(pose, _Cs[pose]); 81 | } 82 | } 83 | 84 | bool WeightsSolver::solve(int iter) { 85 | std::cout 86 | << std::endl 87 | << "Weights Solver [" << iter << "]" << std::endl 88 | << "\tLambda: " << _lambda(iter) << std::endl 89 | << std::endl; 90 | 91 | SolverBase::solve(iter); 92 | 93 | std::mutex logMutex; 94 | 95 | // All of the problems share the same blendshapes, so 96 | // the "A" matrix can be shared with all problems. 97 | appendWeightFit(0, _a); 98 | 99 | auto solverOp = 100 | [this, &logMutex] 101 | (int threadId, size_t poseStart, size_t poseEnd) { 102 | if (_debug && threadId >= 0) { 103 | logMutex.lock(); 104 | std::cout << "Thread [" << threadId << "-WS] Starting: Poses " << poseStart << " -> " << poseEnd << " (" << (poseEnd - poseStart) << ")" << std::endl; 105 | logMutex.unlock(); 106 | } 107 | 108 | WeightsFunctorNumericalDiff data; 109 | 110 | initSolverData(data); 111 | 112 | //appendWeightFit(0, data._a); 113 | 114 | auto pose = (Index) poseStart; 115 | 116 | while (true) { 117 | if (pose >= poseEnd) 118 | break; 119 | 120 | data.a = &_a; 121 | data.c = &_Cs[pose]; 122 | 123 | data.estimateW = _estimateWs[pose]; 124 | data.x = _estimateWs[pose]; 125 | 126 | //copyWeightsTo(_target->weights(pose), data.x); 127 | 128 | Eigen::LevenbergMarquardt lm(data); 129 | 130 | // auto status = lm.minimize(data.x); 131 | 132 | // Use Eigen's (experimental) Levenberg-Marquardt implementation to 133 | // optimize for the blendshape weights for each pose. 134 | auto status = lm.minimizeInit(data.x); 135 | if (status == Eigen::LevenbergMarquardtSpace::ImproperInputParameters) { 136 | std::cerr << "Weights Solver failed to init" << std::endl; 137 | continue; 138 | } 139 | 140 | auto iter = 0; 141 | do { 142 | status = lm.minimizeOneStep(data.x); 143 | 144 | // Hack-ish box constraint 145 | for (auto i = 0; i < data.x.size(); i++) { 146 | data.x(i) = std::clamp(data.x(i), _minWeight, _maxWeight); 147 | } 148 | 149 | iter++; 150 | 151 | } while (status == Eigen::LevenbergMarquardtSpace::Running && iter < _maxIterations); 152 | 153 | copyWeightsTo(data.x, _target->weights(pose)); 154 | 155 | pose++; 156 | } 157 | 158 | if (_debug && threadId >= 0) { 159 | logMutex.lock(); 160 | std::cout << "Thread [" << threadId << "-WS] Complete" << std::endl; 161 | logMutex.unlock(); 162 | } 163 | }; 164 | 165 | if (_useMultithreaded) { 166 | std::vector pool; 167 | const auto size = _target->numPoses(); 168 | const auto numThreads = std::min(size, (size_t) std::thread::hardware_concurrency()); 169 | const auto segmentSize = numThreads == size ? 1 : (size / numThreads) + 1; 170 | auto segmentStart = 0; 171 | 172 | for (auto i = 0; i < numThreads; i++) { 173 | pool.push_back(std::thread(solverOp, i, segmentStart, std::min(segmentStart + segmentSize, size))); 174 | 175 | segmentStart += segmentSize; 176 | } 177 | 178 | for (auto &thread : pool) { 179 | thread.join(); 180 | } 181 | } else { 182 | solverOp(-1, 0, _target->numPoses()); 183 | } 184 | 185 | if (_callback != nullptr) 186 | _callback(iter, _target, _debugPath); 187 | 188 | return true; 189 | } 190 | 191 | void WeightsSolver::initSolverData(WeightsSolver::WeightsFunctor &data) { 192 | // const auto rows = (_target->numVertices() * Vector3::SizeAtCompileTime); 193 | // const auto cols = _target->numBlendshapes() - 1; 194 | 195 | //data._a.resize(rows, cols); 196 | 197 | //data.c.resize(rows); 198 | //data.vDiff.resize(rows); 199 | 200 | data.estimateW.resize(_target->numBlendshapes() - 1); 201 | //data.wDiff.resize(_target->numBlendshapes() - 1); 202 | 203 | data.lambda = _lambda(_iteration); 204 | } 205 | 206 | void WeightsSolver::appendWeightFit(Index pose, MatrixX &a, VectorX &c) { 207 | appendWeightFit(pose, a); 208 | 209 | appendWeightFit(pose, c); 210 | } 211 | 212 | void WeightsSolver::appendWeightFit(Index pose, MatrixX &a) { 213 | for (Index bs = 1; bs < _target->numBlendshapes(); bs++) { 214 | appendWeightFit(pose, bs, a); 215 | } 216 | } 217 | 218 | void WeightsSolver::appendWeightFit(Index pose, Index bs, MatrixX &a) { 219 | auto blendshapeMesh = _target->blendshape(bs).mesh(); 220 | 221 | const auto col = bs - 1; 222 | 223 | for (auto v = 0; v < _target->numVertices(); v++) { 224 | const auto row = v * Eigen::Vector3d::SizeAtCompileTime; 225 | 226 | const auto &p = blendshapeMesh->point(blendshapeMesh->vertex_handle(_target->vertex(v))); 227 | 228 | for (auto i = 0; i < 3; i++) { 229 | a(row + i, col) = p[i]; 230 | } 231 | } 232 | } 233 | 234 | void WeightsSolver::appendWeightFit(Index pose, VectorX &c) { 235 | auto poseMesh = _target->pose(pose).mesh(); 236 | auto neutralMesh = _target->neutral(); 237 | 238 | for (auto v = 0; v < _target->numVertices(); v++) { 239 | const auto row = v * Eigen::Vector3d::SizeAtCompileTime; 240 | 241 | const auto idx = _target->vertex(v); 242 | 243 | const auto &np = neutralMesh->point(neutralMesh->vertex_handle(idx)); 244 | const auto &pp = poseMesh->point(poseMesh->vertex_handle(idx)); 245 | 246 | const auto p = pp - np; 247 | 248 | for (auto i = 0; i < 3; i++) { 249 | c(row + i) = p[i]; 250 | } 251 | } 252 | } 253 | 254 | void WeightsSolver::copyWeightsTo(Weights &weights, VectorX &x) { 255 | x.resize(weights.size() - 1); 256 | 257 | for (auto i = 1; i < weights.size(); i++) 258 | x(i - 1) = weights[i]; 259 | } 260 | 261 | void WeightsSolver::copyWeightsTo(VectorX &x, Weights &weights) { 262 | double diff = 0; 263 | 264 | for (auto i = 0; i < x.rows(); i++) { 265 | const auto w = x(i, 0); 266 | const auto d = w - weights[i]; 267 | diff += (d * d); 268 | 269 | weights[i + 1] = w; 270 | } 271 | 272 | if (_debug) { 273 | std::cout << "Copy Weights: " << std::endl; 274 | std::cout << "Diff: " << std::sqrt(diff) << std::endl; 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /src/ebfr/WeightsSolver.h: -------------------------------------------------------------------------------- 1 | // 2 | // WeightsSolver.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef WeightsSolver_hpp 10 | #define WeightsSolver_hpp 11 | 12 | #include "SolverBase.h" 13 | 14 | #include 15 | 16 | class WeightsSolver : public SolverBase { 17 | public: 18 | struct WeightsFunctor : Eigen::DenseFunctor { 19 | MatrixX *a; 20 | VectorX *c; 21 | //VectorX vDiff; 22 | 23 | VectorX estimateW; 24 | //VectorX wDiff; 25 | 26 | double lambda; 27 | 28 | VectorX x; 29 | 30 | int operator()(const Eigen::VectorXd &x, Eigen::VectorXd &fvec) const; 31 | 32 | int inputs() const; 33 | 34 | int values() const; 35 | }; 36 | 37 | struct WeightsFunctorNumericalDiff : Eigen::NumericalDiff { 38 | }; 39 | 40 | WeightsSolver(); 41 | 42 | void setLambda(const ParameterD &lambda); 43 | 44 | virtual void init(); 45 | 46 | virtual bool solve(int iter); 47 | 48 | private: 49 | const int _maxIterations; 50 | const double _minWeight; 51 | const double _maxWeight; 52 | 53 | ParameterD _lambda; 54 | 55 | std::vector _estimateWs; 56 | 57 | MatrixX _a; 58 | std::vector _Cs; 59 | 60 | StepCallback _callback; 61 | 62 | void initSolverData(WeightsFunctor &data); 63 | 64 | void appendWeightFit(Index Pose, MatrixX &a, VectorX &c); 65 | 66 | void appendWeightFit(Index pose, MatrixX &a); 67 | 68 | void appendWeightFit(Index pose, Index bs, MatrixX &a); 69 | 70 | void appendWeightFit(Index pose, VectorX &c); 71 | 72 | void copyWeightsTo(Weights &weights, VectorX &x); 73 | 74 | void copyWeightsTo(VectorX &x, Weights &weights); 75 | }; 76 | 77 | #endif /* WeightsSolver_hpp */ 78 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // main.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include 10 | 11 | #include "shared/FS.h" 12 | #include "shared/CSV.h" 13 | #include "shared/Timing.h" 14 | #include "shared/SolverUtil.h" 15 | 16 | #include "ebfr/Rig.h" 17 | #include "ebfr/BlendshapeSolver.h" 18 | 19 | #include "Args.h" 20 | 21 | void onVertexStep(int iter, RigPtr rig, const std::string &dir) { 22 | static MeshPtr temp; 23 | if (temp == nullptr) 24 | temp = MakeMesh(rig->neutral()); 25 | 26 | const std::string path = dir + "/bs-" + std::to_string(iter) + "-"; 27 | const std::string ext = ".obj"; 28 | 29 | for (auto bs = 0; bs < rig->numBlendshapes(); bs++) { 30 | AddVertices(rig->blendshape(bs).mesh(), rig->neutral(), 1.0, temp); 31 | 32 | WriteMesh(path + std::to_string(bs) + ext, temp); 33 | } 34 | } 35 | 36 | void onWeightsStep(int iter, RigPtr rig, const std::string &dir) { 37 | static MeshPtr temp; 38 | if (temp == nullptr) 39 | temp = MakeMesh(rig->neutral()); 40 | 41 | //TIMER_START(WritePoses); 42 | 43 | const std::string path = dir + "/pose-" + std::to_string(iter) + "-"; 44 | const std::string ext = ".obj"; 45 | 46 | for (auto pose = 0; pose < rig->numPoses(); pose++) { 47 | const auto &weights = rig->weights(pose); 48 | 49 | CopyVertices(temp, rig->neutral()); 50 | 51 | for (auto bs = 0; bs < rig->numBlendshapes(); bs++) { 52 | AddVertices(temp, rig->blendshape(bs).mesh(), weights[bs]); 53 | } 54 | 55 | WriteMesh(path + std::to_string(pose) + ext, temp); 56 | } 57 | 58 | PoseCSV::Write(JoinPath(dir, "weights-" + std::to_string(iter) + ".csv"), rig->weights()); 59 | 60 | //TIMER_END(WritePoses); 61 | } 62 | 63 | int main(int argc, char *argv[]) { 64 | Args args; 65 | args.read(argc, argv); 66 | 67 | Eigen::initParallel(); 68 | //Eigen::setNbThreads(4); 69 | 70 | TIMER_START(LoadSourceRig); 71 | 72 | auto sourceRig = MakeRig(); 73 | sourceRig->load(args.srcBlendshapeDir, args.srcPoseDir, args.srcWeightsPath, args.vertexMaskPath, false); 74 | 75 | TIMER_END(LoadSourceRig); 76 | 77 | TIMER_START(LoadTargetRig); 78 | 79 | auto targetRig = MakeRig(); 80 | targetRig->load(args.tgtNeutralPath, args.tgtPoseDir, args.tgtWeightsPath, args.vertexMaskPath, true); 81 | 82 | targetRig->generateEmptyBlendshapes(sourceRig->numBlendshapes()); 83 | 84 | TIMER_END(LoadTargetRig); 85 | 86 | const auto estWeights = targetRig->weights(); 87 | targetRig->randomizeWeights(); 88 | 89 | BlendshapeSolver solver; 90 | 91 | solver.setDebugPath(args.debugPath); 92 | solver.setVertexStepCallback(onVertexStep); 93 | solver.setWeightsStepCallback(onWeightsStep); 94 | 95 | solver.setMultithreaded(true); 96 | 97 | if (!solver.setSource(sourceRig)) { 98 | std::cerr << "Failed to set source" << std::endl; 99 | return 1; 100 | } 101 | 102 | if (!solver.setTarget(targetRig)) { 103 | std::cerr << "Failed to set target" << std::endl; 104 | return 1; 105 | } 106 | 107 | if (!solver.solve()) { 108 | std::cerr << "Failed to generate rigging" << std::endl; 109 | return 1; 110 | } 111 | 112 | std::cout << "Writing Final Blendshapes..." << std::endl; 113 | 114 | for (auto i = 1; i < targetRig->numBlendshapes(); i++) { 115 | WriteMesh(JoinPath(args.outputPath, std::to_string(i - 1) + ".obj"), targetRig->blendshape(i).mesh()); 116 | } 117 | 118 | std::cout << "Writing Final Poses..." << std::endl; 119 | 120 | for (auto pose = 0; pose < targetRig->numPoses(); pose++) { 121 | WriteMesh(JoinPath(args.outputPath, "pose-" + std::to_string(pose) + ".obj"), targetRig->generatePose(estWeights[pose])); 122 | } 123 | 124 | std::cout << "Writing Final Weights..." << std::endl; 125 | 126 | PoseCSV::Write(JoinPath(args.outputPath, "poses.csv"), targetRig->weights()); 127 | 128 | return 0; 129 | } 130 | -------------------------------------------------------------------------------- /src/shared/CSV.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // CSV.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "CSV.h" 10 | 11 | #include 12 | #include 13 | 14 | // trim from start (in place) 15 | static inline void ltrim(std::string &s) { 16 | s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { 17 | return !std::isspace(ch); 18 | })); 19 | } 20 | 21 | // trim from end (in place) 22 | static inline void rtrim(std::string &s) { 23 | s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { 24 | return !std::isspace(ch); 25 | }).base(), s.end()); 26 | } 27 | 28 | // trim from both ends (in place) 29 | static inline void trim(std::string &s) { 30 | ltrim(s); 31 | rtrim(s); 32 | } 33 | 34 | CSV::CSV() 35 | : _delim(',') { 36 | 37 | } 38 | 39 | bool CSV::open(const std::string &path) { 40 | _file.open(path); 41 | 42 | if (!_file.is_open()) 43 | return false; 44 | 45 | return readHeader(); 46 | } 47 | 48 | void CSV::close() { 49 | if (_file.is_open()) 50 | _file.close(); 51 | } 52 | 53 | bool CSV::next() { 54 | if (!_file.is_open()) 55 | return false; 56 | 57 | if (_file.eof()) 58 | return false; 59 | 60 | std::getline(_file, _line); 61 | 62 | std::istringstream ss; 63 | ss.str(_line); 64 | 65 | for (auto i = 0; !ss.eof() && i < _numColumns; i++) { 66 | std::getline(ss, _values[i], _delim); 67 | trim(_values[i]); 68 | } 69 | 70 | return true; 71 | } 72 | 73 | const std::string &CSV::value(int index) const { 74 | return _values[index]; 75 | } 76 | 77 | const std::vector &CSV::values() const { 78 | return _values; 79 | } 80 | 81 | bool CSV::values(const std::vector &names, std::vector &values) { 82 | if (_columns.empty()) 83 | return false; 84 | 85 | values.resize(names.size()); 86 | 87 | for (auto i = 0; i < names.size(); i++) 88 | values[i] = _values[_columns[names[i]]]; 89 | 90 | return true; 91 | } 92 | 93 | bool CSV::values(int start, int end, std::vector &values) { 94 | const auto len = end - start; 95 | 96 | values.resize(len); 97 | 98 | for (auto i = 0; i < len; i++) 99 | values[i] = _values[start + i]; 100 | 101 | return true; 102 | } 103 | 104 | bool CSV::readHeader() { 105 | if (_file.eof()) 106 | return false; 107 | 108 | std::getline(_file, _line); 109 | 110 | std::istringstream ss; 111 | ss.str(_line); 112 | 113 | std::string name; 114 | 115 | _numColumns = 0; 116 | 117 | for (auto i = 0; !ss.eof(); i++) { 118 | std::getline(ss, name, _delim); 119 | 120 | _columns[name] = i; 121 | _numColumns++; 122 | } 123 | 124 | _values.resize(_numColumns); 125 | 126 | return true; 127 | } 128 | 129 | bool PoseCSV::Write(const std::string &path, const std::vector> &weights) { 130 | std::ofstream csv; 131 | csv.open(path); 132 | 133 | // Header 134 | csv << "Pose,"; 135 | for (auto i = 0; i < weights.size(); i++) { 136 | csv << i << ","; 137 | } 138 | csv << std::endl; 139 | 140 | for (auto i = 0; i < weights.size(); i++) { 141 | if (i != 0) { 142 | csv << std::endl; 143 | } 144 | 145 | csv << i << ","; 146 | 147 | for (auto w : weights[i]) { 148 | csv << w << ","; 149 | } 150 | } 151 | 152 | csv.close(); 153 | 154 | return true; 155 | } 156 | 157 | PoseCSV::PoseCSV() 158 | : CSV() { 159 | 160 | } 161 | 162 | bool PoseCSV::open(const std::string &path) { 163 | return CSV::open(path); 164 | } 165 | 166 | bool PoseCSV::next() { 167 | return CSV::next(); 168 | } 169 | 170 | bool PoseCSV::values(std::string &name, std::vector &weights) { 171 | auto &v = CSV::values(); 172 | 173 | name = v[0]; 174 | 175 | weights.resize(v.size() - 1); 176 | 177 | for (auto i = 1; i < v.size(); i++) { 178 | weights[i - 1] = v[i].empty() ? 0.0 : std::stod(v[i]); 179 | } 180 | 181 | return true; 182 | } 183 | 184 | -------------------------------------------------------------------------------- /src/shared/CSV.h: -------------------------------------------------------------------------------- 1 | // 2 | // CSV.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef CSV_hpp 10 | #define CSV_hpp 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | class CSV { 20 | public: 21 | CSV(); 22 | 23 | bool open(const std::string &path); 24 | 25 | void close(); 26 | 27 | bool next(); 28 | 29 | const std::vector &values() const; 30 | 31 | const std::string &value(int index) const; 32 | 33 | bool values(const std::vector &names, std::vector &values); 34 | 35 | bool values(int start, int end, std::vector &values); 36 | 37 | private: 38 | char _delim; 39 | 40 | std::ifstream _file; 41 | 42 | int _numColumns; 43 | 44 | std::map _columns; 45 | 46 | std::string _line; 47 | 48 | std::vector _values; 49 | 50 | bool readHeader(); 51 | }; 52 | 53 | class PoseCSV : protected CSV { 54 | public: 55 | static bool Write(const std::string &path, const std::vector> &weights); 56 | 57 | PoseCSV(); 58 | 59 | bool open(const std::string &path); 60 | 61 | bool next(); 62 | 63 | bool values(std::string &name, std::vector &weights); 64 | }; 65 | 66 | #endif /* CSV_hpp */ 67 | -------------------------------------------------------------------------------- /src/shared/FS.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // FS.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "FS.h" 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | bool Exists(const std::string &path) { 20 | struct stat buffer; 21 | 22 | return (stat(path.c_str(), &buffer) == 0); 23 | } 24 | 25 | bool _ListDir(const std::string &dirPath, std::vector *files, std::vector *dirs, 26 | const std::string namePattern = "", bool namesOnly = false) { 27 | DIR *dirp = opendir(dirPath.c_str()); 28 | if (dirp == NULL) 29 | return false; 30 | 31 | struct dirent *dp; 32 | 33 | std::string basePath = dirPath.back() == '/' ? dirPath : (dirPath + "/"); 34 | 35 | const auto hasPattern = !namePattern.empty(); 36 | std::regex pattern(namePattern); 37 | 38 | while ((dp = readdir(dirp)) != NULL) { 39 | if (dp->d_namlen == 0) 40 | continue; 41 | 42 | std::vector *list = nullptr; 43 | 44 | switch (dp->d_type) { 45 | case DT_DIR: 46 | list = dirs; 47 | break; 48 | case DT_REG: 49 | list = files; 50 | break; 51 | } 52 | 53 | if (list != nullptr) { 54 | if (hasPattern) 55 | if (!std::regex_match(dp->d_name, pattern)) 56 | continue; 57 | 58 | if (namesOnly) 59 | list->push_back(dp->d_name); 60 | else 61 | list->push_back(basePath + dp->d_name); 62 | } 63 | } 64 | 65 | closedir(dirp); 66 | 67 | return true; 68 | } 69 | 70 | 71 | bool ListDir(const std::string &dirPath, std::vector &files, std::vector &dirs) { 72 | return _ListDir(dirPath, &files, &dirs); 73 | } 74 | 75 | bool ListFiles(const std::string &dirPath, std::vector &files, const std::string &ext) { 76 | auto success = _ListDir(dirPath, &files, nullptr); 77 | if (!success || ext.empty()) 78 | return success; 79 | 80 | auto endIter = 81 | std::remove_if( 82 | files.begin(), files.end(), 83 | [&ext](const std::string &s) { 84 | return !std::equal(ext.rbegin(), ext.rend(), s.rbegin()); 85 | }); 86 | 87 | const auto numRemoved = files.end() - endIter; 88 | 89 | files.resize(files.size() - numRemoved); 90 | 91 | std::sort(files.begin(), files.end()); 92 | 93 | return true; 94 | } 95 | 96 | bool ListFiles(const std::string &dirPath, std::vector &files) { 97 | const auto dir = BasePath(dirPath); 98 | const auto filePattern = Filename(dirPath); 99 | 100 | auto success = _ListDir(dir, &files, nullptr, filePattern); 101 | if (!success) 102 | return success; 103 | 104 | static std::regex numericSortPattern(".*/(\\d*)[-\\.].*"); 105 | 106 | std::sort(files.begin(), files.end(), 107 | [](const std::string &a, const std::string &b) { 108 | std::smatch aMatch; 109 | std::smatch bMatch; 110 | 111 | if (std::regex_search(a, aMatch, numericSortPattern) && aMatch.size() > 1 112 | && std::regex_search(b, bMatch, numericSortPattern) && bMatch.size() > 1) { 113 | return std::atoi(aMatch.str(1).c_str()) < std::atoi(bMatch.str(1).c_str()); 114 | } 115 | 116 | return a < b; 117 | }); 118 | 119 | return true; 120 | } 121 | 122 | bool ListDirs(const std::string &dirPath, std::vector &dirs) { 123 | return _ListDir(dirPath, nullptr, &dirs); 124 | } 125 | 126 | bool WriteWeights(const std::vector &weights, const std::string &path) { 127 | std::ofstream out; 128 | out.open(path); 129 | 130 | if (!out.is_open()) 131 | return false; 132 | 133 | out << weights.size() << std::endl; 134 | 135 | for (auto w = 0; w < weights.size(); w++) 136 | out << weights[w] << std::endl; 137 | 138 | out.close(); 139 | 140 | return true; 141 | } 142 | 143 | bool WriteWeights(const std::vector> &weights, const std::string &path) { 144 | if (weights.empty()) 145 | return false; 146 | 147 | if (weights[0].empty()) 148 | return false; 149 | 150 | std::ofstream out; 151 | out.open(path); 152 | 153 | if (!out.is_open()) 154 | return false; 155 | 156 | out << weights.size() << "," << weights[0].size() << std::endl; 157 | 158 | for (const auto &ws : weights) { 159 | for (auto w : ws) { 160 | out << w << std::endl; 161 | } 162 | } 163 | 164 | out.close(); 165 | 166 | return true; 167 | } 168 | 169 | //bool WriteFltFile(const ImageMap& img, const std::string& path, bool sparse, bool includeId) 170 | //{ 171 | // std::ofstream file; 172 | // file.open(path); 173 | // 174 | // if (!file.is_open()) 175 | // return false; 176 | // 177 | // if (sparse) 178 | // file << "S"; 179 | // else 180 | // file << "D"; 181 | // 182 | // if (includeId) 183 | // file << "I"; 184 | // 185 | // file << std::endl; 186 | // 187 | // const auto maxDepth = img.maxDepth(); 188 | // 189 | // file << img.width() << "," << img.height() << "," << maxDepth << ",3" << std::endl; 190 | // 191 | // for (auto j = 0; j < img.height(); j++) 192 | // { 193 | // for (auto i = 0; i < img.width(); i++) 194 | // { 195 | // const auto& list = img.get(i, j); 196 | // 197 | // for (auto n = 0; n < list.size(); n++) 198 | // { 199 | // const auto& entry = list[n]; 200 | // 201 | // if (sparse) 202 | // { 203 | // if (entry.p.isZero()) 204 | // continue; 205 | // 206 | // file << i << "," << j << "," << n; 207 | // 208 | // for (auto c = 0; c < 3; c++) 209 | // { 210 | // file << "," << entry.p[c]; 211 | // } 212 | // 213 | // if (includeId) 214 | // { 215 | // file << "," << entry.id; 216 | // } 217 | // 218 | // file << std::endl; 219 | // } 220 | // else 221 | // { 222 | // for (auto c = 0; c < 3; c++) 223 | // { 224 | // file << entry.p[c] << std::endl; 225 | // } 226 | // 227 | // if (includeId) 228 | // { 229 | // file << entry.id << std::endl; 230 | // } 231 | // } 232 | // } 233 | // 234 | // if (!sparse) 235 | // { 236 | // for (auto n = list.size(); n < maxDepth; n++) 237 | // { 238 | // for (auto c = 0; c < 3; c++) 239 | // { 240 | // file << "0" << std::endl; 241 | // } 242 | // 243 | // if (includeId) 244 | // { 245 | // file << "-1" << std::endl; 246 | // } 247 | // } 248 | // } 249 | // } 250 | // } 251 | // 252 | // file.close(); 253 | // 254 | // return true; 255 | //} 256 | // 257 | //bool ReadFltFile(ImageMap& img, const std::string& path) 258 | //{ 259 | // std::ifstream file; 260 | // file.open(path); 261 | // 262 | // if (!file.is_open()) 263 | // { 264 | // std::cerr << "Failed to open FLT file - [" << path << "]" << std::endl; 265 | // return false; 266 | // } 267 | // 268 | // char comma; 269 | // 270 | // char formatType, idType; 271 | // file >> formatType >> idType; 272 | // 273 | // int width, height, depth, components; 274 | // file >> width >> comma >> height >> comma >> depth >> comma >> components; 275 | // 276 | // img.resize(width, height); 277 | // 278 | // bool includesId = idType == 'I'; 279 | // 280 | // if (formatType == 'D') 281 | // { 282 | // for (auto j = 0; j < height; j++) 283 | // { 284 | // for (auto i = 0; i < width; i++) 285 | // { 286 | // for (auto d = 0; d < depth; d++) 287 | // { 288 | // ImageMap::Entry entry; 289 | // 290 | // for (auto c = 0; c < 3; c++) 291 | // { 292 | // file >> entry.p[c]; 293 | // } 294 | // 295 | // if (includesId) 296 | // file >> entry.id; 297 | // 298 | // img.push(i, j, entry.p, entry.id); 299 | // } 300 | // } 301 | // } 302 | // } 303 | // else 304 | // { 305 | // while (!file.eof()) 306 | // { 307 | // int x, y, n; 308 | // file >> x >> comma >> y >> comma >> n; 309 | // 310 | // ImageMap::Entry entry; 311 | // 312 | // for (auto c = 0; c < 3; c++) 313 | // file >> comma >> entry.p[c]; 314 | // 315 | // if (includesId) 316 | // file >> entry.id; 317 | // 318 | // img.push(x, y, entry.p, entry.id); 319 | // } 320 | // } 321 | // 322 | // return true; 323 | //} 324 | -------------------------------------------------------------------------------- /src/shared/FS.h: -------------------------------------------------------------------------------- 1 | // 2 | // FS.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef FS_hpp 10 | #define FS_hpp 11 | 12 | #include 13 | #include 14 | 15 | //#include "ImageMap.h" 16 | 17 | bool Exists(const std::string &path); 18 | 19 | bool ListAll(const std::string &dirPath, std::vector &files, std::vector &dirs); 20 | 21 | bool ListFiles(const std::string &dirPath, std::vector &files); 22 | 23 | bool ListDirs(const std::string &dirPath, std::vector &dirs); 24 | 25 | bool WriteWeights(const std::vector &weights, const std::string &path); 26 | 27 | bool WriteWeights(const std::vector> &weights, const std::string &path); 28 | 29 | //bool WriteFltFile(const ImageMap& img, const std::string& path, bool sparse, bool includeId = false); 30 | // 31 | //bool ReadFltFile(const ImageMap& img, const std::string& path); 32 | 33 | // The base case: we just have a single number. 34 | inline const std::string &JoinPath(const std::string &s) { 35 | return s; 36 | } 37 | 38 | // The recursive case: we take a number, alongside 39 | // some other numbers, and produce their sum. 40 | template 41 | inline std::string JoinPath(const std::string &t, Rest... rest) { 42 | if (t.empty()) 43 | return JoinPath(rest...); 44 | else { 45 | if (t[t.size() - 1] == '/') 46 | return t + JoinPath(rest...); 47 | else 48 | return t + "/" + JoinPath(rest...); 49 | } 50 | } 51 | 52 | inline std::string ReplaceExt(const std::string &path, const std::string &ext) { 53 | const auto index = path.find_last_of('.'); 54 | 55 | if (index == std::string::npos) 56 | return path + "." + ext; 57 | 58 | return path.substr(0, index + 1) + ext; 59 | } 60 | 61 | inline std::string RemoveExt(const std::string &path) { 62 | const auto index = path.find_last_of('.'); 63 | 64 | if (index == std::string::npos) 65 | return path; 66 | 67 | return path.substr(0, index); 68 | } 69 | 70 | inline std::string BasePath(const std::string &path) { 71 | auto index = path.find_last_of('/'); 72 | 73 | if (index == std::string::npos) 74 | return "/"; 75 | 76 | if (index == path.size() - 1) { 77 | index = path.find_last_of('/', index - 1); 78 | 79 | if (index == std::string::npos) 80 | return "/"; 81 | } 82 | 83 | return path.substr(0, index + 1); 84 | } 85 | 86 | inline std::string Filename(const std::string &path, bool withExtension = true) { 87 | const auto index = path.find_last_of('/'); 88 | 89 | if (index == std::string::npos) 90 | return path; 91 | 92 | if (!withExtension) { 93 | const auto extIndex = path.find_last_of('.'); 94 | 95 | return path.substr(index + 1, extIndex - index - 1); 96 | } 97 | 98 | return path.substr(index + 1, path.size() - index - 1); 99 | } 100 | 101 | #endif /* FS_hpp */ 102 | -------------------------------------------------------------------------------- /src/shared/Matrix.h: -------------------------------------------------------------------------------- 1 | // 2 | // Matrix.h 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Matrix_h 10 | #define Matrix_h 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | typedef Eigen::Triplet Triplet; 17 | typedef std::vector TripletList; 18 | 19 | typedef Eigen::SparseMatrix SparseMatrix; 20 | 21 | typedef Eigen::Matrix Matrix3x4; 22 | typedef Eigen::Matrix Matrix3x3; 23 | typedef Eigen::Matrix Matrix3x2; 24 | typedef Eigen::Matrix Matrix2x3; 25 | typedef Eigen::Matrix Matrix2x2; 26 | typedef Eigen::Matrix Matrix9x4; 27 | typedef Eigen::Matrix Matrix6x4; 28 | typedef Eigen::Matrix Matrix9x1; 29 | typedef Eigen::Matrix Matrix6x1; 30 | 31 | typedef Eigen::Vector3d Vector3; 32 | typedef Eigen::Vector4d Vector4; 33 | 34 | typedef Eigen::Matrix MatrixX; 35 | typedef Eigen::Matrix VectorX; 36 | 37 | 38 | #ifndef EPS 39 | #define EPS 0.0001 40 | #endif 41 | 42 | 43 | template 44 | void QRDecompose(const Eigen::Matrix &m, Eigen::Matrix &q, 45 | Eigen::Matrix &r) { 46 | Eigen::Matrix v; 47 | Eigen::Matrix qi; 48 | 49 | r.setZero(); 50 | 51 | for (int j = 0; j < n_cols; j++) { 52 | v = m.col(j); 53 | 54 | for (int i = 0; i < j; i++) { 55 | qi = q.col(i); 56 | 57 | r(i, j) = qi.dot(v); 58 | v = v - r(i, j) * qi; 59 | } 60 | 61 | auto l = v.norm(); 62 | 63 | if (l < EPS) { 64 | r(j, j) = 1; 65 | q.col(j).setZero(); 66 | } else { 67 | r(j, j) = l; 68 | q.col(j) = v / l; 69 | } 70 | } 71 | } 72 | 73 | template 74 | Eigen::Matrix Invert(const Eigen::Matrix &m) { 75 | Eigen::Matrix q; 76 | Eigen::Matrix r; 77 | 78 | QRDecompose(m, q, r); 79 | 80 | return r.inverse() * q.transpose(); 81 | } 82 | 83 | inline std::string Error(Eigen::ComputationInfo info) { 84 | switch (info) { 85 | case Eigen::Success: 86 | return "Success"; 87 | 88 | /** The provided data did not satisfy the prerequisites. */ 89 | case Eigen::NumericalIssue: 90 | return "Numerical Issue"; 91 | 92 | /** Iterative procedure did not converge. */ 93 | case Eigen::NoConvergence: 94 | return "No Convergence"; 95 | 96 | /** The inputs are invalid, or the algorithm has been improperly called. 97 | * When assertions are enabled, such errors trigger an assert. */ 98 | case Eigen::InvalidInput: 99 | return "Invalid Input"; 100 | } 101 | 102 | return "Unknown"; 103 | } 104 | 105 | #endif /* Matrix_h */ 106 | -------------------------------------------------------------------------------- /src/shared/Mesh.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Mesh.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "Mesh.h" 10 | 11 | MeshPtr ReadMesh(const std::string &path, bool exitOnFail) { 12 | auto mesh = MakeMesh(); 13 | 14 | Mesh &meshRef = *mesh; 15 | 16 | meshRef.request_face_normals(); 17 | meshRef.request_vertex_normals(); 18 | //meshRef.request_vertex_texcoord(); 19 | 20 | OpenMesh::IO::Options opts(OpenMesh::IO::Options::VertexNormal | OpenMesh::IO::Options::FaceNormal | 21 | OpenMesh::IO::Options::VertexTexCoord); 22 | if (!OpenMesh::IO::read_mesh(meshRef, path, opts)) { 23 | std::cerr << "Failed to read mesh at [" << path << "]" << std::endl; 24 | 25 | if (exitOnFail) 26 | exit(1); 27 | 28 | return nullptr; 29 | } 30 | 31 | meshRef.update_face_normals(); 32 | meshRef.update_vertex_normals(); 33 | 34 | return mesh; 35 | } 36 | 37 | bool WriteMesh(const std::string &path, MeshPtr mesh) { 38 | if (!OpenMesh::IO::write_mesh(*mesh, path)) { 39 | std::cerr << "Failed to write mesh to [" << path << "]" << std::endl; 40 | return false; 41 | } 42 | 43 | return true; 44 | } 45 | -------------------------------------------------------------------------------- /src/shared/Mesh.h: -------------------------------------------------------------------------------- 1 | // 2 | // Mesh.h 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Mesh_h 10 | #define Mesh_h 11 | 12 | #include 13 | 14 | #include 15 | 16 | #include 17 | 18 | #include 19 | 20 | // OpenMesh Mesh Traits 21 | // Use Double-based structures 22 | struct DoubleTraits : OpenMesh::DefaultTraits { 23 | typedef OpenMesh::Vec3d Point; 24 | typedef OpenMesh::Vec3d Normal; 25 | }; 26 | 27 | typedef OpenMesh::TriMesh_ArrayKernelT Mesh; 28 | 29 | typedef std::shared_ptr MeshPtr; 30 | 31 | inline MeshPtr MakeMesh(MeshPtr copy = nullptr) { 32 | auto mesh = std::make_shared(); 33 | if (copy != nullptr) 34 | *mesh = *copy; 35 | 36 | return mesh; 37 | } 38 | 39 | inline bool isNan(const OpenMesh::Vec3d &v) { 40 | return std::isnan(v[0]) || std::isnan(v[1]) || std::isnan(v[2]); 41 | } 42 | 43 | inline bool isZero(const OpenMesh::Vec3d &v) { 44 | return v[0] == 0.0 && v[1] == 0.0 && v[2] == 0.0; 45 | } 46 | 47 | inline bool isNearZero(const OpenMesh::Vec3d &v, double eps = 0.00001) { 48 | return std::abs(v[0]) < eps && std::abs(v[1]) < eps && std::abs(v[2]) < eps; 49 | } 50 | 51 | inline void FaceVertices(const Mesh &mesh, const Mesh::FaceHandle &face, Mesh::VertexHandle vertices[]) { 52 | size_t i = 0; 53 | for (auto vertIter = mesh.cfv_begin(face), vertEnd = mesh.cfv_end(face); 54 | i < 3 && vertIter != vertEnd; vertIter++, i++) { 55 | vertices[i] = (*vertIter); 56 | } 57 | } 58 | 59 | inline void Adjacent(const Mesh &mesh, const Mesh::FaceHandle &face, Mesh::FaceHandle adjacent[]) { 60 | size_t i = 0; 61 | for (auto adj_iter = mesh.cff_begin(face), adj_end = mesh.cff_end(face); 62 | i < 3 && adj_iter != adj_end; i++, adj_iter++) { 63 | adjacent[i] = *adj_iter; 64 | } 65 | } 66 | 67 | inline void Bounds(const Mesh &mesh, OpenMesh::Vec3d &min, OpenMesh::Vec3d &max) { 68 | min = mesh.point(mesh.vertex_handle(0)); 69 | max = min; 70 | 71 | for (auto vert_iter = mesh.vertices_begin(), vert_end = mesh.vertices_end(); vert_iter != vert_end; vert_iter++) { 72 | const auto &p = mesh.point(*vert_iter); 73 | 74 | min.minimize(p); 75 | max.maximize(p); 76 | } 77 | } 78 | 79 | inline void SetVertices(MeshPtr target, const Mesh::Point &p) { 80 | for (auto vert_iter = target->vertices_begin(), vert_end = target->vertices_end(); 81 | vert_iter != vert_end; vert_iter++) { 82 | const auto vert = *vert_iter; 83 | 84 | target->set_point(vert, p); 85 | } 86 | } 87 | 88 | inline void CopyVertices(MeshPtr target, MeshPtr source) { 89 | const auto *src = source->points(); 90 | 91 | for (auto vert_iter = target->vertices_begin(), vert_end = target->vertices_end(); 92 | vert_iter != vert_end; vert_iter++) { 93 | const auto vert = *vert_iter; 94 | 95 | target->set_point(vert, src[vert.idx()]); 96 | } 97 | } 98 | 99 | inline void CopyVertices(MeshPtr target, MeshPtr source, double weight) { 100 | const auto *src = source->points(); 101 | 102 | for (auto vert_iter = target->vertices_begin(), vert_end = target->vertices_end(); 103 | vert_iter != vert_end; vert_iter++) { 104 | const auto vert = *vert_iter; 105 | 106 | target->set_point(vert, weight * src[vert.idx()]); 107 | } 108 | } 109 | 110 | inline void AddVertices(MeshPtr base, MeshPtr modifier, double weight, MeshPtr dest = nullptr) { 111 | if (dest == nullptr) 112 | dest = base; 113 | 114 | const auto numVertices = std::min(base->n_vertices(), modifier->n_vertices()); 115 | for (auto i = 0; i < numVertices; i++) { 116 | const auto tVert = base->vertex_handle(i); 117 | const auto sVert = modifier->vertex_handle(i); 118 | const auto dVert = dest->vertex_handle(i); 119 | 120 | auto sp = modifier->point(sVert); 121 | sp *= weight; 122 | 123 | auto tp = base->point(tVert); 124 | tp += sp; 125 | 126 | if (isNearZero(tp)) { 127 | tp = Mesh::Point(0, 0, 0); 128 | } 129 | 130 | dest->point(dVert) = tp; 131 | } 132 | } 133 | 134 | MeshPtr ReadMesh(const std::string &path, bool exitOnFail = true); 135 | 136 | bool WriteMesh(const std::string &path, MeshPtr mesh); 137 | 138 | #endif /* Mesh_h */ 139 | -------------------------------------------------------------------------------- /src/shared/SolverUtil.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // SolverUtil.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "SolverUtil.h" 10 | 11 | #include "Util.h" 12 | 13 | void CalculateSurface(const Mesh &ref, const Mesh::FaceHandle &refFace, Matrix3x3 &s) { 14 | ConstructTriangleNormMatrix(ref, refFace, s); 15 | } 16 | 17 | void CalculateInvSurface(const Mesh &ref, const Mesh::FaceHandle &refFace, Matrix3x3 &inv) { 18 | Matrix3x3 Vr; 19 | CalculateSurface(ref, refFace, Vr); 20 | 21 | inv = Vr.inverse(); 22 | } 23 | 24 | void ConstructTriangleNormMatrix(const Mesh &mesh, const Mesh::FaceHandle &face, Matrix3x3 &v) { 25 | Mesh::VertexHandle vertices[3]; 26 | 27 | FaceVertices(mesh, face, vertices); 28 | 29 | const auto v0 = mesh.point(vertices[0]); 30 | const auto v1 = mesh.point(vertices[1]); 31 | const auto v2 = mesh.point(vertices[2]); 32 | 33 | const auto e0 = toEigen(v1 - v0); 34 | const auto e1 = toEigen(v2 - v0); 35 | if (e0.isZero() && e1.isZero()) { 36 | v.setZero(); 37 | } else { 38 | const auto n = e0.cross(e1).normalized(); 39 | 40 | v.col(0) = e0; 41 | v.col(1) = e1; 42 | v.col(2) = n; 43 | } 44 | } 45 | 46 | void CalculateFrames(MeshPtr mesh, std::vector &gradients) { 47 | gradients.resize(mesh->n_faces()); 48 | 49 | for (auto faceIter = mesh->faces_begin(), faceEnd = mesh->faces_end(); faceIter != faceEnd; faceIter++) { 50 | const auto face = *faceIter; 51 | 52 | CalculateSurface(*mesh, face, gradients[face.idx()]); 53 | } 54 | } 55 | 56 | void CalculateInvFrames(MeshPtr mesh, std::vector &gradients) { 57 | gradients.resize(mesh->n_faces()); 58 | 59 | for (auto faceIter = mesh->faces_begin(), faceEnd = mesh->faces_end(); faceIter != faceEnd; faceIter++) { 60 | const auto face = *faceIter; 61 | 62 | CalculateInvSurface(*mesh, face, gradients[face.idx()]); 63 | } 64 | } 65 | 66 | void GenerateEmptyFrames(MeshPtr mesh, std::vector &gradients) { 67 | gradients.resize(mesh->n_faces()); 68 | 69 | for (auto &m : gradients) { 70 | m.setZero(); 71 | } 72 | } -------------------------------------------------------------------------------- /src/shared/SolverUtil.h: -------------------------------------------------------------------------------- 1 | // 2 | // SolverUtil.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef SolverUtil_hpp 10 | #define SolverUtil_hpp 11 | 12 | #include "Mesh.h" 13 | #include "Matrix.h" 14 | 15 | void CalculateSurface(const Mesh &ref, const Mesh::FaceHandle &refFace, Matrix3x3 &s); 16 | 17 | void CalculateInvSurface(const Mesh &ref, const Mesh::FaceHandle &refFace, Matrix3x3 &inv); 18 | 19 | void ConstructTriangleNormMatrix(const Mesh &mesh, const Mesh::FaceHandle &face, Matrix3x3 &v); 20 | 21 | void CalculateFrames(MeshPtr mesh, std::vector &gradients); 22 | 23 | void CalculateInvFrames(MeshPtr mesh, std::vector &gradients); 24 | 25 | void GenerateEmptyFrames(MeshPtr, std::vector &gradients); 26 | 27 | #endif /* SolverUtil_hpp */ 28 | -------------------------------------------------------------------------------- /src/shared/Timing.h: -------------------------------------------------------------------------------- 1 | // 2 | // Timing.h 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Timing_h 10 | #define Timing_h 11 | 12 | #include 13 | #include 14 | 15 | #define TIMER_START(NAME) \ 16 | std::chrono::high_resolution_clock::time_point __TIMER##NAME##_START = std::chrono::high_resolution_clock::now(); 17 | 18 | 19 | #define TIMER_END(NAME) \ 20 | std::chrono::high_resolution_clock::time_point __TIMER##NAME##_END = std::chrono::high_resolution_clock::now(); \ 21 | { \ 22 | auto d = std::chrono::duration_cast( __TIMER##NAME##_END - __TIMER##NAME##_START ).count(); \ 23 | std::cout << "Timer " << #NAME << ": " << (d / 1000.0) << "s" << std::endl; \ 24 | } 25 | 26 | #endif /* Timing_h */ 27 | -------------------------------------------------------------------------------- /src/shared/Util.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Util.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include "Util.h" 10 | -------------------------------------------------------------------------------- /src/shared/Util.h: -------------------------------------------------------------------------------- 1 | // 2 | // Util.hpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #ifndef Util_hpp 10 | #define Util_hpp 11 | 12 | #include "Mesh.h" 13 | #include "Matrix.h" 14 | 15 | inline Vector3 toEigen(const OpenMesh::Vec3d &v) { 16 | return Vector3(v[0], v[1], v[2]); 17 | } 18 | 19 | #endif /* Util_hpp */ 20 | -------------------------------------------------------------------------------- /src/test/gradient.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // gradient.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include 10 | #include 11 | 12 | #include "../shared/Timing.h" 13 | #include "../shared/SolverUtil.h" 14 | 15 | #include "../ebfr/Rig.h" 16 | #include "../ebfr/BlendshapeSolver.h" 17 | 18 | #include "../Args.h" 19 | 20 | int main(int argc, char *argv[]) { 21 | Args args; 22 | args.read(argc, argv); 23 | 24 | Eigen::initParallel(); 25 | //Eigen::setNbThreads(4); 26 | 27 | TIMER_START(LoadSourceRig); 28 | 29 | auto sourceRig = MakeRig(); 30 | sourceRig->load(args.srcBlendshapeDir, args.srcPoseDir, args.srcWeightsPath, args.vertexMaskPath, false); 31 | 32 | TIMER_END(LoadSourceRig); 33 | 34 | TIMER_START(LoadTargetRig); 35 | 36 | auto targetRig = MakeRig(); 37 | targetRig->load(args.tgtNeutralPath, args.tgtPoseDir, args.tgtWeightsPath, args.vertexMaskPath, true); 38 | 39 | targetRig->generateEmptyBlendshapes(sourceRig->numBlendshapes()); 40 | 41 | TIMER_END(LoadTargetRig); 42 | 43 | BlendshapeSolver solver; 44 | 45 | solver.setDebugPath(args.debugPath); 46 | 47 | solver.setMultithreaded(true); 48 | 49 | if (!solver.setSource(sourceRig)) { 50 | std::cerr << "Failed to set source" << std::endl; 51 | return 1; 52 | } 53 | 54 | if (!solver.setTarget(targetRig)) { 55 | std::cerr << "Failed to set target" << std::endl; 56 | return 1; 57 | } 58 | 59 | for (auto iter = 0; iter < 5; iter++) { 60 | const auto iterWeight = (iter + 1) / 5.0f; 61 | 62 | std::cout << "weight: " << iterWeight << std::endl; 63 | 64 | for (auto i = 0; i < sourceRig->numPoses(); i++) { 65 | auto &srcWeights = sourceRig->weights(i); 66 | auto &targetWeights = targetRig->weights(i); 67 | 68 | std::cout << "Pose " << i << ": "; 69 | 70 | for (auto j = 0; j < srcWeights.size(); j++) { 71 | targetWeights[j] = srcWeights[j] * iterWeight; 72 | std::cout << srcWeights[j] << ", "; 73 | } 74 | 75 | std::cout << std::endl; 76 | } 77 | 78 | solver.testGradient(iter); 79 | 80 | auto face = solver.getTarget()->face(0); 81 | 82 | std::cout << "Expected: " << std::endl << solver.getSourceGradients()->blendshapeM[1][face] << std::endl 83 | << "- - -" << std::endl 84 | << "Result: " << std::endl << solver.getTargetGradients()->blendshapeM[1][face] << std::endl 85 | << std::endl << std::endl; 86 | 87 | std::cout << "Expected: " << std::endl << solver.getSourceGradients()->blendshapeM[2][face] << std::endl 88 | << "- - -" << std::endl 89 | << "Result: " << std::endl << solver.getTargetGradients()->blendshapeM[2][face] << std::endl 90 | << std::endl; 91 | } 92 | 93 | return 0; 94 | } 95 | -------------------------------------------------------------------------------- /src/test/posegen.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // posegen.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "../shared/Timing.h" 15 | #include "../shared/SolverUtil.h" 16 | 17 | #include "../ebfr/Rig.h" 18 | 19 | #include 20 | 21 | int main(int argc, char *argv[]) { 22 | cxxopts::Options options("posegen", "Generate poses from blendshapes"); 23 | 24 | options.add_options() 25 | ("blendshapes", "Path to the source reference mesh", cxxopts::value>()) 26 | ("num", "Path to the source reference mesh", cxxopts::value()) 27 | ("output", "Path to save the deformed target mesh to", cxxopts::value()); 28 | 29 | std::vector blendshapeDirs; 30 | std::string outputPath; 31 | int numPoses; 32 | 33 | try { 34 | auto result = options.parse(argc, argv); 35 | 36 | if (!result.count("blendshapes") || !result.count("num") || !result.count("output")) { 37 | std::cout << options.help() << std::endl; 38 | exit(1); 39 | } 40 | 41 | blendshapeDirs = result["blendshapes"].as>(); 42 | numPoses = result["num"].as(); 43 | outputPath = result["output"].as(); 44 | } 45 | catch (const cxxopts::OptionException &e) { 46 | std::cout << "error parsing options: " << e.what() << std::endl; 47 | exit(1); 48 | } 49 | 50 | std::random_device rd{}; 51 | std::mt19937 gen{rd()}; 52 | 53 | std::vector>> groups = { 54 | {{27, 28}, {0, 1}}, // Eyes 55 | {{24, 25}}, // Nose 56 | {{22}, {23}, {17}, {16}}, // Cheeks/Mouth 57 | {{20, 21}, {18, 19}, {14, 15}, {13}, {12}, {11}, {10}, {9}, {4, 5}}, // Mouths 58 | {{9}, {7, 8}, {6}}, // Lower Jaw/Mouth 59 | {{2, 3}} // Brows 60 | }; 61 | 62 | std::vector> remaining; 63 | 64 | remaining.resize(groups.size()); 65 | for (auto i = 0; i < groups.size(); i++) { 66 | const auto num = groups[i].size(); 67 | 68 | auto &list = remaining[i]; 69 | 70 | list.resize(num); 71 | for (auto j = 0; j < num; j++) { 72 | list[j] = j; 73 | } 74 | 75 | std::shuffle(list.begin(), list.end(), gen); 76 | } 77 | 78 | std::normal_distribution<> n{0.6, 0.4}; 79 | 80 | std::vector rigs; 81 | for (const auto &blendshapeDir : blendshapeDirs) { 82 | std::cout << rigs.size() << ": " << blendshapeDir << std::endl; 83 | 84 | std::filesystem::create_directories(outputPath + "/" + std::to_string(rigs.size())); 85 | 86 | auto rig = MakeRig(); 87 | 88 | rig->loadBlendshapes(blendshapeDir); 89 | 90 | rigs.push_back(rig); 91 | } 92 | 93 | std::vector weights(rigs[0]->numBlendshapes() - 1); 94 | 95 | std::ofstream poseWeights; 96 | poseWeights.open(outputPath + "/random.csv", std::ofstream::out | std::ofstream::app); 97 | 98 | poseWeights << "Pose,"; 99 | for (auto i = 0; i < weights.size(); i++) { 100 | poseWeights << i << ","; 101 | } 102 | poseWeights << std::endl; 103 | 104 | for (auto i = 0; i < numPoses; i++) { 105 | for (auto &w : weights) { 106 | w = 0.0; 107 | } 108 | 109 | for (auto j = 0; j < groups.size(); j++) { 110 | const auto &list = groups[j]; 111 | 112 | int index = 0; 113 | 114 | if (remaining[j].empty()) { 115 | std::uniform_int_distribution<> r(-1, list.size() - 1); 116 | 117 | index = r(gen); 118 | } else { 119 | index = remaining[j].back(); 120 | remaining[j].pop_back(); 121 | } 122 | 123 | if (index == -1) { 124 | continue; 125 | } 126 | 127 | const auto &group = list[index]; 128 | 129 | for (auto bs : group) { 130 | weights[bs] = std::clamp(n(gen), 0.25, 1.0); 131 | } 132 | } 133 | 134 | if (i != 0) { 135 | poseWeights << std::endl; 136 | } 137 | 138 | poseWeights << i << ","; 139 | 140 | for (auto w : weights) { 141 | poseWeights << w << ","; 142 | } 143 | 144 | for (auto r = 0; r < rigs.size(); r++) { 145 | auto pose = rigs[r]->generatePose(weights); 146 | 147 | WriteMesh(outputPath + "/" + std::to_string(r) + "/" + std::to_string(i) + ".obj", pose); 148 | } 149 | } 150 | 151 | return 0; 152 | } 153 | -------------------------------------------------------------------------------- /src/test/vertex.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // vertex.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include 10 | #include 11 | 12 | #include "../shared/Timing.h" 13 | #include "../shared/SolverUtil.h" 14 | 15 | #include "../ebfr/Rig.h" 16 | #include "../ebfr/BlendshapeSolver.h" 17 | 18 | #include "../Args.h" 19 | 20 | void onVertexStep(int iter, RigPtr rig, const std::string &dir) { 21 | static MeshPtr temp; 22 | if (temp == nullptr) 23 | temp = MakeMesh(rig->neutral()); 24 | 25 | std::cout 26 | << std::endl 27 | << "Writing Blendshapes [" << iter << "]" 28 | << std::endl; 29 | 30 | TIMER_START(WriteBlendshapes); 31 | 32 | const std::string path = dir + "/bs-" + std::to_string(iter) + "-"; 33 | const std::string ext = ".obj"; 34 | 35 | for (auto bs = 0; bs < rig->numBlendshapes(); bs++) { 36 | AddVertices(rig->blendshape(bs).mesh(), rig->neutral(), 1.0, temp); 37 | 38 | WriteMesh(path + std::to_string(bs) + ext, temp); 39 | } 40 | 41 | TIMER_END(WriteBlendshapes); 42 | } 43 | 44 | int main(int argc, char *argv[]) { 45 | Args args; 46 | args.read(argc, argv); 47 | 48 | Eigen::initParallel(); 49 | //Eigen::setNbThreads(4); 50 | 51 | TIMER_START(LoadSourceRig); 52 | 53 | auto sourceRig = MakeRig(); 54 | sourceRig->load(args.srcBlendshapeDir, args.srcPoseDir, args.srcWeightsPath, args.vertexMaskPath, false); 55 | 56 | TIMER_END(LoadSourceRig); 57 | 58 | TIMER_START(LoadTargetRig); 59 | 60 | auto targetRig = MakeRig(); 61 | targetRig->load(args.tgtNeutralPath, args.tgtPoseDir, args.tgtWeightsPath, args.vertexMaskPath, true); 62 | 63 | targetRig->generateEmptyBlendshapes(sourceRig->numBlendshapes()); 64 | 65 | TIMER_END(LoadTargetRig); 66 | 67 | // Use source weights as the estimated weights for target 68 | // for (auto i = 0; i < targetRig->numPoses(); i++) 69 | // { 70 | // auto& targetPose = targetRig->pose(i); 71 | // auto& sourcePose = sourceRig->pose(i); 72 | // 73 | // std::copy(sourcePose.weights().begin(), sourcePose.weights().end(), std::back_inserter(targetPose.weights())); 74 | // } 75 | 76 | BlendshapeSolver solver; 77 | 78 | solver.setDebugPath(args.debugPath); 79 | solver.setVertexStepCallback(onVertexStep); 80 | 81 | solver.setMultithreaded(true); 82 | 83 | if (!solver.setSource(sourceRig)) { 84 | std::cerr << "Failed to set source" << std::endl; 85 | return 1; 86 | } 87 | 88 | if (!solver.setTarget(targetRig)) { 89 | std::cerr << "Failed to set target" << std::endl; 90 | return 1; 91 | } 92 | 93 | auto target = solver.getTargetGradients(); 94 | auto neutral = sourceRig->neutral(); 95 | auto btemp = MakeMesh(neutral); 96 | 97 | for (auto i = 0; i < sourceRig->numBlendshapes(); i++) { 98 | const auto mesh = sourceRig->blendshape(i).mesh(); 99 | auto &m = target->blendshapeM[i]; 100 | 101 | if (i == 0) { 102 | CalculateFrames(mesh, m); 103 | } else { 104 | AddVertices(mesh, neutral, 1, btemp); 105 | 106 | CalculateFrames(btemp, m); 107 | 108 | const auto &neutralM = target->blendshapeM[0]; 109 | 110 | for (auto j = 0; j < mesh->n_faces(); j++) { 111 | m[j] -= neutralM[j]; 112 | } 113 | } 114 | } 115 | 116 | solver.testVertex(0, 0); 117 | 118 | auto bs = 5; 119 | 120 | solver.testVertex(10, bs); 121 | 122 | auto neutralPoints = solver.getSource()->blendshape(0).mesh()->points(); 123 | auto points = solver.getSource()->blendshape(bs).mesh()->points(); 124 | auto v = 0; 125 | v = solver.getSource()->vertex(0); 126 | std::cout << v << ": " << points[v] << " :: " << points[v] + neutralPoints[v] << std::endl; 127 | v = solver.getSource()->vertex(1); 128 | std::cout << v << ": " << points[v] << " :: " << points[v] + neutralPoints[v] << std::endl; 129 | v = solver.getSource()->vertex(2); 130 | std::cout << v << ": " << points[v] << " :: " << points[v] + neutralPoints[v] << std::endl; 131 | 132 | return 0; 133 | } 134 | -------------------------------------------------------------------------------- /src/test/weights.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // weights.cpp 3 | // ExampleBasedFacialRigging 4 | // 5 | // Created by Kyle on 5/1/21. 6 | // Copyright © 2021 Kyle. All rights reserved. 7 | // 8 | 9 | #include 10 | #include 11 | 12 | #include "../shared/Timing.h" 13 | #include "../shared/SolverUtil.h" 14 | 15 | #include "../ebfr/Rig.h" 16 | #include "../ebfr/BlendshapeSolver.h" 17 | 18 | #include "../Args.h" 19 | 20 | void onWeightsStep(int iter, RigPtr rig, const std::string &dir) { 21 | static MeshPtr temp; 22 | if (temp == nullptr) 23 | temp = MakeMesh(rig->neutral()); 24 | 25 | std::cout 26 | << std::endl 27 | << "Writing Poses [" << iter << "]" 28 | << std::endl; 29 | 30 | TIMER_START(WritePoses); 31 | 32 | const std::string path = dir + "/pose-" + std::to_string(iter) + "-"; 33 | const std::string ext = ".obj"; 34 | 35 | for (auto pose = 0; pose < rig->numPoses(); pose++) { 36 | const auto &weights = rig->weights(pose); 37 | 38 | CopyVertices(temp, rig->neutral()); 39 | 40 | for (auto bs = 0; bs < rig->numBlendshapes(); bs++) { 41 | AddVertices(temp, rig->blendshape(bs).mesh(), weights[bs]); 42 | } 43 | 44 | WriteMesh(path + std::to_string(pose) + ext, temp); 45 | } 46 | 47 | std::ofstream file; 48 | file.open(dir + "/weights-" + std::to_string(iter) + ".csv"); 49 | 50 | for (auto pose = 0; pose < rig->numPoses(); pose++) { 51 | const auto &weights = rig->weights(pose); 52 | 53 | file << pose; 54 | 55 | for (auto w : weights) 56 | file << ", " << w; 57 | 58 | file << std::endl; 59 | } 60 | 61 | file.close(); 62 | 63 | TIMER_END(WritePoses); 64 | } 65 | 66 | int main(int argc, char *argv[]) { 67 | Args args; 68 | args.read(argc, argv); 69 | 70 | Eigen::initParallel(); 71 | //Eigen::setNbThreads(4); 72 | 73 | TIMER_START(LoadSourceRig); 74 | 75 | auto sourceRig = MakeRig(); 76 | sourceRig->load(args.srcBlendshapeDir, args.srcPoseDir, args.srcWeightsPath, args.vertexMaskPath, false); 77 | 78 | TIMER_END(LoadSourceRig); 79 | 80 | TIMER_START(LoadTargetRig); 81 | 82 | auto targetRig = MakeRig(); 83 | targetRig->load(args.tgtNeutralPath, args.tgtPoseDir, args.tgtWeightsPath, args.vertexMaskPath, true); 84 | 85 | targetRig->generateEmptyBlendshapes(sourceRig->numBlendshapes()); 86 | 87 | TIMER_END(LoadTargetRig); 88 | 89 | // Use source weights as the estimated weights for target 90 | // for (auto i = 0; i < targetRig->numPoses(); i++) 91 | // { 92 | // auto& targetPose = targetRig->pose(i); 93 | // auto& sourcePose = sourceRig->pose(i); 94 | // 95 | // std::copy(sourcePose.weights().begin(), sourcePose.weights().end(), std::back_inserter(targetPose.weights())); 96 | // } 97 | 98 | BlendshapeSolver solver; 99 | 100 | solver.setDebugPath(args.debugPath); 101 | solver.setWeightsStepCallback(onWeightsStep); 102 | 103 | solver.setMultithreaded(true); 104 | 105 | if (!solver.setSource(sourceRig)) { 106 | std::cerr << "Failed to set source" << std::endl; 107 | return 1; 108 | } 109 | 110 | if (!solver.setTarget(targetRig)) { 111 | std::cerr << "Failed to set target" << std::endl; 112 | return 1; 113 | } 114 | 115 | for (auto i = 0; i < sourceRig->numPoses(); i++) { 116 | targetRig->pose(i).setMesh(sourceRig->pose(i).mesh()); 117 | 118 | auto &srcWeights = sourceRig->weights(i); 119 | auto &targetWeights = targetRig->weights(i); 120 | 121 | for (auto j = 1; j < srcWeights.size(); j++) { 122 | targetWeights[j] = srcWeights[j] * 0.2; 123 | } 124 | } 125 | 126 | for (auto i = 0; i < sourceRig->numBlendshapes(); i++) { 127 | targetRig->blendshape(i).setMesh(sourceRig->blendshape(i).mesh()); 128 | } 129 | 130 | solver.testWeights(0); 131 | 132 | solver.testWeights(1); 133 | 134 | for (auto i = 0; i < sourceRig->numPoses(); i++) { 135 | auto &srcWeights = sourceRig->weights(i); 136 | auto &targetWeights = targetRig->weights(i); 137 | 138 | std::cout << "Pose: " << i << std::endl; 139 | 140 | for (auto j = 0; j < srcWeights.size(); j++) { 141 | std::cout << srcWeights[j] << " -> " << targetWeights[j] << std::endl; 142 | } 143 | 144 | std::cout << std::endl; 145 | break; 146 | } 147 | 148 | return 0; 149 | } 150 | --------------------------------------------------------------------------------