├── .gitignore ├── Dockerfile ├── FitAdam ├── CMakeLists.txt ├── Shaders │ ├── DepthMapRender.fragmentshader │ ├── DepthMapRender.vertexshader │ ├── Mesh.fragmentshader │ ├── Mesh.vertexshader │ ├── Mesh_texture.fragmentshader │ ├── Mesh_texture.vertexshader │ ├── Mesh_texture_shader.fragmentshader │ ├── Mesh_texture_shader.vertexshader │ ├── NormalMap.fragmentshader │ ├── NormalMap.vertexshader │ ├── Project2D.fragmentshader │ ├── Project2D.vertexshader │ ├── ProjectorOnTexture.fragmentshader │ ├── ProjectorOnTexture.vertexshader │ ├── ProjectorSimulation.fragmentshader │ ├── ProjectorSimulation.vertexshader │ ├── Selection.fragmentshader │ ├── Selection.vertexshader │ ├── SimpleTransform.vertexshader │ ├── SimplestShader.fragmentshader │ ├── SimplestShader.vertexshader │ ├── SingleColor.fragmentshader │ ├── StandardLighting.fragmentshader │ ├── StandardLighting.vertexshader │ ├── TextureFragmentShader.fragmentshader │ └── TransformVertexShader.vertexshader ├── bvh.cpp ├── cmake │ └── Modules │ │ └── FindGFlags.cmake ├── include │ ├── AdamFastCost.h │ ├── BVHWriter.h │ ├── CMeshModelInstance.h │ ├── DCTCost.h │ ├── FKDerivative.h │ ├── FitCost.h │ ├── FitToBody.h │ ├── HandFastCost.h │ ├── KinematicModel.h │ ├── ModelFitter.h │ ├── Renderer.h │ ├── SGSmooth.hpp │ ├── VisualizedData.h │ ├── handm.h │ ├── json │ │ ├── autolink.h │ │ ├── config.h │ │ ├── features.h │ │ ├── forwards.h │ │ ├── json.h │ │ ├── reader.h │ │ ├── value.h │ │ └── writer.h │ ├── meshTrackingProj.h │ ├── pose_to_transforms.h │ ├── rply.h │ ├── rplyfile.h │ ├── simple.h │ ├── totalmodel.h │ └── utils.h ├── model │ ├── adam_v1_plus2.json │ ├── correspondences_nofeet.txt │ ├── mesh_nofeet.obj │ ├── nofeetmesh_byTomas_bottom.obj │ └── regressor_0n1_root.json ├── pythonWrapper.cpp ├── run_fitting.cpp └── src │ ├── AdamFastCost.cpp │ ├── BVHWriter.cpp │ ├── CMeshModelInstance.cpp │ ├── DCTCost.cpp │ ├── FKDerivative.cpp │ ├── FitToBody.cpp │ ├── HandFastCost.cpp │ ├── KinematicModel.cpp │ ├── ModelFitter.cpp │ ├── Renderer.cpp │ ├── SGSmooth.cpp │ ├── handm.cpp │ ├── lib_json │ ├── json_batchallocator.h │ ├── json_internalarray.inl │ ├── json_internalmap.inl │ ├── json_reader.cpp │ ├── json_value.cpp │ ├── json_valueiterator.inl │ ├── json_writer.cpp │ └── sconscript │ ├── librply.so │ ├── meshTrackingProj.cpp │ ├── pose_to_transforms.cpp │ ├── simple.cpp │ ├── totalmodel.cpp │ └── utils.cpp ├── POF ├── calib.json ├── collect_openpose.py ├── data │ ├── Base2DReader.py │ ├── BaseReader.py │ ├── COCOReader.py │ ├── DomeReader.py │ ├── DomeReaderTempConst.py │ ├── GAneratedReader.py │ ├── HumanReader.py │ ├── MultiDataset.py │ ├── OpenposeReader.py │ ├── RHDReader.py │ ├── STBReader.py │ ├── TempConstReader.py │ ├── TsimonDBReader.py │ ├── collect_a4.py │ ├── collect_crop_hand.py │ ├── collect_openpose.py │ ├── collect_stb.py │ └── process_MPII_mask.py ├── nets │ └── CPM.py ├── save_total_sequence.py ├── training_PAF_hand.py ├── training_e2e_PAF.py └── utils │ ├── AdamModel.py │ ├── EvalUtil.py │ ├── PAF.py │ ├── default_PAF_length.py │ ├── default_PAF_lengths.json │ ├── general.py │ ├── handSegNet.py │ ├── keypoint_conversion.py │ ├── load_ckpt.py │ ├── meshWrapper.py │ ├── multigpu.py │ ├── ops.py │ ├── optimization.py │ ├── smoothing.py │ ├── vis_heatmap3d.py │ └── wrapper_hand_model.py ├── README.md ├── data └── .gitignore ├── download.sh └── run_pipeline.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *__pycache__/ 2 | POF/snapshots 3 | POF/mtc_snapshots.zip 4 | */build/ 5 | -------------------------------------------------------------------------------- /FitAdam/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8) 2 | project(FitAdam) 3 | 4 | SET(CMAKE_C_COMPILER ${CMAKE_CXX_COMPILER}) 5 | SET(CMAKE_CXX_STANDARD 11) 6 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 7 | 8 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 9 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wall -Wextra") 10 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 11 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 12 | 13 | set(IGL_INCLUDE_DIRS "/home/donglaix/tools/libigl/include/") 14 | include_directories(${IGL_INCLUDE_DIRS}) 15 | find_package(OpenCV 2.4 REQUIRED) 16 | find_package( PkgConfig ) 17 | pkg_check_modules( EIGEN3 REQUIRED eigen3 ) 18 | find_package(OpenMP REQUIRED) 19 | include_directories(${EIGEN3_INCLUDE_DIRS}) 20 | find_package(Ceres REQUIRED COMPONENTS SuiteSparse) 21 | include_directories(${CERES_INCLUDE_DIRS}) 22 | include_directories("include") 23 | find_package(OpenGL REQUIRED) 24 | find_package(GLUT REQUIRED) 25 | find_package(GLEW REQUIRED) 26 | include_directories( ${OPENGL_INCLUDE_DIRS} ${GLUT_INCLUDE_DIRS} ${GLEW_INCLUDE_DIRS}) 27 | find_package(Boost COMPONENTS system filesystem REQUIRED) 28 | include_directories( ${Boost_INCLUDE_DIRS} ) 29 | 30 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") 31 | find_package(GFlags REQUIRED) 32 | 33 | file(GLOB JSON_SOURCES "src/lib_json/*.cpp") 34 | file(GLOB SOURCES "src/*.cpp") 35 | add_library(MYLIB ${SOURCES} ${JSON_SOURCES}) 36 | 37 | add_executable(run_fitting run_fitting.cpp) 38 | target_link_libraries(run_fitting MYLIB) 39 | target_link_libraries(run_fitting ${OpenCV_LIBS}) 40 | target_link_libraries(run_fitting ${CERES_LIBRARIES}) 41 | target_link_libraries(run_fitting ${OPENGL_LIBRARIES} ${GLUT_LIBRARY} ${GLEW_LIBRARY}) 42 | target_link_libraries(run_fitting ${GFLAGS_LIBRARY} ${Boost_LIBRARIES}) 43 | 44 | add_executable(bvh bvh.cpp) 45 | target_link_libraries(bvh MYLIB) 46 | target_link_libraries(bvh ${OpenCV_LIBS}) 47 | target_link_libraries(bvh ${CERES_LIBRARIES}) 48 | target_link_libraries(bvh ${OPENGL_LIBRARIES} ${GLUT_LIBRARY} ${GLEW_LIBRARY}) 49 | target_link_libraries(bvh ${GFLAGS_LIBRARY} ${Boost_LIBRARIES}) 50 | 51 | add_library(PythonWrapper SHARED pythonWrapper.cpp) 52 | target_link_libraries( PythonWrapper MYLIB ) 53 | target_link_libraries( PythonWrapper ${OpenCV_LIBS} ) 54 | target_link_libraries( PythonWrapper ${CERES_LIBRARIES}) 55 | target_link_libraries( PythonWrapper ${OPENGL_LIBRARIES} ${GLUT_LIBRARY} ${GLEW_LIBRARY}) 56 | -------------------------------------------------------------------------------- /FitAdam/Shaders/DepthMapRender.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | void main() 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /FitAdam/Shaders/DepthMapRender.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | //in vec3 vertex_color; 5 | 6 | //out float depthValue; 7 | //out vec3 color; 8 | 9 | uniform mat4 MVP; 10 | 11 | void main() 12 | { 13 | gl_Position = MVP * vec4(vertex_pos,1); 14 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 core 2 | out vec4 color; 3 | 4 | in vec3 vpos; 5 | in vec3 normal; 6 | in vec3 objectColor; 7 | in float objectAlpha; 8 | 9 | void main() 10 | { 11 | vec3 lightPos = vec3(0,0,0); // set to 0 to ensure good lighting for camera mode + normal mode 12 | vec3 viewPos = vec3(0,0,0); 13 | vec3 lightColor = vec3(1,1,1); 14 | 15 | // Ambient 16 | float ambientStrength = 0.01f; 17 | vec3 ambient = ambientStrength * lightColor; 18 | 19 | // Diffuse 20 | vec3 norm = normalize(normal); 21 | // norm = vec3(0, 1, 0); 22 | vec3 lightDir = normalize(lightPos - vpos); 23 | float diff = max(dot(norm, lightDir), 0.0); 24 | vec3 diffuse = diff * lightColor; 25 | 26 | // Specular 27 | float specularStrength = 0.1f; 28 | vec3 viewDir = normalize(viewPos - vpos); 29 | vec3 reflectDir = reflect(-lightDir, norm); 30 | float spec = pow(max(dot(viewDir, reflectDir), 0.0), 32); 31 | vec3 specular = specularStrength * spec * lightColor; 32 | 33 | vec3 light = ambient + diffuse + specular; 34 | 35 | vec3 result = light * objectColor; 36 | color = vec4(result, objectAlpha); 37 | } 38 | -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_normal; 5 | in vec3 vertex_color; 6 | in float vertex_alpha; 7 | 8 | out vec3 normal; 9 | out vec3 objectColor; 10 | out vec3 vpos; 11 | out float objectAlpha; 12 | 13 | uniform mat4 MVP; 14 | uniform mat4 MV; 15 | 16 | void main() 17 | { 18 | vec4 normal4 = MV*vec4(vertex_normal,0); 19 | normal = vec3(normal4); 20 | objectColor = vertex_color; 21 | //objectColor = objectColor*0.8 22 | //objectColor = vec3(0.8,0.8,0.8); 23 | //color = vertex_normal; 24 | gl_Position = MVP * vec4(vertex_pos,1); 25 | objectAlpha = vertex_alpha; 26 | vpos = vec3(MV * vec4(vertex_pos,1)); 27 | } 28 | -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh_texture.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec2 UV; 4 | 5 | out vec4 fragColor; 6 | 7 | // Values that stay constant for the whole mesh. 8 | uniform sampler2D myTextureSampler; 9 | 10 | 11 | void main() 12 | { 13 | fragColor = texture2D( myTextureSampler, UV ).rgba; 14 | //fragColor = vec4(1,0,1,1); 15 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh_texture.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_normal; 5 | //in vec3 vertex_color; 6 | 7 | in vec2 vertex_uv; 8 | 9 | out vec2 UV; 10 | 11 | uniform mat4 MVP; 12 | uniform mat4 MV; 13 | 14 | void main() 15 | { 16 | //vec4 normal4 = MV*vec4(vertex_normal,0); 17 | //normal = vec3(normal4); 18 | //color = vertex_color; 19 | //color = vertex_normal; 20 | gl_Position = MVP * vec4(vertex_pos,1); 21 | 22 | UV = vertex_uv; 23 | } 24 | -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh_texture_shader.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 core 2 | out vec4 color; 3 | 4 | in vec3 vpos; 5 | in vec3 normal; 6 | in vec2 UV; 7 | 8 | uniform sampler2D myTextureSampler; 9 | void main() 10 | { 11 | vec4 textureColor = texture2D( myTextureSampler, UV ).rgba; 12 | 13 | vec3 objectColor = vec3(0.8,0.8,0.8); 14 | vec3 lightPos = vec3(0,100,0); 15 | vec3 viewPos = vec3(0,0,0); 16 | vec3 lightColor = vec3(1,1,1); 17 | 18 | // Ambient 19 | float ambientStrength = 0.1f; 20 | vec3 ambient = ambientStrength * lightColor; 21 | 22 | // Diffuse 23 | vec3 norm = normalize(normal); 24 | vec3 lightDir = normalize(lightPos - vpos); 25 | float diff = max(dot(norm, lightDir), 0.0); 26 | vec3 diffuse = vec3(textureColor.x,textureColor.y,textureColor.z) ;//diff * lightColor; 27 | 28 | // Specular 29 | float specularStrength = 0.3f; 30 | vec3 viewDir = normalize(viewPos - vpos); 31 | vec3 reflectDir = reflect(-lightDir, norm); 32 | float spec = pow(max(dot(viewDir, reflectDir), 0.0), 32); 33 | vec3 specular = specularStrength * spec * lightColor; 34 | 35 | vec3 result = (ambient + diffuse + specular) * objectColor; 36 | color = vec4(result, 1.0f); 37 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/Mesh_texture_shader.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_normal; 5 | 6 | in vec2 vertex_uv; 7 | out vec2 UV; 8 | 9 | out vec3 normal; 10 | out vec3 color; 11 | out vec3 vpos; 12 | 13 | uniform mat4 MVP; 14 | uniform mat4 MV; 15 | 16 | void main() 17 | { 18 | vec4 normal4 = MV*vec4(vertex_normal,0); 19 | normal = vec3(normal4); 20 | 21 | color = vertex_normal; 22 | gl_Position = MVP * vec4(vertex_pos,1); 23 | 24 | vpos = vec3(MV * vec4(vertex_pos,1)); 25 | 26 | UV = vertex_uv; 27 | } 28 | -------------------------------------------------------------------------------- /FitAdam/Shaders/NormalMap.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 normal; 4 | out vec4 fragColor; 5 | 6 | void main() 7 | { 8 | vec3 color = vec3(normalize(normal)); 9 | color = color * 0.5 + 0.5; 10 | fragColor = vec4(color, 1.0); 11 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/NormalMap.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_normal; 5 | out vec3 normal; 6 | 7 | uniform mat4 MVP; 8 | uniform mat4 MV; 9 | 10 | void main() 11 | { 12 | vec4 normal4 = MV*vec4(vertex_normal,0); 13 | normal = vec3(normal4); 14 | gl_Position = MVP * vec4(vertex_pos,1); 15 | } 16 | -------------------------------------------------------------------------------- /FitAdam/Shaders/Project2D.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 coord2D_mid; 4 | layout (location = 0) out vec4 coord2D_out; 5 | 6 | void main() 7 | { 8 | coord2D_out = vec4(coord2D_mid, 1.0f); 9 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/Project2D.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | layout(location = 0) in vec3 vertex_pos; 4 | layout(location = 1) in vec3 coord2D_in; // 2D coordinate of last frame 5 | 6 | out vec3 coord2D_mid; 7 | 8 | uniform mat4 MVP; 9 | 10 | void main() 11 | { 12 | gl_Position = MVP * vec4(vertex_pos, 1); 13 | coord2D_mid = coord2D_in; 14 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/ProjectorOnTexture.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Interpolated values from the vertex shaders 4 | varying vec2 UV; 5 | varying vec4 vPro ; 6 | varying vec3 Position_worldspace; 7 | varying vec3 Normal_cameraspace; 8 | varying vec3 EyeDirection_cameraspace; 9 | varying vec3 LightDirection1_cameraspace; 10 | varying vec3 LightDirection2_cameraspace; 11 | varying vec3 ProjectorDirection_cameraspace; 12 | 13 | // Values that stay constant for the whole mesh. 14 | uniform sampler2D myTextureSampler; 15 | uniform sampler2D myTextureSampler2; 16 | uniform vec3 LightPosition1_worldspace; 17 | uniform vec3 LightPosition2_worldspace; 18 | uniform vec3 ProjectorPosition_worldspace; 19 | uniform float LightPower; 20 | uniform float ProjectorPower; 21 | uniform mat4 MV; 22 | 23 | void main() 24 | { 25 | // Normal of the computed fragment, in camera space 26 | vec3 n = normalize( Normal_cameraspace ); 27 | 28 | // Eye vector (towards the camera) 29 | vec3 E = normalize(EyeDirection_cameraspace); 30 | 31 | //Map to UV coord + material property 32 | vec3 MaterialDiffuseColor = texture2D( myTextureSampler, UV ).rgb; 33 | vec3 MaterialAmbientColor = vec3(0.1,0.1,0.1) * MaterialDiffuseColor; 34 | vec3 MaterialSpecularColor = vec3(0.3,0.3,0.3); 35 | 36 | // Light emission properties: 37 | vec3 LightColor = vec3(1,1,1); 38 | float distance = length( LightPosition1_worldspace - Position_worldspace ); //distance 39 | vec3 l = normalize( LightDirection1_cameraspace ); // Direction of the light (from the fragment to the light) 40 | float cosTheta = clamp( dot( n,l ), 0,1 ); // Cosine of the angle between the normal and the light direction, clamped above 0 41 | vec3 R = reflect(-l,n); // Direction in which the triangle reflects the light 42 | float cosAlpha = clamp( dot( E,R ), 0,1 ); // Cosine of the angle between the Eye vector and the Reflect vector, clamped to 0 43 | 44 | float distance2 = length( LightPosition2_worldspace - Position_worldspace ); //distance 45 | vec3 l2 = normalize( LightDirection2_cameraspace ); // Direction of the light (from the fragment to the light) 46 | float cosTheta2 = clamp( dot( n,l2 ), 0,1 ); // Cosine of the angle between the normal and the light direction, clamped above 0 47 | vec3 R2 = reflect(-l2,n); // Direction in which the triangle reflects the light 48 | float cosAlpha2 = clamp( dot( E,R2 ), 0,1 ); // Cosine of the angle between the Eye vector and the Reflect vector, clamped to 0 49 | 50 | // Projector emission properties: 51 | float distance3 = length( ProjectorPosition_worldspace - Position_worldspace ); //distance 52 | vec3 l3 = normalize( ProjectorDirection_cameraspace ); // Direction of the projector ray (from the fragment to the light) 53 | float cosTheta3 = clamp( dot( n,l3 ), 0,1 ); // Cosine of the angle between the normal and the projector ray direction, clamped above 0 54 | vec3 R3 = reflect(-l3,n); // Direction in which the triangle reflects the projector ray 55 | float cosAlpha3 = clamp( dot( E,R3 ), 0,1 ); // Cosine of the angle between the Eye vector and the Reflect vector, clamped to 0 56 | 57 | //Projecting color 58 | vec3 ProjectionColor = vec3(1.0, 1.0, 1.0); 59 | if(vPro.w > 0.0) 60 | { 61 | vec2 UV2 = vec2(0.5*vPro.x/vPro.w + 0.5, 0.5*vPro.y/vPro.w + 0.5); 62 | ProjectionColor = texture2D( myTextureSampler2, UV2 ).rgb; 63 | if( UV2.x < 0.0 || UV2.y < 0.0 || UV2.x > 1.0 || UV2.y > 1.0) 64 | ProjectionColor = vec3(0.0, 0.0, 0.0); 65 | } 66 | 67 | gl_FragColor.rgb = MaterialAmbientColor+ 68 | MaterialDiffuseColor *LightColor//*LightPower*cosTheta/(distance*distance) 69 | *LightColor//*LightPower*cosTheta2/(distance2*distance2) 70 | *ProjectionColor;//*ProjectorPower*cosTheta3/ (distance3*distance3); 71 | 72 | //+MaterialSpecularColor * ProjectionColor* LightColor * LightPower * pow(cosAlpha,10) / (distance*distance); 73 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/ProjectorOnTexture.vertexshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Input vertex data, different for all executions of this shader. 4 | attribute vec3 vertexPosition_modelspace; 5 | attribute vec3 vertexNormal_modelspace; 6 | attribute vec2 vertexUV; 7 | 8 | 9 | // Values that stay constant for the whole mesh. 10 | uniform mat4 MVP; 11 | uniform mat4 M; 12 | uniform mat4 V; 13 | uniform mat4 iMV; 14 | uniform mat4 MVPp; 15 | uniform vec3 LightPosition1_worldspace; 16 | uniform vec3 LightPosition2_worldspace; 17 | uniform vec3 ProjectorPosition_worldspace; 18 | 19 | // Output data ; will be interpolated for each fragment. 20 | varying vec2 UV; 21 | varying vec3 Position_worldspace; 22 | varying vec3 Normal_cameraspace; 23 | varying vec3 EyeDirection_cameraspace; 24 | varying vec3 LightDirection1_cameraspace; 25 | varying vec3 LightDirection2_cameraspace; 26 | varying vec3 ProjectorDirection_cameraspace; 27 | varying vec4 vPro; 28 | 29 | void main(){ 30 | 31 | // Output position of the vertex, in camera space : MVP * position 32 | gl_Position = MVP * vec4(vertexPosition_modelspace,1); 33 | 34 | // Position of the vertex, in worldspace : M * position 35 | Position_worldspace = (M * vec4(vertexPosition_modelspace,1)).xyz; 36 | 37 | // Vector that goes from the vertex to the camera, in camera space. In camera space, the camera is at the origin (0,0,0). 38 | vec3 vertexPosition_cameraspace = ( V * M * vec4(vertexPosition_modelspace,1)).xyz; 39 | EyeDirection_cameraspace = vec3(0,0,0) - vertexPosition_cameraspace; 40 | 41 | // Normal of the the vertex, in camera space 42 | Normal_cameraspace = ( iMV* vec4(vertexNormal_modelspace,0)).xyz; 43 | 44 | // Vector that goes from the vertex to the light, in camera space. M is ommited because it's identity. 45 | vec3 LightPosition1_cameraspace = ( V * vec4(LightPosition1_worldspace,1)).xyz; 46 | LightDirection1_cameraspace = LightPosition1_cameraspace + EyeDirection_cameraspace; 47 | 48 | // Vector that goes from the vertex to the light, in camera space. M is ommited because it's identity. 49 | vec3 LightPosition2_cameraspace = ( V * vec4(LightPosition2_worldspace,1)).xyz; 50 | LightDirection2_cameraspace = LightPosition2_cameraspace + EyeDirection_cameraspace; 51 | 52 | // Vector that goes from the vertex to the projector, in camera space. M is ommited because it's identity. 53 | vec3 ProjectorPosition_cameraspace = ( V * vec4(ProjectorPosition_worldspace,1)).xyz; 54 | ProjectorDirection_cameraspace = ProjectorPosition_cameraspace + EyeDirection_cameraspace; 55 | 56 | vPro = MVPp * vec4(vertexPosition_modelspace,1); 57 | 58 | // UV of the vertex. No special space for this one. 59 | UV = vertexUV; 60 | } 61 | 62 | -------------------------------------------------------------------------------- /FitAdam/Shaders/ProjectorSimulation.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Interpolated values from the vertex shaders 4 | varying vec2 UV; 5 | varying vec4 vPro ; 6 | varying vec3 Position_worldspace; 7 | varying vec3 Normal_cameraspace; 8 | varying vec3 EyeDirection_cameraspace; 9 | varying vec3 LightDirection_cameraspace; 10 | varying vec3 ObjectColor; 11 | 12 | // Values that stay constant for the whole mesh. 13 | uniform sampler2D myTextureSampler; 14 | uniform mat4 MV; 15 | uniform vec3 LightPosition_worldspace; 16 | uniform float LightPower; 17 | 18 | void main(){ 19 | // Distance to the light 20 | float distance = length( LightPosition_worldspace - Position_worldspace ); 21 | 22 | // Normal of the computed fragment, in camera space 23 | vec3 n = normalize( Normal_cameraspace ); 24 | 25 | // Direction of the light (from the fragment to the light) 26 | vec3 l = normalize( LightDirection_cameraspace ); 27 | 28 | // Cosine of the angle between the normal and the light direction, clamped above 0 29 | float cosTheta = clamp( dot( n,l ), 0,1 ); 30 | 31 | // Eye vector (towards the camera) 32 | vec3 E = normalize(EyeDirection_cameraspace); 33 | 34 | // Direction in which the triangle reflects the light 35 | vec3 R = reflect(-l,n); 36 | 37 | // Cosine of the angle between the Eye vector and the Reflect vector, clamped to 0 38 | // - Looking into the reflection -> 1 39 | // - Looking elsewhere -> < 1 40 | float cosAlpha = clamp( dot( E,R ), 0,1 ); 41 | 42 | // Material properties 43 | vec3 MaterialDiffuseColor = ObjectColor; 44 | vec3 MaterialAmbientColor = vec3(0.1,0.1,0.1) * MaterialDiffuseColor; 45 | vec3 MaterialSpecularColor = vec3(0.3,0.3,0.3); 46 | 47 | vec3 ProjectionColor = vec3(1.0, 1.0, 1.0); 48 | if(vPro.w > 0.0) 49 | { 50 | //Map to UV coord. 51 | vec2 UV2 = vec2(0.5*vPro.x/vPro.w + 0.5, 0.5*vPro.y/vPro.w + 0.5); 52 | ProjectionColor = texture2D( myTextureSampler, UV2 ).rgb; 53 | if( UV2.x < 0.0 || UV2.y < 0.0 || UV2.x > 1.0 || UV2.y > 1.0) 54 | ProjectionColor = vec3(0.0, 0.0, 0.0); 55 | } 56 | 57 | gl_FragColor.rgb = MaterialAmbientColor; 58 | gl_FragColor.rgb = MaterialDiffuseColor * ProjectionColor * LightPower/ (distance*distance);// * cosTheta ; 59 | //gl_FragColor.rgb += MaterialDiffuseColor * ProjectionColor * LightPower * pow(cosAlpha, 5)/ (distance*distance) ; 60 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/ProjectorSimulation.vertexshader: -------------------------------------------------------------------------------- 1 | 2 | 3 | /* 4 | #version 120 5 | 6 | // Input vertex data, different for all executions of this shader. 7 | attribute vec3 vertexPosition_modelspace; 8 | attribute vec3 vertexNormal_modelspace; 9 | attribute vec2 vertexUV; 10 | attribute vec3 OColor; 11 | 12 | // Values that stay constant for the whole mesh. 13 | uniform mat4 MVP; 14 | uniform mat4 M; 15 | uniform mat4 V; 16 | uniform mat4 iMV; 17 | uniform mat4 MVPp; 18 | uniform vec3 LightPosition_worldspace; 19 | 20 | // Output data ; will be interpolated for each fragment. 21 | varying vec2 UV; 22 | varying vec3 Position_worldspace; 23 | varying vec3 Normal_cameraspace; 24 | varying vec3 EyeDirection_cameraspace; 25 | varying vec3 LightDirection_cameraspace; 26 | varying vec3 ObjectColor; 27 | varying vec4 vPro; 28 | 29 | void main(){ 30 | 31 | // Output position of the vertex, in camera space : MVP * position 32 | gl_Position = MVP * vec4(vertexPosition_modelspace,1); 33 | 34 | // Position of the vertex, in worldspace : M * position 35 | Position_worldspace = (M * vec4(vertexPosition_modelspace,1)).xyz; 36 | 37 | // Vector that goes from the vertex to the camera, in camera space. In camera space, the camera is at the origin (0,0,0). 38 | vec3 vertexPosition_cameraspace = ( V * M * vec4(vertexPosition_modelspace,1)).xyz; 39 | EyeDirection_cameraspace = vec3(0,0,0) - vertexPosition_cameraspace; 40 | 41 | // Vector that goes from the vertex to the light, in camera space. M is ommited because it's identity. 42 | vec3 LightPosition_cameraspace = ( V * vec4(LightPosition_worldspace,1)).xyz; 43 | LightDirection_cameraspace = LightPosition_cameraspace + EyeDirection_cameraspace; 44 | 45 | // Normal of the the vertex, in camera space 46 | //Normal_cameraspace = ( V * M * vec4(vertexNormal_modelspace,0)).xyz; // Only correct if ModelMatrix does not scale the model ! Use its inverse transpose if not. 47 | Normal_cameraspace = ( iMV* vec4(vertexNormal_modelspace,0)).xyz; 48 | 49 | // UV of the vertex. No special space for this one. 50 | UV = vertexUV; 51 | ObjectColor = OColor; 52 | 53 | vPro = MVPp * vec4(vertexPosition_modelspace,1); 54 | } 55 | 56 | */ -------------------------------------------------------------------------------- /FitAdam/Shaders/Selection.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 color; 4 | 5 | //out vec4 fragColor; 6 | layout(location = 0) out vec3 fragColor; 7 | layout(location = 1) out float depth; 8 | 9 | void main() 10 | { 11 | fragColor = color; 12 | depth = gl_FragCoord.z; 13 | } 14 | -------------------------------------------------------------------------------- /FitAdam/Shaders/Selection.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_color; 5 | 6 | out vec3 color; 7 | 8 | uniform mat4 MVP; 9 | 10 | void main() 11 | { 12 | gl_Position = MVP * vec4(vertex_pos,1); 13 | color = vertex_color;//vec4(,1); 14 | } 15 | 16 | /*#version 120 17 | 18 | uniform mat4 gl_ModelViewMatrix; 19 | uniform mat4 gl_ProjectionMatrix; 20 | 21 | varying float zValue; 22 | 23 | void main() 24 | { 25 | //gl_Position = ftransform(); 26 | gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * gl_Vertex; 27 | gl_Position.y = -gl_Position.y; 28 | 29 | gl_FrontColor = gl_Color; 30 | 31 | zValue = gl_Position.z; //-1 to 1 32 | } 33 | */ -------------------------------------------------------------------------------- /FitAdam/Shaders/SimpleTransform.vertexshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Input vertex data, different for all executions of this shader. 4 | attribute vec3 vertexPosition_modelspace; 5 | attribute vec3 vertexColor; 6 | 7 | // Output data ; will be interpolated for each fragment. 8 | varying vec3 fragmentColor; 9 | // Values that stay constant for the whole mesh. 10 | uniform mat4 MVP; 11 | 12 | void main(){ 13 | 14 | // Output position of the vertex, in clip space : MVP * position 15 | gl_Position = MVP * vec4(vertexPosition_modelspace,1); 16 | 17 | // The color of each vertex will be interpolated to produce the color of each fragment 18 | fragmentColor = vertexColor; 19 | } 20 | 21 | -------------------------------------------------------------------------------- /FitAdam/Shaders/SimplestShader.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec4 color; 4 | 5 | out vec4 fragColor; 6 | 7 | void main() 8 | { 9 | fragColor = color; 10 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/SimplestShader.vertexshader: -------------------------------------------------------------------------------- 1 | #version 330 2 | 3 | in vec3 vertex_pos; 4 | in vec3 vertex_color; 5 | 6 | out vec4 color; 7 | 8 | uniform mat4 MVP; 9 | 10 | void main() 11 | { 12 | gl_Position = MVP * vec4(vertex_pos,1); 13 | color = vec4(vertex_color,1); 14 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/SingleColor.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | varying vec3 fragmentColor; 4 | 5 | void main() 6 | { 7 | gl_FragColor = vec4(fragmentColor, 1); 8 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/StandardLighting.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | 4 | 5 | // Interpolated values from the vertex shaders 6 | varying vec3 Position_worldspace; 7 | varying vec3 Normal_cameraspace; 8 | varying vec3 EyeDirection_cameraspace; 9 | varying vec3 LightDirection_cameraspace1; 10 | varying vec3 LightDirection_cameraspace2; 11 | varying vec3 vColor; 12 | 13 | // Values that stay constant for the whole mesh. 14 | uniform mat4 MV; 15 | uniform vec3 LightPosition_worldspace1; 16 | uniform vec3 LightPosition_worldspace2; 17 | uniform float LightPower1; 18 | uniform float LightPower2; 19 | 20 | void main(){ 21 | 22 | // Light emission properties 23 | // You probably want to put them as uniforms 24 | vec3 LightColor = vec3(1,1,1); 25 | 26 | // Material properties 27 | vec3 MaterialDiffuseColor = vColor; 28 | vec3 MaterialAmbientColor = vec3(0.4,0.4,0.4) * MaterialDiffuseColor; 29 | vec3 MaterialSpecularColor = vec3(0.3,0.3,0.3); 30 | 31 | //Light 1 32 | float distance = length( LightPosition_worldspace1 - Position_worldspace ); 33 | 34 | vec3 n = normalize( Normal_cameraspace ); 35 | vec3 l = normalize( LightDirection_cameraspace1 ); 36 | float cosTheta = clamp( dot( n,l ), 0,1 ); 37 | 38 | vec3 E = normalize(EyeDirection_cameraspace); 39 | vec3 R = reflect(-l,n); 40 | float cosAlpha = clamp( dot( E,R ), 0,1 ); 41 | 42 | gl_FragColor.rgb = MaterialAmbientColor + MaterialDiffuseColor * LightColor * LightPower1 * cosTheta / (distance*distance) + 43 | MaterialSpecularColor * LightColor * LightPower1 * pow(cosAlpha,2) / (distance*distance); 44 | 45 | //Light 2 46 | distance = length( LightPosition_worldspace2 - Position_worldspace ); 47 | 48 | n = normalize( Normal_cameraspace ); 49 | l = normalize( LightDirection_cameraspace2 ); 50 | cosTheta = abs(dot(n,l));//clamp( dot( n,l ), 0,1 ); 51 | 52 | E = normalize(EyeDirection_cameraspace); 53 | R = reflect(-l,n); 54 | cosAlpha = abs(dot(E,R));//clamp( dot( E,R ), 0,1 ); 55 | 56 | gl_FragColor.rgb += MaterialAmbientColor + MaterialDiffuseColor * LightColor * LightPower2 * cosTheta / (distance*distance) + 57 | MaterialSpecularColor * LightColor * LightPower2 * pow(cosAlpha,2) / (distance*distance); 58 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/StandardLighting.vertexshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Input vertex data, different for all executions of this shader. 4 | attribute vec3 vertexPosition_modelspace; 5 | attribute vec3 vertexNormal_modelspace; 6 | attribute vec3 vertexColor; 7 | 8 | 9 | // Output data ; will be interpolated for each fragment. 10 | varying vec3 Position_worldspace; 11 | varying vec3 Normal_cameraspace; 12 | varying vec3 EyeDirection_cameraspace; 13 | varying vec3 LightDirection_cameraspace1; 14 | varying vec3 LightDirection_cameraspace2; 15 | varying vec3 vColor; 16 | 17 | // Values that stay constant for the whole mesh. 18 | uniform mat4 MVP; 19 | uniform mat4 V; 20 | uniform mat4 M; 21 | uniform mat4 iMV; 22 | uniform vec3 LightPosition_worldspace1; 23 | uniform vec3 LightPosition_worldspace2; 24 | 25 | void main(){ 26 | 27 | // Output position of the vertex, in clip space : MVP * position 28 | gl_Position = MVP * vec4(vertexPosition_modelspace,1); 29 | 30 | // Position of the vertex, in worldspace : M * position 31 | Position_worldspace = (M * vec4(vertexPosition_modelspace,1)).xyz; 32 | 33 | // Vector that goes from the vertex to the camera, in camera space. 34 | vec3 vertexPosition_cameraspace = ( V * M * vec4(vertexPosition_modelspace,1)).xyz; 35 | EyeDirection_cameraspace = vec3(0,0,0) - vertexPosition_cameraspace; 36 | 37 | // Vector that goes from the vertex to the light, in camera space. M is ommited because it's identity. 38 | vec3 LightPosition_cameraspace = ( V * vec4(LightPosition_worldspace1,1)).xyz; 39 | LightDirection_cameraspace1 = LightPosition_cameraspace + EyeDirection_cameraspace; 40 | 41 | // Vector that goes from the vertex to the light, in camera space. M is ommited because it's identity. 42 | LightPosition_cameraspace = ( V * vec4(LightPosition_worldspace2,1)).xyz; 43 | LightDirection_cameraspace2 = LightPosition_cameraspace + EyeDirection_cameraspace; 44 | 45 | // Normal of the the vertex, in camera space 46 | Normal_cameraspace = ( iMV* vec4(vertexNormal_modelspace,0)).xyz; 47 | 48 | vColor = vertexColor; 49 | } 50 | 51 | -------------------------------------------------------------------------------- /FitAdam/Shaders/TextureFragmentShader.fragmentshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Interpolated values from the vertex shaders 4 | varying vec2 UV; 5 | 6 | // Values that stay constant for the whole mesh. 7 | uniform sampler2D myTextureSampler; 8 | 9 | void main(){ 10 | 11 | // Output color = color of the texture at the specified UV 12 | gl_FragColor = texture2D( myTextureSampler, UV ); 13 | } -------------------------------------------------------------------------------- /FitAdam/Shaders/TransformVertexShader.vertexshader: -------------------------------------------------------------------------------- 1 | #version 120 2 | 3 | // Input vertex data, different for all executions of this shader. 4 | attribute vec3 vertexPosition_modelspace; 5 | attribute vec2 vertexUV; 6 | 7 | // Output data ; will be interpolated for each fragment. 8 | varying vec2 UV; 9 | 10 | // Values that stay constant for the whole mesh. 11 | uniform mat4 MVP; 12 | 13 | void main(){ 14 | 15 | // Output position of the vertex, in clip space : MVP * position 16 | gl_Position = MVP * vec4(vertexPosition_modelspace,1); 17 | 18 | // UV of the vertex. No special space for this one. 19 | UV = vertexUV; 20 | } 21 | 22 | -------------------------------------------------------------------------------- /FitAdam/bvh.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "totalmodel.h" 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "meshTrackingProj.h" 20 | #include "SGSmooth.hpp" 21 | #include "ModelFitter.h" 22 | #include "utils.h" 23 | #include 24 | #include 25 | #include 26 | 27 | #define ROWS 1080 28 | #define COLS 1920 29 | #define FACE_VERIFY_THRESH 0.05 30 | #define PI 3.14159265359 31 | 32 | TotalModel g_total_model; 33 | 34 | int main() 35 | { 36 | // initialize total model 37 | LoadTotalModelFromObj(g_total_model, std::string("model/mesh_nofeet.obj")); 38 | LoadModelColorFromObj(g_total_model, std::string("model/nofeetmesh_byTomas_bottom.obj")); // contain the color information 39 | LoadTotalDataFromJson(g_total_model, std::string("model/adam_v1_plus2.json"), std::string("model/adam_blendshapes_348_delta_norm.json"), std::string("model/correspondences_nofeet.txt")); 40 | LoadCocoplusRegressor(g_total_model, std::string("model/regressor_0n1_root.json")); 41 | 42 | // use the skeleton of the first frame 43 | const std::string param_filename = "../data/example_dance/body_3d_frontal/0001.txt"; 44 | smpl::SMPLParams frame_params; 45 | readFrameParam(param_filename, frame_params); 46 | Eigen::VectorXd J_vec = g_total_model.J_mu_ + g_total_model.dJdc_ * frame_params.m_adam_coeffs; 47 | const Eigen::Matrix J0 = J_vec; 48 | 49 | // assume we already have the estimation result of the first 10 frames of the dancing sequence 50 | std::vector> t; 51 | std::vector> pose; 52 | for (auto i = 1; i <= 10; i++) 53 | { 54 | char frame_param_name[200]; 55 | sprintf(frame_param_name, "../data/example_dance/body_3d_frontal/%04d.txt", i); 56 | readFrameParam(frame_param_name, frame_params); 57 | t.push_back(frame_params.m_adam_t); 58 | pose.push_back(frame_params.m_adam_pose); 59 | } 60 | 61 | BVHWriter bvh(g_total_model.m_parent); 62 | bvh.parseInput(J0, t, pose); 63 | bvh.writeBVH("output.bvh", 1.0 / 30); 64 | 65 | return 0; 66 | } -------------------------------------------------------------------------------- /FitAdam/cmake/Modules/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find GFLAGS 2 | # 3 | # The following variables are optionally searched for defaults 4 | # GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found 5 | # 6 | # The following are set after configuration is done: 7 | # GFLAGS_FOUND 8 | # GFLAGS_INCLUDE_DIRS 9 | # GFLAGS_LIBRARIES 10 | # GFLAGS_LIBRARYRARY_DIRS 11 | 12 | include(FindPackageHandleStandardArgs) 13 | 14 | set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") 15 | 16 | # We are testing only a couple of files in the include directories 17 | if(WIN32) 18 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 19 | PATHS ${GFLAGS_ROOT_DIR}/src/windows) 20 | else() 21 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 22 | PATHS ${GFLAGS_ROOT_DIR}) 23 | endif() 24 | 25 | if(MSVC) 26 | find_library(GFLAGS_LIBRARY_RELEASE 27 | NAMES libgflags 28 | PATHS ${GFLAGS_ROOT_DIR} 29 | PATH_SUFFIXES Release) 30 | 31 | find_library(GFLAGS_LIBRARY_DEBUG 32 | NAMES libgflags-debug 33 | PATHS ${GFLAGS_ROOT_DIR} 34 | PATH_SUFFIXES Debug) 35 | 36 | set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) 37 | else() 38 | find_library(GFLAGS_LIBRARY gflags) 39 | endif() 40 | 41 | find_package_handle_standard_args(GFlags DEFAULT_MSG GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 42 | 43 | 44 | if(GFLAGS_FOUND) 45 | set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 46 | set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) 47 | message(STATUS "Found gflags (include: ${GFLAGS_INCLUDE_DIR}, library: ${GFLAGS_LIBRARY})") 48 | mark_as_advanced(GFLAGS_LIBRARY_DEBUG GFLAGS_LIBRARY_RELEASE 49 | GFLAGS_LIBRARY GFLAGS_INCLUDE_DIR GFLAGS_ROOT_DIR) 50 | endif() 51 | -------------------------------------------------------------------------------- /FitAdam/include/BVHWriter.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | struct BVHData 9 | { 10 | BVHData(std::string _name, double* _offset): name(_name), num_children(0) 11 | { 12 | offset[0] = _offset[0]; 13 | offset[1] = _offset[1]; 14 | offset[2] = _offset[2]; 15 | } 16 | std::string name; 17 | std::array offset; 18 | std::vector> children; 19 | std::vector> euler; 20 | int num_children; 21 | }; 22 | 23 | class BVHWriter 24 | { 25 | public: 26 | BVHWriter(int* _m_parent, const bool unityCompatible = false) : 27 | mUnityCompatible{unityCompatible} 28 | { 29 | std::copy(_m_parent, _m_parent + TotalModel::NUM_JOINTS, m_parent); 30 | } 31 | void parseInput(const Eigen::Matrix& J0, std::vector>& t, std::vector>& pose); 32 | void writeBVH(std::string output_file, double frame_time); 33 | private: 34 | void getHierarchy(const Eigen::Matrix& J0); 35 | void getDynamic(Eigen::Matrix& pose); 36 | void writeData(std::shared_ptr node, int depth); 37 | const bool mUnityCompatible; 38 | int m_parent[TotalModel::NUM_JOINTS]; 39 | int num_frame; 40 | std::vector> trans; 41 | std::string outStr; 42 | std::vector dynamicStr; 43 | std::shared_ptr root; 44 | std::array, TotalModel::NUM_JOINTS> data; 45 | }; 46 | -------------------------------------------------------------------------------- /FitAdam/include/CMeshModelInstance.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #ifndef CMESHMODELINSTANCE 4 | #define CMESHMODELINSTANCE 5 | 6 | struct TotalModel; 7 | 8 | class CMeshModelInstance 9 | { 10 | public: 11 | 12 | enum EnumMeshType 13 | { 14 | MESH_TYPE_UNKNOWN = -1, 15 | MESH_TYPE_SMPL = 0, 16 | MESH_TYPE_FACE, 17 | MESH_TYPE_RHAND, 18 | MESH_TYPE_LHAND, 19 | MESH_TYPE_TOTAL, 20 | MESH_TYPE_ADAM 21 | }; 22 | 23 | CMeshModelInstance() 24 | { 25 | m_meshType = MESH_TYPE_UNKNOWN; 26 | m_humanIdx = -1; 27 | //m_pSourceParam = NULL; 28 | m_sourceParamIdx = std::make_pair(-1, -1); 29 | 30 | m_visType = 0; 31 | 32 | m_frameIdx = -1; 33 | } 34 | 35 | void RecomputeNormal(const TotalModel& model); 36 | void clearMesh(); 37 | void sortFaceDepth(const cv::Point3d angleaxis=cv::Point3d(0, 0, 0)); 38 | 39 | std::vector m_face_vertexIndices; 40 | std::vector m_vertices; 41 | std::vector m_colors; 42 | std::vector m_normals; 43 | std::vector m_uvs; 44 | std::vector m_joints; 45 | std::vector m_joints_regress; //newly regressed joint 46 | std::vector m_alpha; 47 | int m_humanIdx; 48 | int m_visType; 49 | 50 | int m_frameIdx; 51 | 52 | std::pair m_sourceParamIdx; //Where this mesh was generated. g_kinModelManager.m_smpl_fitParms[m_sourceParamIdx.first].m_meshes.m_params[m_sourceParamIdx.secod] 53 | 54 | EnumMeshType m_meshType; 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /FitAdam/include/DCTCost.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | // Perform smoothing by suppressing high frequency components in DCT space. 7 | // Suppressing the highest (num_t - low_comp) components. 8 | class DCTCost: public ceres::CostFunction 9 | { 10 | public: 11 | DCTCost(const uint num_t, const uint low_comp, const uint dim, const uint start_dim, const uint end_dim, const double weight): 12 | num_t_(num_t), low_comp_(low_comp), dim_(dim), start_dim_(start_dim), end_dim_(end_dim), weight_(weight) 13 | { 14 | assert(low_comp_ < num_t_); 15 | assert(end_dim_ > start_dim_); 16 | assert(end_dim_ <= dim); 17 | CostFunction::set_num_residuals((end_dim_ - start_dim_) * (num_t - low_comp)); 18 | CostFunction::mutable_parameter_block_sizes()->clear(); 19 | for (auto i = 0u; i < num_t; i++) 20 | CostFunction::mutable_parameter_block_sizes()->emplace_back(dim_); 21 | } 22 | virtual bool Evaluate(double const* const* parameters, double* residuals, double** jacobians) const; 23 | private: 24 | const uint num_t_, low_comp_, dim_, start_dim_, end_dim_; 25 | const double weight_; 26 | }; 27 | 28 | struct TemporalSmoothCost 29 | { 30 | public: 31 | TemporalSmoothCost(const uint num_t, const uint dim): num_t_(num_t), dim_(dim) 32 | { 33 | assert(num_t > 1); 34 | // CostFunction::set_num_residuals(dim * num_t); 35 | // CostFunction::mutable_parameter_block_sizes()->clear(); 36 | // for (auto i = 0u; i < num_t; i++) 37 | // CostFunction::mutable_parameter_block_sizes()->emplace_back(dim); 38 | } 39 | 40 | template 41 | bool operator()(T const* const* parameters, T* residuals) const { 42 | for (uint d = 0; d < dim_; d++) 43 | { 44 | for (uint t = 0u; t < num_t_; t++) 45 | { 46 | if (t == 0) residuals[d * num_t_ + t] = parameters[t][d] - parameters[t + 1][d]; 47 | else if (t == num_t_ - 1) residuals[d * num_t_ + t] = parameters[t][d] - parameters[t - 1][d]; 48 | else residuals[d * num_t_ + t] = parameters[t][d] + parameters[t][d] - parameters[t - 1][d] - parameters[t + 1][d]; 49 | } 50 | } 51 | return true; 52 | } 53 | private: 54 | const uint num_t_, dim_; 55 | }; 56 | 57 | class TemporalSmoothCostDiff: public ceres::CostFunction 58 | { 59 | public: 60 | TemporalSmoothCostDiff(const uint num_t, const uint dim, const uint start_dim, const uint end_dim): 61 | num_t_(num_t), dim_(dim), start_dim_(start_dim), end_dim_(end_dim) 62 | { 63 | assert(num_t > 1); 64 | assert(end_dim_ > start_dim_); 65 | CostFunction::set_num_residuals((end_dim_ - start_dim_) * num_t); 66 | CostFunction::mutable_parameter_block_sizes()->clear(); 67 | for (auto i = 0u; i < num_t; i++) 68 | CostFunction::mutable_parameter_block_sizes()->emplace_back(dim); 69 | } 70 | virtual bool Evaluate(double const* const* parameters, double* residuals, double** jacobians) const; 71 | 72 | private: 73 | const uint num_t_, dim_, start_dim_, end_dim_; 74 | }; -------------------------------------------------------------------------------- /FitAdam/include/FKDerivative.h: -------------------------------------------------------------------------------- 1 | #ifndef FK_DERIVATIVE 2 | #define FK_DERIVATIVE 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | void AngleAxisToRotationMatrix_Derivative(const double* pose, double* dR_data, const int idj, const int numberColumns=TotalModel::NUM_JOINTS * 3); 10 | void EulerAnglesToRotationMatrix_Derivative(const double* pose, double* dR_data, const int idj, const int numberColumns=TotalModel::NUM_JOINTS * 3); 11 | void Product_Derivative(const double* const A_data, const double* const dA_data, const double* const B_data, 12 | const double* const dB_data, double* dAB_data, const int B_col=3); 13 | void SparseProductDerivative(const double* const A_data, const double* const dA_data, const double* const B_data, 14 | const double* const dB_data, const int colIndex, 15 | const std::vector& parentIndexes, double* dAB_data, const int numberColumns=TotalModel::NUM_JOINTS * 3); 16 | void SparseProductDerivative(const double* const dA_data, const double* const B_data, 17 | const std::vector& parentIndexes, double* dAB_data, const int numberColumns=TotalModel::NUM_JOINTS * 3); 18 | void SparseProductDerivativeConstA(const double* const A_data, const double* const dB_data, 19 | const std::vector& parentIndexes, double* dAB_data, const int numberColumns=TotalModel::NUM_JOINTS * 3); 20 | void SparseAdd(const double* const B_data, const std::vector& parentIndexes, double* A_data, const int numberColumns=TotalModel::NUM_JOINTS * 3); 21 | void SparseSubtract(const double* const B_data, const std::vector& parentIndexes, double* A_data, const int numberColumns=TotalModel::NUM_JOINTS * 3); 22 | 23 | void projection_Derivative(double* dPdI_data, const double* dJdI_data, const int ncol, double* XYZ, const double* pK_, int offsetP, int offsetJ, float weight=1.0f); 24 | 25 | template 26 | void projection_Derivative(Eigen::Map>& dPdI, Eigen::Matrix& dJdI, double* XYZ, const double* pK_, int offsetP, int offsetJ, float weight=1.0f) 27 | { 28 | // Dx/Dt = dx/dX * dX/dt + dx/dY * dY/dt + dx/dZ * dZ/dt 29 | assert(option == Eigen::RowMajor); 30 | const double X = XYZ[0], Y = XYZ[1], Z = XYZ[2]; 31 | const int ncol = dJdI.cols(); 32 | auto* P_row0 = dPdI.data() + offsetP * ncol, P_row1 = dPdI.data() + (offsetP + 1) * ncol; 33 | auto* J_row0 = dJdI.data() + offsetJ * ncol, J_row1 = dJdI.data() + (offsetJ + 1) * ncol, J_row2 = dJdI.data() + (offsetJ + 2) * ncol; 34 | for (int i = 0; i < ncol; i++) 35 | P_row0[i] = weight * ( pK_[0] * J_row0[i] + pK_[1] * J_row1[i] - (pK_[0] * X + pK_[1] * Y) * J_row2[i] / Z ) / Z; 36 | for (int i = 0; i < ncol; i++) 37 | P_row1[i] = weight * pK_[4] * ( J_row1[i] - Y / Z * J_row2[i] ) / Z; 38 | // equivalent to 39 | // dPdI.row(offsetP + 0) = weight * (pK_[0] / Z * dJdI.row(offsetJ + 0) 40 | // + pK_[1] / Z * dJdI.row(offsetJ + 1) 41 | // - (pK_[0] * X + pK_[1] * Y) / Z / Z * dJdI.row(offsetJ + 2)); 42 | // dPdI.row(offsetP + 1) = weight * (pK_[4] / Z * dJdI.row(offsetJ + 1) 43 | // - pK_[4] * Y / Z / Z * dJdI.row(offsetJ + 2)); 44 | } 45 | 46 | #endif -------------------------------------------------------------------------------- /FitAdam/include/FitToBody.h: -------------------------------------------------------------------------------- 1 | #include "handm.h" 2 | #include "totalmodel.h" 3 | #include "ceres/ceres.h" 4 | 5 | void SetSolverOptions(ceres::Solver::Options *options); 6 | 7 | void FitToHandCeres_Right_Naive( 8 | smpl::HandModel &handr_model, 9 | Eigen::MatrixXd &Joints, 10 | Eigen::MatrixXd &surface_constraint, 11 | Eigen::Vector3d& parm_handr_t, 12 | Eigen::Matrix& parm_hand_pose, 13 | Eigen::Matrix& parm_hand_coeffs, 14 | int regressor_type=0, 15 | bool fit_surface=false, 16 | bool euler=true, 17 | float pose_reg=1e-7, 18 | float coeff_reg=0.0f); 19 | 20 | void FitToProjectionCeres( 21 | smpl::HandModel &handr_model, 22 | Eigen::MatrixXd &Joints2d, 23 | const double* K, 24 | Eigen::MatrixXd &PAF, 25 | Eigen::MatrixXd &surface_constraint, 26 | Eigen::Vector3d& parm_handr_t, 27 | Eigen::Matrix& parm_hand_pose, 28 | Eigen::Matrix& parm_hand_coeffs, 29 | int regressor_type=0, 30 | bool fit_surface=false, 31 | bool euler=true, 32 | const double prior_weight=100.0, 33 | const int mode=0); 34 | 35 | void Adam_FitTotalBodyCeres( TotalModel &adam, 36 | smpl::SMPLParams &frame_param, 37 | Eigen::MatrixXd &BodyJoints, 38 | Eigen::MatrixXd &rFoot, //2points 39 | Eigen::MatrixXd &lFoot, //2points 40 | Eigen::MatrixXd &rHandJoints, // 41 | Eigen::MatrixXd &lHandJoints, // 42 | Eigen::MatrixXd &faceJoints); 43 | 44 | void Adam_FitTotalBodyCeres2d( TotalModel &adam, 45 | smpl::SMPLParams &frame_param, 46 | Eigen::MatrixXd &BodyJoints2d, 47 | Eigen::MatrixXd &rFoot2d, //2points 48 | Eigen::MatrixXd &lFoot2d, //2points 49 | Eigen::MatrixXd &rHandJoints2d, // 50 | Eigen::MatrixXd &lHandJoints2d, // 51 | Eigen::MatrixXd &faceJoints2d, 52 | double* calibK); 53 | 54 | void Adam_FitTotalBodyCeres3d2d( TotalModel &adam, 55 | smpl::SMPLParams &frame_param, 56 | Eigen::MatrixXd &BodyJoints, 57 | Eigen::MatrixXd &rFoot, //2points 58 | Eigen::MatrixXd &lFoot, //2points 59 | Eigen::MatrixXd &rHandJoints, // 60 | Eigen::MatrixXd &lHandJoints, // 61 | Eigen::MatrixXd &faceJoints, 62 | double* calibK); 63 | 64 | void Adam_FastFit_Initialize(TotalModel &adam, 65 | smpl::SMPLParams &frame_param, 66 | Eigen::MatrixXd &BodyJoints, 67 | Eigen::MatrixXd &rFoot, 68 | Eigen::MatrixXd &lFoot, 69 | Eigen::MatrixXd &rHandJoints, 70 | Eigen::MatrixXd &lHandJoints, 71 | Eigen::MatrixXd &faceJoints); 72 | 73 | void Adam_FastFit(TotalModel &adam, 74 | smpl::SMPLParams &frame_param, 75 | Eigen::MatrixXd &BodyJoints, 76 | Eigen::MatrixXd &rFoot, 77 | Eigen::MatrixXd &lFoot, 78 | Eigen::MatrixXd &rHandJoints, 79 | Eigen::MatrixXd &lHandJoints, 80 | Eigen::MatrixXd &faceJoints, 81 | bool verbose=false); 82 | 83 | void Adam_Fit_PAF(TotalModel &adam, 84 | smpl::SMPLParams &frame_param, 85 | Eigen::MatrixXd &BodyJoints, 86 | Eigen::MatrixXd &rFoot, 87 | Eigen::MatrixXd &lFoot, 88 | Eigen::MatrixXd &rHandJoints, 89 | Eigen::MatrixXd &lHandJoints, 90 | Eigen::MatrixXd &faceJoints, 91 | Eigen::MatrixXd &PAF, 92 | Eigen::MatrixXd &surface_constraint, 93 | double* K=nullptr, 94 | uint regressor_type=0u, 95 | bool quan=false, 96 | bool fitPAFfirst=false, 97 | bool fit_face_exp=false, 98 | bool euler=true); 99 | 100 | void Adam_Fit_H36M(TotalModel &adam, 101 | smpl::SMPLParams &frame_param, 102 | Eigen::MatrixXd &BodyJoints); 103 | 104 | void Adam_skeletal_refit(TotalModel &adam, 105 | smpl::SMPLParams &frame_param, 106 | Eigen::MatrixXd &BodyJoints, 107 | Eigen::MatrixXd &rFoot, 108 | Eigen::MatrixXd &lFoot, 109 | Eigen::MatrixXd &rHandJoints, 110 | Eigen::MatrixXd &lHandJoints, 111 | Eigen::MatrixXd &faceJoints, 112 | Eigen::MatrixXd &PAF, 113 | uint regressor_type=0u, 114 | bool bFreezeShape=false, 115 | bool euler=true); 116 | 117 | void Adam_skeletal_init(TotalModel &adam, 118 | smpl::SMPLParams &frame_param, 119 | Eigen::MatrixXd &BodyJoints, 120 | Eigen::MatrixXd &rFoot, 121 | Eigen::MatrixXd &lFoot, 122 | Eigen::MatrixXd &rHandJoints, 123 | Eigen::MatrixXd &lHandJoints, 124 | Eigen::MatrixXd &faceJoints, 125 | Eigen::MatrixXd &PAF, 126 | uint regressor_type=0u); 127 | 128 | void Adam_align_mano(TotalModel &adam, 129 | smpl::SMPLParams &frame_param, 130 | Eigen::MatrixXd &surface_constraint); 131 | 132 | void Adam_refit_batch(TotalModel &adam, 133 | std::vector &frame_param, 134 | std::vector &BodyJoints, 135 | std::vector &rFoot, 136 | std::vector &lFoot, 137 | std::vector &rHandJoints, 138 | std::vector &lHandJoints, 139 | std::vector &faceJoints, 140 | std::vector &PAF, 141 | uint regressor_type=0u, 142 | bool bFreezeShape=false, 143 | bool euler=true, 144 | bool bDCT=true); -------------------------------------------------------------------------------- /FitAdam/include/HandFastCost.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "handm.h" 3 | #include 4 | #include "ceres/ceres.h" 5 | #include "ceres/rotation.h" 6 | #include 7 | 8 | class HandFastCost: public ceres::CostFunction 9 | { 10 | public: 11 | HandFastCost(smpl::HandModel& handm, Eigen::MatrixXd &HandJoints, Eigen::MatrixXd &PAF, Eigen::MatrixXd &surface_constraints, 12 | bool fit3d, bool fit2d, bool fitPAF, bool fit_surface=false, const double* K=nullptr, int regressor_type=0, const bool euler=true): 13 | handm_(handm), HandJoints_(HandJoints), PAF_(PAF), surface_constraints(surface_constraints), res_dim(0), num_PAF_constraint(PAF.cols()), 14 | fit3d_(fit3d), fit2d_(fit2d), fitPAF_(fitPAF), fit_surface(fit_surface), K_(K), start_2d_dim(fit3d?3:0), regressor_type(regressor_type), euler_(euler) 15 | { 16 | assert(HandJoints_.rows() == 5); 17 | assert(HandJoints_.cols() == 21); 18 | assert(PAF_.rows() == 3); 19 | assert(PAF_.cols() == 20); 20 | assert(regressor_type >= 0 && regressor_type <= 1); 21 | if (fit3d) res_dim += 3; 22 | if (fit2d) 23 | { 24 | res_dim += 2; 25 | assert(K_); 26 | } 27 | m_nResiduals = res_dim * smpl::HandModel::NUM_JOINTS + (fit_surface ? res_dim * surface_constraints.cols() : 0) + (fitPAF ? 3 * num_PAF_constraint : 0); 28 | start_PAF = res_dim * smpl::HandModel::NUM_JOINTS + (fit_surface ? res_dim * surface_constraints.cols() : 0); 29 | CostFunction::set_num_residuals(m_nResiduals); 30 | auto parameter_block_sizes = CostFunction::mutable_parameter_block_sizes(); 31 | parameter_block_sizes->clear(); 32 | parameter_block_sizes->push_back(3); // Translation 33 | parameter_block_sizes->push_back(smpl::HandModel::NUM_POSE_PARAMETERS); // SMPL Pose 34 | parameter_block_sizes->push_back(smpl::HandModel::NUM_SHAPE_COEFFICIENTS); // SMPL Shape Coefficients 35 | std::cout << "m_nResiduals: " << m_nResiduals << std::endl; 36 | 37 | parentIndexes[handm_.update_inds_(0)].clear(); 38 | parentIndexes[handm_.update_inds_(0)].push_back(handm_.update_inds_(0)); 39 | for(auto i = 1u; i < smpl::HandModel::NUM_JOINTS; i++) 40 | { 41 | parentIndexes[handm_.update_inds_(i)] = std::vector(1, handm_.update_inds_(i)); 42 | while (parentIndexes[handm_.update_inds_(i)].back() != handm_.update_inds_(0)) 43 | parentIndexes[handm_.update_inds_(i)].emplace_back(handm_.parents_[parentIndexes[handm_.update_inds_(i)].back()]); 44 | std::sort(parentIndexes[handm_.update_inds_(i)].begin(), parentIndexes[handm_.update_inds_(i)].end()); 45 | } 46 | 47 | PAF_connection.resize(num_PAF_constraint); 48 | PAF_connection = {{ {{0, 21 - 21, 0, 22 - 21}}, {{0, 22 - 21, 0, 23 - 21}}, {{0, 23 - 21, 0, 24 - 21}}, {{0, 24 - 21, 0, 25 - 21}}, // left hand 49 | {{0, 21 - 21, 0, 26 - 21}}, {{0, 26 - 21, 0, 27 - 21}}, {{0, 27 - 21, 0, 28 - 21}}, {{0, 28 - 21, 0, 29 - 21}}, 50 | {{0, 21 - 21, 0, 30 - 21}}, {{0, 30 - 21, 0, 31 - 21}}, {{0, 31 - 21, 0, 32 - 21}}, {{0, 32 - 21, 0, 33 - 21}}, 51 | {{0, 21 - 21, 0, 34 - 21}}, {{0, 34 - 21, 0, 35 - 21}}, {{0, 35 - 21, 0, 36 - 21}}, {{0, 36 - 21, 0, 37 - 21}}, 52 | {{0, 21 - 21, 0, 38 - 21}}, {{0, 38 - 21, 0, 39 - 21}}, {{0, 39 - 21, 0, 40 - 21}}, {{0, 40 - 21, 0, 41 - 21}}, 53 | }}; 54 | weight_joints = {{ 1, 1, 1, 1, 1, 55 | 1, 1, 1, 1, 56 | 1, 1, 1, 1, 57 | 1, 1, 1, 1, 58 | 1, 1, 1, 1 59 | }}; 60 | 61 | if (regressor_type == 1) 62 | { 63 | total_vertex.clear(); 64 | for (int k = 0; k < handm_.STB_wrist_reg.outerSize(); ++k) 65 | { 66 | for (Eigen::SparseMatrix::InnerIterator it(handm_.STB_wrist_reg, k); it; ++it) 67 | { 68 | total_vertex.push_back(k); 69 | break; // now this vertex is used, go to next vertex 70 | } 71 | } 72 | } 73 | 74 | if (fit_surface) 75 | { 76 | correspondence_vertex_constraints.clear(); 77 | for (auto i = 0; i < surface_constraints.cols(); i++) 78 | { 79 | correspondence_vertex_constraints.push_back(total_vertex.size()); 80 | total_vertex.push_back(surface_constraints(5, i)); 81 | } 82 | weight_vertex.resize(surface_constraints.cols()); 83 | std::fill(weight_vertex.data(), weight_vertex.data() + weight_vertex.size(), 0.0); 84 | } 85 | } 86 | 87 | ~HandFastCost() {} 88 | virtual bool Evaluate(double const* const* parameters, double* residuals, double** jacobians) const; 89 | void ForwardKinematics(double const* p_euler, double const* c, double* J_data, double* dJdP_data, double* dJdc_data) const; 90 | void select_lbs( 91 | std::vector>& MR, 92 | std::vector>& Mt, 93 | std::vector>& dMRdP, 94 | std::vector>& dMRdc, 95 | std::vector>& dMtdP, 96 | std::vector>& dMtdc, 97 | MatrixXdr &outVert, double* dVdP_data, double* dVdc_data) const; 98 | void SparseRegress(const Eigen::SparseMatrix& reg, const double* V_data, const double* dVdP_data, const double* dVdc_data, 99 | double* J_data, double* dJdP_data, double* dJdc_data) const; 100 | // float weight_2d = 1.0f; 101 | float weight_2d[2] = {1.0f, 1.0f}; 102 | float weight_PAF = 50.0f; 103 | std::vector weight_vertex; 104 | 105 | private: 106 | std::vector> PAF_connection; 107 | bool fit3d_, fit2d_, fitPAF_, fit_surface; 108 | int res_dim; 109 | const int start_2d_dim; 110 | int start_PAF; 111 | int m_nResiduals; 112 | const double* K_; 113 | smpl::HandModel& handm_; 114 | Eigen::MatrixXd& HandJoints_; 115 | Eigen::MatrixXd& PAF_; 116 | Eigen::MatrixXd& surface_constraints; 117 | std::array, smpl::HandModel::NUM_JOINTS> parentIndexes; 118 | std::array weight_joints; 119 | const int num_PAF_constraint; 120 | const int regressor_type; 121 | std::vector total_vertex; 122 | std::vector correspondence_vertex_constraints; 123 | const bool euler_; 124 | }; -------------------------------------------------------------------------------- /FitAdam/include/KinematicModel.h: -------------------------------------------------------------------------------- 1 | #include "handm.h" 2 | #include "totalmodel.h" 3 | #include 4 | 5 | #ifndef KINEMATICMODEL 6 | #define KINEMATICMODEL 7 | 8 | void GenerateMesh(CMeshModelInstance& returnMesh, double* resultJoint, smpl::SMPLParams& targetParam, smpl::HandModel& g_handl_model, const int regressor_type=0, const bool euler=true); 9 | void GenerateMesh(CMeshModelInstance& returnMesh, double* resultJoint, smpl::SMPLParams& targetParam, TotalModel& g_total_model, const int regressor_type=0, const bool euler=true); 10 | void GenerateMesh_Fast(CMeshModelInstance& returnMesh, double* resultJoint, smpl::SMPLParams& targetParam, TotalModel& g_total_model, 11 | const Eigen::Matrix &Vt_vec, const Eigen::Matrix &J0_Vec); 12 | 13 | void CopyMesh(const CMeshModelInstance& mesh, VisualizedData& g_vis_data); 14 | 15 | #endif -------------------------------------------------------------------------------- /FitAdam/include/ModelFitter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | class ModelFitter 14 | { 15 | public: 16 | // 21 + 21 + 21 + 70 + 6 17 | const static int NUM_KEYPOINTS_2D = 139; 18 | // 14 + 3 + 6 + 20 + 20 19 | const static int NUM_PAF_VEC = 63; 20 | ModelFitter(const TotalModel& g_total_model); 21 | void setCalibK(const double* K); 22 | void initParameters(const smpl::SMPLParams& frame_params, const uint t=0); 23 | void readOutParameters(smpl::SMPLParams& frame_params, const uint t=0); 24 | void resetTimeStep(const uint t); 25 | void resetFitData(); 26 | void resetCostFunction(); 27 | void runFitting(); 28 | // helper function 29 | void setSurfaceConstraints2D(const std::vector& surfaceConstraint2D, const uint t=0); 30 | void setFitDataNetOutput( 31 | const std::array net_output, 32 | const bool copyBody=true, 33 | const bool copyFoot=true, 34 | const bool copyHand=true, 35 | const bool copyFace=true, 36 | const bool copyPAF=true, 37 | const uint t=0 38 | ); 39 | void setFitDataReconstruction(const std::vector& reconstruction, const uint t=0); 40 | void multiStageFitting(); 41 | 42 | std::vector> pCostFunction; 43 | AdamBodyPoseParamPriorDiff poseRegularizer; // pose regularizer is here to facilitate adjusting weight 44 | ceres::Solver::Options options; 45 | 46 | // some fixed options for AdamFullCost 47 | int regressor_type; 48 | bool fit_face_exp; 49 | bool euler; 50 | // some fixed options for AdamFitData 51 | bool fit3D; 52 | bool fit2D; 53 | bool fitPAF; 54 | bool fit_surface; 55 | bool freezeShape; 56 | bool shareCoeff; 57 | // some options about smoothing 58 | uint DCT_trans_start; 59 | uint DCT_trans_end; 60 | uint DCT_trans_low_comp; 61 | // fit input data 62 | std::vector bodyJoints; // 21 joints SMC order 63 | std::vector rFoot; // large toe, small toe, 64 | std::vector lFoot; 65 | std::vector faceJoints; // 70 joints 66 | std::vector lHandJoints; // 21 joints 67 | std::vector rHandJoints; 68 | std::vector PAF; // 63 PAFs 69 | std::vector surface_constraint; 70 | // weight for regularizers / prior 71 | float wPoseRg, wCoeffRg, wPosePr, wHandPosePr, wFacePr; 72 | private: 73 | void checkTimeStepEqual(); 74 | const TotalModel& adam; 75 | std::vector FitData; 76 | std::array calibK; // camera intrinsics 77 | bool costFunctionInit; 78 | bool calibKInit; 79 | bool fitDataInit; 80 | // parameter buffer for optimization 81 | std::vector> pose; 82 | std::vector> coeffs; 83 | std::vector trans; 84 | std::vector> exp; 85 | // ceres summary variable 86 | ceres::Solver::Summary summary; 87 | // Data for the prior 88 | Eigen::Matrix prior_A; 89 | Eigen::Matrix prior_mu; 90 | Eigen::MatrixXd hand_prior_A; // have to use MatrixXd here, size too large too put on stack 91 | Eigen::Matrix hand_prior_mu; 92 | // regularizer / prior cost 93 | CoeffsParameterNormDiff coeffRegularizer; 94 | std::unique_ptr pPosePriorCost; 95 | std::unique_ptr pHandPosePriorCost; 96 | ceres::NormalPrior facePriorCost; 97 | uint timeStep; 98 | // smooth cost 99 | DCTCost* smooth_cost_trans; 100 | static const uint num_body_joint = 21; 101 | static const uint num_hand_joint = 21; 102 | static const uint num_face_landmark = 70; 103 | }; 104 | 105 | // some separate functions 106 | void fit_single_frame(TotalModel& adam, double* fit_input, double calibK[], smpl::SMPLParams& frame_params, std::vector>::iterator densepose_result, bool fit_surface); 107 | void reconstruct_adam(TotalModel& adam, smpl::SMPLParams& frame_params, std::vector& reconstruction, bool euler=true); 108 | void refit_single_frame(TotalModel& adam, smpl::SMPLParams& frame_params, std::vector& reconstruction, bool bFreezeShape, bool euler); -------------------------------------------------------------------------------- /FitAdam/include/Renderer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "VisualizedData.h" 3 | #include 4 | #include 5 | 6 | // using namespace adam 7 | // { 8 | class Renderer 9 | { 10 | public: 11 | Renderer(int* argc, char** argv); 12 | void RenderHand(VisualizedData& g_visData); 13 | void RenderHandSimple(VisualizedData& g_visData); 14 | void RenderDepthMap(VisualizedData& g_visData); 15 | void RenderProjection(VisualizedData& g_visData); 16 | // static void IdleSaveImage(); 17 | static void RenderAndRead(); 18 | static void RenderAndReadDepthMap(); 19 | static void RenderAndReadProjection(); 20 | void Display(); 21 | static VisualizationOptions options; 22 | void NormalMode(uint position=0u, int width=600, int height=600); 23 | void CameraMode(uint position=0u, int width=1920, int height=1080, double* calibK=NULL); 24 | void OrthoMode(float scale, uint position=0u); 25 | static bool use_color_fbo; 26 | private: 27 | //initialization functions 28 | void simpleInit(); // my simple way of initialization 29 | void InitGraphics(); 30 | static const std::string SHADER_ROOT; 31 | static GLuint g_shaderProgramID[8]; 32 | enum draw_mode {MODE_DRAW_DEFUALT=0, MODE_DRAW_SELECTION_PTCLOUD, MODE_DRAW_DEPTH_RENDER, MODE_DRAW_NORMALMAP, MODE_DRAW_PTCLOUD, MODE_DRAW_MESH, MODE_DRAW_MESH_TEXTURE, MODE_DRAW_PROJECTION}; 33 | void SetShader(const std::string shaderName, GLuint& programId); 34 | GLuint LoadShaderFiles(const char* vertex_file_path, const char* fragment_file_path, bool verbose=false); 35 | 36 | //global variables 37 | static int g_drawMode; 38 | static GLuint g_vao; //Vertex Array: only need one, before starting opengl window 39 | static GLuint g_vertexBuffer, g_uvBuffer, g_normalBuffer, g_indexBuffer; //Vertex Buffer. 40 | static GLuint g_fbo_color, g_fbo_depth, g_fbo_rgbfloat; //frame buffer object 41 | static GLuint g_colorTexture, g_depthTexture, g_rgbfloatTexture, g_imageTexture; 42 | static GLuint g_depth_renderbuffer; 43 | static int g_depthRenderViewPortW, g_depthRenderViewPortH; 44 | static int g_colorTextureBufferWidth, g_colorTextureBufferHeight; 45 | static GLint window_id; 46 | 47 | //pointer to data 48 | static VisualizedData* pData; 49 | 50 | //Call back functions 51 | static void reshape(int w, int h); 52 | // static void CameraModeReshape(int w, int h); 53 | static void SimpleRenderer(); 54 | static void MeshRender(); 55 | static void DepthMapRenderer(); 56 | static void DrawSkeleton(double* joints, uint vis_type, std::vector connMat); 57 | static void SpecialKeys(const int key, const int x, const int y); 58 | static void IdleReadBuffer(); 59 | static void ProjectionRenderer(); 60 | }; 61 | // } 62 | -------------------------------------------------------------------------------- /FitAdam/include/SGSmooth.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __SGSMOOTH_HPP__ 2 | #define __SGSMOOTH_HPP__ 3 | 4 | #include 5 | 6 | // savitzky golay smoothing. 7 | std::vector sg_smooth(const std::vector &v, const int w, const int deg); 8 | //! numerical derivative based on savitzky golay smoothing. 9 | std::vector sg_derivative(const std::vector &v, const int w, 10 | const int deg, const double h=1.0); 11 | 12 | #endif // __SGSMOOTH_HPP__ -------------------------------------------------------------------------------- /FitAdam/include/handm.h: -------------------------------------------------------------------------------- 1 | #ifndef HANDM_H 2 | #define HANDM_H 3 | 4 | #include 5 | #include "simple.h" 6 | #include 7 | #include 8 | #include "CMeshModelInstance.h" 9 | 10 | namespace smpl { 11 | 12 | struct HandModel { 13 | static const int NUM_SHAPE_COEFFICIENTS = 21*3; 14 | static const int NUM_VERTICES = 2068; 15 | static const int NUM_JOINTS = 21; 16 | static const int NUM_POSE_PARAMETERS = NUM_JOINTS*3; 17 | 18 | // Template vertices (vector) 19 | Eigen::Matrix mu_; 20 | 21 | // Triangle faces 22 | Eigen::Matrix faces_; 23 | 24 | // Kinematic tree 25 | Eigen::Matrix update_inds_; //Ordered to trace from parents to children 26 | Eigen::Matrix parents_; 27 | //Eigen::Matrix m_jointmap_pm2model; 28 | Eigen::Matrix m_jointmap_pm2model; // m_jointmap_pm2model(pm_idx) = modelIdx 29 | // 20 0 1 2 3 4 12 13 5 6 14 15 7 8 16 17 9 10 18 19 11 30 | 31 | Eigen::Matrix pose_prior_A_; 32 | Eigen::Matrix pose_prior_mu_; 33 | Eigen::Matrix coeffs_prior_A_; 34 | Eigen::Matrix coeffs_prior_mu_; 35 | 36 | //Eigen::Matrix Mta_; // Bind pose transforms. Original naming of Tomas 37 | Eigen::Matrix m_M_l2pl; // Local coordinate to parent's local coordinate. The parent joint is at the origin 38 | //m_M_l2pl(root): Transform from local coordinate of the root (root joint at origin) 39 | //to world coordinate in binding pose 40 | //Eigen::Matrix MTi_; // Bind pose inverse transforms (with parent). Original Naming of Tomas 41 | Eigen::Matrix m_M_w2l; // World to finger joint local coordinate. The reference Joint is at the origin. 42 | //The world here means the binding location 43 | 44 | //Total Model Alignment 45 | Eigen::Matrix m_T_hand2smpl; 46 | Eigen::Matrix m_M_rootlocal2smplW; 47 | 48 | Eigen::Matrix pose_limits_; // 3*NUM_JOINTS, 2 49 | // Lower and upper bounds 50 | Eigen::Matrix coeffs_limits_; // 3*NUM_JOINTS, 2 51 | // Lower and upper bounds 52 | 53 | //Mesh Model 54 | Eigen::Matrix V_; 55 | Eigen::Matrix F_; 56 | Eigen::Matrix W_; 57 | 58 | Eigen::Matrix uv; //uv map 59 | Eigen::SparseMatrix STB_wrist_reg; // root regressor for STB dataset 60 | 61 | // A model is fully specified by its coefficients, pose, and a translation 62 | Eigen::Matrix coeffs; 63 | Eigen::Matrix pose; 64 | Eigen::Vector3d t; 65 | 66 | bool m_bInit; 67 | 68 | HandModel() { 69 | t.setZero(); 70 | coeffs.setZero(); 71 | pose.setZero(); 72 | m_bInit = false; 73 | } 74 | 75 | EIGEN_MAKE_ALIGNED_OPERATOR_NEW 76 | }; 77 | 78 | void LoadHandModelFromJson( HandModel &handm, const std::string &path ); 79 | 80 | void reconstruct_joints_mesh(const HandModel &handm, 81 | const double *trans_, 82 | const double *coeffs, 83 | const double *pose, 84 | double *outJoints, 85 | double *out_v, 86 | MatrixXdr &dJdc, 87 | MatrixXdr &dJdP, 88 | const int regressor_type=0, 89 | bool euler=true); 90 | 91 | void lbs_hand(const HandModel &handm, double* V, double* out_v); 92 | 93 | void reconstruct_joints(const HandModel &handm, 94 | const double *trans_, 95 | const double *coeffs, 96 | const double *pose, 97 | double *outJoints, 98 | bool euler); 99 | 100 | // template void initMatrix(Eigen::Matrix& m, const Json::Value& value); 101 | // template void initMatrixRowMajor(Eigen::Matrix& m, const Json::Value& value); 102 | 103 | int write_hand_obj(const CMeshModelInstance& mesh, const char* output_filename, const bool flip); 104 | } 105 | 106 | #endif -------------------------------------------------------------------------------- /FitAdam/include/json/autolink.h: -------------------------------------------------------------------------------- 1 | #ifndef JSON_AUTOLINK_H_INCLUDED 2 | # define JSON_AUTOLINK_H_INCLUDED 3 | 4 | # include "config.h" 5 | 6 | # ifdef JSON_IN_CPPTL 7 | # include 8 | # endif 9 | 10 | # if !defined(JSON_NO_AUTOLINK) && !defined(JSON_DLL_BUILD) && !defined(JSON_IN_CPPTL) 11 | # define CPPTL_AUTOLINK_NAME "json" 12 | # undef CPPTL_AUTOLINK_DLL 13 | # ifdef JSON_DLL 14 | # define CPPTL_AUTOLINK_DLL 15 | # endif 16 | # include "autolink.h" 17 | # endif 18 | 19 | #endif // JSON_AUTOLINK_H_INCLUDED 20 | -------------------------------------------------------------------------------- /FitAdam/include/json/config.h: -------------------------------------------------------------------------------- 1 | #ifndef JSON_CONFIG_H_INCLUDED 2 | # define JSON_CONFIG_H_INCLUDED 3 | 4 | /// If defined, indicates that json library is embedded in CppTL library. 5 | //# define JSON_IN_CPPTL 1 6 | 7 | /// If defined, indicates that json may leverage CppTL library 8 | //# define JSON_USE_CPPTL 1 9 | /// If defined, indicates that cpptl vector based map should be used instead of std::map 10 | /// as Value container. 11 | //# define JSON_USE_CPPTL_SMALLMAP 1 12 | /// If defined, indicates that Json specific container should be used 13 | /// (hash table & simple deque container with customizable allocator). 14 | /// THIS FEATURE IS STILL EXPERIMENTAL! 15 | //# define JSON_VALUE_USE_INTERNAL_MAP 1 16 | /// Force usage of standard new/malloc based allocator instead of memory pool based allocator. 17 | /// The memory pools allocator used optimization (initializing Value and ValueInternalLink 18 | /// as if it was a POD) that may cause some validation tool to report errors. 19 | /// Only has effects if JSON_VALUE_USE_INTERNAL_MAP is defined. 20 | //# define JSON_USE_SIMPLE_INTERNAL_ALLOCATOR 1 21 | 22 | /// If defined, indicates that Json use exception to report invalid type manipulation 23 | /// instead of C assert macro. 24 | # define JSON_USE_EXCEPTION 1 25 | 26 | # ifdef JSON_IN_CPPTL 27 | # include 28 | # ifndef JSON_USE_CPPTL 29 | # define JSON_USE_CPPTL 1 30 | # endif 31 | # endif 32 | 33 | # ifdef JSON_IN_CPPTL 34 | # define JSON_API CPPTL_API 35 | # elif defined(JSON_DLL_BUILD) 36 | # define JSON_API __declspec(dllexport) 37 | # elif defined(JSON_DLL) 38 | # define JSON_API __declspec(dllimport) 39 | # else 40 | # define JSON_API 41 | # endif 42 | 43 | #endif // JSON_CONFIG_H_INCLUDED 44 | -------------------------------------------------------------------------------- /FitAdam/include/json/features.h: -------------------------------------------------------------------------------- 1 | #ifndef CPPTL_JSON_FEATURES_H_INCLUDED 2 | # define CPPTL_JSON_FEATURES_H_INCLUDED 3 | 4 | # include "forwards.h" 5 | 6 | namespace Json { 7 | 8 | /** \brief Configuration passed to reader and writer. 9 | * This configuration object can be used to force the Reader or Writer 10 | * to behave in a standard conforming way. 11 | */ 12 | class JSON_API Features 13 | { 14 | public: 15 | /** \brief A configuration that allows all features and assumes all strings are UTF-8. 16 | * - C & C++ comments are allowed 17 | * - Root object can be any JSON value 18 | * - Assumes Value strings are encoded in UTF-8 19 | */ 20 | static Features all(); 21 | 22 | /** \brief A configuration that is strictly compatible with the JSON specification. 23 | * - Comments are forbidden. 24 | * - Root object must be either an array or an object value. 25 | * - Assumes Value strings are encoded in UTF-8 26 | */ 27 | static Features strictMode(); 28 | 29 | /** \brief Initialize the configuration like JsonConfig::allFeatures; 30 | */ 31 | Features(); 32 | 33 | /// \c true if comments are allowed. Default: \c true. 34 | bool allowComments_; 35 | 36 | /// \c true if root must be either an array or an object value. Default: \c false. 37 | bool strictRoot_; 38 | }; 39 | 40 | } // namespace Json 41 | 42 | #endif // CPPTL_JSON_FEATURES_H_INCLUDED 43 | -------------------------------------------------------------------------------- /FitAdam/include/json/forwards.h: -------------------------------------------------------------------------------- 1 | #ifndef JSON_FORWARDS_H_INCLUDED 2 | # define JSON_FORWARDS_H_INCLUDED 3 | 4 | # include "config.h" 5 | 6 | namespace Json { 7 | 8 | // writer.h 9 | class FastWriter; 10 | class StyledWriter; 11 | 12 | // reader.h 13 | class Reader; 14 | 15 | // features.h 16 | class Features; 17 | 18 | // value.h 19 | typedef int Int; 20 | typedef unsigned int UInt; 21 | class StaticString; 22 | class Path; 23 | class PathArgument; 24 | class Value; 25 | class ValueIteratorBase; 26 | class ValueIterator; 27 | class ValueConstIterator; 28 | #ifdef JSON_VALUE_USE_INTERNAL_MAP 29 | class ValueAllocator; 30 | class ValueMapAllocator; 31 | class ValueInternalLink; 32 | class ValueInternalArray; 33 | class ValueInternalMap; 34 | #endif // #ifdef JSON_VALUE_USE_INTERNAL_MAP 35 | 36 | } // namespace Json 37 | 38 | 39 | #endif // JSON_FORWARDS_H_INCLUDED 40 | -------------------------------------------------------------------------------- /FitAdam/include/json/json.h: -------------------------------------------------------------------------------- 1 | #ifndef JSON_JSON_H_INCLUDED 2 | # define JSON_JSON_H_INCLUDED 3 | 4 | # include "autolink.h" 5 | # include "value.h" 6 | # include "reader.h" 7 | # include "writer.h" 8 | # include "features.h" 9 | 10 | #endif // JSON_JSON_H_INCLUDED 11 | -------------------------------------------------------------------------------- /FitAdam/include/json/reader.h: -------------------------------------------------------------------------------- 1 | #ifndef CPPTL_JSON_READER_H_INCLUDED 2 | # define CPPTL_JSON_READER_H_INCLUDED 3 | 4 | # include "features.h" 5 | # include "value.h" 6 | # include 7 | # include 8 | # include 9 | # include 10 | 11 | namespace Json { 12 | 13 | /** \brief Unserialize a JSON document into a Value. 14 | * 15 | */ 16 | class JSON_API Reader 17 | { 18 | public: 19 | typedef char Char; 20 | typedef const Char *Location; 21 | 22 | /** \brief Constructs a Reader allowing all features 23 | * for parsing. 24 | */ 25 | Reader(); 26 | 27 | /** \brief Constructs a Reader allowing the specified feature set 28 | * for parsing. 29 | */ 30 | Reader( const Features &features ); 31 | 32 | /** \brief Read a Value from a JSON document. 33 | * \param document UTF-8 encoded string containing the document to read. 34 | * \param root [out] Contains the root value of the document if it was 35 | * successfully parsed. 36 | * \param collectComments \c true to collect comment and allow writing them back during 37 | * serialization, \c false to discard comments. 38 | * This parameter is ignored if Features::allowComments_ 39 | * is \c false. 40 | * \return \c true if the document was successfully parsed, \c false if an error occurred. 41 | */ 42 | bool parse( const std::string &document, 43 | Value &root, 44 | bool collectComments = true ); 45 | 46 | /** \brief Read a Value from a JSON document. 47 | * \param document UTF-8 encoded string containing the document to read. 48 | * \param root [out] Contains the root value of the document if it was 49 | * successfully parsed. 50 | * \param collectComments \c true to collect comment and allow writing them back during 51 | * serialization, \c false to discard comments. 52 | * This parameter is ignored if Features::allowComments_ 53 | * is \c false. 54 | * \return \c true if the document was successfully parsed, \c false if an error occurred. 55 | */ 56 | bool parse( const char *beginDoc, const char *endDoc, 57 | Value &root, 58 | bool collectComments = true ); 59 | 60 | /// \brief Parse from input stream. 61 | /// \see Json::operator>>(std::istream&, Json::Value&). 62 | bool parse( std::istream &is, 63 | Value &root, 64 | bool collectComments = true ); 65 | 66 | /** \brief Returns a user friendly string that list errors in the parsed document. 67 | * \return Formatted error message with the list of errors with their location in 68 | * the parsed document. An empty string is returned if no error occurred 69 | * during parsing. 70 | */ 71 | std::string getFormatedErrorMessages() const; 72 | 73 | private: 74 | enum TokenType 75 | { 76 | tokenEndOfStream = 0, 77 | tokenObjectBegin, 78 | tokenObjectEnd, 79 | tokenArrayBegin, 80 | tokenArrayEnd, 81 | tokenString, 82 | tokenNumber, 83 | tokenTrue, 84 | tokenFalse, 85 | tokenNull, 86 | tokenArraySeparator, 87 | tokenMemberSeparator, 88 | tokenComment, 89 | tokenError 90 | }; 91 | 92 | class Token 93 | { 94 | public: 95 | TokenType type_; 96 | Location start_; 97 | Location end_; 98 | }; 99 | 100 | class ErrorInfo 101 | { 102 | public: 103 | Token token_; 104 | std::string message_; 105 | Location extra_; 106 | }; 107 | 108 | typedef std::deque Errors; 109 | 110 | bool expectToken( TokenType type, Token &token, const char *message ); 111 | bool readToken( Token &token ); 112 | void skipSpaces(); 113 | bool match( Location pattern, 114 | int patternLength ); 115 | bool readComment(); 116 | bool readCStyleComment(); 117 | bool readCppStyleComment(); 118 | bool readString(); 119 | void readNumber(); 120 | bool readValue(); 121 | bool readObject( Token &token ); 122 | bool readArray( Token &token ); 123 | bool decodeNumber( Token &token ); 124 | bool decodeString( Token &token ); 125 | bool decodeString( Token &token, std::string &decoded ); 126 | bool decodeDouble( Token &token ); 127 | bool decodeUnicodeCodePoint( Token &token, 128 | Location ¤t, 129 | Location end, 130 | unsigned int &unicode ); 131 | bool decodeUnicodeEscapeSequence( Token &token, 132 | Location ¤t, 133 | Location end, 134 | unsigned int &unicode ); 135 | bool addError( const std::string &message, 136 | Token &token, 137 | Location extra = 0 ); 138 | bool recoverFromError( TokenType skipUntilToken ); 139 | bool addErrorAndRecover( const std::string &message, 140 | Token &token, 141 | TokenType skipUntilToken ); 142 | void skipUntilSpace(); 143 | Value ¤tValue(); 144 | Char getNextChar(); 145 | void getLocationLineAndColumn( Location location, 146 | int &line, 147 | int &column ) const; 148 | std::string getLocationLineAndColumn( Location location ) const; 149 | void addComment( Location begin, 150 | Location end, 151 | CommentPlacement placement ); 152 | void skipCommentTokens( Token &token ); 153 | 154 | typedef std::stack Nodes; 155 | Nodes nodes_; 156 | Errors errors_; 157 | std::string document_; 158 | Location begin_; 159 | Location end_; 160 | Location current_; 161 | Location lastValueEnd_; 162 | Value *lastValue_; 163 | std::string commentsBefore_; 164 | Features features_; 165 | bool collectComments_; 166 | }; 167 | 168 | /** \brief Read from 'sin' into 'root'. 169 | 170 | Always keep comments from the input JSON. 171 | 172 | This can be used to read a file into a particular sub-object. 173 | For example: 174 | \code 175 | Json::Value root; 176 | cin >> root["dir"]["file"]; 177 | cout << root; 178 | \endcode 179 | Result: 180 | \verbatim 181 | { 182 | "dir": { 183 | "file": { 184 | // The input stream JSON would be nested here. 185 | } 186 | } 187 | } 188 | \endverbatim 189 | \throw std::exception on parse error. 190 | \see Json::operator<<() 191 | */ 192 | std::istream& operator>>( std::istream&, Value& ); 193 | 194 | } // namespace Json 195 | 196 | #endif // CPPTL_JSON_READER_H_INCLUDED 197 | -------------------------------------------------------------------------------- /FitAdam/include/json/writer.h: -------------------------------------------------------------------------------- 1 | #ifndef JSON_WRITER_H_INCLUDED 2 | # define JSON_WRITER_H_INCLUDED 3 | 4 | # include "value.h" 5 | # include 6 | # include 7 | # include 8 | 9 | namespace Json { 10 | 11 | class Value; 12 | 13 | /** \brief Abstract class for writers. 14 | */ 15 | class JSON_API Writer 16 | { 17 | public: 18 | virtual ~Writer(); 19 | 20 | virtual std::string write( const Value &root ) = 0; 21 | }; 22 | 23 | /** \brief Outputs a Value in JSON format without formatting (not human friendly). 24 | * 25 | * The JSON document is written in a single line. It is not intended for 'human' consumption, 26 | * but may be usefull to support feature such as RPC where bandwith is limited. 27 | * \sa Reader, Value 28 | */ 29 | class JSON_API FastWriter : public Writer 30 | { 31 | public: 32 | FastWriter(); 33 | virtual ~FastWriter(){} 34 | 35 | void enableYAMLCompatibility(); 36 | 37 | public: // overridden from Writer 38 | virtual std::string write( const Value &root ); 39 | 40 | private: 41 | void writeValue( const Value &value ); 42 | 43 | std::string document_; 44 | bool yamlCompatiblityEnabled_; 45 | }; 46 | 47 | /** \brief Writes a Value in JSON format in a human friendly way. 48 | * 49 | * The rules for line break and indent are as follow: 50 | * - Object value: 51 | * - if empty then print {} without indent and line break 52 | * - if not empty the print '{', line break & indent, print one value per line 53 | * and then unindent and line break and print '}'. 54 | * - Array value: 55 | * - if empty then print [] without indent and line break 56 | * - if the array contains no object value, empty array or some other value types, 57 | * and all the values fit on one lines, then print the array on a single line. 58 | * - otherwise, it the values do not fit on one line, or the array contains 59 | * object or non empty array, then print one value per line. 60 | * 61 | * If the Value have comments then they are outputed according to their #CommentPlacement. 62 | * 63 | * \sa Reader, Value, Value::setComment() 64 | */ 65 | class JSON_API StyledWriter: public Writer 66 | { 67 | public: 68 | StyledWriter(); 69 | virtual ~StyledWriter(){} 70 | 71 | public: // overridden from Writer 72 | /** \brief Serialize a Value in JSON format. 73 | * \param root Value to serialize. 74 | * \return String containing the JSON document that represents the root value. 75 | */ 76 | virtual std::string write( const Value &root ); 77 | 78 | private: 79 | void writeValue( const Value &value ); 80 | void writeArrayValue( const Value &value ); 81 | bool isMultineArray( const Value &value ); 82 | void pushValue( const std::string &value ); 83 | void writeIndent(); 84 | void writeWithIndent( const std::string &value ); 85 | void indent(); 86 | void unindent(); 87 | void writeCommentBeforeValue( const Value &root ); 88 | void writeCommentAfterValueOnSameLine( const Value &root ); 89 | bool hasCommentForValue( const Value &value ); 90 | static std::string normalizeEOL( const std::string &text ); 91 | 92 | typedef std::vector ChildValues; 93 | 94 | ChildValues childValues_; 95 | std::string document_; 96 | std::string indentString_; 97 | int rightMargin_; 98 | int indentSize_; 99 | bool addChildValues_; 100 | }; 101 | 102 | /** \brief Writes a Value in JSON format in a human friendly way, 103 | to a stream rather than to a string. 104 | * 105 | * The rules for line break and indent are as follow: 106 | * - Object value: 107 | * - if empty then print {} without indent and line break 108 | * - if not empty the print '{', line break & indent, print one value per line 109 | * and then unindent and line break and print '}'. 110 | * - Array value: 111 | * - if empty then print [] without indent and line break 112 | * - if the array contains no object value, empty array or some other value types, 113 | * and all the values fit on one lines, then print the array on a single line. 114 | * - otherwise, it the values do not fit on one line, or the array contains 115 | * object or non empty array, then print one value per line. 116 | * 117 | * If the Value have comments then they are outputed according to their #CommentPlacement. 118 | * 119 | * \param indentation Each level will be indented by this amount extra. 120 | * \sa Reader, Value, Value::setComment() 121 | */ 122 | class JSON_API StyledStreamWriter 123 | { 124 | public: 125 | StyledStreamWriter( std::string indentation="\t" ); 126 | ~StyledStreamWriter(){} 127 | 128 | public: 129 | /** \brief Serialize a Value in JSON format. 130 | * \param out Stream to write to. (Can be ostringstream, e.g.) 131 | * \param root Value to serialize. 132 | * \note There is no point in deriving from Writer, since write() should not return a value. 133 | */ 134 | void write( std::ostream &out, const Value &root ); 135 | 136 | private: 137 | void writeValue( const Value &value ); 138 | void writeArrayValue( const Value &value ); 139 | bool isMultineArray( const Value &value ); 140 | void pushValue( const std::string &value ); 141 | void writeIndent(); 142 | void writeWithIndent( const std::string &value ); 143 | void indent(); 144 | void unindent(); 145 | void writeCommentBeforeValue( const Value &root ); 146 | void writeCommentAfterValueOnSameLine( const Value &root ); 147 | bool hasCommentForValue( const Value &value ); 148 | static std::string normalizeEOL( const std::string &text ); 149 | 150 | typedef std::vector ChildValues; 151 | 152 | ChildValues childValues_; 153 | std::ostream* document_; 154 | std::string indentString_; 155 | int rightMargin_; 156 | std::string indentation_; 157 | bool addChildValues_; 158 | }; 159 | 160 | std::string JSON_API valueToString( Int value ); 161 | std::string JSON_API valueToString( UInt value ); 162 | std::string JSON_API valueToString( double value ); 163 | std::string JSON_API valueToString( bool value ); 164 | std::string JSON_API valueToQuotedString( const char *value ); 165 | 166 | /// \brief Output using the StyledStreamWriter. 167 | /// \see Json::operator>>() 168 | std::ostream& operator<<( std::ostream&, const Value &root ); 169 | 170 | } // namespace Json 171 | 172 | 173 | 174 | #endif // JSON_WRITER_H_INCLUDED 175 | -------------------------------------------------------------------------------- /FitAdam/include/meshTrackingProj.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | void Tracking_MeshVertex_depthMap( 11 | const bool brox, 12 | const cv::Mat& sourceImg, 13 | const cv::Mat& targetImg, 14 | const cv::Mat_& depthMap, 15 | const double* K, 16 | const std::vector& vertices, 17 | std::vector& target_constraints, 18 | const uint sample_dist=5u); 19 | 20 | bool ComputePtVisibilityUsingDepthMap( 21 | const cv::Point3d pt3d, 22 | const double* K, 23 | const cv::Mat_& depthMap, 24 | cv::Point2d& pt2d); 25 | 26 | cv::Point2d ProjectPt(cv::Point3d pt, const double* K); 27 | 28 | void createVirtualImage( 29 | std::unique_ptr& render, 30 | const double* K, 31 | const CMeshModelInstance& mesh1, 32 | const CMeshModelInstance& mesh2, 33 | const cv::Mat_& depthMap1, // depthmap of mesh1 34 | const cv::Mat& sourceImg, 35 | cv::Mat& resultImg, 36 | cv::Mat& XY, 37 | const bool background=true); 38 | 39 | void getVirtualImageConstraint( 40 | std::unique_ptr& render, 41 | const double* K, 42 | const CMeshModelInstance& mesh1, 43 | const CMeshModelInstance& mesh2, 44 | const cv::Mat_& depthMap1, // depthmap of mesh1 45 | const cv::Mat_& depthMap2, // depthmap of mesh2 46 | const cv::Mat& sourceImg, 47 | const cv::Mat& targetImg, 48 | cv::Mat& resultImg, 49 | std::vector& target_constraints, 50 | const uint sample_dist=1u); 51 | 52 | void downSampleConstraints(std::vector& surface_constraint, const int sample_dist=1, const int height=1080, const int width=1920); 53 | -------------------------------------------------------------------------------- /FitAdam/include/rplyfile.h: -------------------------------------------------------------------------------- 1 | #ifndef RPLY_FILE_H 2 | #define RPLY_FILE_H 3 | /* ---------------------------------------------------------------------- 4 | * RPly library, read/write PLY files 5 | * Diego Nehab, IMPA 6 | * http://www.impa.br/~diego/software/rply 7 | * 8 | * This library is distributed under the MIT License. See notice 9 | * at the end of this file. 10 | * ---------------------------------------------------------------------- */ 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | 16 | /* ---------------------------------------------------------------------- 17 | * Opens a PLY file for reading (fails if file is not a PLY file) 18 | * 19 | * file_pointer: FILE * to file open for reading 20 | * error_cb: error callback function 21 | * idata,pdata: contextual information available to users 22 | * 23 | * Returns 1 if successful, 0 otherwise 24 | * ---------------------------------------------------------------------- */ 25 | p_ply ply_open_from_file(FILE *file_pointer, p_ply_error_cb error_cb, 26 | long idata, void *pdata); 27 | 28 | /* ---------------------------------------------------------------------- 29 | * Creates new PLY file 30 | * 31 | * file_pointer: FILE * to a file open for writing 32 | * storage_mode: file format mode 33 | * error_cb: error callback function 34 | * idata,pdata: contextual information available to users 35 | * 36 | * Returns handle to PLY file if successfull, NULL otherwise 37 | * ---------------------------------------------------------------------- */ 38 | p_ply ply_create_to_file(FILE *file_pointer, e_ply_storage_mode storage_mode, 39 | p_ply_error_cb error_cb, long idata, void *pdata); 40 | 41 | #ifdef __cplusplus 42 | } 43 | #endif 44 | 45 | #endif /* RPLY_FILE_H */ 46 | 47 | /* ---------------------------------------------------------------------- 48 | * Copyright (C) 2003-2015 Diego Nehab. All rights reserved. 49 | * 50 | * Permission is hereby granted, free of charge, to any person obtaining 51 | * a copy of this software and associated documentation files (the 52 | * "Software"), to deal in the Software without restriction, including 53 | * without limitation the rights to use, copy, modify, merge, publish, 54 | * distribute, sublicense, and/or sell copies of the Software, and to 55 | * permit persons to whom the Software is furnished to do so, subject to 56 | * the following conditions: 57 | * 58 | * The above copyright notice and this permission notice shall be 59 | * included in all copies or substantial portions of the Software. 60 | * 61 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 62 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 63 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 64 | * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 65 | * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 66 | * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 67 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 68 | * ---------------------------------------------------------------------- */ 69 | -------------------------------------------------------------------------------- /FitAdam/include/totalmodel.h: -------------------------------------------------------------------------------- 1 | #include "CMeshModelInstance.h" 2 | #include "simple.h" 3 | #include 4 | 5 | #ifndef TOTALMODEL_H 6 | #define TOTALMODEL_H 7 | 8 | struct TotalModelCorresUnit 9 | { 10 | TotalModelCorresUnit() 11 | { 12 | m_sourceMeshType = CMeshModelInstance::MESH_TYPE_UNKNOWN; 13 | }; 14 | 15 | TotalModelCorresUnit(CMeshModelInstance::EnumMeshType type) 16 | { 17 | m_sourceMeshType = type; 18 | }; 19 | 20 | CMeshModelInstance::EnumMeshType m_sourceMeshType;// = CMeshModelInstance::MESH_TYPE_SMPL; 21 | std::vector< std::pair > m_corresWeight; //barycentric, 22 | }; 23 | 24 | struct TotalModelGlueCorresUnit 25 | { 26 | TotalModelGlueCorresUnit() 27 | { 28 | }; 29 | 30 | 31 | std::vector< std::pair > m_sourceVertices_fromSMPL; //barycentric, 32 | int m_targetVertexIdx; //Vertices in FW, LHand, or RHand 33 | }; 34 | 35 | struct TotalModel 36 | { 37 | static const int NUM_VERTICES = 18540; 38 | static const int NUM_FACES = 36946;// 111462; 39 | 40 | static const int NUM_JOINTS = 62; //(SMPL-hands)22 + lhand20 + rhand20 41 | static const int NUM_POSE_PARAMETERS = NUM_JOINTS * 3; 42 | static const int NUM_SHAPE_COEFFICIENTS = 30; 43 | static const int NUM_EXP_BASIS_COEFFICIENTS = 200; //Facial expression 44 | 45 | //Mesh Model 46 | Eigen::Matrix m_vertices; 47 | Eigen::Matrix m_faces; //has zero-based indexing 48 | 49 | Eigen::Matrix m_uvs; //uv map 50 | Eigen::Matrix m_normals; //uv map 51 | Eigen::Matrix m_colors; 52 | 53 | std::vector m_vertexCorresSources; 54 | Eigen::SparseMatrix m_C_face2total; //3*TotalModel::NUM_VERTICES x 3*FaceModel::NUM_VERTICES 55 | Eigen::Matrix m_dVdFaceEx; //precompute: m_C_face2total* g_face_model.U_exp_ 56 | 57 | std::vector m_indicator_noDeltaDefom; //size should be same as vertice size 58 | std::vector m_vertex_laplacianWeight; //size should be same as vertice size 59 | 60 | Eigen::Matrix m_blendW; 61 | Eigen::SparseMatrix m_cocoplus_reg; 62 | Eigen::SparseMatrix m_small_coco_reg; // regressor used by Donglai to implement fast COCO keypoint fitting (SIGGAsia) 63 | 64 | Eigen::Matrix m_kintree_table; 65 | int m_parent[NUM_JOINTS]; 66 | int m_id_to_col[NUM_JOINTS]; 67 | 68 | //Adam to vertex correspondences 69 | Eigen::Matrix m_correspond_adam2face70_face70Idx; //0based indexing 70 | Eigen::Matrix m_correspond_adam2face70_adamIdx; //0based indexing 71 | Eigen::Matrix m_correspond_adam2cocoear_cocobodyIdx; //0based indexing 72 | Eigen::Matrix m_correspond_adam2cocoear_adamIdx; //0based indexing 73 | 74 | //For SMC joints 75 | Eigen::Matrix m_indices_jointConst_adamIdx; //correspondence between smpl and smc (not all joints are included) 76 | Eigen::Matrix m_indices_jointConst_smcIdx; 77 | 78 | 79 | Eigen::Matrix m_correspond_adam2lHand_adamIdx; 80 | Eigen::Matrix m_correspond_adam2lHand_lHandIdx; 81 | 82 | Eigen::Matrix m_correspond_adam2rHand_adamIdx; 83 | Eigen::Matrix m_correspond_adam2rHand_rHandIdx; 84 | 85 | 86 | //Glue constraints 87 | std::vector m_glueConstraint_face; 88 | std::vector m_glueConstraint_lhand; 89 | std::vector m_glueConstraint_rhand; 90 | 91 | //Total Model PCA space 92 | Eigen::Matrix m_shapespace_u; 93 | Eigen::Matrix m_shapespace_Ds; 94 | Eigen::Matrix m_meanshape; 95 | 96 | 97 | //For Adam 98 | Eigen::SparseMatrix m_J_reg; 99 | Eigen::SparseMatrix m_J_reg_smc; 100 | 101 | Eigen::Matrix J_mu_; 102 | Eigen::Matrix dJdc_; 103 | Eigen::VectorXd face_prior_A_exp; 104 | Eigen::Matrix smpl_pose_prior_A; 105 | Eigen::VectorXd smpl_pose_prior_mu; 106 | Eigen::Matrix hand_pose_prior_A; 107 | Eigen::VectorXd hand_pose_prior_mu; 108 | 109 | const std::array h36m_jointConst_smcIdx{{ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 0, 19, 1, 15, 17, 16, 18, 2 }}; 110 | const std::array coco_jointConst_smcIdx{{ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 0, 19, 1, 15, 17, 16, 18, 20 }}; 111 | 112 | bool m_bInit; 113 | 114 | TotalModel() { 115 | m_bInit = false; 116 | } 117 | 118 | EIGEN_MAKE_ALIGNED_OPERATOR_NEW 119 | }; 120 | 121 | void LoadTotalDataFromJson(TotalModel &totalm, const std::string &path, const std::string &pca_path, const std::string &correspondence_path); 122 | void LoadTotalModelFromObj(TotalModel &totalm, const std::string &path); 123 | void LoadModelColorFromObj(TotalModel &totalm, const std::string &path); 124 | void LoadCocoplusRegressor(TotalModel &totalm, const std::string &path); 125 | void adam_reconstruct_Eulers(const TotalModel& smpl, 126 | const double *parm_coeffs, 127 | const double *parm_pose_eulers, 128 | const double *parm_faceEx_coeffs, 129 | double *outVerts, 130 | Eigen::VectorXd &transforms, 131 | const bool euler=true); 132 | void adam_reconstruct_Eulers_Fast(const TotalModel& smpl, 133 | const Eigen::Matrix &Vt, 134 | const Eigen::Matrix &J0, 135 | const double *parm_pose_eulers, 136 | const double *parm_faceEx_coeffs, 137 | double *outVerts, 138 | Eigen::VectorXd &transforms); 139 | Eigen::VectorXd adam_reconstruct_withDerivative_eulers(const TotalModel &mod, 140 | const double *coeffs, 141 | const double *pose, //input: euler angles 142 | const double *parm_faceEx_coeffs, 143 | double *outVerts, //output 144 | MatrixXdr &dVdc, //output 145 | MatrixXdr &dVdP, 146 | MatrixXdr &dVdfc, 147 | MatrixXdr &dTJdc, 148 | MatrixXdr &dTJdP, 149 | bool joint_only, 150 | bool fit_face); //output 151 | void adam_lbs(const TotalModel &totalm, 152 | const double *verts, 153 | const MatrixXdr& T, 154 | double *outVerts); 155 | void adam_lbs(const TotalModel &totalm, 156 | const double *verts, 157 | const MatrixXdr& T, 158 | double *outVerts, //output 159 | const MatrixXdr &dVsdc, 160 | const MatrixXdr &dTdP, 161 | const MatrixXdr &dTdc, 162 | MatrixXdr &dVdc, //output 163 | MatrixXdr &dVdP); //output 164 | int write_adam_obj(const CMeshModelInstance& mesh, const char* filename); 165 | 166 | #endif -------------------------------------------------------------------------------- /FitAdam/include/utils.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | void model_size(const double* const joint, const std::vector& connMat); 6 | cv::Mat alignMeshImage(const cv::Mat& meshImage, const cv::Mat& srcImage); 7 | cv::Mat alignMeshImage(const cv::Mat& meshImage, const cv::Mat& srcImage, const cv::Mat_ depthMap); 8 | cv::Mat alignMeshImageAlpha(const cv::Mat& meshImage, const cv::Mat& srcImage); 9 | -------------------------------------------------------------------------------- /FitAdam/model/regressor_0n1_root.json: -------------------------------------------------------------------------------- 1 | { 2 | "cocoplus_regressor": [ 3 | [20,18540,0], 4 | [0,14352,0.2510911897], 5 | [0,14359,0.4992611864], 6 | [0,14436,0.006261262685], 7 | [0,14442,0.2424633905], 8 | [1,12861,8.2965792e-06], 9 | [1,12949,0.1604132651], 10 | [1,12957,0.316468379], 11 | [1,12964,0.5263159417], 12 | [1,12992,2.53584329e-05], 13 | [2,10747,0.1236176466], 14 | [2,11390,0.002046741735], 15 | [2,11968,3.087832255e-06], 16 | [2,12763,0.8771439245], 17 | [2,13703,9.551470607e-06], 18 | [3,10695,0.8619872127], 19 | [3,11659,2.741350165e-06], 20 | [3,12814,2.998513902e-06], 21 | [3,13442,0.1351881125], 22 | [3,14009,1.161010281e-06], 23 | [4,10792,9.224415723e-07], 24 | [4,10882,0.1529296606], 25 | [4,10890,0.1903185863], 26 | [4,10899,0.4095391585], 27 | [4,10923,0.2498956521], 28 | [5,12353,0.3230676493], 29 | [5,12361,0.434129204], 30 | [5,12438,0.09431584819], 31 | [5,12443,0.1467382589], 32 | [6,14882,0.001369280065], 33 | [6,15571,0.2830697528], 34 | [6,14473,0.2637301077], 35 | [6,14470,0.4237208164], 36 | [6,14608,0.02591381103], 37 | [7,13581,1.127817514e-05], 38 | [7,13582,3.060441246e-05], 39 | [7,13583,0.3689038333], 40 | [7,13586,6.660323456e-06], 41 | [7,13587,0.05226832894], 42 | [7,13630,0.5726834135], 43 | [7,13631,2.051732379e-05], 44 | [8,12555,0.6392643647], 45 | [8,13342,0.0007454234602], 46 | [8,13610,7.506868479e-05], 47 | [8,13737,0.1745794733], 48 | [8,13970,2.922058979e-05], 49 | [8,14028,0.1853539279], 50 | [9,10488,0.6459410232], 51 | [9,11288,1.213969405e-05], 52 | [9,11561,6.667243046e-06], 53 | [9,11694,0.212845671], 54 | [9,11929,4.012223498e-05], 55 | [9,11987,0.1451949286], 56 | [10,11531,0.3660724207], 57 | [10,11532,1.321357636e-05], 58 | [10,11533,1.268680616e-05], 59 | [10,11535,3.092999617e-05], 60 | [10,11536,1.126437107e-05], 61 | [10,11580,0.6364901724], 62 | [10,11583,1.846240024e-05], 63 | [11,16925,0.000760848852], 64 | [11,17614,0.3384045147], 65 | [11,16517,0.341081145], 66 | [11,16514,0.3206651799], 67 | [11,16621,9.870867722e-06], 68 | [12,10439,0.0002499018585], 69 | [12,10486,0.3278316718], 70 | [12,10552,0.001708145677], 71 | [12,10622,0.0005423387901], 72 | [12,12088,0.2370303744], 73 | [12,12511,0.1521129366], 74 | [12,12557,0.2680163088], 75 | [12,12623,0.01076719784], 76 | [12,13716,0.002744808023], 77 | [13,10408,0.1050641384], 78 | [13,7530,0.1276278767], 79 | [13,7528,0.0002094912619], 80 | [13,10442,0.06134766855], 81 | [13,2743,0.4069198135], 82 | [13,12479,0.1622141885], 83 | [13,1223,0.0006709746073], 84 | [13,4939,0.1338824094], 85 | [14,5768,0.239568473], 86 | [14,6328,0.001409349777], 87 | [14,1805,0.3087112149], 88 | [14,5186,0.0004737947967], 89 | [14,3607,0.4482593549], 90 | [15, 6731, 1.0], [16, 4131, 1.0], [17, 6993, 1.0], [18, 4721, 1.0], 91 | [19,10747,4.355192095e-06], 92 | [19,11390,3.958445841e-05], 93 | [19,11968,1.567348881e-06], 94 | [19,12763,0.4043521738], 95 | [19,13703,6.031892008e-06], 96 | [19,10695,0.4903415479], 97 | [19,11659,6.127431135e-06], 98 | [19,12814,7.038538174e-06], 99 | [19,13442,0.1059327754], 100 | [19,14009,1.544101195e-06] 101 | ] 102 | } 103 | -------------------------------------------------------------------------------- /FitAdam/src/BVHWriter.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #define PI 3.1415926 8 | 9 | const char* joint_name[] = {"center", "left_hip", "right_hip", "spine1", "left_knee", "right_knee", "spine2", "left_ankle", "right_ankle", "neck", "left_feet", "right_feet", "head1", 10 | "left_armpit", "right_armpit", "head2", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow", "left_wrist", "right_wrist", 11 | "left_thumb1", "left_thumb2", "left_thumb3", "left_thumb4", "left_index1", "left_index2", "left_index3", "left_index4", 12 | "left_middle1", "left_middle2", "left_middle3", "left_middle4", "left_ring1", "left_ring2", "left_ring3", "left_ring4", 13 | "left_little1", "left_little2", "left_little3", "left_little4", 14 | "right_thumb1", "right_thumb2", "right_thumb3", "right_thumb4", "right_index1", "right_index2", "right_index3", "right_index4", 15 | "right_middle1", "right_middle2", "right_middle3", "right_middle4", "right_ring1", "right_ring2", "right_ring3", "right_ring4", 16 | "right_little1", "right_little2", "right_little3", "right_little4" 17 | }; 18 | 19 | void RotationMatrixToEulerAngle(const Eigen::Matrix& R, std::array& angle) 20 | { 21 | if (R(2, 0) < 1.0) 22 | { 23 | if (R(2, 0) > -1.0) 24 | { 25 | angle.at(0) = atan2(R(2, 1), R(2, 2)); 26 | angle.at(1) = -asin(R(2, 0)); 27 | angle.at(2) = atan2(R(1, 0), R(0, 0)); 28 | } 29 | else 30 | { 31 | angle.at(0) = 0.0; 32 | angle.at(1) = PI / 2; 33 | angle.at(2) = -atan2(R(1, 2), R(1, 1)); 34 | } 35 | } 36 | else 37 | { 38 | angle.at(0) = 0.0; 39 | angle.at(1) = -PI / 2; 40 | angle.at(2) = -atan2(R(1, 2), R(1, 1)); 41 | } 42 | 43 | for (int i = 0; i < 3; i++) angle.at(i) = angle.at(i) * 180 / PI; 44 | } 45 | 46 | void BVHWriter::parseInput(const Eigen::Matrix& J0, std::vector>& t, std::vector>& pose) 47 | { 48 | assert(t.size() == pose.size()); 49 | this->num_frame = t.size(); 50 | this->trans = t; 51 | 52 | getHierarchy(J0); 53 | for (int time = 0; time < this->num_frame; time++) 54 | { 55 | Eigen::Matrix& pose_frame = pose[time]; 56 | getDynamic(pose_frame); 57 | } 58 | if (mUnityCompatible) 59 | { 60 | // add additional joints 61 | // left_hand 62 | std::shared_ptr left_hand_node = std::make_shared(std::string("left_hand"), this->data[20]->offset.data()); 63 | for (int time = 0; time < this->num_frame; time++) 64 | { 65 | std::array left_hand_angle = {0.0, 0.0, 0.0}; 66 | left_hand_node->euler.push_back(left_hand_angle); 67 | } 68 | left_hand_node->children = this->data[20]->children; 69 | this->data[20]->children.clear(); 70 | this->data[20]->children.push_back(left_hand_node); 71 | for(int i = 0; i < 3; i++) this->data[20]->offset[i] = 0.0; 72 | 73 | // right_hand 74 | std::shared_ptr right_hand_node = std::make_shared(std::string("right_hand"), this->data[21]->offset.data()); 75 | for (int time = 0; time < this->num_frame; time++) 76 | { 77 | std::array right_hand_angle = {0.0, 0.0, 0.0}; 78 | right_hand_node->euler.push_back(right_hand_angle); 79 | } 80 | right_hand_node->children = this->data[21]->children; 81 | this->data[21]->children.clear(); 82 | this->data[21]->children.push_back(right_hand_node); 83 | for(int i = 0; i < 3; i++) this->data[21]->offset[i] = 0.0; 84 | } 85 | } 86 | 87 | void BVHWriter::writeBVH(std::string output_file, double frame_time) 88 | { 89 | outStr.clear(); 90 | for (int i = 0; i < this->num_frame; i++) dynamicStr.push_back(std::string()); 91 | 92 | outStr += "HIERARCHY\n"; 93 | outStr += (std::string("ROOT ") + this->root->name); 94 | outStr += "\n{\n"; 95 | outStr += "\tOFFSET 0.0 0.0 0.0\n"; 96 | outStr += "\tCHANNELS 6 Xposition Yposition Zposition Zrotation Yrotation Xrotation\n"; 97 | 98 | for (int i = 0; i < this->num_frame; i++) 99 | dynamicStr[i] += std::to_string(this->trans[i](0) + this->root->offset[0]) + " " + std::to_string(this->trans[i](1) + this->root->offset[1]) + " " + std::to_string(this->trans[i](2) + this->root->offset[2]); 100 | if (mUnityCompatible) 101 | for (int i = 0; i < this->num_frame; i++) 102 | dynamicStr[i] += " 0.0 0.0 180.0"; 103 | else 104 | for (int i = 0; i < this->num_frame; i++) 105 | dynamicStr[i] += " 0.0 0.0 0.0"; 106 | 107 | writeData(this->root, 0); 108 | 109 | outStr += "}\n"; 110 | outStr += "MOTION\n"; 111 | outStr += ("Frames: " + std::to_string(this->num_frame) + "\n"); 112 | outStr += ("Frame Time: " + std::to_string(frame_time) + "\n"); 113 | 114 | for (int i = 0; i < this->num_frame; i++) 115 | outStr += (dynamicStr[i] + "\n"); 116 | 117 | std::ofstream out(output_file.c_str(), std::ios::out); 118 | out << outStr; 119 | out.close(); 120 | } 121 | 122 | void BVHWriter::getHierarchy(const Eigen::Matrix& J0) 123 | { 124 | int idj = 0; 125 | double offset[3] = {J0(3 * idj), J0(3 * idj + 1), J0(3 * idj + 2)}; 126 | this->root = std::make_shared(std::string(joint_name[idj]), offset); 127 | this->data[idj] = this->root; 128 | 129 | for (idj = 1; idj < TotalModel::NUM_JOINTS; idj++) 130 | { 131 | int idp = this->m_parent[idj]; 132 | offset[0] = J0(3 * idj) - J0(3 * idp); 133 | offset[1] = J0(3 * idj + 1) - J0(3 * idp + 1); 134 | offset[2] = J0(3 * idj + 2) - J0(3 * idp + 2); 135 | std::shared_ptr node = std::make_shared(std::string(joint_name[idj]), offset); 136 | this->data.at(idp)->children.push_back(node); 137 | this->data.at(idj) = node; 138 | } 139 | } 140 | 141 | void BVHWriter::getDynamic(Eigen::Matrix& pose) 142 | { 143 | int idj = 0; 144 | Eigen::Matrix R; 145 | std::array angle; 146 | ceres::AngleAxisToRotationMatrix(pose.data() + 3 * idj, R.data()); 147 | // Eigen::Matrix RT; 148 | // RT << R.transpose(); 149 | RotationMatrixToEulerAngle(R, angle); 150 | this->root->euler.push_back(angle); 151 | 152 | for (idj = 1; idj < TotalModel::NUM_JOINTS; idj++) 153 | { 154 | ceres::AngleAxisToRotationMatrix(pose.data() + 3 * idj, R.data()); 155 | // ceres::EulerAnglesToRotationMatrix(pose.data() + 3 * idj, 3, R.data()); 156 | // RT << R.transpose(); 157 | std::array angle; 158 | RotationMatrixToEulerAngle(R, angle); 159 | this->data[idj]->euler.push_back(angle); 160 | } 161 | } 162 | 163 | void BVHWriter::writeData(std::shared_ptr node, int depth) 164 | { 165 | if (node->children.size() > 0) 166 | { 167 | for (auto& child: node->children) 168 | { 169 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 170 | outStr += "JOINT "; 171 | outStr += (child->name + "\n"); 172 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 173 | outStr += "{\n"; 174 | for (int i = 0; i < depth + 2; i++) outStr += "\t"; 175 | outStr += "OFFSET "; 176 | if (depth == 0) 177 | { // root node 178 | outStr += "0.0 0.0 0.0\n"; 179 | } 180 | else 181 | { 182 | outStr += std::to_string(node->offset[0]) + " " + std::to_string(node->offset[1]) + " " + std::to_string(node->offset[2]) + "\n"; 183 | } 184 | for (int i = 0; i < depth + 2; i++) outStr += "\t"; 185 | outStr += "CHANNELS 3 Zrotation Yrotation Xrotation\n"; 186 | for (int j = 0; j < this->num_frame; j++) 187 | dynamicStr[j] += (" " + std::to_string(node->euler[j][2]) + " " + std::to_string(node->euler[j][1]) + " " + std::to_string(node->euler[j][0])); 188 | writeData(child, depth + 1); 189 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 190 | outStr += "}\n"; 191 | } 192 | } 193 | else 194 | { 195 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 196 | outStr += "End Site\n"; 197 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 198 | outStr += "{\n"; 199 | for (int i = 0; i < depth + 2; i++) outStr += "\t"; 200 | outStr += "OFFSET "; 201 | outStr += std::to_string(node->offset[0]) + " " + std::to_string(node->offset[1]) + " " + std::to_string(node->offset[2]) + "\n"; 202 | for (int i = 0; i < depth + 1; i++) outStr += "\t"; 203 | outStr += "}\n"; 204 | } 205 | } -------------------------------------------------------------------------------- /FitAdam/src/DCTCost.cpp: -------------------------------------------------------------------------------- 1 | #include "DCTCost.h" 2 | #define PI 3.14159265358979323846 3 | 4 | bool DCTCost::Evaluate(double const* const* parameters, double* residuals, double** jacobians) const 5 | { 6 | // E_d(B) = 1 / |num_t| || weight * DCT(input)||^2 7 | const double inv_sqrt_num_t = 1. / sqrt(num_t_); 8 | const int num_dim = end_dim_ - start_dim_; 9 | for (uint d = 0; d < num_dim; d++) 10 | { 11 | // compute DCT components for dim d 12 | for (uint k = low_comp_; k < num_t_; k++) 13 | { 14 | // compute the k-th DCT component 15 | auto& f_k = residuals[k - low_comp_ + d * (num_t_ - low_comp_)]; 16 | f_k = 0; 17 | for (uint t = 0u; t < num_t_; t++) 18 | { 19 | // f_k += parameters[t][d] * cos(k * (t + 0.5) * PI / num_t_) * (2 - 2 * cos(k * PI / num_t_)); 20 | f_k += parameters[t][d + start_dim_] * cos(k * (t + 0.5) * PI / num_t_); 21 | } 22 | f_k = f_k * weight_ * inv_sqrt_num_t; 23 | } 24 | } 25 | 26 | if (jacobians) 27 | { 28 | for (uint t = 0u; t < num_t_; t++) 29 | { 30 | if (jacobians[t]) 31 | { 32 | std::fill(jacobians[t], jacobians[t] + (num_t_ - low_comp_) * num_dim * dim_, 0.0); 33 | for (uint d = 0; d < num_dim; d++) 34 | { 35 | for (uint k = low_comp_; k < num_t_; k++) 36 | { 37 | // current residual is k - low_comp_ + d * (num_t_ - low_comp_) 38 | // jacobians[t][(k - low_comp_ + d * (num_t_ - low_comp_)) * dim_ + d] = cos(k * (t + 0.5) * PI / num_t_) * weight_ * inv_sqrt_num_t * (2 - 2 * cos(k * PI / num_t_)); 39 | jacobians[t][(k - low_comp_ + d * (num_t_ - low_comp_)) * dim_ + d + start_dim_] = cos(k * (t + 0.5) * PI / num_t_) * weight_ * inv_sqrt_num_t; 40 | } 41 | } 42 | } 43 | } 44 | } 45 | return true; 46 | } 47 | 48 | bool TemporalSmoothCostDiff::Evaluate(double const* const* parameters, double* residuals, double** jacobians) const 49 | { 50 | // Multiply by difference matrix 51 | const int num_dim = end_dim_ - start_dim_; 52 | for (uint d = 0; d < num_dim; d++) 53 | { 54 | for (uint t = 0u; t < num_t_; t++) 55 | { 56 | if (t == 0) residuals[d * num_t_ + t] = parameters[t][d + start_dim_] - parameters[t + 1][d + start_dim_]; 57 | else if (t == num_t_ - 1) residuals[d * num_t_ + t] = parameters[t][d + start_dim_] - parameters[t - 1][d + start_dim_]; 58 | else residuals[d * num_t_ + t] = 2 * parameters[t][d + start_dim_] - parameters[t - 1][d + start_dim_] - parameters[t + 1][d + start_dim_]; 59 | } 60 | } 61 | 62 | if (jacobians) 63 | { 64 | for (uint t = 0u; t < num_t_; t++) 65 | { 66 | if (jacobians[t]) 67 | { 68 | std::fill(jacobians[t], jacobians[t] + num_t_ * num_dim * dim_, 0.0); 69 | if (t == 0) 70 | { 71 | for (uint d = 0; d < num_dim; d++) 72 | { 73 | jacobians[t][(d * num_t_ + t) * dim_ + d + start_dim_] = 1; 74 | jacobians[t + 1][(d * num_t_ + t) * dim_ + d + start_dim_] = -1; 75 | } 76 | } 77 | else if (t == num_t_ - 1) 78 | { 79 | for (uint d = 0; d < num_dim; d++) 80 | { 81 | jacobians[t][(d * num_t_ + t) * dim_ + d + start_dim_] = 1; 82 | jacobians[t - 1][(d * num_t_ + t) * dim_ + d + start_dim_] = -1; 83 | } 84 | } 85 | else 86 | { 87 | for (uint d = 0; d < num_dim; d++) 88 | { 89 | jacobians[t][(d * num_t_ + t) * dim_ + d + start_dim_] = 2; 90 | jacobians[t - 1][(d * num_t_ + t) * dim_ + d + start_dim_] = -1; 91 | jacobians[t + 1][(d * num_t_ + t) * dim_ + d + start_dim_] = -1; 92 | } 93 | } 94 | } 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /FitAdam/src/lib_json/json_batchallocator.h: -------------------------------------------------------------------------------- 1 | #ifndef JSONCPP_BATCHALLOCATOR_H_INCLUDED 2 | # define JSONCPP_BATCHALLOCATOR_H_INCLUDED 3 | 4 | # include 5 | # include 6 | 7 | # ifndef JSONCPP_DOC_EXCLUDE_IMPLEMENTATION 8 | 9 | namespace Json { 10 | 11 | /* Fast memory allocator. 12 | * 13 | * This memory allocator allocates memory for a batch of object (specified by 14 | * the page size, the number of object in each page). 15 | * 16 | * It does not allow the destruction of a single object. All the allocated objects 17 | * can be destroyed at once. The memory can be either released or reused for future 18 | * allocation. 19 | * 20 | * The in-place new operator must be used to construct the object using the pointer 21 | * returned by allocate. 22 | */ 23 | template 25 | class BatchAllocator 26 | { 27 | public: 28 | typedef AllocatedType Type; 29 | 30 | BatchAllocator( unsigned int objectsPerPage = 255 ) 31 | : freeHead_( 0 ) 32 | , objectsPerPage_( objectsPerPage ) 33 | { 34 | // printf( "Size: %d => %s\n", sizeof(AllocatedType), typeid(AllocatedType).name() ); 35 | assert( sizeof(AllocatedType) * objectPerAllocation >= sizeof(AllocatedType *) ); // We must be able to store a slist in the object free space. 36 | assert( objectsPerPage >= 16 ); 37 | batches_ = allocateBatch( 0 ); // allocated a dummy page 38 | currentBatch_ = batches_; 39 | } 40 | 41 | ~BatchAllocator() 42 | { 43 | for ( BatchInfo *batch = batches_; batch; ) 44 | { 45 | BatchInfo *nextBatch = batch->next_; 46 | free( batch ); 47 | batch = nextBatch; 48 | } 49 | } 50 | 51 | /// allocate space for an array of objectPerAllocation object. 52 | /// @warning it is the responsability of the caller to call objects constructors. 53 | AllocatedType *allocate() 54 | { 55 | if ( freeHead_ ) // returns node from free list. 56 | { 57 | AllocatedType *object = freeHead_; 58 | freeHead_ = *(AllocatedType **)object; 59 | return object; 60 | } 61 | if ( currentBatch_->used_ == currentBatch_->end_ ) 62 | { 63 | currentBatch_ = currentBatch_->next_; 64 | while ( currentBatch_ && currentBatch_->used_ == currentBatch_->end_ ) 65 | currentBatch_ = currentBatch_->next_; 66 | 67 | if ( !currentBatch_ ) // no free batch found, allocate a new one 68 | { 69 | currentBatch_ = allocateBatch( objectsPerPage_ ); 70 | currentBatch_->next_ = batches_; // insert at the head of the list 71 | batches_ = currentBatch_; 72 | } 73 | } 74 | AllocatedType *allocated = currentBatch_->used_; 75 | currentBatch_->used_ += objectPerAllocation; 76 | return allocated; 77 | } 78 | 79 | /// Release the object. 80 | /// @warning it is the responsability of the caller to actually destruct the object. 81 | void release( AllocatedType *object ) 82 | { 83 | assert( object != 0 ); 84 | *(AllocatedType **)object = freeHead_; 85 | freeHead_ = object; 86 | } 87 | 88 | private: 89 | struct BatchInfo 90 | { 91 | BatchInfo *next_; 92 | AllocatedType *used_; 93 | AllocatedType *end_; 94 | AllocatedType buffer_[objectPerAllocation]; 95 | }; 96 | 97 | // disabled copy constructor and assignement operator. 98 | BatchAllocator( const BatchAllocator & ); 99 | void operator =( const BatchAllocator &); 100 | 101 | static BatchInfo *allocateBatch( unsigned int objectsPerPage ) 102 | { 103 | const unsigned int mallocSize = sizeof(BatchInfo) - sizeof(AllocatedType)* objectPerAllocation 104 | + sizeof(AllocatedType) * objectPerAllocation * objectsPerPage; 105 | BatchInfo *batch = static_cast( malloc( mallocSize ) ); 106 | batch->next_ = 0; 107 | batch->used_ = batch->buffer_; 108 | batch->end_ = batch->buffer_ + objectsPerPage; 109 | return batch; 110 | } 111 | 112 | BatchInfo *batches_; 113 | BatchInfo *currentBatch_; 114 | /// Head of a single linked list within the allocated space of freeed object 115 | AllocatedType *freeHead_; 116 | unsigned int objectsPerPage_; 117 | }; 118 | 119 | 120 | } // namespace Json 121 | 122 | # endif // ifndef JSONCPP_DOC_INCLUDE_IMPLEMENTATION 123 | 124 | #endif // JSONCPP_BATCHALLOCATOR_H_INCLUDED 125 | 126 | -------------------------------------------------------------------------------- /FitAdam/src/lib_json/sconscript: -------------------------------------------------------------------------------- 1 | Import( 'env buildLibrary' ) 2 | 3 | buildLibrary( env, Split( """ 4 | json_reader.cpp 5 | json_value.cpp 6 | json_writer.cpp 7 | """ ), 8 | 'json' ) 9 | -------------------------------------------------------------------------------- /FitAdam/src/librply.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CMU-Perceptual-Computing-Lab/MonocularTotalCapture/c35ca466269eb60ba61b5f51fbfb116f95f760fc/FitAdam/src/librply.so -------------------------------------------------------------------------------- /FitAdam/src/utils.cpp: -------------------------------------------------------------------------------- 1 | #include "utils.h" 2 | #include 3 | #include 4 | #include 5 | 6 | void model_size(const double* const joint, const std::vector& connMat) 7 | { 8 | double lhand_size = 0, body_size = 0, rhand_size = 0; 9 | for (auto i = 0u; i < connMat.size(); i += 2) 10 | { 11 | const double length2 = (joint[3*connMat[i]] - joint[3*connMat[i+1]]) * (joint[3*connMat[i]] - joint[3*connMat[i+1]]) 12 | + (joint[3*connMat[i] + 1] - joint[3*connMat[i+1] + 1]) * (joint[3*connMat[i] + 1] - joint[3*connMat[i+1] + 1]) 13 | + (joint[3*connMat[i] + 2] - joint[3*connMat[i+1] + 2]) * (joint[3*connMat[i] + 2] - joint[3*connMat[i+1] + 2]); 14 | const double length = sqrt(length2); 15 | if ((i >= 4 && i < 8) || (i >= 10 && i < 14) || (i >= 18 && i < 22) || (i >= 24 && i < 28)) 16 | body_size += length; 17 | else if (i >= 36 && i < 76) 18 | lhand_size += length; 19 | else if (i >= 76) 20 | rhand_size += length; 21 | } 22 | std::cout << "body size: " << body_size << " lhand size: " << lhand_size << " rhand size: " << rhand_size << std::endl; 23 | } 24 | 25 | cv::Mat alignMeshImage(const cv::Mat& meshImage, const cv::Mat& srcImage) 26 | { 27 | assert(meshImage.cols == srcImage.cols && meshImage.rows == srcImage.rows); 28 | assert(meshImage.type() == CV_8UC3); 29 | assert(srcImage.type() == CV_8UC3 || srcImage.type() == CV_8UC1); 30 | cv::Mat mask_array[3]; 31 | cv::Mat ret = srcImage.clone(); 32 | cv::Mat bgmask, fgmask; 33 | cv::Mat foreGround = meshImage.clone(); 34 | cv::compare(meshImage, cv::Scalar(255, 255, 255), bgmask, cv::CMP_EQ); // background mask, 255, 3 channels 35 | cv::split(bgmask, mask_array); 36 | bgmask = mask_array[0]; 37 | bgmask.mul(mask_array[1]); 38 | bgmask.mul(mask_array[2]); 39 | cv::bitwise_not(bgmask, fgmask); 40 | bgmask = bgmask / 255; 41 | fgmask = fgmask / 255; 42 | if (srcImage.type() == CV_8UC3) 43 | { 44 | mask_array[0] = mask_array[1] = mask_array[2] = bgmask; 45 | cv::merge(mask_array, 3, bgmask); 46 | mask_array[0] = mask_array[1] = mask_array[2] = fgmask; 47 | cv::merge(mask_array, 3, fgmask); 48 | } 49 | else 50 | cv::cvtColor(foreGround, foreGround, CV_BGR2GRAY); 51 | 52 | ret.convertTo(ret, CV_32F); 53 | foreGround.convertTo(foreGround, CV_32F); 54 | bgmask.convertTo(bgmask, CV_32F); 55 | fgmask.convertTo(fgmask, CV_32F); 56 | 57 | cv::multiply(ret, bgmask, ret); 58 | cv::multiply(foreGround, fgmask, foreGround); 59 | cv::add(ret, foreGround, ret); 60 | 61 | ret.convertTo(ret, srcImage.type()); 62 | return ret; 63 | } 64 | 65 | cv::Mat alignMeshImage(const cv::Mat& meshImage, const cv::Mat& srcImage, const cv::Mat_ depthMap) 66 | { 67 | assert(meshImage.cols == srcImage.cols && meshImage.rows == srcImage.rows); 68 | assert(meshImage.type() == CV_8UC3); 69 | assert(srcImage.type() == CV_8UC3 || srcImage.type() == CV_8UC1); 70 | assert(meshImage.cols == depthMap.cols && meshImage.rows == depthMap.rows); 71 | 72 | cv::Mat ret = srcImage.clone(); 73 | cv::Mat bgmask, fgmask, foreGround; 74 | cv::compare(depthMap, cv::Scalar(999), fgmask, cv::CMP_LT); // fg region is 255 75 | fgmask = fgmask / 255; 76 | bgmask = 1 - fgmask; 77 | 78 | if (srcImage.type() == CV_8UC3) 79 | { 80 | cv::Mat mask_array[3] = {bgmask, bgmask, bgmask}; 81 | cv::merge(mask_array, 3, bgmask); 82 | mask_array[0] = mask_array[1] = mask_array[2] = fgmask; 83 | cv::merge(mask_array, 3, fgmask); 84 | } 85 | else 86 | cv::cvtColor(foreGround, foreGround, CV_BGR2GRAY); 87 | 88 | ret.convertTo(ret, CV_32F); 89 | meshImage.convertTo(foreGround, CV_32F); 90 | fgmask.convertTo(fgmask, CV_32F); 91 | bgmask.convertTo(bgmask, CV_32F); 92 | 93 | cv::multiply(ret, bgmask, ret); 94 | cv::multiply(foreGround, fgmask, foreGround); 95 | cv::add(ret, foreGround, ret); 96 | 97 | ret.convertTo(ret, srcImage.type()); 98 | return ret; 99 | } 100 | 101 | cv::Mat alignMeshImageAlpha(const cv::Mat& meshImage, const cv::Mat& srcImage) 102 | { 103 | // meshImage should be RGBA, srcImage should be RGB 104 | assert(meshImage.type() == CV_8UC4); 105 | assert(srcImage.type() == CV_8UC1 || srcImage.type() == CV_8UC3); 106 | 107 | cv::Mat mask_array[4]; 108 | cv::Mat foreGround; 109 | cv::split(meshImage, mask_array); 110 | cv::Mat fgmask = mask_array[3]; 111 | fgmask.convertTo(fgmask, CV_32F); 112 | fgmask = fgmask / 255.0f; 113 | // for (auto y = 0; y < fgmask.rows; y++) 114 | // for (auto x = 0; x < fgmask.cols; x++) 115 | // { 116 | // if (fgmask.at(y, x) != 0.0) 117 | // fgmask.at(y, x) = 1; 118 | // } 119 | cv::Mat bgmask = 1.0f - fgmask; 120 | cv::merge(mask_array, 3, foreGround); 121 | cv::cvtColor(foreGround, foreGround, CV_RGB2BGR); 122 | foreGround.convertTo(foreGround, CV_32F); 123 | cv::Mat ret = srcImage.clone(); 124 | ret.convertTo(ret, CV_32F); 125 | 126 | if (srcImage.type() == CV_8UC3) 127 | { 128 | cv::Mat mask_array[3] = {bgmask, bgmask, bgmask}; 129 | cv::merge(mask_array, 3, bgmask); 130 | mask_array[0] = mask_array[1] = mask_array[2] = fgmask; 131 | cv::merge(mask_array, 3, fgmask); 132 | } 133 | else 134 | cv::cvtColor(foreGround, foreGround, CV_BGR2GRAY); 135 | 136 | cv::multiply(ret, bgmask, ret); 137 | cv::multiply(foreGround, fgmask, foreGround); 138 | cv::add(ret, foreGround, ret); 139 | ret.convertTo(ret, srcImage.type()); 140 | 141 | return ret; 142 | } -------------------------------------------------------------------------------- /POF/calib.json: -------------------------------------------------------------------------------- 1 | {"K":[[2000, 0, 960],[0, 2000, 540],[0,0,1]]} 2 | -------------------------------------------------------------------------------- /POF/collect_openpose.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import numpy.linalg as nl 4 | import json 5 | import pickle 6 | import argparse 7 | 8 | map_body25_to_body19 = list(range(8)) + list(range(9, 25)) # total of 24 9 | 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('--seqName', '-n', type=str) 12 | parser.add_argument('--rootDir', '-r', type=str) 13 | parser.add_argument('--count', '-c', type=int) 14 | args = parser.parse_args() 15 | 16 | seqName = args.seqName 17 | root = args.rootDir 18 | 19 | calib_file = os.path.join(root, 'calib.json') 20 | with open(calib_file) as f: 21 | calib_data = json.load(f) 22 | 23 | frameRange = range(1, args.count + 1) 24 | person_idx = -1 25 | 26 | bs = [] 27 | ls = [] 28 | rs = [] 29 | fs = [] 30 | img_dirs = [] 31 | frame_indices = [] 32 | 33 | for i in frameRange: 34 | img_file = os.path.join(root, "raw_image", '{}_{:08d}.png'.format(seqName, i)) 35 | assert os.path.isfile(img_file) 36 | annot_2d = os.path.join(root, 'openpose_result', '{}_{:08d}_keypoints.json'.format(seqName, i)) 37 | assert os.path.exists(annot_2d) 38 | with open(annot_2d) as f: 39 | data = json.load(f) 40 | 41 | # ideally there should be only one person 42 | assert len(data['people']) == 1 43 | ip = 0 44 | 45 | joint2d = np.array(data["people"][ip]["pose_keypoints_2d"]).reshape(-1, 3) 46 | left_hand2d = np.array(data["people"][ip]["hand_left_keypoints_2d"]).reshape(-1, 3) 47 | right_hand2d = np.array(data["people"][ip]["hand_right_keypoints_2d"]).reshape(-1, 3) 48 | face2d = np.array(data["people"][ip]["face_keypoints_2d"]).reshape(-1, 3) 49 | 50 | bs.append(joint2d[map_body25_to_body19]) 51 | fs.append(face2d) 52 | ls.append(left_hand2d) 53 | rs.append(right_hand2d) 54 | img_dirs.append(img_file) 55 | frame_indices.append(i) 56 | 57 | img_dirs = np.array(img_dirs) 58 | bs = np.array(bs) 59 | ls = np.array(ls) 60 | rs = np.array(rs) 61 | fs = np.array(fs) 62 | frame_indices = np.array(frame_indices) 63 | 64 | print('Openpose output collected: data dimension:') 65 | print((len(ls), len(rs), len(fs), len(bs), len(img_dirs), len(frame_indices))) 66 | 67 | with open('{}/{}.pkl'.format(root, seqName), 'wb') as f: 68 | pickle.dump((bs, ls, rs, fs, img_dirs, calib_data, frame_indices), f) 69 | -------------------------------------------------------------------------------- /POF/data/COCOReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | import numpy as np 4 | import json 5 | from data.Base2DReader import Base2DReader 6 | from utils.keypoint_conversion import COCO_to_main, MPII_to_main 7 | 8 | 9 | class COCOReader(Base2DReader): 10 | def __init__(self, name='COCO', mode='training', objtype=0, shuffle=True, batch_size=1, crop_noise=False): 11 | super(COCOReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 12 | 13 | self.name = name 14 | assert name in ('COCO', 'MPII') 15 | assert mode in ('training', 'evaluation') 16 | 17 | if name == 'COCO': 18 | self.image_root = '/media/posefs3b/Users/gines/openpose_train/dataset/COCO/cocoapi/images/train2017/' 19 | self.mask_root = '/media/posefs3b/Users/gines/openpose_train/dataset/COCO/cocoapi/images/mask2017/train2017/' 20 | 21 | assert mode == 'training' 22 | path_to_db = '/media/posefs3b/Users/gines/openpose_train/dataset/COCO/json/COCO.json' 23 | 24 | with open(path_to_db) as f: 25 | db_data = json.load(f) 26 | 27 | img_dirs = [] 28 | mask_dirs = [] 29 | human = {'body': [], 'body_valid': [], 'other_bbox': []} 30 | for i, image_data in enumerate(db_data['root']): 31 | # bounding box test 32 | # discard the image if this bounding box overlaps with any other bounding box 33 | bbox = np.array(image_data['bbox'], dtype=np.float32) 34 | bbox[2:] += bbox[:2] 35 | 36 | if type(image_data['bbox_other']) != dict and len(image_data['bbox_other']) > 0: 37 | bbox_other = np.array(image_data['bbox_other'], dtype=np.float32).reshape(-1, 4) 38 | bbox_other[:, 2:] += bbox_other[:, :2] 39 | # xmin = np.maximum(bbox_other[:, 0], bbox[0]) 40 | # ymin = np.maximum(bbox_other[:, 1], bbox[1]) 41 | # xmax = np.minimum(bbox_other[:, 2], bbox[2]) 42 | # ymax = np.minimum(bbox_other[:, 3], bbox[3]) 43 | # overlap_cond = np.logical_and(xmin < xmax, ymin < ymax).any() 44 | # if overlap_cond: 45 | # continue 46 | 47 | zero_left = np.zeros([20 - bbox_other.shape[0], 4]) 48 | bbox_other = np.concatenate([bbox_other, zero_left], axis=0).astype(np.int32) 49 | 50 | else: 51 | bbox_other = np.zeros([20, 4], dtype=np.int32) 52 | 53 | body = np.array(image_data['joint_self'], dtype=int) 54 | if np.sum(body[:, 2] == 1) <= 3: 55 | continue 56 | 57 | img_dirs.append(os.path.join(self.image_root, image_data['img_paths'])) 58 | mask_dirs.append(os.path.join(self.mask_root, image_data['img_paths'][:-3] + 'png')) 59 | 60 | neck = (body[5:6, :2] + body[6:7, :2]) / 2 61 | heattop = np.zeros((1, 2), dtype=int) 62 | chest = 0.25 * (body[5:6, :2] + body[6:7, :2] + body[11:12, :2] + body[12:13, :2]) 63 | neck_valid = np.logical_and(body[5:6, 2] == 1, body[6:7, 2] == 1) 64 | heattop_valid = np.zeros((1,), dtype=bool) 65 | chest_valid = np.logical_and(body[5:6, 2] == 1, body[6:7, 2] == 1) * np.logical_and(body[11:12, 2] == 1, body[12:13, 2] == 1) 66 | body2d = np.concatenate([body[:, :2], neck, heattop, chest], axis=0) 67 | valid = np.concatenate([body[:, 2] == 1, neck_valid, heattop_valid, chest_valid]) 68 | 69 | human['body'].append(body2d.astype(np.float32)) 70 | human['body_valid'].append(valid.astype(bool)) 71 | human['other_bbox'].append(bbox_other) 72 | 73 | human['img_dirs'] = img_dirs 74 | human['mask_dirs'] = mask_dirs 75 | order_dict = COCO_to_main 76 | 77 | elif name == 'MPII': 78 | self.image_root = '/media/posefs3b/Datasets/MPI/images/' 79 | self.mask_root = '/media/posefs3b/Users/donglaix/mpii_mask/' 80 | path_to_db = 'data/MPII_collected.json' 81 | with open(path_to_db) as f: 82 | db_data = json.load(f) 83 | total_num = len(db_data['img_paths']) 84 | human = {'body': [], 'body_valid': [], 'other_bbox': []} 85 | img_dirs = [] 86 | mask_dirs = [] 87 | for i in range(total_num): 88 | if (mode == 'training' and not db_data['is_train'][i]) and (mode == 'evaluation' and db_data['is_train'][i]): 89 | continue 90 | body = np.array(db_data['joint_self'][i], dtype=int) 91 | if np.sum(body[:, 2] == 1) <= 3: 92 | continue 93 | img_dirs.append(os.path.join(self.image_root, db_data['img_paths'][i])) 94 | mask_dirs.append(os.path.join(self.mask_root, '{:05d}.png'.format(i))) 95 | body = np.concatenate([body, np.zeros([1, 3], dtype=int)], axis=0) 96 | human['body'].append(body[:, :2].astype(np.float32)) 97 | human['body_valid'].append(body[:, 2].astype(bool)) 98 | human['img_dirs'] = img_dirs 99 | human['mask_dirs'] = mask_dirs 100 | order_dict = MPII_to_main 101 | 102 | else: 103 | raise NotImplementedError 104 | 105 | self.register_tensor(human, order_dict) 106 | self.num_samples = len(self.tensor_dict['img_dirs']) 107 | 108 | def get(self): 109 | if self.name == 'COCO': 110 | d = super(COCOReader, self).get(withPAF=True, read_image=True, imw=640, imh=640) 111 | elif self.name == 'MPII': 112 | d = super(COCOReader, self).get(withPAF=True, read_image=True, imw=1920, imh=1080) 113 | else: 114 | raise NotImplementedError 115 | return d 116 | 117 | 118 | if __name__ == '__main__': 119 | dataset = COCOReader(name='COCO', mode='training', shuffle=False, objtype=0, crop_noise=False) 120 | data_dict = dataset.get() 121 | 122 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) 123 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 124 | sess.run(tf.global_variables_initializer()) 125 | tf.train.start_queue_runners(sess=sess) 126 | 127 | import matplotlib.pyplot as plt 128 | import utils.general 129 | from utils.PAF import plot_all_PAF, plot_PAF 130 | 131 | for i in range(dataset.num_samples): 132 | image_crop, image, body2d, body_valid, img_dir, mask, mask_crop, PAF, PAF_type = \ 133 | sess.run([data_dict['image_crop'], data_dict['image'], data_dict['keypoint_uv_local'], data_dict['body_valid'], data_dict['img_dir'], data_dict['mask'], 134 | data_dict['mask_crop'], data_dict['PAF'], data_dict['PAF_type']]) 135 | print ('{}: {}'.format(i, img_dir[0].decode())) 136 | body2d = np.squeeze(body2d) 137 | body_valid = np.squeeze(body_valid) 138 | image_crop = np.squeeze((image_crop + 0.5) * 255).astype(np.uint8) 139 | image = np.squeeze((image + 0.5) * 255).astype(np.uint8) 140 | mask = np.squeeze(mask) 141 | mask_crop = np.squeeze(mask_crop) 142 | PAF = np.squeeze(PAF) 143 | 144 | mask_image = np.stack([mask] * 3, axis=2) 145 | mask_crop_image = np.stack([mask_crop] * 3, axis=2) 146 | 147 | fig = plt.figure(1) 148 | ax1 = fig.add_subplot(231) 149 | plt.imshow(image_crop) 150 | utils.general.plot2d(ax1, body2d, valid_idx=body_valid) 151 | 152 | ax2 = fig.add_subplot(232) 153 | plt.imshow(image) 154 | 155 | ax3 = fig.add_subplot(233) 156 | plt.gray() 157 | plt.imshow(mask_image) 158 | 159 | ax4 = fig.add_subplot(234) 160 | plt.gray() 161 | plt.imshow(mask_crop_image) 162 | 163 | ax5 = fig.add_subplot(235) 164 | PAF_img, img_z = plot_all_PAF(PAF, 3) 165 | ax5.imshow(PAF_img) 166 | 167 | ax6 = fig.add_subplot(236) 168 | ax6.imshow(img_z) 169 | 170 | plt.show() 171 | -------------------------------------------------------------------------------- /POF/data/GAneratedReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from data.Base2DReader import Base2DReader 3 | import os 4 | import pickle 5 | import numpy as np 6 | from utils.keypoint_conversion import GAnerated_to_main as order_dict 7 | 8 | 9 | class GAneratedReader(Base2DReader): 10 | def __init__(self, mode='training', objtype=1, shuffle=False, batch_size=1, crop_noise=False): 11 | super(GAneratedReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 12 | assert mode == 'training' 13 | assert objtype == 1 14 | self.name = 'GAnerated' 15 | 16 | self.image_root = '/media/posefs1b/Users/donglaix/hand_datasets/GANeratedHands_Release/data/' # GANerated 17 | self.path_to_db = '/media/posefs1b/Users/donglaix/hand_datasets/GANeratedHands_Release/data/collected_data.pkl' 18 | 19 | human2d = {'left_hand': [], 'right_hand': [], 'left_hand_valid': [], 'right_hand_valid': []} 20 | 21 | with open(self.path_to_db, 'rb') as f: 22 | db_data = pickle.load(f) 23 | # load a tuple of 3 elements: list of img dirs, array of 2D joint, array of 3D joint 24 | 25 | img_dirs = [os.path.join(self.image_root, _) for _ in db_data[0]] 26 | human2d['right_hand'] = np.zeros((len(img_dirs), 21, 2), dtype=np.float32) 27 | human2d['right_hand_valid'] = np.zeros((len(img_dirs), 21), dtype=bool) 28 | human2d['right_hand_3d'] = np.zeros((len(img_dirs), 21, 3), dtype=np.float32) 29 | human2d['left_hand'] = db_data[1].astype(np.float32) 30 | human2d['left_hand_valid'] = np.ones((len(img_dirs), 21), dtype=bool) 31 | human2d['left_hand_3d'] = db_data[2].astype(np.float32) 32 | human2d['img_dirs'] = img_dirs 33 | 34 | self.num_samples = len(img_dirs) 35 | self.register_tensor(human2d, order_dict) 36 | 37 | def get(self): 38 | d = super(GAneratedReader, self).get(imw=256, imh=256) 39 | return d 40 | 41 | 42 | if __name__ == '__main__': 43 | d = GAneratedReader() 44 | d.rotate_augmentation = True 45 | d.blur_augmentation = True 46 | data_dict = d.get() 47 | 48 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) 49 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 50 | sess.run(tf.global_variables_initializer()) 51 | tf.train.start_queue_runners(sess=sess) 52 | 53 | import matplotlib.pyplot as plt 54 | from mpl_toolkits.mplot3d import Axes3D 55 | import utils.general 56 | from utils.vis_heatmap3d import vis_heatmap3d 57 | from utils.PAF import plot_PAF, PAF_to_3D, plot_all_PAF 58 | 59 | validation_images = [] 60 | 61 | for i in range(d.num_samples): 62 | print('{}/{}'.format(i + 1, d.num_samples)) 63 | values = \ 64 | sess.run([data_dict['image_crop'], data_dict['img_dir'], data_dict['keypoint_uv_local'], data_dict['hand_valid'], data_dict['scoremap2d'], 65 | data_dict['PAF'], data_dict['mask_crop'], data_dict['keypoint_xyz_origin']]) 66 | image_crop, img_dir, hand2d, hand_valid, hand2d_heatmap, PAF, mask_crop, hand3d = [np.squeeze(_) for _ in values] 67 | 68 | image_name = img_dir.item().decode() 69 | image_v = ((image_crop + 0.5) * 255).astype(np.uint8) 70 | 71 | hand2d_detected = utils.general.detect_keypoints2d(hand2d_heatmap)[:21, :] 72 | hand3d_detected, _ = PAF_to_3D(hand2d_detected, PAF, objtype=1) 73 | hand3d_detected = hand3d_detected[:21, :] 74 | 75 | fig = plt.figure(1) 76 | ax1 = fig.add_subplot(231) 77 | plt.imshow(image_v) 78 | utils.general.plot2d(ax1, hand2d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 79 | utils.general.plot2d(ax1, hand2d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 80 | 81 | ax2 = fig.add_subplot(232, projection='3d') 82 | utils.general.plot3d(ax2, hand3d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 83 | max_range = 0.5 * (np.amax(hand3d_detected, axis=0) - np.amin(hand3d_detected, axis=0)).max() 84 | center = 0.5 * (np.amax(hand3d_detected, axis=0) + np.amin(hand3d_detected, axis=0)) 85 | ax2.set_xlabel('X Label') 86 | ax2.set_ylabel('Y Label') 87 | ax2.set_zlabel('Z Label') 88 | ax2.set_xlim(center[0] - max_range, center[0] + max_range) 89 | ax2.set_ylim(center[1] - max_range, center[1] + max_range) 90 | ax2.set_zlim(center[2] - max_range, center[2] + max_range) 91 | 92 | ax3 = fig.add_subplot(233, projection='3d') 93 | utils.general.plot3d(ax3, hand3d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 94 | max_range = 0.5 * (np.amax(hand3d, axis=0) - np.amin(hand3d, axis=0)).max() 95 | center = 0.5 * (np.amax(hand3d, axis=0) + np.amin(hand3d, axis=0)) 96 | ax3.set_xlabel('X Label') 97 | ax3.set_ylabel('Y Label') 98 | ax3.set_zlabel('Z Label') 99 | ax3.set_xlim(center[0] - max_range, center[0] + max_range) 100 | ax3.set_ylim(center[1] - max_range, center[1] + max_range) 101 | ax3.set_zlim(center[2] - max_range, center[2] + max_range) 102 | 103 | xy, z = plot_all_PAF(PAF, 3) 104 | ax4 = fig.add_subplot(234) 105 | ax4.imshow(xy) 106 | 107 | ax5 = fig.add_subplot(235) 108 | ax5.imshow(z) 109 | 110 | plt.show() 111 | -------------------------------------------------------------------------------- /POF/data/MultiDataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | class MultiDataset(object): 5 | # A class to combine multi dataset input 6 | def __init__(self, db_list): 7 | assert type(db_list) == list and len(db_list) >= 1 8 | self.db_list = db_list 9 | 10 | def get(self, name_wanted): 11 | data_list = [] 12 | for i, db in enumerate(self.db_list): 13 | data = db.get() 14 | data_list.append(data) 15 | 16 | ret_data = {} 17 | for name in name_wanted: 18 | ret_data[name] = tf.concat([d[name] for d in data_list], axis=0) 19 | 20 | return ret_data 21 | 22 | 23 | def combineMultiDataset(data_list, name_wanted): 24 | # data_list is a list of data_dict 25 | ret_data = {} 26 | for name in name_wanted: 27 | ret_data[name] = tf.concat([d[name] for d in data_list], axis=0) 28 | 29 | return ret_data 30 | 31 | 32 | if __name__ == '__main__': 33 | pass 34 | -------------------------------------------------------------------------------- /POF/data/OpenposeReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from data.BaseReader import BaseReader 3 | import numpy as np 4 | import pickle 5 | from utils.keypoint_conversion import a4_to_main as order_dict 6 | import json 7 | import os 8 | 9 | 10 | class OpenposeReader(BaseReader): 11 | 12 | def __init__(self, seqName, mode='evaluation', objtype=0, shuffle=False, batch_size=1, crop_noise=False): 13 | super(OpenposeReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 14 | assert mode == 'evaluation' 15 | assert objtype == 2 16 | self.image_root = '/media/posefs1b/Users/donglaix/siggasia018/{}/'.format(seqName) 17 | assert os.path.isdir(self.image_root) 18 | 19 | path_to_db = './data/{}.pkl'.format(seqName) 20 | 21 | with open(path_to_db, 'rb') as f: 22 | db_data = pickle.load(f) 23 | 24 | human3d = {} 25 | num_samples = len(db_data[0]) 26 | K = np.array(db_data[5]['K'], dtype=np.float32) 27 | K = np.expand_dims(K, axis=0) 28 | K = np.tile(K, (num_samples, 1, 1)) 29 | human3d['K'] = K 30 | human3d['openpose_body'] = db_data[0].astype(np.float32)[:, :18, :] 31 | # duplicate the neck for head top and chest 32 | human3d['openpose_body'] = np.concatenate((human3d['openpose_body'], human3d['openpose_body'][:, 1:2, :], human3d['openpose_body'][:, 1:2, :]), axis=1) 33 | human3d['openpose_body_score'] = db_data[0][:, :18, 2].astype(np.float32) 34 | # duplicate the neck for head top and chest 35 | human3d['openpose_body_score'] = np.concatenate((human3d['openpose_body_score'], human3d['openpose_body_score'][:, 1:2], human3d['openpose_body_score'][:, 1:2]), axis=1) 36 | human3d['openpose_lhand'] = db_data[1].astype(np.float32) 37 | human3d['openpose_lhand_score'] = db_data[1][:, :, 2].astype(np.float32) 38 | human3d['openpose_rhand'] = db_data[2].astype(np.float32) 39 | human3d['openpose_rhand_score'] = db_data[2][:, :, 2].astype(np.float32) 40 | human3d['openpose_face'] = db_data[3].astype(np.float32) 41 | human3d['openpose_face_score'] = db_data[3][:, :, 2].astype(np.float32) 42 | human3d['openpose_foot'] = db_data[0].astype(np.float32)[:, 18:, :] 43 | human3d['openpose_foot_score'] = db_data[0].astype(np.float32)[:, 18:, 2] 44 | human3d['img_dirs'] = np.core.defchararray.add(np.array([self.image_root]), db_data[4]) 45 | 46 | human3d['body_valid'] = np.ones((num_samples, 20), dtype=bool) 47 | human3d['left_hand_valid'] = np.ones((num_samples, 21), dtype=bool) 48 | human3d['right_hand_valid'] = np.ones((num_samples, 21), dtype=bool) 49 | 50 | # dummy values 51 | R = np.eye(3, dtype=np.float32) 52 | R = np.expand_dims(R, axis=0) 53 | R = np.tile(R, (num_samples, 1, 1)) 54 | human3d['R'] = R 55 | t = np.ones((3,), dtype=np.float32) 56 | t = np.expand_dims(t, axis=0) 57 | t = np.tile(t, (num_samples, 1)) 58 | human3d['t'] = t 59 | dc = np.zeros((5,), dtype=np.float32) 60 | dc = np.expand_dims(dc, axis=0) 61 | dc = np.tile(dc, (num_samples, 1)) 62 | human3d['distCoef'] = dc 63 | 64 | human3d['body'] = np.zeros((num_samples, 21, 3), dtype=np.float32) 65 | human3d['left_hand'] = np.zeros((num_samples, 21, 3), dtype=np.float32) 66 | human3d['right_hand'] = np.zeros((num_samples, 21, 3), dtype=np.float32) 67 | 68 | for key, val in human3d.items(): 69 | if 'openpose' in key and 'score' not in key: 70 | # valid = val[:, :, 2] > 0.05 71 | valid = val[:, :, 2] > 0.0 72 | val[:, :, 0] *= valid 73 | val[:, :, 1] *= valid 74 | human3d[key] = val[:, :, :2] 75 | 76 | self.register_tensor(human3d, order_dict) 77 | self.num_samples = len(self.tensor_dict['img_dirs']) 78 | 79 | def get(self, imw=1920, imh=1080): 80 | d = super(OpenposeReader, self).get(withPAF=False, bbox2d=1, imw=imw, imh=imh) 81 | return d 82 | 83 | 84 | if __name__ == '__main__': 85 | d = OpenposeReader(mode='evaluation', seqName='test3', shuffle=False, objtype=2, crop_noise=False) 86 | data_dict = d.get() 87 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) 88 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 89 | sess.run(tf.global_variables_initializer()) 90 | tf.train.start_queue_runners(sess=sess) 91 | 92 | import matplotlib.pyplot as plt 93 | from mpl_toolkits.mplot3d import Axes3D 94 | import utils.general 95 | from utils.vis_heatmap3d import vis_heatmap3d 96 | 97 | validation_images = [] 98 | 99 | for i in range(d.num_samples): 100 | print('{}/{}'.format(i + 1, d.num_samples)) 101 | bimage_crop, image, body2d, body2d_local, foot2d = sess.run([data_dict['bimage_crop'], data_dict['image'], data_dict['openpose_body'], data_dict['body_uv_local'], data_dict['openpose_foot']]) 102 | foot2d = np.squeeze(foot2d) 103 | image_v = ((image[0] + 0.5) * 255).astype(np.uint8) 104 | image_crop_v = ((bimage_crop[0] + 0.5) * 255).astype(np.uint8) 105 | 106 | fig = plt.figure() 107 | ax1 = fig.add_subplot(121) 108 | plt.imshow(image_v) 109 | plt.scatter(foot2d[:, 0], foot2d[:, 1]) 110 | for i in range(4): 111 | plt.text(int(foot2d[i, 0]), int(foot2d[i, 1]), str(i)) 112 | utils.general.plot2d(ax1, body2d[0]) 113 | 114 | ax2 = fig.add_subplot(122) 115 | plt.imshow(image_crop_v) 116 | utils.general.plot2d(ax2, body2d_local[0]) 117 | plt.show() 118 | -------------------------------------------------------------------------------- /POF/data/RHDReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import pickle 3 | from data.BaseReader import BaseReader 4 | import os 5 | import numpy as np 6 | 7 | 8 | class RHDReader(BaseReader): 9 | def __init__(self, mode='training', objtype=1, shuffle=False, batch_size=1, crop_noise=False): 10 | assert objtype == 1 11 | super(RHDReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 12 | assert mode in ('training', 'evaluation') 13 | self.name = 'RHD' 14 | 15 | self.image_root = '/media/posefs0c/Users/donglaix/Experiments/RHD_published_v2/{}/'.format(mode) 16 | path_to_db = os.path.join(self.image_root, 'anno_{}.pickle'.format(mode)) 17 | 18 | with open(path_to_db, 'rb') as f: 19 | db_data = pickle.load(f) 20 | 21 | human3d = {'K': [], 'R': [], 't': [], 'distCoef': [], 'left_hand': [], 'left_hand_valid': [], 'right_hand': [], 'right_hand_valid': []} 22 | img_dirs = [] 23 | mask_dirs = [] 24 | for i, data in db_data.items(): 25 | img_dir = os.path.join(self.image_root, 'color', '{:05d}.png'.format(i)) 26 | 27 | if data['uv_vis'][:21, 2].all(): 28 | # add the left hand 29 | img_dirs.append(img_dir) 30 | human3d['R'].append(np.eye(3, dtype=np.float32)) 31 | human3d['t'].append(np.zeros((3,), dtype=np.float32)) 32 | human3d['distCoef'].append(np.zeros((5,), dtype=np.float32)) 33 | human3d['K'].append(data['K'].astype(np.float32)) 34 | 35 | human3d['left_hand'].append(data['xyz'][:21, :].astype(np.float32)) 36 | human3d['right_hand'].append(np.zeros((21, 3), dtype=np.float32)) 37 | human3d['left_hand_valid'].append(np.ones((21,), dtype=bool)) 38 | human3d['right_hand_valid'].append(np.zeros((21,), dtype=bool)) 39 | 40 | mask_dir = os.path.join(self.image_root, 'mask_sep', 'left_{:05d}.png'.format(i)) 41 | mask_dirs.append(mask_dir) 42 | 43 | if data['uv_vis'][21:, 2].all(): 44 | # add the right hand 45 | img_dirs.append(img_dir) 46 | human3d['R'].append(np.eye(3, dtype=np.float32)) 47 | human3d['t'].append(np.zeros((3,), dtype=np.float32)) 48 | human3d['distCoef'].append(np.zeros((5,), dtype=np.float32)) 49 | human3d['K'].append(data['K'].astype(np.float32)) 50 | 51 | human3d['right_hand'].append(data['xyz'][21:, :].astype(np.float32)) 52 | human3d['left_hand'].append(np.zeros((21, 3), dtype=np.float32)) 53 | human3d['left_hand_valid'].append(np.zeros((21,), dtype=bool)) 54 | human3d['right_hand_valid'].append(np.ones((21,), dtype=bool)) 55 | 56 | mask_dir = os.path.join(self.image_root, 'mask_sep', 'right_{:05d}.png'.format(i)) 57 | mask_dirs.append(mask_dir) 58 | 59 | human3d['img_dirs'] = img_dirs 60 | # human3d['mask_dirs'] = mask_dirs 61 | self.register_tensor(human3d, {}) # pass in an empty dict because no order needs to be changed 62 | self.num_samples = len(img_dirs) 63 | 64 | def get(self): 65 | d = super(RHDReader, self).get(imw=320, imh=320) 66 | return d 67 | 68 | 69 | if __name__ == '__main__': 70 | d = RHDReader(mode='training', shuffle=True, objtype=1, crop_noise=True) 71 | d.rotate_augmentation = True 72 | d.blur_augmentation = True 73 | data_dict = d.get() 74 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.05) 75 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 76 | sess.run(tf.global_variables_initializer()) 77 | tf.train.start_queue_runners(sess=sess) 78 | 79 | import matplotlib.pyplot as plt 80 | from mpl_toolkits.mplot3d import Axes3D 81 | import utils.general 82 | from utils.PAF import plot_PAF, PAF_to_3D, plot_all_PAF 83 | 84 | for i in range(d.num_samples): 85 | print('{}/{}'.format(i + 1, d.num_samples)) 86 | values = \ 87 | sess.run([data_dict['image_crop'], data_dict['img_dir'], data_dict['keypoint_uv_local'], data_dict['scoremap2d'], 88 | data_dict['PAF'], data_dict['mask_crop'], data_dict['keypoint_xyz_local'], data_dict['keypoint_uv_origin'], data_dict['image']]) 89 | image_crop, img_dir, hand2d, hand2d_heatmap, PAF, mask_crop, hand3d, hand2d_origin, image_full = [np.squeeze(_) for _ in values] 90 | image_v = ((image_crop + 0.5) * 255).astype(np.uint8) 91 | image_full_v = ((image_full + 0.5) * 255).astype(np.uint8) 92 | 93 | hand2d_detected = utils.general.detect_keypoints2d(hand2d_heatmap)[:21, :] 94 | hand3d_detected, _ = PAF_to_3D(hand2d_detected, PAF, objtype=1) 95 | hand3d_detected = hand3d_detected[:21, :] 96 | hand_valid = np.ones((21,), dtype=bool) 97 | 98 | fig = plt.figure(1) 99 | ax1 = fig.add_subplot(241) 100 | plt.imshow(image_v) 101 | utils.general.plot2d(ax1, hand2d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 102 | utils.general.plot2d(ax1, hand2d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 103 | for j in range(21): 104 | plt.text(hand2d[j, 0], hand2d[j, 1], str(j)) 105 | 106 | ax2 = fig.add_subplot(242, projection='3d') 107 | utils.general.plot3d(ax2, hand3d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 108 | ax2.set_xlabel('X Label') 109 | ax2.set_ylabel('Y Label') 110 | ax2.set_zlabel('Z Label') 111 | plt.axis('equal') 112 | 113 | ax3 = fig.add_subplot(243, projection='3d') 114 | utils.general.plot3d(ax3, hand3d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 115 | ax3.set_xlabel('X Label') 116 | ax3.set_ylabel('Y Label') 117 | ax3.set_zlabel('Z Label') 118 | plt.axis('equal') 119 | 120 | xy, z = plot_all_PAF(PAF, 3) 121 | ax4 = fig.add_subplot(244) 122 | ax4.imshow(xy) 123 | 124 | ax5 = fig.add_subplot(245) 125 | ax5.imshow(z) 126 | 127 | ax6 = fig.add_subplot(246) 128 | ax6.imshow(image_full_v) 129 | utils.general.plot2d(ax6, hand2d_origin, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 130 | 131 | ax7 = fig.add_subplot(247) 132 | mask_3c = np.stack([mask_crop] * 3, axis=2) 133 | ax7.imshow(mask_3c) 134 | 135 | ax8 = fig.add_subplot(248) 136 | ax8.imshow((mask_3c * image_v).astype(np.uint8)) 137 | 138 | plt.show() 139 | -------------------------------------------------------------------------------- /POF/data/STBReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | import numpy as np 4 | from data.BaseReader import BaseReader 5 | import pickle 6 | from data.collect_stb import PATH_TO_DATASET, K, Rl, Rr, tl, tr, TRAIN_SEQS, TEST_SEQS 7 | from utils.keypoint_conversion import STB_to_main 8 | 9 | 10 | class STBReader(BaseReader): 11 | def __init__(self, mode='training', objtype=1, shuffle=False, batch_size=1, crop_noise=False): 12 | assert objtype == 1 13 | super(STBReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 14 | assert mode in ('training', 'evaluation') 15 | self.name = 'STB' 16 | 17 | self.image_root = PATH_TO_DATASET 18 | path_to_db = './data/stb_collected.pkl' 19 | with open(path_to_db, 'rb') as f: 20 | db_data = pickle.load(f) 21 | 22 | if mode == 'training': 23 | mode_data = db_data[0] 24 | SEQS = TRAIN_SEQS 25 | else: 26 | mode_data = db_data[1] 27 | SEQS = TEST_SEQS 28 | assert mode_data.shape[0] == len(SEQS) * 1500 29 | 30 | hand3d = np.tile(mode_data, [2, 1, 1]).astype(np.float32) 31 | hand3d[:, 0] = 2 * hand3d[:, 0] - hand3d[:, 9] 32 | self.num_samples = hand3d.shape[0] 33 | Ks = np.array([K] * self.num_samples, dtype=np.float32) 34 | Rs = np.array([Rl] * mode_data.shape[0] + [Rr] * mode_data.shape[0], dtype=np.float32) 35 | ts = np.array([tl] * mode_data.shape[0] + [tr] * mode_data.shape[0], dtype=np.float32) 36 | distCoef = np.zeros([self.num_samples, 5], dtype=np.float32) 37 | left_hand_valid = np.ones([self.num_samples, 21], dtype=bool) 38 | img_dirs = [os.path.join(self.image_root, seq, 'BB_left_{}.png').format(i) for seq in SEQS for i in range(1500)] + \ 39 | [os.path.join(self.image_root, seq, 'BB_right_{}.png'.format(i)) for seq in SEQS for i in range(1500)] 40 | 41 | human3d = {'K': Ks, 'R': Rs, 't': ts, 'distCoef': distCoef, 42 | 'left_hand': hand3d, 'left_hand_valid': left_hand_valid, 'img_dirs': img_dirs, 43 | 'right_hand': np.zeros([self.num_samples, 21, 3], dtype=np.float32), 'right_hand_valid': np.zeros([self.num_samples, 21], dtype=bool)} 44 | self.register_tensor(human3d, STB_to_main) 45 | 46 | def get(self): 47 | d = super(STBReader, self).get(imw=640, imh=480) 48 | return d 49 | 50 | 51 | if __name__ == '__main__': 52 | d = STBReader(mode='evaluation', shuffle=True, objtype=1, crop_noise=True) 53 | data_dict = d.get() 54 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) 55 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 56 | sess.run(tf.global_variables_initializer()) 57 | tf.train.start_queue_runners(sess=sess) 58 | 59 | import matplotlib.pyplot as plt 60 | from mpl_toolkits.mplot3d import Axes3D 61 | import utils.general 62 | from utils.PAF import plot_PAF, PAF_to_3D, plot_all_PAF 63 | 64 | for i in range(d.num_samples): 65 | print('{}/{}'.format(i + 1, d.num_samples)) 66 | values = \ 67 | sess.run([data_dict['image_crop'], data_dict['img_dir'], data_dict['keypoint_uv_local'], data_dict['left_hand_valid'], data_dict['scoremap2d'], 68 | data_dict['PAF'], data_dict['mask_crop'], data_dict['keypoint_xyz_local'], data_dict['keypoint_uv_origin'], data_dict['image']]) 69 | image_crop, img_dir, hand2d, hand_valid, hand2d_heatmap, PAF, mask_crop, hand3d, hand2d_origin, image_full = [np.squeeze(_) for _ in values] 70 | image_v = ((image_crop + 0.5) * 255).astype(np.uint8) 71 | image_full_v = ((image_full + 0.5) * 255).astype(np.uint8) 72 | 73 | hand2d_detected = utils.general.detect_keypoints2d(hand2d_heatmap)[:21, :] 74 | hand3d_detected, _ = PAF_to_3D(hand2d_detected, PAF, objtype=1) 75 | hand3d_detected = hand3d_detected[:21, :] 76 | 77 | fig = plt.figure(1) 78 | ax1 = fig.add_subplot(231) 79 | plt.imshow(image_v) 80 | utils.general.plot2d(ax1, hand2d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 81 | utils.general.plot2d(ax1, hand2d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 82 | for j in range(21): 83 | plt.text(hand2d[j, 0], hand2d[j, 1], str(j)) 84 | 85 | ax2 = fig.add_subplot(232, projection='3d') 86 | utils.general.plot3d(ax2, hand3d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 87 | ax2.set_xlabel('X Label') 88 | ax2.set_ylabel('Y Label') 89 | ax2.set_zlabel('Z Label') 90 | plt.axis('equal') 91 | 92 | ax3 = fig.add_subplot(233, projection='3d') 93 | utils.general.plot3d(ax3, hand3d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 94 | ax3.set_xlabel('X Label') 95 | ax3.set_ylabel('Y Label') 96 | ax3.set_zlabel('Z Label') 97 | plt.axis('equal') 98 | 99 | xy, z = plot_all_PAF(PAF, 3) 100 | ax4 = fig.add_subplot(234) 101 | ax4.imshow(xy) 102 | 103 | ax5 = fig.add_subplot(235) 104 | ax5.imshow(z) 105 | 106 | ax6 = fig.add_subplot(236) 107 | ax6.imshow(image_full_v) 108 | utils.general.plot2d(ax6, hand2d_origin, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 109 | 110 | plt.show() 111 | -------------------------------------------------------------------------------- /POF/data/TsimonDBReader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import json 4 | from data.Base2DReader import Base2DReader 5 | import os 6 | from utils.keypoint_conversion import tsimon_to_main as order_dict 7 | 8 | 9 | class TsimonDBReader(Base2DReader): 10 | def __init__(self, mode='training', objtype=1, shuffle=False, batch_size=1, crop_noise=False): 11 | super(TsimonDBReader, self).__init__(objtype, shuffle, batch_size, crop_noise) 12 | assert mode == 'training' 13 | assert objtype == 1 14 | self.name = 'Tsimon' 15 | 16 | self.image_root = '/media/posefs0c/Users/donglaix/tsimon/' 17 | self.path_to_db = ['/media/posefs0c/Users/donglaix/tsimon/hands_v12.json', '/media/posefs0c/Users/donglaix/tsimon/hands_v13.json', '/media/posefs0c/Users/donglaix/tsimon/hands_v143.json'] 18 | 19 | human2d = {'left_hand': [], 'right_hand': [], 'left_hand_valid': [], 'right_hand_valid': []} 20 | img_dirs = [] 21 | 22 | for filename in self.path_to_db: 23 | with open(filename) as f: 24 | filedata = json.load(f) 25 | for ihand, hand_data in enumerate(filedata['root']): 26 | joint2d = np.array(hand_data['joint_self']) 27 | human2d['right_hand'].append(joint2d[:, :2].astype(np.float32)) 28 | human2d['right_hand_valid'].append(joint2d[:, 2].astype(bool)) 29 | human2d['left_hand'].append(np.zeros((21, 2), dtype=np.float32)) 30 | human2d['left_hand_valid'].append(np.zeros((21,), dtype=bool)) 31 | 32 | img_dir = os.path.join(self.image_root, '/'.join(hand_data['img_paths'].split('/')[5:])) 33 | img_dirs.append(img_dir) 34 | 35 | human2d['img_dirs'] = img_dirs 36 | self.num_samples = len(img_dirs) 37 | self.register_tensor(human2d, order_dict) 38 | 39 | def get(self): 40 | d = super(TsimonDBReader, self).get(imw=1920, imh=1080) 41 | return d 42 | 43 | 44 | if __name__ == '__main__': 45 | dataset = TsimonDBReader(mode='training', shuffle=True, objtype=1, crop_noise=True) 46 | data_dict = dataset.get() 47 | 48 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1) 49 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) 50 | sess.run(tf.global_variables_initializer()) 51 | tf.train.start_queue_runners(sess=sess) 52 | 53 | import matplotlib.pyplot as plt 54 | from mpl_toolkits.mplot3d import Axes3D 55 | import utils.general 56 | from utils.PAF import plot_PAF, PAF_to_3D, plot_all_PAF 57 | 58 | for i in range(dataset.num_samples): 59 | print('{}/{}'.format(i + 1, dataset.num_samples)) 60 | values = \ 61 | sess.run([data_dict['image_crop'], data_dict['img_dir'], data_dict['keypoint_uv_local'], data_dict['scoremap2d'], 62 | data_dict['PAF'], data_dict['mask_crop'], data_dict['keypoint_uv_origin'], data_dict['image'], 63 | data_dict['left_hand_valid'], data_dict['right_hand_valid']]) 64 | image_crop, img_dir, hand2d, hand2d_heatmap, PAF, mask_crop, hand2d_origin, image_full, left_hand_valid, right_hand_valid \ 65 | = [np.squeeze(_) for _ in values] 66 | image_v = ((image_crop + 0.5) * 255).astype(np.uint8) 67 | image_full_v = ((image_full + 0.5) * 255).astype(np.uint8) 68 | 69 | hand2d_detected = utils.general.detect_keypoints2d(hand2d_heatmap)[:21, :] 70 | hand_valid = right_hand_valid 71 | 72 | fig = plt.figure(1) 73 | ax1 = fig.add_subplot(241) 74 | plt.imshow(image_v) 75 | utils.general.plot2d(ax1, hand2d, type_str='hand', valid_idx=hand_valid, color=np.array([1.0, 0.0, 0.0])) 76 | utils.general.plot2d(ax1, hand2d_detected, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 77 | for j in range(21): 78 | plt.text(hand2d[j, 0], hand2d[j, 1], str(j)) 79 | 80 | xy, z = plot_all_PAF(PAF, 3) 81 | ax4 = fig.add_subplot(244) 82 | ax4.imshow(xy) 83 | 84 | ax6 = fig.add_subplot(246) 85 | ax6.imshow(image_full_v) 86 | utils.general.plot2d(ax6, hand2d_origin, type_str='hand', valid_idx=hand_valid, color=np.array([0.0, 0.0, 1.0])) 87 | 88 | ax7 = fig.add_subplot(247) 89 | mask_3c = np.stack([mask_crop] * 3, axis=2) 90 | ax7.imshow(mask_3c) 91 | 92 | ax8 = fig.add_subplot(248) 93 | ax8.imshow((mask_3c * image_v).astype(np.uint8)) 94 | 95 | plt.show() 96 | -------------------------------------------------------------------------------- /POF/data/collect_crop_hand.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import numpy as np 4 | from utils.general import plot2d_cv2 5 | import cv2 6 | 7 | map_index = np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=int) 8 | 9 | 10 | def project(joints, K, R=None, t=None, distCoef=None): 11 | """ Perform Projection. 12 | joints: N * 3 13 | """ 14 | x = joints.T 15 | if R is not None: 16 | x = np.dot(R, x) 17 | if t is not None: 18 | x = x + t.reshape(3, 1) 19 | 20 | xp = x[:2, :] / x[2, :] 21 | 22 | if distCoef is not None: 23 | X2 = xp[0, :] * xp[0, :] 24 | Y2 = xp[1, :] * xp[1, :] 25 | XY = X2 * Y2 26 | R2 = X2 + Y2 27 | R4 = R2 * R2 28 | R6 = R4 * R2 29 | 30 | dc = distCoef 31 | radial = 1.0 + dc[0] * R2 + dc[1] * R4 + dc[4] * R6 32 | tan_x = 2.0 * dc[2] * XY + dc[3] * (R2 + 2.0 * X2) 33 | tan_y = 2.0 * dc[3] * XY + dc[2] * (R2 + 2.0 * Y2) 34 | 35 | xp[0, :] = radial * xp[0, :] + tan_x 36 | xp[1, :] = radial * xp[1, :] + tan_y 37 | 38 | pt = np.dot(K[:2, :2], xp) + K[:2, 2].reshape((2, 1)) 39 | 40 | return pt.T, x.T 41 | 42 | 43 | if __name__ == '__main__': 44 | image_root = '/media/posefs0c/panopticdb/' 45 | save_root = '/media/posefs1b/Users/donglaix/clean_a4_hand/crop_hand_new/' 46 | with open('./data/a4_collected.pkl', 'rb') as f: 47 | data = pickle.load(f) 48 | with open('./data/camera_data_a4.pkl', 'rb') as f: 49 | cam_data = pickle.load(f) 50 | 51 | for set_name, set_data in data.items(): 52 | for i, sample_data in enumerate(set_data): 53 | print ('processing {} {} / {}'.format(set_name, i, len(set_data))) 54 | seqName = sample_data['seqName'] 55 | frame_str = sample_data['frame_str'] 56 | if 'left_hand' in sample_data: 57 | joints = np.array(sample_data['left_hand']['landmarks']).reshape(-1, 3) 58 | joints = joints[map_index] 59 | count_img = 0 60 | for c in np.random.permutation(31): 61 | if count_img == 3: # enough 62 | break 63 | if c not in sample_data['left_hand']['2D']: 64 | continue 65 | if sum(sample_data['left_hand']['2D'][c]['insideImg']) < 15 or \ 66 | sum(sample_data['left_hand']['2D'][c]['occluded']) > 5 or (sample_data['left_hand']['2D'][c]['occluded'] == 1): 67 | continue 68 | count_img += 1 69 | joint2d, _ = project(joints, cam_data[seqName][c]['K'], cam_data[seqName][c]['R'], cam_data[seqName][c]['t'], cam_data[seqName][c]['distCoef']) 70 | img_name = '{}/{}/hdImgs/{}/{}/00_{:02d}_{}.jpg'.format(image_root, 'a4', seqName, frame_str, c, frame_str) 71 | img = cv2.imread(img_name) 72 | assert img is not None 73 | 74 | x1 = np.amin(joint2d[:, 0]) 75 | x2 = np.amax(joint2d[:, 0]) 76 | y1 = np.amin(joint2d[:, 1]) 77 | y2 = np.amax(joint2d[:, 1]) 78 | cx = (x1 + x2) / 2 79 | cy = (y1 + y2) / 2 80 | size = max(x2 - x1, y2 - y1) 81 | scale = 200 / (1.5 * size) 82 | M = np.array([[scale, 0, (100 - scale * cx)], 83 | [0, scale, (100 - scale * cy)]], dtype=float) 84 | target_img = cv2.warpAffine(img, M, (200, 200)) 85 | tjoint2d = (joint2d - np.array([cx, cy])) * scale + 100 86 | plot2d_cv2(target_img, tjoint2d, 'hand', s=3, use_color=True) 87 | filename = '{}#{}#left#{:02d}.png'.format(seqName, frame_str, c) 88 | cv2.imwrite(os.path.join(save_root, filename), target_img) 89 | 90 | if 'right_hand' in sample_data: 91 | joints = np.array(sample_data['right_hand']['landmarks']).reshape(-1, 3) 92 | joints = joints[map_index] 93 | count_img = 0 94 | for c in np.random.permutation(31): 95 | if count_img == 3: # enough 96 | break 97 | if c not in sample_data['right_hand']['2D']: 98 | continue 99 | if sum(sample_data['right_hand']['2D'][c]['insideImg']) < 15 or \ 100 | sum(sample_data['right_hand']['2D'][c]['occluded']) > 5 or (sample_data['right_hand']['2D'][c]['occluded'] == 1): 101 | continue 102 | count_img += 1 103 | joint2d, _ = project(joints, cam_data[seqName][c]['K'], cam_data[seqName][c]['R'], cam_data[seqName][c]['t'], cam_data[seqName][c]['distCoef']) 104 | img_name = '{}/{}/hdImgs/{}/{}/00_{:02d}_{}.jpg'.format(image_root, 'a4', seqName, frame_str, c, frame_str) 105 | img = cv2.imread(img_name) 106 | assert img is not None 107 | 108 | x1 = np.amin(joint2d[:, 0]) 109 | x2 = np.amax(joint2d[:, 0]) 110 | y1 = np.amin(joint2d[:, 1]) 111 | y2 = np.amax(joint2d[:, 1]) 112 | cx = (x1 + x2) / 2 113 | cy = (y1 + y2) / 2 114 | size = max(x2 - x1, y2 - y1) 115 | scale = 200 / (1.5 * size) 116 | M = np.array([[scale, 0, (100 - scale * cx)], 117 | [0, scale, (100 - scale * cy)]], dtype=float) 118 | target_img = cv2.warpAffine(img, M, (200, 200)) 119 | tjoint2d = (joint2d - np.array([cx, cy])) * scale + 100 120 | plot2d_cv2(target_img, tjoint2d, 'hand', s=3, use_color=True) 121 | filename = '{}#{}#righ#{:02d}.png'.format(seqName, frame_str, c) 122 | cv2.imwrite(os.path.join(save_root, filename), target_img) 123 | -------------------------------------------------------------------------------- /POF/data/collect_openpose.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import numpy.linalg as nl 4 | import json 5 | import pickle 6 | 7 | map_body25_to_body19 = list(range(8)) + list(range(9, 25)) # total of 24 8 | 9 | seqName = 'Dexter_Grasp2' 10 | # root = '/home/donglaix/Documents/Experiments/{}'.format(seqName) 11 | root = '/media/posefs1b/Users/donglaix/siggasia018/{}/'.format(seqName) 12 | 13 | calib_file = os.path.join(root, 'calib.json') 14 | with open(calib_file) as f: 15 | calib_data = json.load(f) 16 | 17 | start = 0 18 | end = 648 19 | frameRange = range(start, end) 20 | person_idx = -1 21 | # -1 for most obvious person, -2 for second obvious person 22 | 23 | bs = [] 24 | ls = [] 25 | rs = [] 26 | fs = [] 27 | img_dirs = [] 28 | 29 | for i in frameRange: 30 | # img_file = os.path.join('openpose_image', '{}_{:012d}.jpg'.format(seqName, i)) if os.path.exists(os.path.join(root, 'openpose_image', '{}_{:012d}.jpg'.format(seqName, i))) \ 31 | # else os.path.join('openpose_image', '{}_{:012d}.png'.format(seqName, i)) # Openpose run on images 32 | img_file = os.path.join('openpose_image', '{}_{:012d}_rendered.png'.format(seqName, i)) # Openpose run on video 33 | assert os.path.exists(os.path.join(root, img_file)) 34 | annot_2d = os.path.join(root, 'openpose_result', '{}_{:012d}_keypoints.json'.format(seqName, i)) 35 | assert os.path.exists(annot_2d) 36 | with open(annot_2d) as f: 37 | data = json.load(f) 38 | scores = [] 39 | areas = [] 40 | for ip in range(len(data["people"])): 41 | joint2d = np.array(data["people"][ip]["pose_keypoints_2d"]).reshape(-1, 3) 42 | left_hand2d = np.array(data["people"][ip]["hand_left_keypoints_2d"]).reshape(-1, 3) 43 | right_hand2d = np.array(data["people"][ip]["hand_right_keypoints_2d"]).reshape(-1, 3) 44 | face2d = np.array(data["people"][ip]["face_keypoints_2d"]).reshape(-1, 3) 45 | score = np.sum(joint2d[:, 2]) + np.sum(left_hand2d[:, 2]) + np.sum(right_hand2d[:, 2]) + np.sum(face2d[:, 2]) 46 | scores.append(score) 47 | joint_valid = (joint2d[:, 0] > 0.0) * (joint2d[:, 1] > 0.0) 48 | joint_nonzero = joint2d[joint_valid, :][:, :2] 49 | mx, my = joint_nonzero.min(axis=0) 50 | Mx, My = joint_nonzero.max(axis=0) 51 | areas.append((Mx - mx) * (My - my)) 52 | scores = np.array(scores) 53 | areas = np.array(areas) 54 | 55 | idx = np.argsort(scores) 56 | # idx = np.argsort(areas) 57 | ip = idx[person_idx] 58 | 59 | joint2d = np.array(data["people"][ip]["pose_keypoints_2d"]).reshape(-1, 3) 60 | left_hand2d = np.array(data["people"][ip]["hand_left_keypoints_2d"]).reshape(-1, 3) 61 | right_hand2d = np.array(data["people"][ip]["hand_right_keypoints_2d"]).reshape(-1, 3) 62 | face2d = np.array(data["people"][ip]["face_keypoints_2d"]).reshape(-1, 3) 63 | final_body = joint2d[map_body25_to_body19] 64 | final_left = left_hand2d 65 | final_right = right_hand2d 66 | final_face = face2d 67 | 68 | bs.append(final_body) 69 | fs.append(final_face) 70 | ls.append(final_left) 71 | rs.append(final_right) 72 | img_dirs.append(img_file) 73 | 74 | img_dirs = np.array(img_dirs) 75 | bs = np.array(bs) 76 | ls = np.array(ls) 77 | rs = np.array(rs) 78 | fs = np.array(fs) 79 | 80 | print((len(ls), len(rs), len(fs), len(bs), len(img_dirs))) 81 | 82 | with open('{}.pkl'.format(seqName), 'wb') as f: 83 | pickle.dump((bs, ls, rs, fs, img_dirs, calib_data), f) 84 | -------------------------------------------------------------------------------- /POF/data/collect_stb.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from scipy.io import loadmat 3 | import os 4 | import numpy as np 5 | 6 | PATH_TO_DATASET = '/media/posefs0c/Users/donglaix/Experiments/StereoHandTracking/' 7 | TEST_SEQS = ['B1Counting', 'B1Random'] 8 | TRAIN_SEQS = ['B2Counting', 'B2Random', 'B3Counting', 'B3Random', 'B4Counting', 'B4Random', 'B5Counting', 'B5Random', 'B6Counting', 'B6Random'] 9 | 10 | K = np.diag([822.79041, 822.79041, 1.0]).astype(np.float32) 11 | K[0, 2] = 318.47345 12 | K[1, 2] = 250.31296 13 | 14 | base = 120.054 15 | Rl = np.eye(3, dtype=np.float32) 16 | Rr = np.eye(3, dtype=np.float32) 17 | tl = np.zeros((3,), dtype=np.float32) 18 | tr = np.array([-base, 0, 0], dtype=np.float32) 19 | 20 | if __name__ == '__main__': 21 | assert os.path.isdir(PATH_TO_DATASET) 22 | # collect the testing sequences 23 | all_test_data = np.zeros((0, 21, 3), dtype=np.float32) 24 | for test_seq in TEST_SEQS: 25 | mat_path = os.path.join(PATH_TO_DATASET, 'labels', test_seq + '_BB.mat') 26 | mat_data = loadmat(mat_path) 27 | mat_data = np.transpose(mat_data['handPara'], (2, 1, 0)) 28 | all_test_data = np.concatenate((all_test_data, mat_data), axis=0) 29 | 30 | all_train_data = np.zeros((0, 21, 3), dtype=np.float32) 31 | for train_seq in TRAIN_SEQS: 32 | mat_path = os.path.join(PATH_TO_DATASET, 'labels', train_seq + '_BB.mat') 33 | mat_data = loadmat(mat_path) 34 | mat_data = np.transpose(mat_data['handPara'], (2, 1, 0)) 35 | all_train_data = np.concatenate((all_train_data, mat_data), axis=0) 36 | 37 | with open('stb_collected.pkl', 'wb') as f: 38 | pickle.dump((all_train_data, all_test_data), f) 39 | -------------------------------------------------------------------------------- /POF/data/process_MPII_mask.py: -------------------------------------------------------------------------------- 1 | # Run this script with OpenCV2 2 | import cv2 3 | import numpy as np 4 | import os 5 | import json 6 | 7 | source_dir = '/media/posefs3b/Users/gines/mpii_mask' 8 | target_dir = '/media/posefs3b/Users/donglaix/mpii_mask' 9 | 10 | if __name__ == '__main__': 11 | path_to_db = './MPII_collected.json' 12 | with open(path_to_db) as f: 13 | db_data = json.load(f) 14 | total_num = len(db_data['img_paths']) 15 | for i in range(total_num): 16 | print ('processing image {} / {}'.format(i, total_num)) 17 | bbox = np.array(db_data['bbox'][i], dtype=np.float32) 18 | bbox_other = np.array(db_data['bbox_other'][i], dtype=np.float32).reshape(-1, 4) 19 | x = (bbox[0] + bbox[2]) / 2 20 | y = (bbox[1] + bbox[3]) / 2 21 | img_path = db_data['img_paths'][i] 22 | source_mask = os.path.join('/media/posefs3b/Users/gines/mpii_mask', img_path) 23 | mask = cv2.imread(source_mask) 24 | mask = (mask[:, :, 0] >= 128).astype(np.uint8) # the stored data are 0 ~ 255, convert to bool 25 | contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) 26 | belong = [] 27 | for cnt in contours: 28 | x1 = np.amin(cnt[:, 0, 0]) 29 | x2 = np.amax(cnt[:, 0, 0]) 30 | y1 = np.amin(cnt[:, 0, 1]) 31 | y2 = np.amax(cnt[:, 0, 1]) 32 | if x < x1 or x > x2 or y < y1 or y > y2: 33 | belong.append(False) 34 | continue 35 | # the center is inside this contour, now check the all other bounding boxes 36 | xo = (bbox_other[:, 0] + bbox_other[:, 2]) / 2 37 | yo = (bbox_other[:, 1] + bbox_other[:, 3]) / 2 38 | if ((xo >= x1) * (xo <= x2) * (yo >= y1) * (yo <= y2)).any(): # the center of any other bounding boxes fall inside 39 | belong.append(False) 40 | else: 41 | belong.append(True) # the center of current bbox is in and others are not in. 42 | assert len(belong) == len(contours) 43 | new_mask = np.ones(mask.shape, dtype=np.uint8) 44 | cv2.drawContours(new_mask, [cnt for TF, cnt in zip(belong, contours) if not TF], -1, 0, -1) 45 | cv2.imwrite(os.path.join(target_dir, '{:05d}.png'.format(i)), new_mask) 46 | -------------------------------------------------------------------------------- /POF/utils/EvalUtil.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def calc_auc(x, y): 5 | """ Given x and y values it calculates the approx. integral and normalizes it: area under curve""" 6 | integral = np.trapz(y, x) 7 | norm = np.trapz(np.ones_like(y), x) 8 | return integral / norm 9 | 10 | 11 | class EvalUtil: 12 | """ Util class for evaluation networks. 13 | """ 14 | def __init__(self, num_kp=21): 15 | # init empty data storage 16 | self.data = list() 17 | self.num_kp = num_kp 18 | for _ in range(num_kp): 19 | self.data.append(list()) 20 | 21 | def feed(self, keypoint_gt, keypoint_vis, keypoint_pred): 22 | """ Used to feed data to the class. Stores the euclidean distance between gt and pred, when it is visible. """ 23 | keypoint_gt = np.squeeze(keypoint_gt) 24 | keypoint_pred = np.squeeze(keypoint_pred) 25 | keypoint_vis = np.squeeze(keypoint_vis).astype('bool') 26 | 27 | assert len(keypoint_gt.shape) == 2 28 | assert len(keypoint_pred.shape) == 2 29 | assert len(keypoint_vis.shape) == 1 30 | 31 | # calc euclidean distance 32 | diff = keypoint_gt - keypoint_pred 33 | euclidean_dist = np.sqrt(np.sum(np.square(diff), axis=1)) 34 | 35 | num_kp = keypoint_gt.shape[0] 36 | for i in range(num_kp): 37 | if keypoint_vis[i]: 38 | self.data[i].append(euclidean_dist[i]) 39 | 40 | def _get_pck(self, kp_id, threshold): 41 | """ Returns pck for one keypoint for the given threshold. """ 42 | if len(self.data[kp_id]) == 0: 43 | return None 44 | 45 | data = np.array(self.data[kp_id]) 46 | pck = np.mean((data <= threshold).astype('float')) 47 | return pck 48 | 49 | def _get_epe(self, kp_id): 50 | """ Returns end point error for one keypoint. """ 51 | if len(self.data[kp_id]) == 0: 52 | return None, None 53 | 54 | data = np.array(self.data[kp_id]) 55 | epe_mean = np.mean(data) 56 | epe_median = np.median(data) 57 | return epe_mean, epe_median 58 | 59 | def get_measures(self, val_min, val_max, steps): 60 | """ Outputs the average mean and median error as well as the pck score. """ 61 | thresholds = np.linspace(val_min, val_max, steps) 62 | thresholds = np.array(thresholds) 63 | norm_factor = np.trapz(np.ones_like(thresholds), thresholds) 64 | 65 | # init mean measures 66 | epe_mean_all = list() 67 | epe_median_all = list() 68 | auc_all = list() 69 | pck_curve_all = list() 70 | 71 | # Create one plot for each part 72 | for part_id in range(self.num_kp): 73 | # mean/median error 74 | mean, median = self._get_epe(part_id) 75 | 76 | if mean is None: 77 | # there was no valid measurement for this keypoint 78 | continue 79 | 80 | epe_mean_all.append(mean) 81 | epe_median_all.append(median) 82 | 83 | # pck/auc 84 | pck_curve = list() 85 | for t in thresholds: 86 | pck = self._get_pck(part_id, t) 87 | pck_curve.append(pck) 88 | 89 | pck_curve = np.array(pck_curve) 90 | pck_curve_all.append(pck_curve) 91 | auc = np.trapz(pck_curve, thresholds) 92 | auc /= norm_factor 93 | auc_all.append(auc) 94 | 95 | epe_mean_all = np.mean(np.array(epe_mean_all)) 96 | epe_median_all = np.mean(np.array(epe_median_all)) 97 | auc_all = np.mean(np.array(auc_all)) 98 | pck_curve_all = np.mean(np.array(pck_curve_all), 0) # mean only over keypoints 99 | 100 | return epe_mean_all, epe_median_all, auc_all, pck_curve_all, thresholds 101 | -------------------------------------------------------------------------------- /POF/utils/default_PAF_length.py: -------------------------------------------------------------------------------- 1 | from utils.AdamModel import AdamModel 2 | from utils.PAF import PAFConnection 3 | import tensorflow as tf 4 | import numpy as np 5 | import json 6 | 7 | if __name__ == '__main__': 8 | adam = AdamModel() 9 | adam_joints = adam.reconstruct() 10 | sess = tf.Session() 11 | V_vec, joints_v = sess.run([adam.mean_shape, adam_joints]) 12 | sess.close() 13 | joints_v = joints_v.reshape(adam.num_joints, 3) 14 | V = V_vec.reshape(adam.num_vertices, 3) 15 | 16 | coords3d = np.zeros([19, 3], dtype=np.float64) 17 | coords3d[1] = joints_v[12] 18 | coords3d[2] = joints_v[17] 19 | coords3d[3] = joints_v[19] 20 | coords3d[4] = joints_v[21] 21 | coords3d[5] = joints_v[16] 22 | coords3d[6] = joints_v[18] 23 | coords3d[7] = joints_v[20] 24 | coords3d[8] = joints_v[2] 25 | coords3d[9] = joints_v[5] 26 | coords3d[10] = joints_v[8] 27 | coords3d[11] = joints_v[1] 28 | coords3d[12] = joints_v[4] 29 | coords3d[13] = joints_v[7] 30 | coords3d[0] = V[8130] 31 | coords3d[16] = V[10088] 32 | coords3d[17] = V[6970] 33 | coords3d[18] = V[1372] 34 | coords3d[14] = V[9707] 35 | coords3d[15] = V[2058] 36 | 37 | PAF_lengths = [[], []] 38 | for conn in PAFConnection[0]: 39 | vector = coords3d[conn[1]] - coords3d[conn[0]] 40 | length = np.sqrt(vector.dot(vector)) 41 | PAF_lengths[0].append(length) 42 | 43 | coords3d_hand = np.zeros([21, 3], dtype=np.float64) 44 | coords3d_hand[0] = joints_v[20] 45 | coords3d_hand[1] = joints_v[25] 46 | coords3d_hand[2] = joints_v[24] 47 | coords3d_hand[3] = joints_v[23] 48 | coords3d_hand[4] = joints_v[22] 49 | 50 | coords3d_hand[5] = joints_v[29] 51 | coords3d_hand[6] = joints_v[28] 52 | coords3d_hand[7] = joints_v[27] 53 | coords3d_hand[8] = joints_v[26] 54 | 55 | coords3d_hand[9] = joints_v[33] 56 | coords3d_hand[10] = joints_v[32] 57 | coords3d_hand[11] = joints_v[31] 58 | coords3d_hand[12] = joints_v[30] 59 | 60 | coords3d_hand[13] = joints_v[37] 61 | coords3d_hand[14] = joints_v[36] 62 | coords3d_hand[15] = joints_v[35] 63 | coords3d_hand[16] = joints_v[34] 64 | 65 | coords3d_hand[17] = joints_v[41] 66 | coords3d_hand[18] = joints_v[40] 67 | coords3d_hand[19] = joints_v[39] 68 | coords3d_hand[20] = joints_v[38] 69 | 70 | for conn in PAFConnection[1]: 71 | vector = coords3d_hand[conn[1]] - coords3d_hand[conn[0]] 72 | length = np.sqrt(vector.dot(vector)) 73 | PAF_lengths[1].append(length) 74 | 75 | with open('utils/default_PAF_lengths.json', 'w') as f: 76 | json.dump(PAF_lengths, f) 77 | -------------------------------------------------------------------------------- /POF/utils/default_PAF_lengths.json: -------------------------------------------------------------------------------- 1 | [[61.28923935624357, 38.03693171978008, 40.66363399518575, 60.679825386808595, 38.18667071420241, 41.3615808694613, 18.957141424446892, 25.143386172197538, 25.712162007789665, 22.287498884704696, 21.319498724407605, 24.63918887771886, 25.207377058133446, 22.537877881631474, 21.160039251405873, 5.828056400602892, 5.8724851096257265, 11.214240916123442, 11.03611941191539, 28.662122402181176, 50.745876178875385, 49.75236946222326, 84.35916414296307, 84.26425909340333, 78.3348641437306, 79.13285849223925], [5.208600563984615, 3.144690659755568, 3.0561653414502263, 2.216972605209146, 9.331936013182958, 4.0258752590669555, 2.1169128304880362, 1.9055776978186252, 8.788702335389496, 4.299818791238955, 2.325951226949148, 2.0265296027594606, 8.51392968631289, 3.7378351747514733, 2.221473468848724, 1.9115007695922452, 8.132973574679031, 3.053337606188647, 1.5647095464888916, 1.4466426394407983]] -------------------------------------------------------------------------------- /POF/utils/handSegNet.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import pickle 3 | import os 4 | from utils.ops import NetworkOps as ops 5 | 6 | 7 | class handSegNet: 8 | def __init__(self): 9 | pass 10 | 11 | def init_sess(self, sess): 12 | file_name = './weights/handsegnet-rhd.pickle' 13 | exclude_var_list = [] 14 | assert os.path.exists(file_name), "File not found." 15 | with open(file_name, 'rb') as fi: 16 | weight_dict = pickle.load(fi) 17 | weight_dict = {k: v for k, v in weight_dict.items() if not any([x in k for x in exclude_var_list])} 18 | if len(weight_dict) > 0: 19 | init_op, init_feed = tf.contrib.framework.assign_from_values(weight_dict) 20 | sess.run(init_op, init_feed) 21 | print('Loaded %d variables from %s' % (len(weight_dict), file_name)) 22 | 23 | def inference_detection(self, image, train=False): 24 | """ HandSegNet: Detects the hand in the input image by segmenting it. 25 | 26 | Inputs: 27 | image: [B, H, W, 3] tf.float32 tensor, Image with mean subtracted 28 | train: bool, True in case weights should be trainable 29 | 30 | Outputs: 31 | scoremap_list_large: list of [B, 256, 256, 2] tf.float32 tensor, Scores for the hand segmentation classes 32 | """ 33 | with tf.variable_scope('HandSegNet'): 34 | scoremap_list = list() 35 | layers_per_block = [2, 2, 4, 4] 36 | out_chan_list = [64, 128, 256, 512] 37 | pool_list = [True, True, True, False] 38 | 39 | # learn some feature representation, that describes the image content well 40 | x = image 41 | for block_id, (layer_num, chan_num, pool) in enumerate(zip(layers_per_block, out_chan_list, pool_list), 1): 42 | for layer_id in range(layer_num): 43 | x = ops.conv_relu(x, 'conv%d_%d' % (block_id, layer_id + 1), kernel_size=3, stride=1, out_chan=chan_num, trainable=train) 44 | if pool: 45 | x = ops.max_pool(x, 'pool%d' % block_id) 46 | 47 | x = ops.conv_relu(x, 'conv5_1', kernel_size=3, stride=1, out_chan=512, trainable=train) 48 | encoding = ops.conv_relu(x, 'conv5_2', kernel_size=3, stride=1, out_chan=128, trainable=train) 49 | 50 | # use encoding to detect initial scoremap 51 | x = ops.conv_relu(encoding, 'conv6_1', kernel_size=1, stride=1, out_chan=512, trainable=train) 52 | scoremap = ops.conv(x, 'conv6_2', kernel_size=1, stride=1, out_chan=2, trainable=train) 53 | scoremap_list.append(scoremap) 54 | 55 | # upsample to full size 56 | s = image.get_shape().as_list() 57 | scoremap_list_large = [tf.image.resize_images(x, (s[1], s[2])) for x in scoremap_list] 58 | 59 | return scoremap_list_large 60 | -------------------------------------------------------------------------------- /POF/utils/keypoint_conversion.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import numpy.linalg as nl 3 | from utils.general import connMat 4 | 5 | a4_to_main = { 6 | 'body': np.array([1, 0, 9, 10, 11, 3, 4, 5, 12, 13, 14, 6, 7, 8, 17, 15, 18, 16, 19, 20], dtype=np.int64), # convert to order of openpose 7 | '1_body': np.array([1, 0, 9, 10, 11, 3, 4, 5, 12, 13, 14, 6, 7, 8, 17, 15, 18, 16, 19, 20], dtype=np.int64), # convert to order of openpose 8 | '2_body': np.array([1, 0, 9, 10, 11, 3, 4, 5, 12, 13, 14, 6, 7, 8, 17, 15, 18, 16, 19, 20], dtype=np.int64), # convert to order of openpose 9 | 'left_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 10 | '1_left_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 11 | '2_left_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 12 | 'right_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 13 | '1_right_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 14 | '2_right_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), # convert to order of freiburg 15 | 'openpose_lhand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 16 | 'openpose_rhand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 17 | 'openpose_lhand_score': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 18 | 'openpose_rhand_score': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 19 | } 20 | 21 | human36m_to_main = { 22 | 'body': np.array([9, 8, 14, 15, 16, 11, 12, 13, 4, 5, 6, 1, 2, 3, 17, 17, 17, 17, 10, 17], dtype=np.int64) 23 | } 24 | 25 | mpi3d_to_main = { 26 | 'body': np.array([6, 5, 14, 15, 16, 9, 10, 11, 23, 24, 25, 18, 19, 20, 28, 28, 28, 28, 7], dtype=np.int64) 27 | } 28 | 29 | adam_to_main = { 30 | 'body': np.array([12, 17, 19, 21, 16, 18, 20, 2, 5, 8, 1, 4, 7], dtype=np.int64), 31 | 'select_body_main': np.arange(1, 14, dtype=np.int64) 32 | } 33 | 34 | COCO_to_main = { 35 | 'body': np.array([0, 17, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3, 18, 19], dtype=np.int64), 36 | 'body_valid': np.array([0, 17, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3, 18, 19], dtype=np.int64), 37 | 'all_body': np.array([0, 17, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3, 18, 19], dtype=np.int64), 38 | 'all_body_valid': np.array([0, 17, 6, 8, 10, 5, 7, 9, 12, 14, 16, 11, 13, 15, 2, 1, 4, 3, 18, 19], dtype=np.int64) 39 | } 40 | 41 | SMPL_to_main = { # actually COCOPLUS regressor to main 42 | 'body': np.array([14, 12, 8, 7, 6, 9, 10, 11, 2, 1, 0, 3, 4, 5, 16, 15, 18, 17, 13], dtype=np.int64) 43 | } 44 | 45 | STB_to_main = { 46 | 'left_hand': np.array([0, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], dtype=np.int64) 47 | } 48 | 49 | MPII_to_main = { 50 | 'body': np.array([16, 8, 12, 11, 10, 13, 14, 15, 2, 1, 0, 3, 4, 5, 16, 16, 16, 16, 9], dtype=np.int64), 51 | 'body_valid': np.array([16, 8, 12, 11, 10, 13, 14, 15, 2, 1, 0, 3, 4, 5, 16, 16, 16, 16, 9], dtype=np.int64) 52 | } 53 | 54 | tsimon_to_main = { 55 | 'left_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 56 | 'right_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 57 | 'left_hand_valid': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 58 | 'right_hand_valid': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 59 | } 60 | 61 | GAnerated_to_main = { 62 | 'left_hand': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 63 | 'left_hand_valid': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 64 | 'left_hand_3d': np.array([0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19, 18, 17], dtype=np.int64), 65 | 'right_hand': np.arange(21, dtype=np.int64), 66 | 'right_hand_valid': np.arange(21, dtype=np.int64), 67 | 'right_hand_3d': np.arange(21, dtype=np.int64) 68 | } 69 | 70 | std_body_size = 267.807 71 | std_hand_size = (82.2705 + 79.8843) / 2 72 | 73 | 74 | def compute_size(joint3d, type_str): 75 | """ use this to compute size for scaling: joints are in main order. 76 | """ 77 | length = 0.0 78 | for ic, conn in enumerate(connMat[type_str]): 79 | if type_str == 'body': 80 | if ic in (2, 3, 5, 6, 8, 9, 11, 12): 81 | length += nl.norm(joint3d[conn[0]] - joint3d[conn[1]]) 82 | else: 83 | assert type_str == 'hand' 84 | length += nl.norm(joint3d[conn[0]] - joint3d[conn[1]]) 85 | return length 86 | 87 | 88 | def main_to_a4(joint): 89 | assert joint.shape[0] == 20 90 | output = np.zeros((21, joint.shape[1]), dtype=joint.dtype) 91 | for io, ic in enumerate(a4_to_main['body']): 92 | output[ic, :] = joint[io, :] 93 | output[2, :] = (output[6, :] + output[12, :]) / 2 94 | return output 95 | 96 | 97 | def main_to_a4_hand(joint): 98 | assert joint.shape[0] == 21 99 | output = np.zeros(joint.shape, dtype=joint.dtype) 100 | output[0] = joint[0] 101 | for i in (1, 5, 9, 13, 17): 102 | output[i:i + 4] = joint[i + 3:i - 1:-1] 103 | return output 104 | 105 | 106 | def assemble_total_3d(body, lhand, rhand): 107 | len_b = compute_size(body, 'body') 108 | if len_b > 0: 109 | sbody = (std_body_size / len_b) * body 110 | else: 111 | sbody = body 112 | len_l = compute_size(lhand, 'hand') 113 | if len_l > 0: 114 | slhand = (std_hand_size / len_l) * lhand 115 | else: 116 | slhand = lhand 117 | len_r = compute_size(rhand, 'hand') 118 | if len_r > 0: 119 | srhand = (std_hand_size / len_r) * rhand 120 | else: 121 | srhand = rhand 122 | 123 | sbody = main_to_a4(sbody) 124 | slhand = main_to_a4_hand(slhand) 125 | srhand = main_to_a4_hand(srhand) 126 | 127 | slhand_invalid = (slhand[:, 0] == 0) * (slhand[:, 1] == 0) * (slhand[:, 2] == 0) 128 | srhand_invalid = (srhand[:, 0] == 0) * (srhand[:, 1] == 0) * (srhand[:, 2] == 0) 129 | 130 | if not slhand[0].any(): 131 | slhand_invalid[:] = True 132 | if not srhand[0].any(): 133 | srhand_invalid[:] = True 134 | 135 | lhand_idx_a4 = 5 136 | rhand_idx_a4 = 11 137 | 138 | shift_lhand = sbody[lhand_idx_a4] - slhand[0] 139 | shift_rhand = sbody[rhand_idx_a4] - srhand[0] 140 | 141 | slhand += shift_lhand 142 | srhand += shift_rhand 143 | 144 | slhand[slhand_invalid] = 0 145 | srhand[srhand_invalid] = 0 146 | 147 | return np.concatenate([sbody, slhand, srhand], axis=0), std_body_size / len_b 148 | 149 | 150 | def assemble_total_2d(body_2d, lhand_2d, rhand_2d): 151 | keypoint_list = [] 152 | for i, item in enumerate((body_2d, lhand_2d, rhand_2d)): 153 | keypoint = item['uv_local'] 154 | keypoint = (keypoint - 184) / item['scale2d'] + item['crop_center2d'] 155 | valid = item['valid'] 156 | keypoint = keypoint * np.stack([valid, valid], axis=1) # remove those invalid values 157 | if i == 0: 158 | keypoint = main_to_a4(keypoint) 159 | else: 160 | keypoint = main_to_a4_hand(keypoint) 161 | keypoint_list.append(keypoint) 162 | 163 | ret = np.concatenate(keypoint_list, axis=0) 164 | ret[np.isnan(ret)] = 0.0 # nan when the whole joint is zero 165 | return ret 166 | 167 | 168 | def main_to_human36m(joint): 169 | # except 9, 10 in human36m 170 | out = np.zeros((17, 3), dtype=joint.dtype) 171 | for im, ih in enumerate(human36m_to_main['body']): 172 | if ih == 17: # virtual zero joint 173 | continue 174 | out[ih] = np.copy(joint[im, :]) 175 | out[0] = (out[1] + out[4]) / 2 # middle hip 176 | out[7] = (out[1] + out[4] + out[11] + out[14]) / 4 # abdomen (average of l/r hip, l/r shoulder) 177 | return out 178 | -------------------------------------------------------------------------------- /POF/utils/load_ckpt.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.python import pywrap_tensorflow 3 | 4 | 5 | def load_weights_from_snapshot(session, checkpoint_path, discard_list=None, rename_dict=None): 6 | """ Loads weights from a snapshot except the ones indicated with discard_list. Others are possibly renamed. """ 7 | reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path) 8 | var_to_shape_map = reader.get_variable_to_shape_map() 9 | 10 | # Remove everything from the discard list 11 | if discard_list is not None: 12 | num_disc = 0 13 | var_to_shape_map_new = dict() 14 | for k, v in var_to_shape_map.items(): 15 | good = True 16 | for dis_str in discard_list: 17 | if dis_str in k: 18 | good = False 19 | 20 | if good: 21 | var_to_shape_map_new[k] = v 22 | else: 23 | num_disc += 1 24 | var_to_shape_map = dict(var_to_shape_map_new) 25 | print('Discarded %d items' % num_disc) 26 | 27 | # rename everything according to rename_dict 28 | num_rename = 0 29 | var_to_shape_map_new = dict() 30 | for name in var_to_shape_map.keys(): 31 | new_name = name 32 | if rename_dict is not None: 33 | for rename_str in rename_dict.keys(): 34 | if rename_str in name: 35 | new_name = new_name.replace(rename_str, rename_dict[rename_str], 1) # my modification: replace no more than once 36 | num_rename += 1 37 | var_to_shape_map_new[new_name] = reader.get_tensor(name) 38 | var_to_shape_map = dict(var_to_shape_map_new) 39 | 40 | init_op, init_feed = tf.contrib.framework.assign_from_values(var_to_shape_map) 41 | session.run(init_op, init_feed) 42 | print('Initialized %d variables from %s.' % (len(var_to_shape_map), checkpoint_path)) 43 | 44 | 45 | def load_weights_to_dict(checkpoint_path, discard_list=None, rename_dict=None): 46 | """ Loads weights from a snapshot except the ones indicated with discard_list. Others are possibly renamed. """ 47 | reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path) 48 | var_to_shape_map = reader.get_variable_to_shape_map() 49 | 50 | # Remove everything from the discard list 51 | if discard_list is not None: 52 | num_disc = 0 53 | var_to_shape_map_new = dict() 54 | for k, v in var_to_shape_map.items(): 55 | good = True 56 | for dis_str in discard_list: 57 | if dis_str in k: 58 | good = False 59 | 60 | if good: 61 | var_to_shape_map_new[k] = v 62 | else: 63 | num_disc += 1 64 | var_to_shape_map = dict(var_to_shape_map_new) 65 | print('Discarded %d items' % num_disc) 66 | 67 | # rename everything according to rename_dict 68 | num_rename = 0 69 | var_to_shape_map_new = dict() 70 | for name in var_to_shape_map.keys(): 71 | new_name = name 72 | if rename_dict is not None: 73 | for rename_str in rename_dict.keys(): 74 | if rename_str in name: 75 | new_name = new_name.replace(rename_str, rename_dict[rename_str]) 76 | num_rename += 1 77 | var_to_shape_map_new[new_name] = reader.get_tensor(name) 78 | var_to_shape_map = dict(var_to_shape_map_new) 79 | 80 | return var_to_shape_map 81 | -------------------------------------------------------------------------------- /POF/utils/multigpu.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def average_gradients(tower_grads): 5 | 6 | average_grads = [] 7 | for grad_and_vars in zip(*tower_grads): 8 | # Note that each grad_and_vars looks like the following: 9 | # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) 10 | 11 | grads = [] 12 | for g, _ in grad_and_vars: 13 | 14 | # Add 0 dimension to the gradients to represent the tower. 15 | expanded_g = tf.expand_dims(g, 0) 16 | 17 | # Append on a 'tower' dimension which we will average over below. 18 | grads.append(expanded_g) 19 | 20 | # Average over the 'tower' dimension. 21 | grad = tf.concat(axis=0, values=grads) 22 | grad = tf.reduce_mean(grad, 0) 23 | 24 | # Keep in mind that the Variables are redundant because they are shared 25 | # across towers. So .. we will just return the first tower's pointer to 26 | # the Variable. 27 | v = grad_and_vars[0][1] 28 | grad_and_var = (grad, v) 29 | average_grads.append(grad_and_var) 30 | return average_grads 31 | -------------------------------------------------------------------------------- /POF/utils/optimization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def transReProjectionLoss(t, X0, K, uv): 5 | assert t.shape == (3,) 6 | assert len(X0.shape) == 2 and X0.shape[1] == 3 7 | assert K.shape == (3, 3) 8 | assert len(uv.shape) == 2 and uv.shape[1] == 2 9 | 10 | X = X0 + t[np.newaxis, :] 11 | x = X.dot(K.T) 12 | x /= x[:, 2][:, np.newaxis] 13 | 14 | return np.sum(np.square(x[:, :2] - uv)) 15 | -------------------------------------------------------------------------------- /POF/utils/smoothing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import factorial 3 | 4 | 5 | def savitzky_golay(y, window_size, order, deriv=0, rate=1): 6 | r"""Smooth (and optionally differentiate) data with a Savitzky-Golay filter. 7 | The Savitzky-Golay filter removes high frequency noise from data. 8 | It has the advantage of preserving the original shape and 9 | features of the signal better than other types of filtering 10 | approaches, such as moving averages techniques. 11 | Parameters 12 | ---------- 13 | y : array_like, shape (N,) 14 | the values of the time history of the signal. 15 | window_size : int 16 | the length of the window. Must be an odd integer number. 17 | order : int 18 | the order of the polynomial used in the filtering. 19 | Must be less then `window_size` - 1. 20 | deriv: int 21 | the order of the derivative to compute (default = 0 means only smoothing) 22 | Returns 23 | ------- 24 | ys : ndarray, shape (N) 25 | the smoothed signal (or it's n-th derivative). 26 | Notes 27 | ----- 28 | The Savitzky-Golay is a type of low-pass filter, particularly 29 | suited for smoothing noisy data. The main idea behind this 30 | approach is to make for each point a least-square fit with a 31 | polynomial of high order over a odd-sized window centered at 32 | the point. 33 | Examples 34 | -------- 35 | t = np.linspace(-4, 4, 500) 36 | y = np.exp( -t**2 ) + np.random.normal(0, 0.05, t.shape) 37 | ysg = savitzky_golay(y, window_size=31, order=4) 38 | import matplotlib.pyplot as plt 39 | plt.plot(t, y, label='Noisy signal') 40 | plt.plot(t, np.exp(-t**2), 'k', lw=1.5, label='Original signal') 41 | plt.plot(t, ysg, 'r', label='Filtered signal') 42 | plt.legend() 43 | plt.show() 44 | References 45 | ---------- 46 | .. [1] A. Savitzky, M. J. E. Golay, Smoothing and Differentiation of 47 | Data by Simplified Least Squares Procedures. Analytical 48 | Chemistry, 1964, 36 (8), pp 1627-1639. 49 | .. [2] Numerical Recipes 3rd Edition: The Art of Scientific Computing 50 | W.H. Press, S.A. Teukolsky, W.T. Vetterling, B.P. Flannery 51 | Cambridge University Press ISBN-13: 9780521880688 52 | """ 53 | 54 | try: 55 | window_size = np.abs(np.int(window_size)) 56 | order = np.abs(np.int(order)) 57 | except ValueError: 58 | raise ValueError("window_size and order have to be of type int") 59 | if window_size % 2 != 1 or window_size < 1: 60 | raise TypeError("window_size size must be a positive odd number") 61 | if window_size < order + 2: 62 | raise TypeError("window_size is too small for the polynomials order") 63 | order_range = range(order + 1) 64 | half_window = (window_size - 1) // 2 65 | # precompute coefficients 66 | b = np.mat([[k**i for i in order_range] for k in range(-half_window, half_window + 1)]) 67 | m = np.linalg.pinv(b).A[deriv] * rate**deriv * factorial(deriv) 68 | # pad the signal at the extremes with 69 | # values taken from the signal itself 70 | firstvals = y[0] - np.abs(y[1:half_window + 1][::-1] - y[0]) 71 | lastvals = y[-1] + np.abs(y[-half_window - 1:-1][::-1] - y[-1]) 72 | y = np.concatenate((firstvals, y, lastvals)) 73 | return np.convolve(m[::-1], y, mode='valid') 74 | -------------------------------------------------------------------------------- /POF/utils/vis_heatmap3d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from mpl_toolkits.mplot3d import Axes3D 4 | from matplotlib.widgets import Slider 5 | import utils.general 6 | 7 | 8 | class vis_heatmap3d(object): 9 | def __init__(self, fig, ax, heatmap, keypoints=None, type_str=None): 10 | assert len(heatmap.shape) == 4 11 | self.fig = fig 12 | self.idx = 0 13 | self.threshold = 0.5 14 | self.heatmap = heatmap 15 | self.ax = ax 16 | self.keypoints = keypoints 17 | self.type_str = type_str 18 | 19 | axcolor = 'lightgoldenrodyellow' 20 | axx = plt.axes([0.25, 0.1, 0.65, 0.03], axisbg=axcolor) 21 | 22 | self.slider_threshold = Slider(axx, 'threshold', 0.0, 1.0, valinit=0.5) 23 | self.slider_threshold.on_changed(self.update) 24 | 25 | def draw(self): 26 | self.ax.clear() 27 | if self.keypoints is not None: 28 | utils.general.plot3d(self.ax, self.keypoints, self.type_str) 29 | active_map = self.heatmap[:, :, :, self.idx] 30 | Z, Y, X = np.where(active_map >= self.threshold) 31 | colors = [(1 - s) * np.array([0., 0., 1.], dtype=float) for s in active_map[Z, Y, X]] 32 | self.ax.scatter(X, Y, Z, color=colors) 33 | 34 | def update(self, val): 35 | self.threshold = self.slider_threshold.val 36 | self.draw() 37 | self.fig.canvas.draw_idle() 38 | -------------------------------------------------------------------------------- /POF/utils/wrapper_hand_model.py: -------------------------------------------------------------------------------- 1 | # Don't use anaconda for this 2 | import ctypes 3 | import os 4 | from PIL import Image, ImageOps 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | 9 | class wrapper_hand_model(object): 10 | def __init__(self, lib_file='./utils/libPythonWrapper.so', model_file='./utils/hand2_l_all_uv.json'): 11 | self.lib = ctypes.cdll.LoadLibrary(lib_file) 12 | 13 | self.fit_hand3d = self.lib.fit_hand3d 14 | self.fit_hand3d.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.c_char_p, ctypes.POINTER(ctypes.c_double), 15 | ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.c_int, ctypes.c_bool] 16 | self.fit_hand3d.restype = None 17 | 18 | self.Opengl_visualize = self.lib.Opengl_visualize 19 | self.Opengl_visualize.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), 20 | ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.c_bool, ctypes.c_uint, ctypes.c_int, ctypes.c_bool, ctypes.c_bool] 21 | self.Opengl_visualize.restype = None 22 | 23 | self.fit_hand2d = self.lib.fit_hand2d 24 | self.fit_hand2d.argtypes = [ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.c_char_p, 25 | ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.c_int, ctypes.c_bool, 26 | ctypes.c_double, ctypes.c_int] 27 | self.fit_hand2d.restype = None 28 | 29 | self.extract_fit_result = self.lib.extract_fit_result 30 | self.extract_fit_result.argtypes = [ctypes.c_char_p, ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), 31 | ctypes.POINTER(ctypes.c_double), ctypes.POINTER(ctypes.c_double), ctypes.c_int, ctypes.c_bool] 32 | self.extract_fit_result.restype = None 33 | 34 | self.lib.set_calibK.argtypes = [ctypes.POINTER(ctypes.c_double)] 35 | self.lib.set_calibK.restype = None 36 | 37 | self.cmodel_file = ctypes.create_string_buffer(model_file.encode('ascii')) 38 | self.ctarget_array = (ctypes.c_double * 63)() 39 | self.ctrans = (ctypes.c_double * 3)() 40 | self.ccoeff = (ctypes.c_double * 63)() 41 | self.cpose = (ctypes.c_double * 63)() 42 | self.cret_bytes = (ctypes.c_ubyte * (600 * 600 * 3))() 43 | self.ctarget2d_array = (ctypes.c_double * 42)() 44 | self.calibK = (ctypes.c_double * 9)() 45 | self.cret_bytes_cam = (ctypes.c_ubyte * (1080 * 1920 * 4))() 46 | 47 | self.PAF_array = (ctypes.c_double * (20 * 3))() 48 | 49 | def reset_value(self): 50 | self.ctrans[:] = [0.0, 0.0, 2.0] 51 | self.ccoeff[:] = [1.0 for _ in range(63)] 52 | self.cpose[:] = [0.0 for _ in range(63)] 53 | 54 | def fit3d(self, joint3d, regressor_type=0, euler=True): 55 | assert joint3d.shape == (21, 3) 56 | self.ctarget_array[:] = joint3d.reshape(-1).tolist() 57 | self.fit_hand3d(self.ctarget_array, self.cmodel_file, self.cpose, self.ccoeff, self.ctrans, regressor_type, euler) 58 | trans = np.array(self.ctrans[:]) 59 | pose = np.array(self.cpose[:]).reshape(-1, 3) 60 | coeff = np.array(self.ccoeff[:]).reshape(-1, 3) 61 | return trans, pose, coeff 62 | 63 | def fit2d(self, joint2d, calibK, PAF, regressor_type=0, euler=True, prior_weight=100.0, mode=0): 64 | assert joint2d.shape == (21, 2) and calibK.shape == (3, 3) 65 | self.ctarget2d_array[:] = joint2d.reshape(-1).tolist() 66 | self.calibK[:] = calibK.reshape(-1).tolist() 67 | assert PAF.size == len(self.PAF_array[:]) 68 | self.PAF_array[:] = PAF.reshape(-1).tolist() 69 | self.fit_hand2d(self.ctarget2d_array, self.calibK, self.PAF_array, self.cmodel_file, self.cpose, self.ccoeff, self.ctrans, 70 | regressor_type, euler, prior_weight, mode) 71 | trans = np.array(self.ctrans[:]) 72 | pose = np.array(self.cpose[:]).reshape(-1, 3) 73 | coeff = np.array(self.ccoeff[:]).reshape(-1, 3) 74 | return trans, pose, coeff 75 | 76 | def render(self, cameraMode=False, target=True, first_render=False, position=0, regressor_type=0, stay=False, euler=True): 77 | if cameraMode: 78 | read_buffer = self.cret_bytes_cam 79 | else: 80 | read_buffer = self.cret_bytes 81 | if target: 82 | if first_render: 83 | self.Opengl_visualize(self.cmodel_file, read_buffer, self.cpose, self.ccoeff, self.ctrans, self.ctarget_array, ctypes.c_bool(cameraMode), position, regressor_type, stay, euler) 84 | self.Opengl_visualize(self.cmodel_file, read_buffer, self.cpose, self.ccoeff, self.ctrans, self.ctarget_array, ctypes.c_bool(cameraMode), position, regressor_type, stay, euler) 85 | else: 86 | if first_render: 87 | self.Opengl_visualize(self.cmodel_file, read_buffer, self.cpose, self.ccoeff, self.ctrans, None, ctypes.c_bool(cameraMode), position, regressor_type, stay, euler) 88 | self.Opengl_visualize(self.cmodel_file, read_buffer, self.cpose, self.ccoeff, self.ctrans, None, ctypes.c_bool(cameraMode), position, regressor_type, stay, euler) 89 | img = bytes(read_buffer) 90 | if not cameraMode: 91 | img = Image.frombytes("RGBA", (600, 600), img) 92 | else: 93 | img = Image.frombytes("RGBA", (1920, 1080), img) 94 | img = ImageOps.flip(img) 95 | return img 96 | 97 | def set_calibK(self, K): 98 | self.calibK[:] = K.reshape(-1).tolist() 99 | self.lib.set_calibK(self.calibK) 100 | 101 | 102 | if __name__ == '__main__': 103 | import numpy as np 104 | wrapper = wrapper_hand_model("/home/donglaix/Documents/Experiments/hand_model/build/libPythonWrapper.so") 105 | joint3d = np.array([-33.3889, -173.355, -36.0744, -35.0518, -173.959, -37.7108, -36.5972, -176.126, -40.7544, -37.4367, -178.032, -43.6272, -38.7743, -178.843, -45.5877, -36.4731, -180.718, -38.2183, -37.0009, -181.596, -42.4443, -37.4651, -181.437, -45.0006, -37.7732, -181.458, -47.0573, -34.2598, -180.606, -38.3926, -35.2143, - 106 | 180.671, -43.2699, -36.3031, -179.876, -45.6931, -37.1902, -179.438, -47.745, -32.0926, -179.69, -38.4972, -33.7518, -179.847, -42.8798, -34.9357, -179.212, -45.3947, -35.7699, -178.853, -47.3468, -30.3247, -178.334, -39.2571, -31.8778, -178.837, -42.4667, -33.003, -178.501, -44.2697, -33.8762, -178.325, -45.8248]).reshape(-1, 3) 107 | joint2d = np.array([1284.646, 254.091, 1296.991, 248.479, 1319.012, 231.635, 1339.5621, 217.027, 1354.4766, 209.81, 1300.0491, 200.093, 1330.055, 192.596, 1348.5556, 192.777, 1363.3952, 191.943, 1299.7998, 202.764, 1334.6115, 108 | 200.494, 1352.7438, 204.628, 1368.139, 206.547, 1299.2785, 210.884, 1330.8779, 207.547, 1349.5700, 210.478, 1364.09, 211.918, 1303.6187, 221.421, 1326.7478, 216.127, 1340.2151, 217.196, 1351.8205, 217.42]).reshape(-1, 2) 109 | K = np.array([[1633.34, 0, 942.256], [0, 1628.84, 557.344], [0, 0, 1]]) 110 | wrapper.fit3d(joint3d) 111 | wrapper.fit2d(joint2d, K) 112 | img = wrapper.render(cameraMode=False, first_render=True) 113 | # print(trans) 114 | # print(pose) 115 | # print(coeff) 116 | plt.imshow(img) 117 | plt.show() 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Monocular Total Capture 2 | Code for CVPR19 paper "Monocular Total Capture: Posing Face, Body and Hands in the Wild" 3 | 4 | ![Teaser Image](https://xiangdonglai.github.io/MTC_teaser.jpg) 5 | 6 | Project website: [] 7 | 8 | # Dependencies 9 | This code is tested on a Ubuntu 16.04 machine with a GTX 1080Ti GPU, with the following dependencies. 10 | 1. ffmpeg 11 | 2. Python 3.5 (with TensorFlow 1.5.0, OpenCV, Matplotlib, packages installed with pip3) 12 | 3. cmake >= 2.8 13 | 4. OpenCV 2.4.13 (compiled from source with CUDA 9.0, CUDNN 7.0) 14 | 5. Ceres-Solver 1.13.0 (with SuiteSparse) 15 | 6. OpenGL, GLUT, GLEW 16 | 7. libigl 17 | 8. wget 18 | 9. OpenPose 19 | 20 | # Installation 21 | 1. git clone this repository; suppose the main directory is ${ROOT} on your local machine. 22 | 2. "cd ${ROOT}" 23 | 3. "bash download.sh" 24 | 4. git clone OpenPose and compile. Suppose the main directory of OpenPose is ${openposeDir}, such that the compiled binary is at ${openposeDir}/build/examples/openpose/openpose.bin 25 | 5. Edit ${ROOT}/run_pipeline.sh: set line 13 to you ${openposeDir} 26 | 4. Edit ${ROOT}/FitAdam/CMakeLists.txt: set line 13 to the "include" directory of libigl (this is a header only library) 27 | 5. "cd ${ROOT}/FitAdam/ && mkdir build && cd build" 28 | 6. "cmake .." 29 | 7. "make -j12" 30 | 31 | # Usage 32 | 1. Suppose the video to be tested is named "${seqName}.mp4". Place it in "${ROOT}/${seqName}/${seqName}.mp4". 33 | 2. If the camera intrinsics is known, put it in "${ROOT}/${seqName}/calib.json" (refer to "POF/calib.json" for example); otherwise, a default camera intrinsics will be used. 34 | 3. In ${ROOT}, run "bash run_pipeline.sh ${seqName}"; if the subject in the video shows only upper body, run "bash run_pipeline.sh ${seqName} -f". 35 | 36 | # Docker Image 37 | 1. Install NVIDIA Docker 38 | 2. Build the docker image 39 | ``` 40 | docker build . --tag mtc 41 | ``` 42 | 3. Running the docker image: 43 | ``` 44 | xhost local:root 45 | docker run --gpus 0 -it -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY -e XAUTHORITY -e NVIDIA_DRIVER_CAPABILITIES=all mtc 46 | ``` 47 | 4. Once inside (should be in /opt/mtc by default): 48 | ``` 49 | bash run_pipeline.sh example_speech -f 50 | ``` 51 | Tested on Ubuntu 16.04 and 18.04 with Titan Xp and Titan X Maxwell (External w/Razer Core). 52 | 53 | # Examples 54 | "download.sh" automatically download 2 example videos to test. After successful installation run 55 | ``` 56 | bash run_pipeline.sh example_dance 57 | ``` 58 | or 59 | ``` 60 | bash run_pipeline.sh example_speech -f 61 | ``` 62 | 63 | # License and Citation 64 | This code can only be used for **non-commercial research purposes**. If you use this code in your research, please cite the following papers. 65 | ``` 66 | @inproceedings{xiang2019monocular, 67 | title={Monocular total capture: Posing face, body, and hands in the wild}, 68 | author={Xiang, Donglai and Joo, Hanbyul and Sheikh, Yaser}, 69 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 70 | year={2019} 71 | } 72 | 73 | @inproceedings{joo2018total, 74 | title={Total capture: A 3d deformation model for tracking faces, hands, and bodies}, 75 | author={Joo, Hanbyul and Simon, Tomas and Sheikh, Yaser}, 76 | booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, 77 | year={2018} 78 | } 79 | ``` 80 | 81 | Some part of this code is modified from [lmb-freiburg/hand3d](https://github.com/lmb-freiburg/hand3d). 82 | 83 | # Adam Model 84 | We use the deformable human model [**Adam**](http://www.cs.cmu.edu/~hanbyulj/totalcapture/) in this code. 85 | 86 | **The relationship between Adam and SMPL:** The body part of Adam is derived from [SMPL](http://smpl.is.tue.mpg.de/license_body) model by Loper et al. 2015. It follows SMPL's body joint hierarchy, but uses a different joint regressor. Adam does not contain the original SMPL model's shape and pose blendshapes, but uses its own version trained from Panoptic Studio database. 87 | 88 | **The relationship between Adam and FaceWarehouse:** The face part of Adam is derived from [FaceWarehouse](http://kunzhou.net/zjugaps/facewarehouse/). In particular, the mesh topology of face of Adam is a modified version of the learned model from FaceWarehouse dataset. Adam does not contain the blendshapes of the original FaceWarehouse data, and facial expression of Adam model is unavailable due to copyright issues. 89 | 90 | The Adam model is shared for research purpose only, and cannot be used for commercial purpose. Redistributing the original or modified version of Adam is also not allowed without permissions. 91 | 92 | # Special Notice 93 | 1. In our code, the output of ceres::AngleAxisToRotationMatrix is always a RowMajor matrix, while the function is designed for a ColMajor matrix. To account for this, please treat our output pose parameters as the opposite value. In other words, before exporting our pose parameter to other softwares, please multiply them by -1. 94 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /download.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | cd POF/ 3 | wget domedb.perception.cs.cmu.edu/data/mtc/mtc_snapshots.zip 4 | unzip mtc_snapshots.zip 5 | if [ ! -d snapshots ]; then 6 | echo "Pretrained model not extracted! Please check your setting." 7 | else 8 | echo "Pretrained model successfully downloaded." 9 | fi 10 | cd ../FitAdam/model 11 | wget domedb.perception.cs.cmu.edu/data/mtc/adam_blendshapes_348_delta_norm.json 12 | cd ../include/ 13 | wget domedb.perception.cs.cmu.edu/data/mtc/InitializeAdamData.h 14 | cd ../../data/ 15 | mkdir example_dance && cd example_dance 16 | wget domedb.perception.cs.cmu.edu/data/mtc/example_dance.mp4 17 | cd ../ 18 | mkdir example_speech && cd example_speech 19 | wget domedb.perception.cs.cmu.edu/data/mtc/example_speech.mp4 20 | -------------------------------------------------------------------------------- /run_pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -e 4 | # input param 5 | # $1: sequence name 6 | # $2: whether the video is upper body only (false by default, enable by -f) 7 | seqName=$1 8 | upperBody=$2 9 | 10 | # Assume that you already have a video in $dataDir/(seqName)/(seqName).mp4 11 | dataDir=./data/ 12 | # Git clone openpose to ../openpose and compile with cmake 13 | openposeDir=../openpose/ 14 | 15 | # convert to absolute path 16 | MTCDir=$(readlink -f .) 17 | dataDir=$(readlink -f $dataDir) 18 | openposeDir=$(readlink -f $openposeDir) 19 | 20 | if [ ! -f $dataDir/$seqName/calib.json ]; then 21 | echo "Camera intrinsics not specified, use default." 22 | cp -v POF/calib.json $dataDir/$seqName 23 | fi 24 | 25 | # use ffmpeg to extract image frames 26 | cd $dataDir/$seqName 27 | if [ ! -d raw_image ]; then 28 | mkdir raw_image 29 | ffmpeg -i $seqName.mp4 raw_image/${seqName}_%08d.png 30 | fi 31 | 32 | # run OpenPose on image frames 33 | if [ ! -d openpose_result ]; then 34 | mkdir openpose_result 35 | cd $openposeDir 36 | ./build/examples/openpose/openpose.bin --face --hand --image_dir $dataDir/$seqName/raw_image --write_json $dataDir/$seqName/openpose_result --render_pose 0 --display 0 -model_pose BODY_25 --number_people_max 1 37 | fi 38 | 39 | # merge openpose results into a single file 40 | cd $MTCDir 41 | numFrame=$(ls $dataDir/$seqName/openpose_result/$seqName_* | wc -l) 42 | python3 POF/collect_openpose.py -n $seqName -r $dataDir/$seqName -c $numFrame 43 | 44 | # run POF generation code 45 | cd POF 46 | if [ ! -d $dataDir/$seqName/net_output/ ]; then 47 | python3 save_total_sequence.py -s $seqName -p $dataDir/$seqName $upperBody 48 | # python3 save_total_sequence.py -s $seqName -p $dataDir/$seqName --end-index 10 $upperBody 49 | else 50 | echo "POF results exist. Skip POF generation." 51 | fi 52 | 53 | # run Adam Fitting 54 | cd $MTCDir/FitAdam/ 55 | if [ ! -f ./build/run_fitting ]; then 56 | echo "C++ project not correctly compiled. Please check your setting." 57 | fi 58 | ./build/run_fitting --root_dirs $dataDir --seqName $seqName --start 1 --end $((numFrame + 1)) --stage 1 --imageOF 59 | # ./build/run_fitting --root_dirs $dataDir --seqName $seqName --start 1 --end 11 --stage 1 --imageOF 60 | --------------------------------------------------------------------------------