├── VideoFace3D ├── renderer │ ├── __init__.py │ ├── __pycache__ │ │ ├── render.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── lightning.cpython-37.pyc │ │ └── weak_projection.cpython-37.pyc │ ├── weak_projection.py │ ├── lightning.py │ └── render.py ├── SFS │ ├── SFSNet │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── mask.cpython-37.pyc │ │ │ ├── model.cpython-37.pyc │ │ │ ├── utils.cpython-37.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── functions.cpython-37.pyc │ │ ├── utils.py │ │ ├── functions.py │ │ ├── model.py │ │ └── mask.py │ ├── __init__.py │ ├── __pycache__ │ │ ├── SfS.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ └── SfS.py ├── face_track │ ├── lib │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── utils.cpython-37.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── face_utils.cpython-37.pyc │ │ ├── face_utils.py │ │ └── utils.py │ ├── src │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── sort.cpython-37.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── kalman_tracker.cpython-37.pyc │ │ │ └── data_association.cpython-37.pyc │ │ ├── data_association.py │ │ ├── sort.py │ │ └── kalman_tracker.py │ ├── align │ │ ├── __init__.py │ │ ├── det1.npy │ │ ├── det2.npy │ │ ├── det3.npy │ │ └── __pycache__ │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── detect_face.cpython-37.pyc │ ├── __init__.py │ ├── __pycache__ │ │ ├── tracker.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ └── tracker.py ├── segmentation │ ├── __init__.py │ ├── __pycache__ │ │ ├── segment.cpython-37.pyc │ │ └── __init__.cpython-37.pyc │ ├── faceparsing │ │ ├── __pycache__ │ │ │ ├── model.cpython-37.pyc │ │ │ └── resnet.cpython-37.pyc │ │ ├── resnet.py │ │ └── model.py │ └── segment.py ├── landmark_detect │ ├── __init__.py │ ├── __pycache__ │ │ ├── ddfa_io.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── ddfa_ddfa.cpython-37.pyc │ │ ├── detector.cpython-37.pyc │ │ ├── ddfa_params.cpython-37.pyc │ │ ├── ddfa_inference.cpython-37.pyc │ │ ├── ddfa_landmarks.cpython-37.pyc │ │ ├── ddfa_mobilenet.cpython-37.pyc │ │ ├── dlib_landmark.cpython-37.pyc │ │ └── ddfa_estimate_pose.cpython-37.pyc │ ├── dlib_landmark.py │ ├── ddfa_params.py │ ├── ddfa_estimate_pose.py │ ├── detector.py │ ├── ddfa_io.py │ ├── ddfa_landmarks.py │ ├── ddfa_ddfa.py │ ├── ddfa_mobilenet.py │ └── ddfa_inference.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── fitting.cpython-37.pyc │ │ ├── face_model.cpython-37.pyc │ │ ├── ddfa_predict.cpython-37.pyc │ │ └── shape_predict.cpython-37.pyc │ ├── shape_predict.py │ ├── ddfa_predict.py │ └── face_model.py ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── Global.cpython-37.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── geometry.cpython-37.pyc │ │ ├── video_utils.cpython-37.pyc │ │ ├── visualization.cpython-37.pyc │ │ └── temporal_smooth.cpython-37.pyc │ ├── Global.py │ ├── video_utils.py │ ├── visualization.py │ ├── temporal_smooth.py │ └── geometry.py ├── __pycache__ │ └── __init__.cpython-37.pyc └── __init__.py ├── examples ├── example_pics │ ├── 1.png │ └── 2.png ├── example_results │ ├── sfs.png │ ├── 0_vis.png │ ├── 1_vis.png │ ├── sfs_1.png │ ├── sfs_2.png │ ├── 0_input.png │ ├── 0_mask.png │ ├── 1_input.png │ ├── 1_mask.png │ ├── fitting.png │ ├── landmark.png │ ├── lanmark_2D.png │ ├── lanmark_3D.png │ ├── 0_mask_prob.png │ ├── 1_mask_prob.png │ ├── segmentation.png │ ├── fitting_fast_0.png │ └── fitting_fast_1.png ├── example_landmarks_single_image.py ├── examples_segementation_single_image.py ├── example_sfs_single_image.py ├── example_fitting_single_image.py └── example_video.py ├── setup.py └── README.md /VideoFace3D/renderer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/__init__.py: -------------------------------------------------------------------------------- 1 | from .SfS import * -------------------------------------------------------------------------------- /VideoFace3D/face_track/__init__.py: -------------------------------------------------------------------------------- 1 | from .tracker import * -------------------------------------------------------------------------------- /VideoFace3D/segmentation/__init__.py: -------------------------------------------------------------------------------- 1 | from .segment import * -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__init__.py: -------------------------------------------------------------------------------- 1 | from .detector import * -------------------------------------------------------------------------------- /examples/example_pics/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_pics/1.png -------------------------------------------------------------------------------- /examples/example_pics/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_pics/2.png -------------------------------------------------------------------------------- /VideoFace3D/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .face_model import * 2 | from .fitting import * 3 | from .shape_predict import * -------------------------------------------------------------------------------- /examples/example_results/sfs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/sfs.png -------------------------------------------------------------------------------- /examples/example_results/0_vis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/0_vis.png -------------------------------------------------------------------------------- /examples/example_results/1_vis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/1_vis.png -------------------------------------------------------------------------------- /examples/example_results/sfs_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/sfs_1.png -------------------------------------------------------------------------------- /examples/example_results/sfs_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/sfs_2.png -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/det1.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/align/det1.npy -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/det2.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/align/det2.npy -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/det3.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/align/det3.npy -------------------------------------------------------------------------------- /examples/example_results/0_input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/0_input.png -------------------------------------------------------------------------------- /examples/example_results/0_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/0_mask.png -------------------------------------------------------------------------------- /examples/example_results/1_input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/1_input.png -------------------------------------------------------------------------------- /examples/example_results/1_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/1_mask.png -------------------------------------------------------------------------------- /examples/example_results/fitting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/fitting.png -------------------------------------------------------------------------------- /examples/example_results/landmark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/landmark.png -------------------------------------------------------------------------------- /examples/example_results/lanmark_2D.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/lanmark_2D.png -------------------------------------------------------------------------------- /examples/example_results/lanmark_3D.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/lanmark_3D.png -------------------------------------------------------------------------------- /VideoFace3D/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .Global import * 2 | from .visualization import * 3 | from .geometry import * 4 | from .video_utils import * -------------------------------------------------------------------------------- /examples/example_results/0_mask_prob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/0_mask_prob.png -------------------------------------------------------------------------------- /examples/example_results/1_mask_prob.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/1_mask_prob.png -------------------------------------------------------------------------------- /examples/example_results/segmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/segmentation.png -------------------------------------------------------------------------------- /examples/example_results/fitting_fast_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/fitting_fast_0.png -------------------------------------------------------------------------------- /examples/example_results/fitting_fast_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/examples/example_results/fitting_fast_1.png -------------------------------------------------------------------------------- /VideoFace3D/SFS/__pycache__/SfS.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/__pycache__/SfS.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/Global.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/Global.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__pycache__/mask.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/SFSNet/__pycache__/mask.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/models/__pycache__/fitting.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/models/__pycache__/fitting.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/renderer/__pycache__/render.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/renderer/__pycache__/render.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/geometry.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/geometry.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/SFSNet/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/SFSNet/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/__pycache__/tracker.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/__pycache__/tracker.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/models/__pycache__/face_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/models/__pycache__/face_model.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/renderer/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/renderer/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/renderer/__pycache__/lightning.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/renderer/__pycache__/lightning.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/video_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/video_utils.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/SFSNet/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/__pycache__/functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/SFS/SFSNet/__pycache__/functions.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/lib/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/__pycache__/sort.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/src/__pycache__/sort.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/models/__pycache__/ddfa_predict.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/models/__pycache__/ddfa_predict.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/models/__pycache__/shape_predict.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/models/__pycache__/shape_predict.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/segmentation/__pycache__/segment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/segmentation/__pycache__/segment.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/visualization.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/visualization.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/lib/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/src/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_io.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_io.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/segmentation/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/segmentation/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/utils/__pycache__/temporal_smooth.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/utils/__pycache__/temporal_smooth.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/align/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/__pycache__/face_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/lib/__pycache__/face_utils.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_ddfa.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_ddfa.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/detector.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/detector.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/renderer/__pycache__/weak_projection.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/renderer/__pycache__/weak_projection.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/align/__pycache__/detect_face.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/align/__pycache__/detect_face.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_params.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_params.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/__pycache__/kalman_tracker.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/src/__pycache__/kalman_tracker.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_inference.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_inference.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_landmarks.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_landmarks.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_mobilenet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_mobilenet.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/dlib_landmark.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/dlib_landmark.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/segmentation/faceparsing/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/segmentation/faceparsing/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/__pycache__/data_association.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/face_track/src/__pycache__/data_association.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/segmentation/faceparsing/__pycache__/resnet.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/segmentation/faceparsing/__pycache__/resnet.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/__pycache__/ddfa_estimate_pose.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lstcutong/video-face-3d/HEAD/VideoFace3D/landmark_detect/__pycache__/ddfa_estimate_pose.cpython-37.pyc -------------------------------------------------------------------------------- /VideoFace3D/__init__.py: -------------------------------------------------------------------------------- 1 | from .face_track import * 2 | from .landmark_detect import * 3 | from .models import * 4 | from .renderer import * 5 | from .utils import * 6 | from .SFS import * 7 | from .segmentation import * -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/dlib_landmark.py: -------------------------------------------------------------------------------- 1 | import dlib 2 | import cv2 3 | import numpy as np 4 | 5 | def detect_landmark_dlib_2D(image_path, predictor): 6 | image = cv2.imread(image_path) 7 | detector = dlib.get_frontal_face_detector() 8 | gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 9 | 10 | landmarks = [] 11 | 12 | dets = detector(gray, 1) 13 | for face in dets: 14 | single_face = [] 15 | shape = predictor(image, face) 16 | for pt in shape.parts(): 17 | pt_pos = (pt.x, pt.y) 18 | 19 | single_face.append(np.array(pt_pos)) 20 | 21 | landmarks.append(np.array(single_face)) 22 | return landmarks -------------------------------------------------------------------------------- /examples/example_landmarks_single_image.py: -------------------------------------------------------------------------------- 1 | import VideoFace3D as vf3d 2 | import cv2 3 | import os 4 | def example_landmarks(): 5 | ld_2d = vf3d.FaceLandmarkDetector("2D") 6 | ld_3d = vf3d.FaceLandmarkDetector("3D") 7 | image_path = "./example_pics/1.png" 8 | 9 | l2d = ld_2d.detect_face_landmark(image_path) 10 | l3d = ld_3d.detect_face_landmark(image_path) 11 | 12 | im2d = vf3d.draw_landmarks(cv2.imread(image_path), l2d, colors=[vf3d.ComfortableColor().mint_d.to_bgr()]) 13 | im3d = vf3d.draw_landmarks(cv2.imread(image_path), l3d, colors=[vf3d.ComfortableColor().mint_d.to_bgr()]) 14 | 15 | cv2.imwrite("./example_results/lanmark_2D.png", im2d) 16 | cv2.imwrite("./example_results/lanmark_3D.png", im3d) 17 | 18 | if __name__ == '__main__': 19 | example_landmarks() -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | files =["data/*"] 4 | 5 | setup( 6 | description='utils for video face data preprocess, include video face tracking, landmark detecting, morphable face model fitting', 7 | author='Luo Shoutong', 8 | author_email='MF1933071@smail.nju.edu.cn', 9 | license='Magic', 10 | version='0.1.0', 11 | name='VideoFace3D', 12 | include_package_data = True, 13 | packages=find_packages(), 14 | ) 15 | 16 | ''' 17 | "./data/BFM_front_idx.mat", 18 | "./data/BFM_model_front.mat", 19 | "./data/FaceReconModel.pb", 20 | "./data/phase1_wpdc_vdc.pth.tar", 21 | "./data/shape_predictor_68_face_landmarks.dat", 22 | "./data/similarity_Lm3D_all.mat", 23 | "./data/tri.mat", 24 | "./data/keypoints_sim.npy", 25 | "./data/Model_PAF.pkl", 26 | "./data/param_whitening.pkl", 27 | "./data/pncc_code.npy", 28 | "./data/u_exp.npy", 29 | "./data/u_shp.npy", 30 | "./data/w_exp_sim.npy", 31 | "./data/w_shp_sim.npy" 32 | ''' -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | from __future__ import absolute_import, division, print_function 3 | import cv2 4 | import numpy as np 5 | 6 | 7 | def _convert(src, max_value): 8 | # find min and max 9 | _min = np.min(src) 10 | _max = np.max(src) 11 | # scale to (0, max_value) 12 | dst = (src - _min) / (_max - _min + 1e-10) 13 | dst *= max_value 14 | return dst 15 | 16 | 17 | def convert(src, dtype=np.uint8, max_value=255.0): 18 | # type: (np.ndarray, object, float) -> np.ndarray 19 | # copy src 20 | dst = src.copy() 21 | if src.ndim == 2: 22 | dst = _convert(dst, max_value) 23 | elif src.ndim == 3: 24 | dst = cv2.cvtColor(dst, cv2.COLOR_BGR2LAB) 25 | light_channel = _convert(dst[0], max_value) 26 | dst[0, ...] = light_channel 27 | dst = cv2.cvtColor(dst, cv2.COLOR_LAB2BGR)*255 28 | else: 29 | raise RuntimeError("src/utils.py(30): src.ndim should be 2 or 3") 30 | return dst.astype(dtype) 31 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/face_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def judge_side_face(facial_landmarks): 5 | wide_dist = np.linalg.norm(facial_landmarks[0] - facial_landmarks[1]) 6 | high_dist = np.linalg.norm(facial_landmarks[0] - facial_landmarks[3]) 7 | dist_rate = high_dist / wide_dist 8 | 9 | # cal std 10 | vec_A = facial_landmarks[0] - facial_landmarks[2] 11 | vec_B = facial_landmarks[1] - facial_landmarks[2] 12 | vec_C = facial_landmarks[3] - facial_landmarks[2] 13 | vec_D = facial_landmarks[4] - facial_landmarks[2] 14 | dist_A = np.linalg.norm(vec_A) 15 | dist_B = np.linalg.norm(vec_B) 16 | dist_C = np.linalg.norm(vec_C) 17 | dist_D = np.linalg.norm(vec_D) 18 | 19 | # cal rate 20 | high_rate = dist_A / dist_C 21 | width_rate = dist_C / dist_D 22 | high_ratio_variance = np.fabs(high_rate - 1.1) # smaller is better 23 | width_ratio_variance = np.fabs(width_rate - 1) 24 | 25 | return dist_rate, high_ratio_variance, width_ratio_variance 26 | -------------------------------------------------------------------------------- /VideoFace3D/utils/Global.py: -------------------------------------------------------------------------------- 1 | SMOOTH_METHODS_OPTIMIZE = 1 2 | SMOOTH_METHODS_MEDUIM = 2 3 | SMOOTH_METHODS_MEAN = 3 4 | SMOOTH_METHODS_GAUSSIAN = 4 5 | SMOOTH_METHODS_DCNN = 5 6 | 7 | import os.path as osp 8 | import os 9 | 10 | def make_abs_path(d): 11 | return osp.join(osp.dirname(osp.realpath(__file__)), d) 12 | 13 | project_dir = os.path.dirname(os.path.abspath(__file__)) 14 | 15 | d = make_abs_path("../data") 16 | #d = "/home/magic/lst/codes/MorphableFaceFitting/MorphableModel" 17 | 18 | BFM_FRONT_MODEL_PATH = osp.join(d, "BFM_model_front.mat") 19 | SINGLE_IMAGE_RECON_MODEL_PATH = osp.join(d, "FaceReconModel.pb") 20 | SIMILARITY_LM3D_ALL_MODEL_PATH = osp.join(d, "similarity_Lm3D_all.mat") 21 | FRONT_FACE_INDEX_PATH = osp.join(d, "BFM_front_idx.mat") 22 | CHECKPOINT_FP_PATH = osp.join(d, "phase1_wpdc_vdc.pth.tar") 23 | DLIB_LANDMARK_MODEL_PATH = osp.join(d, "shape_predictor_68_face_landmarks.dat") 24 | TRI_PATH = osp.join(d, "tri.mat") 25 | SFSNET_PATH = osp.join(d, "SfSNet.pth") 26 | BISENET_MODEL_PATH = osp.join(d, "bisenet.pth") -------------------------------------------------------------------------------- /VideoFace3D/renderer/weak_projection.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import torch 4 | 5 | 6 | def weak_projection(vertices, K, R, t, orig_size): 7 | ''' 8 | Calculate weak_projective transformation of vertices given a projection matrix 9 | Input parameters: 10 | K: batch_size * 3 * 3 scale matrix 11 | R, t: batch_size * 3 * 3, batch_size * 1 * 3 extrinsic calibration parameters 12 | 13 | Returns: For each point [X,Y,Z] in world coordinates [u,v,z] where u,v are the coordinates of the projection in 14 | pixels and z is the depth 15 | ''' 16 | 17 | # instead of P*x we compute x'*P' 18 | vertices = torch.matmul(vertices, R.transpose(2,1)) + t 19 | x, y, z = vertices[:, :, 0], vertices[:, :, 1], vertices[:, :, 2] 20 | 21 | vertices = torch.stack([x, y, torch.ones_like(z)], dim=-1) 22 | vertices = torch.matmul(vertices, K.transpose(1,2)) 23 | u, v = vertices[:, :, 0], vertices[:, :, 1] 24 | # map u,v from [0, img_size] to [-1, 1] to use by the renderer 25 | u = 2 * (u - orig_size / 2.) / orig_size 26 | v = 2 * (v - orig_size / 2.) / orig_size 27 | vertices = torch.stack([u, v, z], dim=-1) 28 | return vertices 29 | -------------------------------------------------------------------------------- /examples/examples_segementation_single_image.py: -------------------------------------------------------------------------------- 1 | import VideoFace3D as vf3d 2 | import cv2 3 | import shutil 4 | import os 5 | import numpy as np 6 | 7 | def example_segmentation(): 8 | image_path = ["./example_pics/1.png", 9 | "./example_pics/2.png"] 10 | 11 | segs = vf3d.FaceSegmentation() 12 | lmd = vf3d.FaceLandmarkDetector("3D") 13 | 14 | for id, imp in enumerate(image_path): 15 | lds = lmd.detect_face_landmark(imp) 16 | new_img, new_lds = vf3d.alignment_and_crop(imp, lds[0]) 17 | new_img = new_img[0] 18 | 19 | cv2.imwrite("./cache.png", new_img) 20 | 21 | org_parsing, mask, mask_prob = segs.create_face_mask("./cache.png") 22 | vis_parsing = segs.visualize(org_parsing, "./cache.png") 23 | mask = (mask * 255).astype(np.uint8) 24 | mask_prob = (mask_prob * 255).astype(np.uint8) 25 | inputs = cv2.imread("./cache.png") 26 | 27 | 28 | cv2.imwrite("./example_results/{}_mask.png".format(id), mask) 29 | cv2.imwrite("./example_results/{}_mask_prob.png".format(id), mask_prob) 30 | cv2.imwrite("./example_results/{}_vis.png".format(id), vis_parsing) 31 | cv2.imwrite("./example_results/{}_input.png".format(id), inputs) 32 | 33 | os.remove("./cache.png") 34 | 35 | if __name__ == '__main__': 36 | example_segmentation() -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_params.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | import os.path as osp 5 | import numpy as np 6 | from .ddfa_io import _load 7 | 8 | 9 | def make_abs_path(d): 10 | return osp.join(osp.dirname(osp.realpath(__file__)), d) 11 | 12 | 13 | d = make_abs_path('../data') 14 | keypoints = _load(osp.join(d, 'keypoints_sim.npy')) 15 | w_shp = _load(osp.join(d, 'w_shp_sim.npy')) 16 | w_exp = _load(osp.join(d, 'w_exp_sim.npy')) # simplified version 17 | meta = _load(osp.join(d, 'param_whitening.pkl')) 18 | # param_mean and param_std are used for re-whitening 19 | param_mean = meta.get('param_mean') 20 | param_std = meta.get('param_std') 21 | u_shp = _load(osp.join(d, 'u_shp.npy')) 22 | u_exp = _load(osp.join(d, 'u_exp.npy')) 23 | u = u_shp + u_exp 24 | w = np.concatenate((w_shp, w_exp), axis=1) 25 | w_base = w[keypoints] 26 | w_norm = np.linalg.norm(w, axis=0) 27 | w_base_norm = np.linalg.norm(w_base, axis=0) 28 | 29 | # for inference 30 | dim = w_shp.shape[0] // 3 31 | u_base = u[keypoints].reshape(-1, 1) 32 | w_shp_base = w_shp[keypoints] 33 | w_exp_base = w_exp[keypoints] 34 | std_size = 120 35 | 36 | # for paf (pac) 37 | paf = _load(osp.join(d, 'Model_PAF.pkl')) 38 | u_filter = paf.get('mu_filter') 39 | w_filter = paf.get('w_filter') 40 | w_exp_filter = paf.get('w_exp_filter') 41 | 42 | # pncc code (mean shape) 43 | pncc_code = _load(osp.join(d, 'pncc_code.npy')) 44 | -------------------------------------------------------------------------------- /examples/example_sfs_single_image.py: -------------------------------------------------------------------------------- 1 | import VideoFace3D as vf3d 2 | import cv2 3 | import numpy as np 4 | 5 | def SfSTest(): 6 | fsp = vf3d.FaceSfSPipline() 7 | lmd = vf3d.FaceLandmarkDetector("3D") 8 | 9 | image_path = [ 10 | "./example_pics/1.png", 11 | "./example_pics/2.png" 12 | ] 13 | 14 | ncount = 0 15 | for im_path in image_path: 16 | ncount += 1 17 | lds = lmd.detect_face_landmark(im_path) 18 | new_img, new_lds = vf3d.alignment_and_crop(im_path, lds[0]) 19 | new_img = new_img[0] 20 | 21 | norm, albedo, light = fsp.disentangle(new_img) 22 | 23 | Irec, Ishd = vf3d.create_shading_recon(norm, albedo, light) 24 | 25 | from PIL import Image 26 | Irec = np.array(Image.fromarray((Irec.clip(0,1)*255).astype(np.uint8)).resize((224,224),Image.ANTIALIAS)) 27 | norm = np.array(Image.fromarray((norm.clip(0,1)*255).astype(np.uint8)).resize((224,224),Image.ANTIALIAS)) 28 | albedo = np.array(Image.fromarray((albedo.clip(0,1)*255).astype(np.uint8)).resize((224,224),Image.ANTIALIAS)) 29 | Ishd = np.array(Image.fromarray((Ishd.clip(0,1)*255).astype(np.uint8)).resize((224,224),Image.ANTIALIAS)) 30 | 31 | 32 | im = np.column_stack([new_img, albedo, norm, Ishd, Irec]) 33 | cv2.imwrite("./example_results/sfs_{}.png".format(ncount), im) 34 | 35 | if __name__ == '__main__': 36 | SfSTest() -------------------------------------------------------------------------------- /VideoFace3D/models/shape_predict.py: -------------------------------------------------------------------------------- 1 | from VideoFace3D.utils.Global import * 2 | import torch 3 | from VideoFace3D.landmark_detect import ddfa_mobilenet as mobilenet_v1 4 | import dlib 5 | from VideoFace3D.models.ddfa_predict import * 6 | import sys 7 | from scipy import io 8 | 9 | 10 | class FaceShapePredict(): 11 | def __init__(self, cuda=True): 12 | self.device = torch.device("cuda" if torch.cuda.is_available() and cuda else "cpu") 13 | self.__load_3D_static_data_file__() 14 | self.front_face_index = io.loadmat(FRONT_FACE_INDEX_PATH)["idx"][:, 0].astype(np.int) - 1 15 | 16 | def __load_3D_static_data_file__(self): 17 | checkpoint_fp = CHECKPOINT_FP_PATH 18 | 19 | arch = 'mobilenet_1' 20 | 21 | checkpoint = torch.load(checkpoint_fp, map_location=lambda storage, loc: storage)['state_dict'] 22 | self.model = getattr(mobilenet_v1, arch)(num_classes=62) # 62 = 12(pose) + 40(shape) +10(expression) 23 | 24 | model_dict = self.model.state_dict() 25 | # because the model is trained by multiple gpus, prefix module should be removed 26 | for k in checkpoint.keys(): 27 | model_dict[k.replace('module.', '')] = checkpoint[k] 28 | self.model.load_state_dict(model_dict) 29 | 30 | self.model.to(self.device) 31 | 32 | dlib_landmark_model = DLIB_LANDMARK_MODEL_PATH 33 | self.face_regressor = dlib.shape_predictor(dlib_landmark_model) 34 | 35 | def predict_shape(self, image_path, rects=None): 36 | vertices, color = detect_landmark_ddfa_3D_shape(image_path, self.model, self.face_regressor, self.device, 37 | rects=rects) 38 | 39 | return vertices[:, self.front_face_index, :], color[:, self.front_face_index, :] 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # video-face-3d 2 | 3 | 人脸工具包大杂烩，主要整合了近年来的SOTA的一些方法，包括视频人脸追踪，68关键点检测，三维人脸3DMM参数标注，shape-from-shading等 4 | 5 | 6 | 7 | ## 依赖 8 | 9 | - pytorch 0.4.1以上 10 | - tensorflow 1.0 以上 11 | - neural_renderer_pytorch 12 | 13 | 14 | 15 | ## 安装 16 | 17 | 安装本代码前请先确保安装了pytorch,tensorflow以及neural_renderer_pytorch 18 | 19 | - neural_renderer_pytorch安装说明 20 | 21 | - 如果你是Linux或者Mac用户，请直接`pip install neural_renderer_pytorch` 22 | 23 | - 如果你是Windows10用户，请参考[neural_renderer-win10](https://github.com/lstcutong/neural_renderer_pytorch-win10) 进行安装 24 | 25 | 下载本代码依赖的静态资源文件，解压到`/VideoFace3D/data`，下载链接[data.zip](https://pan.baidu.com/s/1-btsHsuPlFR4U_GO0Yxavg)，密码 rle5 26 | 27 | 推荐使用`develop`模式安装代码 `python setup.py develop` 28 | 29 | 30 | 31 | ## Examples 32 | 33 | - 人脸关键点检测-单图像结果 34 | 35 | ``` 36 | cd examples 37 | python example_landmarks_single_image.py 38 | ``` 39 | 40 | 41 | 42 | - 3DMM参数化模型拟合-单图像结果 43 | 44 | ``` 45 | cd examples 46 | python example_fitting_single_image.py 47 | ``` 48 | 49 | 50 | 51 | - segmetation 单图像结果 52 | 53 | ``` 54 | cd examples 55 | python examples_segementation_single_image.py 56 | ``` 57 | 58 | 59 | 60 | - SFS- shape-from-shading 单图像结果 61 | 62 | ``` 63 | cd examples 64 | python example_sfs_single_image.py 65 | ``` 66 | 67 | 68 | 69 | ## TODO 70 | 71 | - example视频 72 | - api说明 73 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_estimate_pose.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | """ 5 | Reference: https://github.com/YadiraF/PRNet/blob/master/utils/estimate_pose.py 6 | """ 7 | 8 | from math import cos, sin, atan2, asin, sqrt 9 | import numpy as np 10 | from VideoFace3D.landmark_detect.ddfa_params import param_mean, param_std 11 | 12 | 13 | def parse_pose(param): 14 | param = param * param_std + param_mean 15 | Ps = param[:12].reshape(3, -1) # camera matrix 16 | # R = P[:, :3] 17 | s, R, t3d = P2sRt(Ps) 18 | P = np.concatenate((R, t3d.reshape(3, -1)), axis=1) # without scale 19 | # P = Ps / s 20 | pose = matrix2angle(R) # yaw, pitch, roll 21 | # offset = p_[:, -1].reshape(3, 1) 22 | return P, pose 23 | 24 | def get_rotate_matrix(param): 25 | param = param * param_std + param_mean 26 | Ps = param[:12].reshape(3, -1) # camera matrix 27 | R = Ps[:, :3] 28 | 29 | return R 30 | 31 | def matrix2angle(R): 32 | ''' compute three Euler angles from a Rotation Matrix. Ref: http://www.gregslabaugh.net/publications/euler.pdf 33 | Args: 34 | R: (3,3). rotation matrix 35 | Returns: 36 | x: yaw 37 | y: pitch 38 | z: roll 39 | ''' 40 | # assert(isRotationMatrix(R)) 41 | 42 | if R[2, 0] != 1 and R[2, 0] != -1: 43 | x = asin(R[2, 0]) 44 | y = atan2(R[2, 1] / cos(x), R[2, 2] / cos(x)) 45 | z = atan2(R[1, 0] / cos(x), R[0, 0] / cos(x)) 46 | 47 | else: # Gimbal lock 48 | z = 0 # can be anything 49 | if R[2, 0] == -1: 50 | x = np.pi / 2 51 | y = z + atan2(R[0, 1], R[0, 2]) 52 | else: 53 | x = -np.pi / 2 54 | y = -z + atan2(-R[0, 1], -R[0, 2]) 55 | 56 | return x, y, z 57 | 58 | 59 | def P2sRt(P): 60 | ''' decompositing camera matrix P. 61 | Args: 62 | P: (3, 4). Affine Camera Matrix. 63 | Returns: 64 | s: scale factor. 65 | R: (3, 3). rotation matrix. 66 | t2d: (2,). 2d translation. 67 | ''' 68 | t3d = P[:, 3] 69 | R1 = P[0:1, :3] 70 | R2 = P[1:2, :3] 71 | s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2.0 72 | r1 = R1 / np.linalg.norm(R1) 73 | r2 = R2 / np.linalg.norm(R2) 74 | r3 = np.cross(r1, r2) 75 | 76 | R = np.concatenate((r1, r2, r3), 0) 77 | return s, R, t3d 78 | 79 | 80 | def main(): 81 | pass 82 | 83 | 84 | if __name__ == '__main__': 85 | main() 86 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/detector.py: -------------------------------------------------------------------------------- 1 | from VideoFace3D.utils.Global import * 2 | import torch 3 | from VideoFace3D.landmark_detect import ddfa_mobilenet as mobilenet_v1 4 | import dlib 5 | from VideoFace3D.landmark_detect.ddfa_landmarks import detect_landmark_ddfa_3D 6 | from VideoFace3D.landmark_detect.dlib_landmark import detect_landmark_dlib_2D 7 | import sys 8 | 9 | 10 | class LandmarkDetector(): 11 | def __init__(self, cuda=True): 12 | self.device = torch.device("cuda" if torch.cuda.is_available() and cuda else "cpu") 13 | 14 | 15 | class FaceLandmarkDetector(LandmarkDetector): 16 | def __init__(self, mode="2D", cuda=True): 17 | super(FaceLandmarkDetector, self).__init__(cuda) 18 | self.mode = mode 19 | 20 | if self.mode == "2D": 21 | self.__load_2D_static_data_file__() 22 | elif self.mode == "3D": 23 | self.__load_3D_static_data_file__() 24 | else: 25 | raise Exception("Please choose between '2D' or '3D'") 26 | 27 | def __load_2D_static_data_file__(self): 28 | dlib_landmark_model = DLIB_LANDMARK_MODEL_PATH 29 | self.face_regressor = dlib.shape_predictor(dlib_landmark_model) 30 | 31 | def __load_3D_static_data_file__(self): 32 | checkpoint_fp = CHECKPOINT_FP_PATH 33 | 34 | arch = 'mobilenet_1' 35 | 36 | checkpoint = torch.load(checkpoint_fp, map_location=lambda storage, loc: storage)['state_dict'] 37 | self.model = getattr(mobilenet_v1, arch)(num_classes=62) # 62 = 12(pose) + 40(shape) +10(expression) 38 | 39 | model_dict = self.model.state_dict() 40 | # because the model is trained by multiple gpus, prefix module should be removed 41 | for k in checkpoint.keys(): 42 | model_dict[k.replace('module.', '')] = checkpoint[k] 43 | self.model.load_state_dict(model_dict) 44 | 45 | self.model.to(self.device) 46 | 47 | dlib_landmark_model = DLIB_LANDMARK_MODEL_PATH 48 | self.face_regressor = dlib.shape_predictor(dlib_landmark_model) 49 | 50 | def detect_face_landmark(self, image_path, rects=None): 51 | if self.mode == "3D": 52 | landmarks = detect_landmark_ddfa_3D(image_path, self.model, self.face_regressor, self.device, rects=rects) 53 | else: 54 | landmarks = detect_landmark_dlib_2D(image_path, self.face_regressor) 55 | 56 | if len(landmarks) == 0: 57 | return [] 58 | else: 59 | return landmarks 60 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/lib/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | import uuid 5 | from operator import itemgetter 6 | 7 | import cv2 8 | from VideoFace3D.utils.Global import * 9 | 10 | log_file_root_path = os.path.join(project_dir, 'logs') 11 | log_time = time.strftime('%Y_%m_%d_%H_%M', time.localtime(time.time())) 12 | 13 | 14 | def mkdir(path): 15 | path.strip() 16 | path.rstrip('\\') 17 | isExists = os.path.exists(path) 18 | if not isExists: 19 | os.makedirs(path) 20 | 21 | 22 | def save_to_file(root_dic, tracker): 23 | filter_face_addtional_attribute_list = [] 24 | for item in tracker.face_addtional_attribute: 25 | if item[2] < 1.4 and item[4] < 1: # recommended thresold value 26 | filter_face_addtional_attribute_list.append(item) 27 | if len(filter_face_addtional_attribute_list) > 0: 28 | score_reverse_sorted_list = sorted(filter_face_addtional_attribute_list, key=itemgetter(4)) 29 | mkdir(root_dic) 30 | cv2.imwrite("{0}/{1}.jpg".format(root_dic, str(uuid.uuid1())), score_reverse_sorted_list[0][0]) 31 | 32 | 33 | class Logger: 34 | 35 | def __init__(self, module_name="MOT"): 36 | super().__init__() 37 | path_join = os.path.join(log_file_root_path, module_name) 38 | mkdir(path_join) 39 | 40 | self.logger = logging.getLogger(module_name) 41 | self.logger.setLevel(logging.INFO) 42 | log_file = os.path.join(path_join, '{}.log'.format(log_time)) 43 | if not self.logger.handlers: 44 | fh = logging.FileHandler(log_file, encoding='utf-8') 45 | fh.setLevel(logging.INFO) 46 | 47 | ch = logging.StreamHandler() 48 | ch.setLevel(logging.INFO) 49 | formatter = logging.Formatter( 50 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(threadName)s - %(process)d ") 51 | ch.setFormatter(formatter) 52 | fh.setFormatter(formatter) 53 | self.logger.addHandler(ch) 54 | self.logger.addHandler(fh) 55 | 56 | def error(self, msg, *args, **kwargs): 57 | if self.logger is not None: 58 | self.logger.error(msg, *args, **kwargs) 59 | 60 | def info(self, msg, *args, **kwargs): 61 | if self.logger is not None: 62 | self.logger.info(msg, *args, **kwargs) 63 | 64 | def warn(self, msg, *args, **kwargs): 65 | if self.logger is not None: 66 | self.logger.warning(msg, *args, **kwargs) 67 | 68 | def warning(self, msg, *args, **kwargs): 69 | if self.logger is not None: 70 | self.logger.warning(msg, *args, **kwargs) 71 | 72 | def exception(self, msg, *args, exc_info=True, **kwargs): 73 | if self.logger is not None: 74 | self.logger.exception(msg, *args, exc_info=True, **kwargs) 75 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/data_association.py: -------------------------------------------------------------------------------- 1 | """ 2 | As implemented in https://github.com/abewley/sort but with some modifications 3 | 4 | For each detected item, it computes the intersection over union (IOU) w.r.t. each tracked object. (IOU matrix) 5 | Then, it applies the Hungarian algorithm (via linear_assignment) to assign each det. item to the best possible 6 | tracked item (i.e. to the one with max. IOU). 7 | 8 | Note: a more recent approach uses a Deep Association Metric instead. 9 | see https://github.com/nwojke/deep_sort 10 | """ 11 | 12 | import numpy as np 13 | from numba import jit 14 | from sklearn.utils.linear_assignment_ import linear_assignment 15 | 16 | 17 | @jit 18 | def iou(bb_test, bb_gt): 19 | """ 20 | Computes IUO between two bboxes in the form [x1,y1,x2,y2] 21 | """ 22 | xx1 = np.maximum(bb_test[0], bb_gt[0]) 23 | yy1 = np.maximum(bb_test[1], bb_gt[1]) 24 | xx2 = np.minimum(bb_test[2], bb_gt[2]) 25 | yy2 = np.minimum(bb_test[3], bb_gt[3]) 26 | w = np.maximum(0., xx2 - xx1) 27 | h = np.maximum(0., yy2 - yy1) 28 | wh = w * h 29 | o = wh / ((bb_test[2] - bb_test[0]) * (bb_test[3] - bb_test[1]) 30 | + (bb_gt[2] - bb_gt[0]) * (bb_gt[3] - bb_gt[1]) - wh) 31 | return (o) 32 | 33 | 34 | def associate_detections_to_trackers(detections, trackers, iou_threshold=0.25): 35 | """ 36 | Assigns detections to tracked object (both represented as bounding boxes) 37 | 38 | Returns 3 lists of matches, unmatched_detections and unmatched_trackers 39 | """ 40 | if len(trackers) == 0: 41 | return np.empty((0, 2), dtype=int), np.arange(len(detections)), np.empty((0, 5), dtype=int) 42 | iou_matrix = np.zeros((len(detections), len(trackers)), dtype=np.float32) 43 | 44 | for d, det in enumerate(detections): 45 | for t, trk in enumerate(trackers): 46 | iou_matrix[d, t] = iou(det, trk) 47 | '''The linear assignment module tries to minimise the total assignment cost. 48 | In our case we pass -iou_matrix as we want to maximise the total IOU between track predictions and the frame detection.''' 49 | matched_indices = linear_assignment(-iou_matrix) 50 | 51 | unmatched_detections = [] 52 | for d, det in enumerate(detections): 53 | if d not in matched_indices[:, 0]: 54 | unmatched_detections.append(d) 55 | unmatched_trackers = [] 56 | for t, trk in enumerate(trackers): 57 | if t not in matched_indices[:, 1]: 58 | unmatched_trackers.append(t) 59 | 60 | # filter out matched with low IOU 61 | matches = [] 62 | for m in matched_indices: 63 | if iou_matrix[m[0], m[1]] < iou_threshold: 64 | unmatched_detections.append(m[0]) 65 | unmatched_trackers.append(m[1]) 66 | else: 67 | matches.append(m.reshape(1, 2)) 68 | if len(matches) == 0: 69 | matches = np.empty((0, 2), dtype=int) 70 | else: 71 | matches = np.concatenate(matches, axis=0) 72 | 73 | return matches, np.array(unmatched_detections), np.array(unmatched_trackers) 74 | -------------------------------------------------------------------------------- /examples/example_fitting_single_image.py: -------------------------------------------------------------------------------- 1 | import time 2 | import torch 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import VideoFace3D as vf3d 6 | 7 | def render_single_batch_image(annot, image_size): 8 | device = torch.device("cuda") 9 | 10 | facemodel = vf3d.FaceModelBFM() 11 | 12 | idi, ex, tex, r, t, s, gamma = annot["id"], annot["exp"], annot["tex"], annot["r"], annot["t"], annot["s"], annot[ 13 | "gamma"] 14 | 15 | s = torch.from_numpy(s).unsqueeze(2).to(device).float() 16 | t = torch.from_numpy(t).unsqueeze(1).to(device).float() 17 | r = torch.from_numpy(r).to(device).float() 18 | gamma = torch.from_numpy(gamma).to(device).float() 19 | 20 | batch = len(s) 21 | R = vf3d.euler2rot(r) 22 | K = torch.Tensor.repeat(torch.eye(3).unsqueeze(0), (batch, 1, 1)).to(device) * s 23 | 24 | shape = facemodel.shape_formation(torch.from_numpy(idi).to(device), torch.from_numpy(ex).to(device)) 25 | texture = facemodel.texture_formation(torch.from_numpy(tex).to(device)) 26 | 27 | triangles = torch.Tensor.repeat((torch.from_numpy(facemodel.tri) - 1).long().unsqueeze(0), (batch, 1, 1)).to(device) 28 | 29 | 30 | renderer = vf3d.Renderer(image_size=image_size, K=K, R=R, t=t, near=0.1, far=10000, light_mode="SH", SH_Coeff=gamma) 31 | rgb, depth, silh = renderer(shape, triangles, texture) 32 | rgb = rgb.detach().cpu().numpy().transpose((0, 2, 3, 1)) * 255 33 | rgb = np.clip(rgb, 0, 255).astype(np.uint8) 34 | return rgb[:, :, :, ::-1] 35 | 36 | 37 | def render_and_save_result(image, annot, save_path): 38 | render_im = render_single_batch_image(annot, 224) 39 | all_im = [] 40 | for i in range(len(image)): 41 | im = image[i] 42 | re_im = render_im[i] 43 | all_im.append(np.column_stack([im, re_im])) 44 | 45 | all_im = np.row_stack(all_im) 46 | plt.clf() 47 | fig = plt.figure(figsize=(10, 10)) 48 | plt.imshow(all_im[:, :, ::-1]) 49 | #plt.show() 50 | plt.savefig(save_path) 51 | 52 | 53 | def ModelFitting(): 54 | image_path = [ 55 | "./example_pics/1.png", 56 | "./example_pics/2.png" 57 | ] 58 | 59 | #accurate_fitting = FaceFittingPipline(image_path, "accurate", show_mid=False) 60 | fast_fitting = vf3d.FaceFittingPipline("fast", show_mid=False, checking=False) 61 | landmark_detect = vf3d.FaceLandmarkDetector("3D") 62 | #t0 = time.time() 63 | #result_accu = accurate_fitting.start_fiiting() 64 | #t1 = time.time() 65 | 66 | t2 = time.time() 67 | result_fast = fast_fitting.start_fiiting(image_path,landmark_detect) 68 | t3 = time.time() 69 | for i in range(len(result_fast)): 70 | render_and_save_result(result_fast[i][0], result_fast[i][2], "./example_results/fitting_fast_{}.png".format(i)) 71 | #for i in range(len(result_accu)): 72 | # render_and_save_result(result_accu[i][0],result_accu[i][2],"./accu_{}.png".format(i)) 73 | #print("accurate fitting using time:{}".format(t1-t0)) 74 | print("fast fitting using time:{}".format(t3 - t2)) 75 | 76 | if __name__ == '__main__': 77 | ModelFitting() 78 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/SfS.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import numpy as np 4 | import cv2 5 | import torch 6 | 7 | from VideoFace3D.SFS.SFSNet.functions import create_shading_recon 8 | from VideoFace3D.SFS.SFSNet.mask import MaskGenerator 9 | from VideoFace3D.SFS.SFSNet.model import SfSNet 10 | from VideoFace3D.SFS.SFSNet.utils import convert 11 | from VideoFace3D.utils.Global import SFSNET_PATH 12 | from VideoFace3D.utils.geometry import * 13 | from PIL import Image 14 | 15 | class SfSPipline(): 16 | def __init__(self, cuda=True): 17 | self.device = torch.device("cuda") if cuda and torch.cuda.is_available() else torch.device("cpu") 18 | 19 | 20 | class FaceSfSPipline(SfSPipline): 21 | def __init__(self, cuda=True): 22 | super(FaceSfSPipline, self).__init__(cuda) 23 | 24 | self.net = SfSNet() 25 | self.net.eval() 26 | self.net.load_state_dict(torch.load(SFSNET_PATH)) 27 | self.net.to(self.device) 28 | 29 | self.M = 128 30 | 31 | def disentangle(self, image, ldmark=None): 32 | ''' 33 | 34 | :param image: [h,w,3] bgr 35 | :param ldmark: [68,2] 36 | :return: norm[224,224,3], albedo[224,224,3], light[27] 37 | ''' 38 | 39 | if ldmark is not None: 40 | cv2.imwrite("./cache.png", image) 41 | im, kp = alignment_and_crop("./cache.png", ldmark) 42 | 43 | os.remove("./cache.png") 44 | else: 45 | im = np.array([image]) 46 | 47 | im = Image.fromarray(im[0]).resize((self.M, self.M), Image.ANTIALIAS) 48 | #im = cv2.resize(im[0], (self.M, self.M)) 49 | 50 | im = np.float32(im) / 255.0 51 | # from (128, 128, 3) to (1, 3, 128, 128) 52 | im = np.transpose(im, [2, 0, 1]) 53 | im = np.expand_dims(im, 0) 54 | 55 | # get the normal, albedo and light parameter 56 | normal, albedo, light = self.net(torch.from_numpy(im).to(self.device)) 57 | 58 | n_out = normal.cpu().detach().numpy() 59 | al_out = albedo.cpu().detach().numpy() 60 | light_out = light.cpu().detach().numpy() 61 | 62 | n_out = np.squeeze(n_out, 0) 63 | n_out = np.transpose(n_out, [1, 2, 0]) 64 | # from [1, 3, 128, 128] to [128, 128, 3] 65 | al_out = np.squeeze(al_out, 0) 66 | al_out = np.transpose(al_out, [1, 2, 0]) 67 | # from [1, 27] to [27, 1] 68 | light_out = np.transpose(light_out, [1, 0]) 69 | 70 | n_out2 = n_out[:, :, (2, 1, 0)] 71 | n_out2 = 2 * n_out2 - 1 72 | nr = np.sqrt(np.sum(n_out2 ** 2, axis=2)) # nr=sqrt(sum(n_out2.^2,3)) 73 | nr = np.expand_dims(nr, axis=2) 74 | n_out2 = n_out2 / np.repeat(nr, 3, axis=2) 75 | 76 | al_out2 = al_out[:, :, (2, 1, 0)] 77 | # Note: n_out2, al_out2, light_out is the actual output 78 | 79 | al_out2 = cv2.cvtColor(al_out2, cv2.COLOR_RGB2BGR) 80 | n_out2 = cv2.cvtColor(n_out2, cv2.COLOR_RGB2BGR) 81 | 82 | #al_out2 = cv2.resize(al_out2, (224,224)) 83 | #n_out2 = cv2.resize(n_out2, (224,224)) 84 | 85 | return n_out2, al_out2, light_out -------------------------------------------------------------------------------- /examples/example_video.py: -------------------------------------------------------------------------------- 1 | import VideoFace3D as vf3d 2 | import cv2 3 | import torch 4 | import numpy as np 5 | import os 6 | 7 | segmenter = vf3d.FaceSegmentation() 8 | land_2d = vf3d.FaceLandmarkDetector("2D") 9 | land_3d = vf3d.FaceLandmarkDetector("3D") 10 | 11 | comfortable_colors = vf3d.ComfortableColor() 12 | colors = [ 13 | comfortable_colors.sun_flower.to_bgr(), 14 | comfortable_colors.blue_jeans.to_bgr(), 15 | comfortable_colors.lavander.to_bgr(), 16 | comfortable_colors.bitter_sweet.to_bgr(), 17 | comfortable_colors.aqua.to_bgr()] 18 | 19 | 20 | def get_video_fps(video_path): 21 | cap = cv2.VideoCapture(video_path) 22 | fps = cap.get(cv2.CAP_PROP_FPS) 23 | cap.release() 24 | return fps 25 | 26 | 27 | def do_results_tracking(video_path): 28 | global colors 29 | tracker = vf3d.FaceTracker(echo=True) 30 | print("start tracking ...") 31 | tracking_info = tracker.start_track(video_path) 32 | print("tracking done...") 33 | single_face_bbox = [] 34 | all_frames = [] 35 | for frame, people in tracking_info: 36 | single_face_bbox.append(people[0][0]) 37 | all_frames.append(frame) 38 | 39 | single_face_bbox = np.array(single_face_bbox) # [seq_len, 4] 40 | single_face_bbox = vf3d.video_temporal_smooth_constrains(single_face_bbox.T).T 41 | single_face_bbox = list(single_face_bbox.astype(np.int)) 42 | 43 | all_frame_bboxs = [] 44 | for i, bbox in enumerate(single_face_bbox): 45 | frame_bbox = vf3d.draw_bbox(all_frames[i], [bbox], colors=colors) 46 | all_frame_bboxs.append(frame_bbox) 47 | vf3d.progressbar(i + 1, len(single_face_bbox), prefix="plot tracking...") 48 | 49 | return all_frames, all_frame_bboxs, single_face_bbox 50 | 51 | 52 | def do_results_landmark_detection(all_frames, bboxs, landmark_detector): 53 | # land_3d = vf3d.FaceLandmarkDetector("3D") 54 | cache_image_path = "./example_results/cache.png" 55 | 56 | ldmarks = [] 57 | for i, frame in enumerate(all_frames): 58 | cv2.imwrite(cache_image_path, frame) 59 | lds = landmark_detector.detect_face_landmark(cache_image_path, [bboxs[i]])[0] 60 | ldmarks.append(lds) 61 | vf3d.progressbar(i + 1, len(all_frames), prefix="detect landmarks...") 62 | 63 | os.remove(cache_image_path) 64 | 65 | ldmarks = np.array(ldmarks) # [seq, num_point, 2] 66 | seq, num_point, _ = ldmarks.shape 67 | ldmarks = ldmarks.reshape((seq, -1)) 68 | ldmarks = vf3d.video_temporal_smooth_constrains(ldmarks.T).T 69 | ldmarks = ldmarks.reshape((seq, num_point, -1)) 70 | 71 | all_frame_ldmarks = [] 72 | for i in range(len(ldmarks)): 73 | frame_ldmarks = vf3d.draw_landmarks(all_frames[i], [ldmarks[i]], colors=colors) 74 | all_frame_ldmarks.append(frame_ldmarks) 75 | vf3d.progressbar(i + 1, len(all_frames), prefix="plot landmarks...") 76 | return all_frame_ldmarks, ldmarks 77 | 78 | 79 | if __name__ == '__main__': 80 | video_path = r"E:\datasets\300VW_Dataset_2015_12_14\521\vid.avi" 81 | fps = get_video_fps(video_path) 82 | 83 | all_frames, all_frame_bboxs, all_bboxs = do_results_tracking(video_path) 84 | all_frame_ldmarks, all_ldmarks = do_results_landmark_detection(all_frames, all_bboxs, land_3d) 85 | vf3d.frames2video("./example_results/cache1.mp4", all_frame_ldmarks, fps=fps) 86 | -------------------------------------------------------------------------------- /VideoFace3D/models/ddfa_predict.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as transforms 3 | 4 | import numpy as np 5 | import cv2 6 | import dlib 7 | from VideoFace3D.landmark_detect.ddfa_ddfa import ToTensorGjz, NormalizeGjz, str2bool 8 | 9 | from VideoFace3D.landmark_detect.ddfa_inference import get_suffix, parse_roi_box_from_landmark, crop_img, \ 10 | predict_68pts, dump_to_ply, dump_vertex, \ 11 | draw_landmarks, predict_dense, parse_roi_box_from_bbox, get_colors, write_obj_with_colors 12 | from VideoFace3D.landmark_detect.ddfa_estimate_pose import parse_pose, get_rotate_matrix 13 | 14 | STD_SIZE = 120 15 | 16 | 17 | def detect_landmark_ddfa_3D_shape(image_path, model, face_regressor, device, bbox_init="one", rects=None): 18 | model.eval() 19 | 20 | transform = transforms.Compose([ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)]) 21 | 22 | img_ori = cv2.imread(image_path) 23 | dlib_landmarks = True if rects is None else False 24 | if rects is None: 25 | face_detector = dlib.get_frontal_face_detector() 26 | gray = cv2.cvtColor(img_ori, cv2.COLOR_BGR2GRAY) 27 | rects = face_detector(gray, 1) 28 | 29 | if len(rects) == 0: 30 | return [] 31 | pts_res = [] 32 | Ps = [] # Camera matrix collection 33 | poses = [] # pose collection, [todo: validate it] 34 | vertices_lst = [] # store multiple face vertices 35 | ind = 0 36 | suffix = get_suffix(image_path) 37 | all_vertices = [] 38 | all_colors = [] 39 | for rect in rects: 40 | 41 | if dlib_landmarks: 42 | pts = face_regressor(img_ori, rect).parts() 43 | pts = np.array([[pt.x, pt.y] for pt in pts]).T 44 | roi_box = parse_roi_box_from_landmark(pts) 45 | else: 46 | roi_box = rect 47 | img = crop_img(img_ori, roi_box) 48 | 49 | # forward: one step 50 | img = cv2.resize(img, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 51 | input = transform(img).unsqueeze(0).to(device) 52 | with torch.no_grad(): 53 | param = model(input) 54 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 55 | # 68 pts 56 | pts68 = predict_68pts(param, roi_box) 57 | 58 | # two-step for more accurate bbox to crop face 59 | if bbox_init == 'two': 60 | roi_box = parse_roi_box_from_landmark(pts68) 61 | img_step2 = crop_img(img_ori, roi_box) 62 | img_step2 = cv2.resize(img_step2, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 63 | input = transform(img_step2).unsqueeze(0).to(device) 64 | with torch.no_grad(): 65 | param = model(input) 66 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 67 | 68 | pts68 = predict_68pts(param, roi_box) 69 | 70 | pts_res.append(pts68.transpose(1, 0)[:, 0:2]) 71 | P, pose = parse_pose(param) 72 | 73 | Ps.append(P) 74 | poses.append(pose) 75 | 76 | vertices = predict_dense(param, roi_box) 77 | colors = get_colors(img_ori, vertices) 78 | 79 | vertices = predict_dense(param, roi_box, rotate=False) 80 | #R = get_rotate_matrix(param) 81 | 82 | #vertices = np.linalg.inv(R) @ vertices 83 | 84 | all_vertices.append(vertices.transpose((1, 0))) 85 | all_colors.append(colors[:, ::-1]) 86 | return np.array(all_vertices), np.array(all_colors) 87 | -------------------------------------------------------------------------------- /VideoFace3D/models/face_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from scipy.io import loadmat 4 | from VideoFace3D.utils.Global import * 5 | 6 | class FaceModelBFM(): 7 | def __init__(self): 8 | self.model_path = BFM_FRONT_MODEL_PATH 9 | model = loadmat(self.model_path) 10 | self.meanshape = model['meanshape'] # mean face shape 11 | self.idBase = model['idBase'] # identity basis 12 | self.exBase = model['exBase'] # expression basis 13 | self.meantex = model['meantex'] # mean face texture 14 | self.texBase = model['texBase'] # texture basis 15 | # point_buf can be used for calculating each vertex's norm once face's norm is computed 16 | self.point_buf = model[ 17 | 'point_buf'] # adjacent face index for each vertex, starts from 1 (only used for calculating face normal) 18 | # tri can be used for calculating each face's norm 19 | self.tri = model['tri'][:,::-1].copy() # vertex index for each triangle face, starts from 1 20 | self.keypoints = np.squeeze(model['keypoints']).astype(np.int32) - 1 # 68 face landmark index, starts from 0 21 | self.keypoints_org = self.keypoints.copy() 22 | self.transform_keypoints_index() 23 | 24 | def shape_formation(self, id_param, ex_param): 25 | ''' 26 | 27 | :param id_param: [batch, dim] 28 | :param ex_param: [batch, dim] 29 | :return: 30 | ''' 31 | batch = len(id_param) 32 | 33 | idBase = torch.from_numpy(self.idBase).float().to(id_param.device) 34 | ms = torch.from_numpy(self.meanshape.reshape(-1)).float().to(id_param.device) 35 | exBase = torch.from_numpy(self.exBase).float().to(id_param.device) 36 | 37 | 38 | shape = ms.unsqueeze(1) + idBase @ id_param.float().transpose(1, 0) + exBase @ ex_param.float().transpose(1, 0) 39 | shape = shape.transpose(1, 0).reshape((batch, -1, 3)) 40 | 41 | face_shape = shape - torch.mean(torch.reshape(ms, [1, -1, 3]), dim=1) 42 | 43 | return face_shape 44 | 45 | def texture_formation(self, tex_param): 46 | batch = len(tex_param) 47 | texBase = torch.from_numpy(self.texBase).float().to(tex_param.device) 48 | mt = torch.from_numpy(self.meantex.reshape(-1)).float().to(tex_param.device) 49 | 50 | tex = mt.unsqueeze(1) + texBase @ tex_param.transpose(1, 0) 51 | tex = tex.transpose(1, 0) 52 | 53 | tex = tex.reshape((batch, -1, 3)) 54 | return tex 55 | 56 | # keypoint index in BFM_front_face model is not the same as defined in dlib 57 | # this function makes them be same 58 | # default using dlib index 59 | def transform_keypoints_index(self): 60 | kp_idx = self.keypoints.reshape(-1) 61 | coutour = list(kp_idx[0:17][::-1]) 62 | nose_u = list(kp_idx[27:31]) 63 | nose_d = list(kp_idx[31:36][::-1]) 64 | 65 | eye_cor_order = np.array([45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40]) 66 | mouse_cor_order = np.array([54, 53, 52, 51, 50, 49, 48, 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65]) 67 | 68 | mouse = list(kp_idx[mouse_cor_order]) 69 | eyes = list(kp_idx[eye_cor_order]) 70 | eye_brow = list(kp_idx[17:27][::-1]) 71 | kp_idx = coutour + eye_brow + nose_u + nose_d + eyes + mouse 72 | 73 | self.keypoints = np.array(kp_idx).reshape(-1).astype(np.int64) 74 | 75 | def get_triangle_and_kp68_index(self): 76 | kp_idx = self.keypoints 77 | 78 | return torch.from_numpy(self.tri.astype(np.int32) - 1).unsqueeze(0).cuda(), torch.from_numpy( 79 | kp_idx.astype(np.int64)).cuda() -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/sort.py: -------------------------------------------------------------------------------- 1 | """ 2 | As implemented in https://github.com/abewley/sort but with some modifications 3 | """ 4 | 5 | from __future__ import print_function 6 | 7 | import numpy as np 8 | from VideoFace3D.face_track.src.data_association import associate_detections_to_trackers 9 | from VideoFace3D.face_track.src.kalman_tracker import KalmanBoxTracker 10 | 11 | #logger = utils.Logger("MOT") 12 | 13 | 14 | class Sort: 15 | 16 | def __init__(self, max_age=1, min_hits=3): 17 | """ 18 | Sets key parameters for SORT 19 | """ 20 | self.max_age = max_age 21 | self.min_hits = min_hits 22 | self.trackers = [] 23 | self.frame_count = 0 24 | 25 | def update(self, dets, img_size, root_dic, addtional_attribute_list, predict_num): 26 | """ 27 | Params: 28 | dets - a numpy array of detections in the format [[x,y,w,h,score],[x,y,w,h,score],...] 29 | Requires: this method must be called once for each frame even with empty detections. 30 | Returns the a similar array, where the last column is the object ID. 31 | 32 | NOTE:as in practical realtime MOT, the detector doesn't run on every single frame 33 | """ 34 | self.frame_count += 1 35 | # get predicted locations from existing trackers. 36 | trks = np.zeros((len(self.trackers), 5)) 37 | to_del = [] 38 | ret = [] 39 | for t, trk in enumerate(trks): 40 | pos = self.trackers[t].predict() # kalman predict ,very fast ,<1ms 41 | trk[:] = [pos[0], pos[1], pos[2], pos[3], 0] 42 | if np.any(np.isnan(pos)): 43 | to_del.append(t) 44 | trks = np.ma.compress_rows(np.ma.masked_invalid(trks)) 45 | for t in reversed(to_del): 46 | self.trackers.pop(t) 47 | if dets != []: 48 | matched, unmatched_dets, unmatched_trks = associate_detections_to_trackers(dets, trks) 49 | 50 | # update matched trackers with assigned detections 51 | for t, trk in enumerate(self.trackers): 52 | if t not in unmatched_trks: 53 | d = matched[np.where(matched[:, 1] == t)[0], 0] 54 | trk.update(dets[d, :][0]) 55 | trk.face_addtional_attribute.append(addtional_attribute_list[d[0]]) 56 | 57 | # create and initialise new trackers for unmatched detections 58 | for i in unmatched_dets: 59 | trk = KalmanBoxTracker(dets[i, :]) 60 | trk.face_addtional_attribute.append(addtional_attribute_list[i]) 61 | #logger.info("new Tracker: {0}".format(trk.id + 1)) 62 | self.trackers.append(trk) 63 | 64 | i = len(self.trackers) 65 | for trk in reversed(self.trackers): 66 | if dets == []: 67 | trk.update([]) 68 | d = trk.get_state() 69 | if (trk.time_since_update < 1) and (trk.hit_streak >= self.min_hits or self.frame_count <= self.min_hits): 70 | ret.append(np.concatenate((d, [trk.id + 1])).reshape(1, -1)) # +1 as MOT benchmark requires positive 71 | i -= 1 72 | # remove dead tracklet 73 | if trk.time_since_update >= self.max_age or trk.predict_num >= predict_num or d[2] < 0 or d[3] < 0 or d[0] > img_size[1] or d[1] > img_size[0]: 74 | if len(trk.face_addtional_attribute) >= 5: 75 | pass 76 | #utils.save_to_file(root_dic, trk) 77 | #logger.info('remove tracker: {0}'.format(trk.id + 1)) 78 | self.trackers.pop(i) 79 | if len(ret) > 0: 80 | return np.concatenate(ret) 81 | return np.empty((0, 5)) 82 | -------------------------------------------------------------------------------- /VideoFace3D/face_track/src/kalman_tracker.py: -------------------------------------------------------------------------------- 1 | """ 2 | As implemented in https://github.com/abewley/sort but with some modifications 3 | """ 4 | 5 | import numpy as np 6 | from filterpy.kalman import KalmanFilter 7 | 8 | '''Motion Model''' 9 | 10 | 11 | class KalmanBoxTracker(object): 12 | """ 13 | This class represents the internal state of individual tracked objects observed as bbox. 14 | """ 15 | count = 0 16 | 17 | def __init__(self, bbox): 18 | """ 19 | Initialises a tracker using initial bounding box. 20 | """ 21 | # define constant velocity model 22 | self.kf = KalmanFilter(dim_x=7, dim_z=4) 23 | self.kf.F = np.array( 24 | [[1, 0, 0, 0, 1, 0, 0], [0, 1, 0, 0, 0, 1, 0], [0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 1, 0, 0, 0], 25 | [0, 0, 0, 0, 1, 0, 0], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0, 1]]) 26 | self.kf.H = np.array( 27 | [[1, 0, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0]]) 28 | 29 | self.kf.R[2:, 2:] *= 10. 30 | self.kf.P[4:, 4:] *= 1000. # give high uncertainty to the unobservable initial velocities 31 | self.kf.P *= 10. 32 | self.kf.Q[-1, -1] *= 0.01 33 | self.kf.Q[4:, 4:] *= 0.01 34 | 35 | self.kf.x[:4] = convert_bbox_to_z(bbox) 36 | self.time_since_update = 0 37 | self.id = KalmanBoxTracker.count 38 | KalmanBoxTracker.count += 1 39 | self.history = [] 40 | self.hits = 0 41 | self.hit_streak = 0 42 | self.age = 0 43 | 44 | self.predict_num = 0 # 解决画面中无人脸检测到时而导致的原有追踪器人像预测的漂移bug 45 | 46 | # addtional fields 47 | self.face_addtional_attribute = [] 48 | 49 | def update(self, bbox): 50 | """ 51 | Updates the state vector with observed bbox. 52 | """ 53 | self.time_since_update = 0 54 | self.history = [] 55 | self.hits += 1 56 | self.hit_streak += 1 57 | if bbox != []: 58 | self.kf.update(convert_bbox_to_z(bbox)) 59 | self.predict_num = 0 60 | else: 61 | self.predict_num += 1 62 | 63 | def predict(self): 64 | """ 65 | Advances the state vector and returns the predicted bounding box estimate. 66 | """ 67 | if (self.kf.x[6] + self.kf.x[2]) <= 0: 68 | self.kf.x[6] *= 0.0 69 | self.kf.predict() 70 | self.age += 1 71 | if self.time_since_update > 0: 72 | self.hit_streak = 0 73 | self.time_since_update += 1 74 | self.history.append(convert_x_to_bbox(self.kf.x)) 75 | return self.history[-1][0] 76 | 77 | def get_state(self): 78 | """ 79 | Returns the current bounding box estimate. 80 | """ 81 | return convert_x_to_bbox(self.kf.x)[0] 82 | 83 | 84 | def convert_bbox_to_z(bbox): 85 | """ 86 | Takes a bounding box in the form [x1,y1,x2,y2] and returns z in the form 87 | [x,y,s,r] where x,y is the centre of the box and s is the scale/area and r is 88 | the aspect ratio 89 | """ 90 | w = bbox[2] - bbox[0] 91 | h = bbox[3] - bbox[1] 92 | x = bbox[0] + w / 2. 93 | y = bbox[1] + h / 2. 94 | s = w * h # scale is just area 95 | r = w / float(h) 96 | return np.array([x, y, s, r]).reshape((4, 1)) 97 | 98 | 99 | def convert_x_to_bbox(x, score=None): 100 | """ 101 | Takes a bounding box in the centre form [x,y,s,r] and returns it in the form 102 | [x1,y1,x2,y2] where x1,y1 is the top left and x2,y2 is the bottom right 103 | """ 104 | w = np.sqrt(x[2] * x[3]) 105 | h = x[2] / w 106 | if score is None: 107 | return np.array([x[0] - w / 2., x[1] - h / 2., x[0] + w / 2., x[1] + h / 2.]).reshape((1, 4)) 108 | else: 109 | return np.array([x[0] - w / 2., x[1] - h / 2., x[0] + w / 2., x[1] + h / 2., score]).reshape((1, 5)) 110 | -------------------------------------------------------------------------------- /VideoFace3D/utils/video_utils.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import sys 4 | from VideoFace3D.utils.Global import * 5 | from VideoFace3D.utils.temporal_smooth import * 6 | 7 | def id_generator(number, id_len=4): 8 | number = str(number) 9 | assert len(number) < id_len 10 | 11 | return "0" * (id_len - len(number)) + number 12 | 13 | def progressbar(current, total, num=40, prefix=""): 14 | sys.stdout.write("\r{} {}/{} |{}{}| {:.2f}%".format(prefix, current, total, 15 | "*" * int(num * current / total), 16 | " " * (num - int(num * current / total)), 17 | 100 * current / total)) 18 | sys.stdout.flush() 19 | if current == total: 20 | print("") 21 | 22 | def str2seconds(time): 23 | try: 24 | 25 | h, m, s = time.split(":")[0], time.split(":")[1], time.split(":")[2] 26 | h, m, s = int(h), int(m), int(s) 27 | assert h >= 0 28 | assert 0 <= m < 60 29 | assert 0 <= s < 60 30 | seconds = h * 3600 + m * 60 + s 31 | return int(seconds) 32 | except: 33 | assert False, "wrong time format" 34 | #sys.exit(0) 35 | 36 | def extract_frame_from_video(video_path, save_path=None, ret_frame=True, time_start="default", time_end="default"): 37 | start_frame, end_frame = 0, 0 38 | 39 | if save_path is not None: 40 | if not os.path.exists(save_path): 41 | os.makedirs(save_path) 42 | 43 | cap = cv2.VideoCapture(video_path) 44 | 45 | fps = cap.get(5) 46 | frame_nums = cap.get(7) 47 | total_seconds = int(frame_nums / fps) 48 | 49 | if time_start == "default": 50 | start_frame = 0 51 | else: 52 | start_frame = int(frame_nums * (str2seconds(time_start) / total_seconds)) 53 | if time_end == "default": 54 | end_frame = frame_nums 55 | else: 56 | tmp = int(frame_nums * (str2seconds(time_end) / total_seconds)) 57 | if tmp > frame_nums: 58 | end_frame = frame_nums 59 | else: 60 | end_frame = tmp 61 | 62 | assert start_frame <= end_frame 63 | 64 | iters = int(end_frame - start_frame) 65 | 66 | cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) 67 | 68 | all_frames = [] 69 | for count in range(iters): 70 | ret, frame = cap.read() 71 | if frame is None: 72 | break 73 | if ret_frame: 74 | all_frames.append(frame) 75 | if save_path is not None: 76 | cv2.imwrite(os.path.join(save_path, "{}.png".format(id_generator(count, 7))), frame) 77 | progressbar(count+1, iters, prefix="extract") 78 | if ret_frame: 79 | return all_frames 80 | else: 81 | return None 82 | 83 | def frames2video(save_path, frames, fps=24): 84 | base_folder = os.path.split(save_path)[0] 85 | if not os.path.exists(base_folder): 86 | os.makedirs(base_folder) 87 | 88 | H, W = frames[0].shape[0:2] 89 | img_size = (W, H) 90 | 91 | fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G') 92 | video_writer = cv2.VideoWriter(save_path, fourcc, fps, img_size) 93 | 94 | num = 0 95 | for frame in frames: 96 | video_writer.write(frame) 97 | num += 1 98 | progressbar(num, len(frames), prefix="write video") 99 | 100 | video_writer.release() 101 | 102 | 103 | def video_temporal_smooth_constrains(ref_param, method=SMOOTH_METHODS_GAUSSIAN): 104 | if method == SMOOTH_METHODS_DCNN: 105 | return smooth_DCNN(ref_param) 106 | 107 | if method == SMOOTH_METHODS_GAUSSIAN: 108 | return smooth_gaussian_filter(ref_param) 109 | 110 | if method == SMOOTH_METHODS_MEAN: 111 | return smooth_mean_filter(ref_param) 112 | 113 | if method == SMOOTH_METHODS_MEDUIM: 114 | return smooth_medium_filter(ref_param) 115 | 116 | if method == SMOOTH_METHODS_OPTIMIZE: 117 | return smooth_optimize(ref_param) 118 | -------------------------------------------------------------------------------- /VideoFace3D/utils/visualization.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import random 3 | import numpy as np 4 | 5 | class CColor(): 6 | def __init__(self, rgb_string): 7 | self.rgb_string = rgb_string 8 | 9 | def to_rgb(self): 10 | (r, g, b) = string2rgb(self.rgb_string) 11 | return (r, g, b) 12 | 13 | def to_bgr(self): 14 | (r, g, b) = string2rgb(self.rgb_string) 15 | return (b, g, r) 16 | 17 | 18 | class ComfortableColor(): 19 | def __init__(self): 20 | self.grape_fruit = CColor("#ED5565") 21 | self.grape_fruit_d = CColor("#DA4453") 22 | self.sun_flower = CColor("#FFCE54") 23 | self.sun_flower_d = CColor("#F6BB42") 24 | self.mint = CColor("#48CFAD") 25 | self.mint_d = CColor("#37BC9B") 26 | self.blue_jeans = CColor("#5D9CEC") 27 | self.blue_jeans_d = CColor("#4A89DC") 28 | self.pink_rose = CColor("#EC89C0") 29 | self.pink_rose_d = CColor("#D770AD") 30 | self.bitter_sweet = CColor("#FC6E51") 31 | self.bitter_sweet_d = CColor("#E9573F") 32 | self.grass = CColor("#A0D468") 33 | self.grass_d = CColor("#8CC152") 34 | self.aqua = CColor("#4FC1E9") 35 | self.aqua_d = CColor("#3BAFDA") 36 | self.lavander = CColor("#AC92EC") 37 | self.lavander_d = CColor("#967ADC") 38 | self.light_gray = CColor("#F5F7FA") 39 | self.light_gray_d = CColor("#E6E9ED") 40 | self.medium_gray = CColor("#CCD1D9") 41 | self.medium_gray_d = CColor("#AAB2BD") 42 | self.dark_gray = CColor("#656D78") 43 | self.dark_gray_d = CColor("#434A54") 44 | 45 | def string2hex(string): 46 | hex = 0 47 | for i in range(len(string)): 48 | if string[i] in ["{}".format(i) for i in range(10)]: 49 | hex += int(string[i]) * 16 ** (len(string) - i - 1) 50 | elif string[i].upper() == "A": 51 | hex += 10 * 16 ** (len(string) - i - 1) 52 | elif string[i].upper() == "B": 53 | hex += 11 * 16 ** (len(string) - i - 1) 54 | elif string[i].upper() == "C": 55 | hex += 12 * 16 ** (len(string) - i - 1) 56 | elif string[i].upper() == "D": 57 | hex += 13 * 16 ** (len(string) - i - 1) 58 | elif string[i].upper() == "E": 59 | hex += 14 * 16 ** (len(string) - i - 1) 60 | elif string[i].upper() == "F": 61 | hex += 15 * 16 ** (len(string) - i - 1) 62 | return int(hex) 63 | 64 | 65 | def string2rgb(string): 66 | r, g, b = string[1:3], string[3:5], string[5:7] 67 | return (string2hex(r), string2hex(g), string2hex(b)) 68 | 69 | 70 | def draw_landmarks(image, landmarks, plot_index=False, colors=None): 71 | image1 = image.copy() 72 | for i in range(len(landmarks)): 73 | lm_num = len(landmarks[i]) 74 | if colors is not None: 75 | color = colors[i % len(colors)] 76 | else: 77 | color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)) 78 | for j in range(lm_num): 79 | x, y = int(landmarks[i][j][0]), int(landmarks[i][j][1]) 80 | image1 = cv2.circle(image1, (x, y), radius=3, thickness=2, color=color) 81 | if plot_index: 82 | image1 = cv2.putText(image1, str(j), (x - 5, y - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.3, 83 | color, 84 | thickness=1) 85 | return image1 86 | 87 | 88 | def draw_bbox(image, bboxs, colors=None): 89 | image1 = image.copy() 90 | for i in range(len(bboxs)): 91 | if colors is not None: 92 | color = colors[i % len(colors)] 93 | else: 94 | color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)) 95 | image1 = cv2.rectangle(image1, (bboxs[i][0], bboxs[i][1]), (bboxs[i][2], bboxs[i][3]), color[i]) 96 | return image1 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /VideoFace3D/utils/temporal_smooth.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | import copy 5 | 6 | 7 | # from networks import TimeSmoothNetWork 8 | class Model(nn.Module): 9 | def __init__(self, H, W, initation=None): 10 | super(Model, self).__init__() 11 | 12 | self.H = H 13 | self.W = W 14 | self.l1 = nn.Linear(self.W, self.H) 15 | 16 | if initation is not None: 17 | self.l1.weight.data = torch.from_numpy(initation) 18 | 19 | self.smo_param = self.l1.weight 20 | 21 | def divergence(self, x): 22 | g_x = x[:, 1:self.W] - x[:, 0:self.W - 1] 23 | 24 | g_xx = g_x[:, 1:self.W - 1] - g_x[:, 0:self.W - 2] 25 | 26 | return g_xx 27 | 28 | def gradient(self, x): 29 | g_x = x[:, 1:self.W] - x[:, 0:self.W - 1] 30 | return g_x 31 | 32 | def forward(self, ref_param): 33 | sim_loss = torch.sum((ref_param - self.smo_param.float()) ** 2) 34 | 35 | smo_loss = torch.norm(self.divergence(self.smo_param.float()), p=2) 36 | smo_loss2 = torch.norm(self.gradient(self.smo_param.float()), p=2) 37 | return sim_loss, smo_loss 38 | 39 | 40 | ''' 41 | 提供5种平滑方式 42 | 均值平滑，中值平滑，高斯平滑，，基于优化的平滑，卷积网络平滑 43 | 输入: ref_param 参考参数，类型 ndarray, shape:[param_num,frames] 44 | 输出: new_param 平滑参数, 类型 ndarray, shape:[param_num,frames] 45 | ''' 46 | 47 | 48 | def smooth_optimize(ref_param): 49 | ref_param = ref_param.astype(np.float32) 50 | H, W = ref_param.shape 51 | 52 | model = Model(H, W, initation=ref_param).cuda() 53 | ref_param = torch.from_numpy(ref_param).float().cuda() 54 | 55 | iterations = 300 56 | optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) 57 | 58 | # print(ref_param.shape) 59 | 60 | for it in range(iterations): 61 | sim_loss, smo_loss = model(ref_param) 62 | 63 | loss = 1 * sim_loss + 1.3 * smo_loss 64 | 65 | optimizer.zero_grad() 66 | loss.backward() 67 | optimizer.step() 68 | 69 | new_param = model.smo_param.cpu().detach().numpy() 70 | 71 | return new_param 72 | 73 | 74 | def smooth_medium_filter(ref_param, k=5): 75 | assert k % 2 == 1, "未实现偶数步长" 76 | H, W = ref_param.shape 77 | 78 | s = int(k / 2) 79 | 80 | new_param = copy.deepcopy(ref_param) 81 | for i in range(0, W): 82 | start = np.maximum(0, i - s) 83 | end = np.minimum(W, i + s + 1) 84 | 85 | new_param[:, i] = np.median(ref_param[:, start:end], axis=1) 86 | 87 | return new_param 88 | 89 | 90 | def smooth_mean_filter(ref_param, k=5): 91 | assert k % 2 == 1, "未实现偶数步长" 92 | H, W = ref_param.shape 93 | 94 | s = int(k / 2) 95 | 96 | new_param = copy.deepcopy(ref_param) 97 | for i in range(0, W): 98 | start = np.maximum(0, i - s) 99 | end = np.minimum(W, i + s + 1) 100 | 101 | new_param[:, i] = np.mean(ref_param[:, start:end], axis=1) 102 | 103 | return new_param 104 | 105 | 106 | def smooth_gaussian_filter(ref_param, k=5): 107 | miu, sigma = 0, 1 108 | assert k % 2 == 1, "未实现偶数步长" 109 | H, W = ref_param.shape 110 | 111 | center = int(k / 2) 112 | x = np.array([i - center for i in range(k)]) 113 | 114 | weights = np.exp(-(x - miu) ** 2 / (2 * sigma ** 2)) / (sigma * np.sqrt(2 * np.pi)) 115 | weights = weights / np.sum(weights) 116 | 117 | new_param = copy.deepcopy(ref_param) 118 | 119 | for i in range(center, W - center): 120 | start = np.maximum(0, i - center) 121 | end = np.minimum(W, i + center + 1) 122 | 123 | new_param[:, i] = ref_param[:, start:end] @ weights 124 | 125 | return new_param 126 | 127 | 128 | def smooth_DCNN(ref_param): 129 | pass 130 | 131 | 132 | def calculate_smooth_loss(x): 133 | H, W = x.shape 134 | 135 | g_x = x[:, 1:W] - x[:, 0:W - 1] 136 | 137 | g_xx = g_x[:, 1:W - 1] - g_x[:, 0:W - 2] 138 | 139 | return np.sum(g_xx ** 2) 140 | # return np.sum(np.abs(g_xx)) 141 | 142 | 143 | def calculate_sim_loss(x, y): 144 | return np.sum((x - y) ** 2) 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /VideoFace3D/segmentation/faceparsing/resnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.utils.model_zoo as modelzoo 8 | 9 | # from modules.bn import InPlaceABNSync as BatchNorm2d 10 | 11 | resnet18_url = 'https://download.pytorch.org/models/resnet18-5c106cde.pth' 12 | 13 | 14 | def conv3x3(in_planes, out_planes, stride=1): 15 | """3x3 convolution with padding""" 16 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, 17 | padding=1, bias=False) 18 | 19 | 20 | class BasicBlock(nn.Module): 21 | def __init__(self, in_chan, out_chan, stride=1): 22 | super(BasicBlock, self).__init__() 23 | self.conv1 = conv3x3(in_chan, out_chan, stride) 24 | self.bn1 = nn.BatchNorm2d(out_chan) 25 | self.conv2 = conv3x3(out_chan, out_chan) 26 | self.bn2 = nn.BatchNorm2d(out_chan) 27 | self.relu = nn.ReLU(inplace=True) 28 | self.downsample = None 29 | if in_chan != out_chan or stride != 1: 30 | self.downsample = nn.Sequential( 31 | nn.Conv2d(in_chan, out_chan, 32 | kernel_size=1, stride=stride, bias=False), 33 | nn.BatchNorm2d(out_chan), 34 | ) 35 | 36 | def forward(self, x): 37 | residual = self.conv1(x) 38 | residual = F.relu(self.bn1(residual)) 39 | residual = self.conv2(residual) 40 | residual = self.bn2(residual) 41 | 42 | shortcut = x 43 | if self.downsample is not None: 44 | shortcut = self.downsample(x) 45 | 46 | out = shortcut + residual 47 | out = self.relu(out) 48 | return out 49 | 50 | 51 | def create_layer_basic(in_chan, out_chan, bnum, stride=1): 52 | layers = [BasicBlock(in_chan, out_chan, stride=stride)] 53 | for i in range(bnum-1): 54 | layers.append(BasicBlock(out_chan, out_chan, stride=1)) 55 | return nn.Sequential(*layers) 56 | 57 | 58 | class Resnet18(nn.Module): 59 | def __init__(self): 60 | super(Resnet18, self).__init__() 61 | self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, 62 | bias=False) 63 | self.bn1 = nn.BatchNorm2d(64) 64 | self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) 65 | self.layer1 = create_layer_basic(64, 64, bnum=2, stride=1) 66 | self.layer2 = create_layer_basic(64, 128, bnum=2, stride=2) 67 | self.layer3 = create_layer_basic(128, 256, bnum=2, stride=2) 68 | self.layer4 = create_layer_basic(256, 512, bnum=2, stride=2) 69 | self.init_weight() 70 | 71 | def forward(self, x): 72 | x = self.conv1(x) 73 | x = F.relu(self.bn1(x)) 74 | x = self.maxpool(x) 75 | 76 | x = self.layer1(x) 77 | feat8 = self.layer2(x) # 1/8 78 | feat16 = self.layer3(feat8) # 1/16 79 | feat32 = self.layer4(feat16) # 1/32 80 | return feat8, feat16, feat32 81 | 82 | def init_weight(self): 83 | state_dict = modelzoo.load_url(resnet18_url) 84 | self_state_dict = self.state_dict() 85 | for k, v in state_dict.items(): 86 | if 'fc' in k: continue 87 | self_state_dict.update({k: v}) 88 | self.load_state_dict(self_state_dict) 89 | 90 | def get_params(self): 91 | wd_params, nowd_params = [], [] 92 | for name, module in self.named_modules(): 93 | if isinstance(module, (nn.Linear, nn.Conv2d)): 94 | wd_params.append(module.weight) 95 | if not module.bias is None: 96 | nowd_params.append(module.bias) 97 | elif isinstance(module, nn.BatchNorm2d): 98 | nowd_params += list(module.parameters()) 99 | return wd_params, nowd_params 100 | 101 | 102 | if __name__ == "__main__": 103 | net = Resnet18() 104 | x = torch.randn(16, 3, 224, 224) 105 | out = net(x) 106 | print(out[0].size()) 107 | print(out[1].size()) 108 | print(out[2].size()) 109 | net.get_params() 110 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_io.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | import os 5 | import numpy as np 6 | import torch 7 | import pickle 8 | import scipy.io as sio 9 | 10 | 11 | def mkdir(d): 12 | """only works on *nix system""" 13 | if not os.path.isdir(d) and not os.path.exists(d): 14 | os.system('mkdir -p {}'.format(d)) 15 | 16 | 17 | def _get_suffix(filename): 18 | """a.jpg -> jpg""" 19 | pos = filename.rfind('.') 20 | if pos == -1: 21 | return '' 22 | return filename[pos + 1:] 23 | 24 | 25 | def _load(fp): 26 | suffix = _get_suffix(fp) 27 | if suffix == 'npy': 28 | return np.load(fp) 29 | elif suffix == 'pkl': 30 | return pickle.load(open(fp, 'rb')) 31 | 32 | 33 | def _dump(wfp, obj): 34 | suffix = _get_suffix(wfp) 35 | if suffix == 'npy': 36 | np.save(wfp, obj) 37 | elif suffix == 'pkl': 38 | pickle.dump(obj, open(wfp, 'wb')) 39 | else: 40 | raise Exception('Unknown Type: {}'.format(suffix)) 41 | 42 | 43 | def _load_tensor(fp, mode='cpu'): 44 | if mode.lower() == 'cpu': 45 | return torch.from_numpy(_load(fp)) 46 | elif mode.lower() == 'gpu': 47 | return torch.from_numpy(_load(fp)).cuda() 48 | 49 | 50 | def _tensor_to_cuda(x): 51 | if x.is_cuda: 52 | return x 53 | else: 54 | return x.cuda() 55 | 56 | 57 | def _load_gpu(fp): 58 | return torch.from_numpy(_load(fp)).cuda() 59 | 60 | 61 | def load_bfm(model_path): 62 | suffix = _get_suffix(model_path) 63 | if suffix == 'mat': 64 | C = sio.loadmat(model_path) 65 | model = C['model_refine'] 66 | model = model[0, 0] 67 | 68 | model_new = {} 69 | w_shp = model['w'].astype(np.float32) 70 | model_new['w_shp_sim'] = w_shp[:, :40] 71 | w_exp = model['w_exp'].astype(np.float32) 72 | model_new['w_exp_sim'] = w_exp[:, :10] 73 | 74 | u_shp = model['mu_shape'] 75 | u_exp = model['mu_exp'] 76 | u = (u_shp + u_exp).astype(np.float32) 77 | model_new['mu'] = u 78 | model_new['tri'] = model['tri'].astype(np.int32) - 1 79 | 80 | # flatten it, pay attention to index value 81 | keypoints = model['keypoints'].astype(np.int32) - 1 82 | keypoints = np.concatenate((3 * keypoints, 3 * keypoints + 1, 3 * keypoints + 2), axis=0) 83 | 84 | model_new['keypoints'] = keypoints.T.flatten() 85 | 86 | # 87 | w = np.concatenate((w_shp, w_exp), axis=1) 88 | w_base = w[keypoints] 89 | w_norm = np.linalg.norm(w, axis=0) 90 | w_base_norm = np.linalg.norm(w_base, axis=0) 91 | 92 | dim = w_shp.shape[0] // 3 93 | u_base = u[keypoints].reshape(-1, 1) 94 | w_shp_base = w_shp[keypoints] 95 | w_exp_base = w_exp[keypoints] 96 | 97 | model_new['w_norm'] = w_norm 98 | model_new['w_base_norm'] = w_base_norm 99 | model_new['dim'] = dim 100 | model_new['u_base'] = u_base 101 | model_new['w_shp_base'] = w_shp_base 102 | model_new['w_exp_base'] = w_exp_base 103 | 104 | _dump(model_path.replace('.mat', '.pkl'), model_new) 105 | return model_new 106 | else: 107 | return _load(model_path) 108 | 109 | 110 | def load_obj(obj_file): 111 | sents = open(obj_file).read().split("\n") 112 | vertices, color, tri = [], [], [] 113 | for s in sents: 114 | p = s.split(" ") 115 | if p[0] == 'v': 116 | vertices.append([float(p[1]), float(p[2]), float(p[3])]) 117 | if len(p) == 7: 118 | color.append([int(p[4]), int(p[5]), int(p[6])]) 119 | if p[0] == 'c': 120 | color.append([int(p[4]), int(p[5]), int(p[6])]) 121 | if p[0] == 'f': 122 | tri.append([int(p[1]), int(p[2]), int(p[3])]) 123 | 124 | return np.array(vertices), np.array(color)[:, ::-1], np.array(tri) - 1 125 | 126 | 127 | def write_obj_with_colors(obj_name, vertices, triangles, colors): 128 | triangles = triangles.copy() # meshlab start with 1 129 | 130 | if obj_name.split('.')[-1] != 'obj': 131 | obj_name = obj_name + '.obj' 132 | 133 | # write obj 134 | with open(obj_name, 'w') as f: 135 | # write vertices & colors 136 | for i in range(vertices.shape[0]): 137 | s = 'v {:.4f} {:.4f} {:.4f} {} {} {}\n'.format(vertices[i, 0], vertices[i, 1], vertices[i, 2], colors[i, 2], 138 | colors[i, 1], colors[i, 0]) 139 | f.write(s) 140 | 141 | # write f: ver ind/ uv ind 142 | for i in range(triangles.shape[0]): 143 | s = 'f {} {} {}\n'.format(triangles[i, 0], triangles[i, 1], triangles[i, 2]) 144 | f.write(s) 145 | 146 | 147 | _load_cpu = _load 148 | _numpy_to_tensor = lambda x: torch.from_numpy(x) 149 | _tensor_to_numpy = lambda x: x.cpu() 150 | _numpy_to_cuda = lambda x: _tensor_to_cuda(torch.from_numpy(x)) 151 | _cuda_to_tensor = lambda x: x.cpu() 152 | _cuda_to_numpy = lambda x: x.cpu().numpy() 153 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_landmarks.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.transforms as transforms 3 | 4 | import numpy as np 5 | import cv2 6 | import dlib 7 | from VideoFace3D.landmark_detect.ddfa_ddfa import ToTensorGjz, NormalizeGjz, str2bool 8 | 9 | from VideoFace3D.landmark_detect.ddfa_inference import get_suffix, parse_roi_box_from_landmark, crop_img, \ 10 | predict_68pts, dump_to_ply, dump_vertex, \ 11 | draw_landmarks, predict_dense, parse_roi_box_from_bbox, get_colors, write_obj_with_colors 12 | from VideoFace3D.landmark_detect.ddfa_estimate_pose import parse_pose 13 | 14 | STD_SIZE = 120 15 | 16 | 17 | def detect_landmark_ddfa_3D(image_path, model, face_regressor, device, bbox_init="one", rects=None): 18 | model.eval() 19 | 20 | transform = transforms.Compose([ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)]) 21 | 22 | img_ori = cv2.imread(image_path) 23 | dlib_landmarks = True if rects is None else False 24 | if rects is None: 25 | face_detector = dlib.get_frontal_face_detector() 26 | gray = cv2.cvtColor(img_ori, cv2.COLOR_BGR2GRAY) 27 | rects = face_detector(gray, 1) 28 | 29 | if len(rects) == 0: 30 | return [] 31 | pts_res = [] 32 | Ps = [] # Camera matrix collection 33 | poses = [] # pose collection, [todo: validate it] 34 | vertices_lst = [] # store multiple face vertices 35 | ind = 0 36 | suffix = get_suffix(image_path) 37 | for rect in rects: 38 | 39 | 40 | if dlib_landmarks: 41 | pts = face_regressor(img_ori, rect).parts() 42 | pts = np.array([[pt.x, pt.y] for pt in pts]).T 43 | roi_box = parse_roi_box_from_landmark(pts) 44 | else: 45 | roi_box = rect 46 | img = crop_img(img_ori, roi_box) 47 | 48 | # forward: one step 49 | img = cv2.resize(img, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 50 | input = transform(img).unsqueeze(0).to(device) 51 | with torch.no_grad(): 52 | param = model(input) 53 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 54 | # 68 pts 55 | pts68 = predict_68pts(param, roi_box) 56 | 57 | # two-step for more accurate bbox to crop face 58 | if bbox_init == 'two': 59 | roi_box = parse_roi_box_from_landmark(pts68) 60 | img_step2 = crop_img(img_ori, roi_box) 61 | img_step2 = cv2.resize(img_step2, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 62 | input = transform(img_step2).unsqueeze(0).to(device) 63 | with torch.no_grad(): 64 | param = model(input) 65 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 66 | 67 | pts68 = predict_68pts(param, roi_box) 68 | 69 | pts_res.append(pts68.transpose(1, 0)[:, 0:2]) 70 | P, pose = parse_pose(param) 71 | Ps.append(P) 72 | poses.append(pose) 73 | 74 | vertices = predict_dense(param, roi_box) 75 | # colors = get_colors(img_ori, vertices) 76 | return pts_res 77 | 78 | 79 | ''' 80 | def detect_landmark_ddfa_3D(image_path, model, rects, face_regressor, device, bbox_init="one"): 81 | model.eval() 82 | 83 | transform = transforms.Compose([ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)]) 84 | 85 | img_ori = cv2.imread(image_path) 86 | gray = cv2.cvtColor(img_ori, cv2.COLOR_BGR2GRAY) 87 | rects = face_detector(gray, 1) 88 | 89 | if len(rects) == 0: 90 | return [] 91 | pts_res = [] 92 | Ps = [] # Camera matrix collection 93 | poses = [] # pose collection, [todo: validate it] 94 | vertices_lst = [] # store multiple face vertices 95 | ind = 0 96 | suffix = get_suffix(image_path) 97 | for rect in rects: 98 | 99 | pts = face_regressor(img_ori, rect).parts() 100 | pts = np.array([[pt.x, pt.y] for pt in pts]).T 101 | roi_box = parse_roi_box_from_landmark(pts) 102 | 103 | img = crop_img(img_ori, roi_box) 104 | 105 | # forward: one step 106 | img = cv2.resize(img, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 107 | input = transform(img).unsqueeze(0).to(device) 108 | with torch.no_grad(): 109 | param = model(input) 110 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 111 | # 68 pts 112 | pts68 = predict_68pts(param, roi_box) 113 | 114 | # two-step for more accurate bbox to crop face 115 | if bbox_init == 'two': 116 | roi_box = parse_roi_box_from_landmark(pts68) 117 | img_step2 = crop_img(img_ori, roi_box) 118 | img_step2 = cv2.resize(img_step2, dsize=(STD_SIZE, STD_SIZE), interpolation=cv2.INTER_LINEAR) 119 | input = transform(img_step2).unsqueeze(0).to(device) 120 | with torch.no_grad(): 121 | param = model(input) 122 | param = param.squeeze().cpu().numpy().flatten().astype(np.float32) 123 | 124 | pts68 = predict_68pts(param, roi_box) 125 | 126 | pts_res.append(pts68.transpose(1, 0)[:, 0:2]) 127 | P, pose = parse_pose(param) 128 | Ps.append(P) 129 | poses.append(pose) 130 | 131 | vertices = predict_dense(param, roi_box) 132 | # colors = get_colors(img_ori, vertices) 133 | return pts_res 134 | ''' -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_ddfa.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | import os.path as osp 5 | from pathlib import Path 6 | import numpy as np 7 | 8 | import torch 9 | import torch.utils.data as data 10 | import cv2 11 | import pickle 12 | import argparse 13 | from VideoFace3D.landmark_detect.ddfa_io import _numpy_to_tensor, _load_cpu, _load_gpu 14 | from VideoFace3D.landmark_detect.ddfa_params import * 15 | 16 | 17 | def _parse_param(param): 18 | """Work for both numpy and tensor""" 19 | p_ = param[:12].reshape(3, -1) 20 | p = p_[:, :3] 21 | offset = p_[:, -1].reshape(3, 1) 22 | alpha_shp = param[12:52].reshape(-1, 1) 23 | alpha_exp = param[52:].reshape(-1, 1) 24 | return p, offset, alpha_shp, alpha_exp 25 | 26 | 27 | def reconstruct_vertex(param, whitening=True, dense=False, transform=True, rotate=True): 28 | """Whitening param -> 3d vertex, based on the 3dmm param: u_base, w_shp, w_exp 29 | dense: if True, return dense vertex, else return 68 sparse landmarks. All dense or sparse vertex is transformed to 30 | image coordinate space, but without alignment caused by face cropping. 31 | transform: whether transform to image space 32 | """ 33 | if len(param) == 12: 34 | param = np.concatenate((param, [0] * 50)) 35 | if whitening: 36 | if len(param) == 62: 37 | param = param * param_std + param_mean 38 | else: 39 | param = np.concatenate((param[:11], [0], param[11:])) 40 | param = param * param_std + param_mean 41 | 42 | p, offset, alpha_shp, alpha_exp = _parse_param(param) 43 | 44 | if dense: 45 | if rotate: 46 | vertex = p @ (u + w_shp @ alpha_shp + w_exp @ alpha_exp).reshape(3, -1, order='F') + offset 47 | else: 48 | vertex = (u + w_shp @ alpha_shp + w_exp @ alpha_exp).reshape(3, -1, order='F') 49 | 50 | if transform: 51 | # transform to image coordinate space 52 | vertex[1, :] = std_size + 1 - vertex[1, :] 53 | else: 54 | """For 68 pts""" 55 | if rotate: 56 | vertex = p @ (u_base + w_shp_base @ alpha_shp + w_exp_base @ alpha_exp).reshape(3, -1, order='F') + offset 57 | else: 58 | vertex = (u_base + w_shp_base @ alpha_shp + w_exp_base @ alpha_exp).reshape(3, -1, order='F') 59 | 60 | if transform: 61 | # transform to image coordinate space 62 | vertex[1, :] = std_size + 1 - vertex[1, :] 63 | 64 | return vertex 65 | 66 | 67 | def img_loader(path): 68 | return cv2.imread(path, cv2.IMREAD_COLOR) 69 | 70 | 71 | def str2bool(v): 72 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 73 | return True 74 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 75 | return False 76 | else: 77 | raise argparse.ArgumentTypeError('Boolean value expected') 78 | 79 | 80 | class AverageMeter(object): 81 | """Computes and stores the average and current value""" 82 | 83 | def __init__(self): 84 | self.reset() 85 | 86 | def reset(self): 87 | self.val = 0 88 | self.avg = 0 89 | self.sum = 0 90 | self.count = 0 91 | 92 | def update(self, val, n=1): 93 | self.val = val 94 | self.sum += val * n 95 | self.count += n 96 | self.avg = self.sum / self.count 97 | 98 | 99 | class ToTensorGjz(object): 100 | def __call__(self, pic): 101 | if isinstance(pic, np.ndarray): 102 | img = torch.from_numpy(pic.transpose((2, 0, 1))) 103 | return img.float() 104 | 105 | def __repr__(self): 106 | return self.__class__.__name__ + '()' 107 | 108 | 109 | class NormalizeGjz(object): 110 | def __init__(self, mean, std): 111 | self.mean = mean 112 | self.std = std 113 | 114 | def __call__(self, tensor): 115 | tensor.sub_(self.mean).div_(self.std) 116 | return tensor 117 | 118 | 119 | class DDFADataset(data.Dataset): 120 | def __init__(self, root, filelists, param_fp, transform=None, **kargs): 121 | self.root = root 122 | self.transform = transform 123 | self.lines = Path(filelists).read_text().strip().split('\n') 124 | self.params = _numpy_to_tensor(_load_cpu(param_fp)) 125 | self.img_loader = img_loader 126 | 127 | def _target_loader(self, index): 128 | target = self.params[index] 129 | 130 | return target 131 | 132 | def __getitem__(self, index): 133 | path = osp.join(self.root, self.lines[index]) 134 | img = self.img_loader(path) 135 | 136 | target = self._target_loader(index) 137 | 138 | if self.transform is not None: 139 | img = self.transform(img) 140 | return img, target 141 | 142 | def __len__(self): 143 | return len(self.lines) 144 | 145 | 146 | class DDFATestDataset(data.Dataset): 147 | def __init__(self, filelists, root='', transform=None): 148 | self.root = root 149 | self.transform = transform 150 | self.lines = Path(filelists).read_text().strip().split('\n') 151 | 152 | def __getitem__(self, index): 153 | path = osp.join(self.root, self.lines[index]) 154 | img = img_loader(path) 155 | 156 | if self.transform is not None: 157 | img = self.transform(img) 158 | return img 159 | 160 | def __len__(self): 161 | return len(self.lines) 162 | -------------------------------------------------------------------------------- /VideoFace3D/renderer/lightning.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import math 3 | 4 | 5 | def Illumination_SH(face_texture, norm, gamma): 6 | ''' 7 | 8 | :param face_texture: [batch, face_num, 3] 9 | :param norm: [batch, face_num, 3] 10 | :param gamma: [batch, 27] 11 | :return: 12 | ''' 13 | pi = 3.1415926 14 | num_vertex = face_texture.shape[1] 15 | batch = len(face_texture) 16 | 17 | init_lit = torch.Tensor([0.8, 0, 0, 0, 0, 0, 0, 0, 0]).to(gamma.device) 18 | gamma = torch.reshape(gamma, [-1, 3, 9]) 19 | gamma = gamma + torch.reshape(init_lit, [1, 1, 9]) 20 | 21 | # parameter of 9 SH function 22 | a0 = torch.Tensor([pi]).to(gamma.device) 23 | a1 = torch.Tensor([2 * pi / math.sqrt(3.0)]).to(gamma.device) 24 | a2 = torch.Tensor([2 * pi / math.sqrt(8.0)]).to(gamma.device) 25 | c0 = torch.Tensor([1 / math.sqrt(4 * pi)]).to(gamma.device) 26 | c1 = torch.Tensor([math.sqrt(3.0) / math.sqrt(4 * pi)]).to(gamma.device) 27 | c2 = torch.Tensor([3 * math.sqrt(5.0) / math.sqrt(12 * pi)]).to(gamma.device) 28 | 29 | Y0 = torch.Tensor.repeat(torch.reshape(a0 * c0, [1, 1, 1]), [batch, num_vertex, 1]) 30 | Y1 = torch.reshape(-a1 * c1 * norm[:, :, 1], [batch, num_vertex, 1]) 31 | Y2 = torch.reshape(a1 * c1 * norm[:, :, 2], [batch, num_vertex, 1]) 32 | Y3 = torch.reshape(-a1 * c1 * norm[:, :, 0], [batch, num_vertex, 1]) 33 | Y4 = torch.reshape(a2 * c2 * norm[:, :, 0] * norm[:, :, 1], [batch, num_vertex, 1]) 34 | Y5 = torch.reshape(-a2 * c2 * norm[:, :, 1] * norm[:, :, 2], [batch, num_vertex, 1]) 35 | Y6 = torch.reshape(a2 * c2 * 0.5 / math.sqrt(3.0) * (3 * norm[:, :, 2] ** 2 - 1), [batch, num_vertex, 1]) 36 | Y7 = torch.reshape(-a2 * c2 * norm[:, :, 0] * norm[:, :, 2], [batch, num_vertex, 1]) 37 | Y8 = torch.reshape(a2 * c2 * 0.5 * (norm[:, :, 0] ** 2 - norm[:, :, 1] ** 2), [batch, num_vertex, 1]) 38 | 39 | Y = torch.cat([Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7, Y8], dim=2) 40 | 41 | # Y shape:[batch,N,9]. 42 | 43 | # [batch,N,9] * [batch,9,1] = [batch,N] 44 | lit_r = torch.squeeze(torch.matmul(Y, torch.unsqueeze(gamma[:, 0, :], 2)), 2) 45 | lit_g = torch.squeeze(torch.matmul(Y, torch.unsqueeze(gamma[:, 1, :], 2)), 2) 46 | lit_b = torch.squeeze(torch.matmul(Y, torch.unsqueeze(gamma[:, 2, :], 2)), 2) 47 | 48 | # shape:[batch,N,3] 49 | 50 | face_color_r = (lit_r * face_texture[:, :, 0]).unsqueeze(2) 51 | face_color_g = (lit_g * face_texture[:, :, 1]).unsqueeze(2) 52 | face_color_b = (lit_b * face_texture[:, :, 2]).unsqueeze(2) 53 | 54 | face_color = torch.cat([face_color_r, face_color_g, face_color_b], dim=2) 55 | lighting = torch.cat([lit_r.unsqueeze(2), lit_g.unsqueeze(2), lit_b.unsqueeze(2)], dim=2) * 128 56 | return face_color, lighting 57 | 58 | 59 | import torch 60 | import torch.nn.functional as F 61 | import numpy as np 62 | 63 | def lighting_phong(faces, textures, intensity_ambient=0.5, intensity_directional=0.5, 64 | color_ambient=(1, 1, 1), color_directional=(1, 1, 1), direction=(0, 1, 0)): 65 | 66 | bs, nf = faces.shape[:2] 67 | device = faces.device 68 | 69 | # arguments 70 | # make sure to convert all inputs to float tensors 71 | if isinstance(color_ambient, tuple) or isinstance(color_ambient, list): 72 | color_ambient = torch.tensor(color_ambient, dtype=torch.float32, device=device) 73 | elif isinstance(color_ambient, np.ndarray): 74 | color_ambient = torch.from_numpy(color_ambient).float().to(device) 75 | if isinstance(color_directional, tuple) or isinstance(color_directional, list): 76 | color_directional = torch.tensor(color_directional, dtype=torch.float32, device=device) 77 | elif isinstance(color_directional, np.ndarray): 78 | color_directional = torch.from_numpy(color_directional).float().to(device) 79 | if isinstance(direction, tuple) or isinstance(direction, list): 80 | direction = torch.tensor(direction, dtype=torch.float32, device=device) 81 | elif isinstance(direction, np.ndarray): 82 | direction = torch.from_numpy(direction).float().to(device) 83 | if color_ambient.ndimension() == 1: 84 | color_ambient = color_ambient[None, :] 85 | if color_directional.ndimension() == 1: 86 | color_directional = color_directional[None, :] 87 | if direction.ndimension() == 1: 88 | direction = direction[None, :] 89 | 90 | # create light 91 | light = torch.zeros(bs, nf, 3, dtype=torch.float32).to(device) 92 | 93 | # ambient light 94 | if intensity_ambient != 0: 95 | light += intensity_ambient * color_ambient[:, None, :] 96 | 97 | # directional light 98 | if intensity_directional != 0: 99 | faces = faces.reshape((bs * nf, 3, 3)) 100 | v10 = faces[:, 0] - faces[:, 1] 101 | v12 = faces[:, 2] - faces[:, 1] 102 | # pytorch normalize divides by max(norm, eps) instead of (norm+eps) in chainer 103 | normals = F.normalize(torch.cross(v10, v12), eps=1e-5) 104 | normals = normals.reshape((bs, nf, 3)) 105 | 106 | if direction.ndimension() == 2: 107 | direction = direction[:, None, :] 108 | cos = F.relu(torch.sum(normals * direction, dim=2)) 109 | # may have to verify that the next line is correct 110 | light += intensity_directional * (color_directional[:, None, :] * cos[:, :, None]) 111 | 112 | # apply 113 | light = light[:,:,None, None, None, :] 114 | textures *= light 115 | return textures 116 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/functions.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | import numpy as np 3 | import sys 4 | from matplotlib.path import Path 5 | 6 | 7 | def create_shading_recon(n_out2, al_out2, light_out): 8 | """ 9 | :type n_out2: np.ndarray 10 | :type al_out2: np.ndarray 11 | :type light_out: np.ndarray 12 | :return: 13 | """ 14 | M = n_out2.shape[0] 15 | No1 = np.reshape(n_out2, (M * M, 3)) 16 | tex1 = np.reshape(al_out2, (M * M, 3)) 17 | 18 | la = lambertian_attenuation(3) 19 | HN1 = normal_harmonics(No1.T, la) 20 | 21 | HS1r = np.matmul(HN1, light_out[0:9]) 22 | HS1g = np.matmul(HN1, light_out[9:18]) 23 | HS1b = np.matmul(HN1, light_out[18:27]) 24 | 25 | HS1 = np.zeros(shape=(M, M, 3), dtype=np.float32) 26 | HS1[:, :, 0] = np.reshape(HS1r, (M, M)) 27 | HS1[:, :, 1] = np.reshape(HS1g, (M, M)) 28 | HS1[:, :, 2] = np.reshape(HS1b, (M, M)) 29 | Tex1 = np.reshape(tex1, (M, M, 3)) * HS1 30 | 31 | IRen0 = Tex1 32 | Shd = (200 / 255.0) * HS1 # 200 is added instead of 255 so that not to scale the shading to all white 33 | Ishd0 = Shd 34 | return [IRen0, Ishd0] 35 | 36 | 37 | def lambertian_attenuation(n): 38 | # a = [.8862; 1.0233; .4954]; 39 | a = [np.pi * i for i in [1.0, 2 / 3.0, .25]] 40 | if n > 3: 41 | sys.stderr.write('don\'t record more than 3 attenuation') 42 | exit(-1) 43 | o = a[0:n] 44 | return o 45 | 46 | 47 | def normal_harmonics(N, att): 48 | """ 49 | Return the harmonics evaluated at surface normals N, attenuated by att. 50 | :param N: 51 | :param att: 52 | :return: 53 | 54 | Normals can be scaled surface normals, in which case value of each 55 | harmonic at each point is scaled by albedo. 56 | Harmonics written as polynomials 57 | 0,0 1/sqrt(4*pi) 58 | 1,0 z*sqrt(3/(4*pi)) 59 | 1,1e x*sqrt(3/(4*pi)) 60 | 1,1o y*sqrt(3/(4*pi)) 61 | 2,0 (2*z.^2 - x.^2 - y.^2)/2 * sqrt(5/(4*pi)) 62 | 2,1e x*z * 3*sqrt(5/(12*pi)) 63 | 2,1o y*z * 3*sqrt(5/(12*pi)) 64 | 2,2e (x.^2-y.^2) * 3*sqrt(5/(48*pi)) 65 | 2,2o x*y * 3*sqrt(5/(12*pi)) 66 | """ 67 | xs = N[0, :].T 68 | ys = N[1, :].T 69 | zs = N[2, :].T 70 | a = np.sqrt(xs ** 2 + ys ** 2 + zs ** 2) 71 | denom = (a == 0) + a 72 | # %x = xs./a; y = ys./a; z = zs./a; 73 | x = xs / denom 74 | y = ys / denom 75 | z = zs / denom 76 | 77 | x2 = x * x 78 | y2 = y * y 79 | z2 = z * z 80 | xy = x * y 81 | xz = x * z 82 | yz = y * z 83 | 84 | H1 = att[0] * (1 / np.sqrt(4 * np.pi)) * a 85 | H2 = att[1] * (np.sqrt(3 / (4 * np.pi))) * zs 86 | H3 = att[1] * (np.sqrt(3 / (4 * np.pi))) * xs 87 | H4 = att[1] * (np.sqrt(3 / (4 * np.pi))) * ys 88 | H5 = att[2] * (1 / 2.0) * (np.sqrt(5 / (4 * np.pi))) * ((2 * z2 - x2 - y2) * a) 89 | H6 = att[2] * (3 * np.sqrt(5 / (12 * np.pi))) * (xz * a) 90 | H7 = att[2] * (3 * np.sqrt(5 / (12 * np.pi))) * (yz * a) 91 | H8 = att[2] * (3 * np.sqrt(5 / (48 * np.pi))) * ((x2 - y2) * a) 92 | H9 = att[2] * (3 * np.sqrt(5 / (12 * np.pi))) * (xy * a) 93 | H = [H1, H2, H3, H4, H5, H6, H7, H8, H9] 94 | 95 | # --------add by wang ----------- 96 | H = [np.expand_dims(h, axis=1) for h in H] 97 | H = np.concatenate(H, -1) 98 | # -------------end--------------- 99 | return H 100 | 101 | 102 | def create_mask_fiducial(fiducials, Image): 103 | """ 104 | create mask use fiducials of Image 105 | :param fiducials: the 68 landmarks detected using dlib 106 | :type fiducials np.ndarray 107 | :param Image: a 3-channel image 108 | :type Image np.ndarray 109 | :return: 110 | """ 111 | # fiducals is 2x68 112 | fiducials = np.float32(fiducials) 113 | border_fid = fiducials[:, 0:17] 114 | face_fid = fiducials[:, 17:] 115 | 116 | c1 = np.array([border_fid[0, 0], face_fid[1, 2]]) # left 117 | c2 = np.array([border_fid[0, 16], face_fid[1, 7]]) # right 118 | eye = np.linalg.norm(face_fid[:, 22] - face_fid[:, 25]) 119 | c3 = face_fid[:, 2] 120 | c3[1] = c3[1] - 0.3 * eye 121 | c4 = face_fid[:, 7] 122 | c4[1] = c4[1] - 0.3 * eye 123 | 124 | border = [c1, border_fid, c2, c4, c3] 125 | border = [item.reshape(2, -1) for item in border] 126 | border = np.hstack(border) 127 | 128 | M = Image.shape[0] # row -> y 129 | N = Image.shape[1] # col -> x 130 | 131 | y = np.arange(0, M, step=1, dtype=np.float32) 132 | x = np.arange(0, N, step=1, dtype=np.float32) 133 | X, Y = np.meshgrid(x, y) 134 | 135 | _in, _on = inpolygon(X, Y, border[0, :].T, border[1, :].T) 136 | 137 | mask = np.round(np.reshape(_in | _on, [M, N])) 138 | mask = 255 * np.uint8(mask) 139 | mask = np.repeat(np.expand_dims(mask, -1), 3, axis=-1) 140 | return mask 141 | 142 | 143 | def inpolygon(xq, yq, xv, yv): 144 | """ 145 | reimplement inpolygon in matlab 146 | :type xq: np.ndarray 147 | :type yq: np.ndarray 148 | :type xv: np.ndarray 149 | :type yv: np.ndarray 150 | """ 151 | # http://blog.sina.com.cn/s/blog_70012f010102xnel.html 152 | # merge xy and yv into vertices 153 | vertices = np.vstack((xv, yv)).T 154 | # define a Path object 155 | path = Path(vertices) 156 | # merge X and Y into test_points 157 | test_points = np.hstack([xq.reshape(xq.size, -1), yq.reshape(yq.size, -1)]) 158 | # get mask of test_points in path 159 | _in = path.contains_points(test_points) 160 | # get mask of test_points in path(include the points on path) 161 | _in_on = path.contains_points(test_points, radius=-1e-10) 162 | # get the points on path 163 | _on = _in ^ _in_on 164 | return _in_on, _on 165 | 166 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_mobilenet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | """ 7 | Creates a MobileNet Model as defined in: 8 | Andrew G. Howard Menglong Zhu Bo Chen, et.al. (2017). 9 | MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications. 10 | Copyright (c) Yang Lu, 2017 11 | 12 | Modified By cleardusk 13 | """ 14 | import math 15 | import torch.nn as nn 16 | 17 | __all__ = ['mobilenet_2', 'mobilenet_1', 'mobilenet_075', 'mobilenet_05', 'mobilenet_025'] 18 | 19 | 20 | class DepthWiseBlock(nn.Module): 21 | def __init__(self, inplanes, planes, stride=1, prelu=False): 22 | super(DepthWiseBlock, self).__init__() 23 | inplanes, planes = int(inplanes), int(planes) 24 | self.conv_dw = nn.Conv2d(inplanes, inplanes, kernel_size=3, padding=1, stride=stride, groups=inplanes, 25 | bias=False) 26 | self.bn_dw = nn.BatchNorm2d(inplanes) 27 | self.conv_sep = nn.Conv2d(inplanes, planes, kernel_size=1, stride=1, padding=0, bias=False) 28 | self.bn_sep = nn.BatchNorm2d(planes) 29 | if prelu: 30 | self.relu = nn.PReLU() 31 | else: 32 | self.relu = nn.ReLU(inplace=True) 33 | 34 | def forward(self, x): 35 | out = self.conv_dw(x) 36 | out = self.bn_dw(out) 37 | out = self.relu(out) 38 | 39 | out = self.conv_sep(out) 40 | out = self.bn_sep(out) 41 | out = self.relu(out) 42 | 43 | return out 44 | 45 | 46 | class MobileNet(nn.Module): 47 | def __init__(self, widen_factor=1.0, num_classes=1000, prelu=False, input_channel=3): 48 | """ Constructor 49 | Args: 50 | widen_factor: config of widen_factor 51 | num_classes: number of classes 52 | """ 53 | super(MobileNet, self).__init__() 54 | 55 | block = DepthWiseBlock 56 | self.conv1 = nn.Conv2d(input_channel, int(32 * widen_factor), kernel_size=3, stride=2, padding=1, 57 | bias=False) 58 | 59 | self.bn1 = nn.BatchNorm2d(int(32 * widen_factor)) 60 | if prelu: 61 | self.relu = nn.PReLU() 62 | else: 63 | self.relu = nn.ReLU(inplace=True) 64 | 65 | self.dw2_1 = block(32 * widen_factor, 64 * widen_factor, prelu=prelu) 66 | self.dw2_2 = block(64 * widen_factor, 128 * widen_factor, stride=2, prelu=prelu) 67 | 68 | self.dw3_1 = block(128 * widen_factor, 128 * widen_factor, prelu=prelu) 69 | self.dw3_2 = block(128 * widen_factor, 256 * widen_factor, stride=2, prelu=prelu) 70 | 71 | self.dw4_1 = block(256 * widen_factor, 256 * widen_factor, prelu=prelu) 72 | self.dw4_2 = block(256 * widen_factor, 512 * widen_factor, stride=2, prelu=prelu) 73 | 74 | self.dw5_1 = block(512 * widen_factor, 512 * widen_factor, prelu=prelu) 75 | self.dw5_2 = block(512 * widen_factor, 512 * widen_factor, prelu=prelu) 76 | self.dw5_3 = block(512 * widen_factor, 512 * widen_factor, prelu=prelu) 77 | self.dw5_4 = block(512 * widen_factor, 512 * widen_factor, prelu=prelu) 78 | self.dw5_5 = block(512 * widen_factor, 512 * widen_factor, prelu=prelu) 79 | self.dw5_6 = block(512 * widen_factor, 1024 * widen_factor, stride=2, prelu=prelu) 80 | 81 | self.dw6 = block(1024 * widen_factor, 1024 * widen_factor, prelu=prelu) 82 | 83 | self.avgpool = nn.AdaptiveAvgPool2d(1) 84 | self.fc = nn.Linear(int(1024 * widen_factor), num_classes) 85 | 86 | for m in self.modules(): 87 | if isinstance(m, nn.Conv2d): 88 | n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels 89 | m.weight.data.normal_(0, math.sqrt(2. / n)) 90 | elif isinstance(m, nn.BatchNorm2d): 91 | m.weight.data.fill_(1) 92 | m.bias.data.zero_() 93 | 94 | def forward(self, x): 95 | x = self.conv1(x) 96 | x = self.bn1(x) 97 | x = self.relu(x) 98 | 99 | x = self.dw2_1(x) 100 | x = self.dw2_2(x) 101 | x = self.dw3_1(x) 102 | x = self.dw3_2(x) 103 | x = self.dw4_1(x) 104 | x = self.dw4_2(x) 105 | x = self.dw5_1(x) 106 | x = self.dw5_2(x) 107 | x = self.dw5_3(x) 108 | x = self.dw5_4(x) 109 | x = self.dw5_5(x) 110 | x = self.dw5_6(x) 111 | x = self.dw6(x) 112 | 113 | x = self.avgpool(x) 114 | x = x.view(x.size(0), -1) 115 | x = self.fc(x) 116 | 117 | return x 118 | 119 | 120 | def mobilenet(widen_factor=1.0, num_classes=1000): 121 | """ 122 | Construct MobileNet. 123 | widen_factor=1.0 for mobilenet_1 124 | widen_factor=0.75 for mobilenet_075 125 | widen_factor=0.5 for mobilenet_05 126 | widen_factor=0.25 for mobilenet_025 127 | """ 128 | model = MobileNet(widen_factor=widen_factor, num_classes=num_classes) 129 | return model 130 | 131 | 132 | def mobilenet_2(num_classes=62, input_channel=3): 133 | model = MobileNet(widen_factor=2.0, num_classes=num_classes, input_channel=input_channel) 134 | return model 135 | 136 | 137 | def mobilenet_1(num_classes=62, input_channel=3): 138 | model = MobileNet(widen_factor=1.0, num_classes=num_classes, input_channel=input_channel) 139 | return model 140 | 141 | 142 | def mobilenet_075(num_classes=62, input_channel=3): 143 | model = MobileNet(widen_factor=0.75, num_classes=num_classes, input_channel=input_channel) 144 | return model 145 | 146 | 147 | def mobilenet_05(num_classes=62, input_channel=3): 148 | model = MobileNet(widen_factor=0.5, num_classes=num_classes, input_channel=input_channel) 149 | return model 150 | 151 | 152 | def mobilenet_025(num_classes=62, input_channel=3): 153 | model = MobileNet(widen_factor=0.25, num_classes=num_classes, input_channel=input_channel) 154 | return model 155 | -------------------------------------------------------------------------------- /VideoFace3D/segmentation/segment.py: -------------------------------------------------------------------------------- 1 | from VideoFace3D.segmentation.faceparsing.model import BiSeNet 2 | import torch 3 | from VideoFace3D.utils.Global import BISENET_MODEL_PATH 4 | import torchvision.transforms as transforms 5 | from PIL import Image 6 | import numpy as np 7 | import cv2 8 | 9 | class FaceSegmentation(): 10 | def __init__(self, cuda=True): 11 | self.device = torch.device("cuda") if cuda and torch.cuda.is_available() else torch.device("cpu") 12 | self.model = BiSeNet(n_classes=19) 13 | self.model.load_state_dict(torch.load(BISENET_MODEL_PATH)) 14 | self.model = self.model.to(self.device) 15 | self.model.eval() 16 | self.to_tensor = transforms.Compose([ 17 | transforms.ToTensor(), 18 | transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), 19 | ]) 20 | self.stride = 1 21 | 22 | self.bg = 0 23 | self.skin = 1 24 | self.l_brow = 2 25 | self.r_brow = 3 26 | self.l_eye = 4 27 | self.r_eye = 5 28 | self.eye_g = 6 29 | self.l_ear = 7 30 | self.r_ear = 8 31 | self.ear_r = 9 32 | self.nose = 10 33 | self.mouth = 11 34 | self.u_lip = 12 35 | self.l_lip = 13 36 | self.neck = 14 37 | self.neck_l = 15 38 | self.cloth = 16 39 | self.hair = 17 40 | self.hat = 18 41 | 42 | self.skins = [self.skin] 43 | self.eyes = [self.l_brow, self.r_brow, self.l_eye, self.r_eye, self.eye_g] 44 | self.noses = [self.nose] 45 | self.mouths = [self.mouth, self.u_lip, self.l_lip] 46 | self.ears = [self.l_ear, self.r_ear, self.ear_r] 47 | self.necks = [self.neck, self.neck_l] 48 | self.cloths = [self.cloth] 49 | self.hairs = [self.hair] 50 | self.hats = [self.hat] 51 | 52 | def create_face_mask(self, image_path, 53 | skin=True, 54 | eye=True, 55 | nose=True, 56 | mouth=True, 57 | ear=False, 58 | neck=False, 59 | cloth=False, 60 | hair=False, 61 | hat=False 62 | ): 63 | ''' 64 | 65 | :return: mask, mask_probability 66 | ''' 67 | img = Image.open(image_path) 68 | org_w, org_h = img.size 69 | image = img.resize((512, 512), Image.BILINEAR) 70 | img = self.to_tensor(image) 71 | img = torch.unsqueeze(img, 0) 72 | img = img.to(self.device) 73 | out = self.model(img)[0].squeeze(0).cpu().detach().numpy() 74 | 75 | parsing_anno = out.argmax(0) 76 | 77 | vis_parsing_anno = parsing_anno.copy().astype(np.uint8) 78 | vis_parsing_anno = cv2.resize(vis_parsing_anno, (org_h, org_w), fx=self.stride, fy=self.stride, interpolation=cv2.INTER_NEAREST) 79 | org_parsing = vis_parsing_anno.copy() 80 | mask_prob = out - out.min(0) 81 | mask_prob_exp = np.exp(mask_prob) 82 | mask_prob = mask_prob_exp / mask_prob_exp.sum(0) 83 | mask_prob = mask_prob.max(0) 84 | 85 | mask_prob = cv2.resize(mask_prob, (org_h, org_w), fx=self.stride, fy=self.stride) 86 | 87 | mask = np.zeros((org_w, org_h)) 88 | 89 | if skin: 90 | for p in self.skins: 91 | mask += vis_parsing_anno == p 92 | if eye: 93 | for p in self.eyes: 94 | mask += vis_parsing_anno == p 95 | if nose: 96 | for p in self.noses: 97 | mask += vis_parsing_anno == p 98 | if mouth: 99 | for p in self.mouths: 100 | mask += vis_parsing_anno == p 101 | if ear: 102 | for p in self.ears: 103 | mask += vis_parsing_anno == p 104 | if neck: 105 | for p in self.necks: 106 | mask += vis_parsing_anno == p 107 | if cloth: 108 | for p in self.cloths: 109 | mask += vis_parsing_anno == p 110 | if hair: 111 | for p in self.hairs: 112 | mask += vis_parsing_anno == p 113 | if hat: 114 | for p in self.hats: 115 | mask += vis_parsing_anno == p 116 | 117 | return org_parsing, mask, mask_prob * mask 118 | 119 | def visualize(self, parsing_anno, image_path): 120 | stride = 1 121 | part_colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], 122 | [255, 0, 85], [255, 0, 170], 123 | [0, 255, 0], [85, 255, 0], [170, 255, 0], 124 | [0, 255, 85], [0, 255, 170], 125 | [0, 0, 255], [85, 0, 255], [170, 0, 255], 126 | [0, 85, 255], [0, 170, 255], 127 | [255, 255, 0], [255, 255, 85], [255, 255, 170], 128 | [255, 0, 255], [255, 85, 255], [255, 170, 255], 129 | [0, 255, 255], [85, 255, 255], [170, 255, 255]] 130 | 131 | 132 | im = cv2.imread(image_path) 133 | vis_im = im.copy().astype(np.uint8) 134 | vis_parsing_anno = parsing_anno.copy().astype(np.uint8) 135 | vis_parsing_anno = cv2.resize(vis_parsing_anno, None, fx=stride, fy=stride, interpolation=cv2.INTER_NEAREST) 136 | vis_parsing_anno_color = np.zeros((vis_parsing_anno.shape[0], vis_parsing_anno.shape[1], 3)) + 255 137 | 138 | num_of_class = np.max(vis_parsing_anno) 139 | 140 | for pi in range(1, num_of_class + 1): 141 | index = np.where(vis_parsing_anno == pi) 142 | vis_parsing_anno_color[index[0], index[1], :] = part_colors[pi] 143 | 144 | vis_parsing_anno_color = vis_parsing_anno_color.astype(np.uint8) 145 | # print(vis_parsing_anno_color.shape, vis_im.shape) 146 | vis_im = cv2.addWeighted(cv2.cvtColor(vis_im, cv2.COLOR_RGB2BGR), 0.4, vis_parsing_anno_color, 0.6, 0) 147 | return vis_im -------------------------------------------------------------------------------- /VideoFace3D/face_track/tracker.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | from time import time 4 | import sys 5 | 6 | from VideoFace3D.face_track.align import detect_face 7 | import cv2 8 | import numpy as np 9 | import tensorflow as tf 10 | from VideoFace3D.face_track.lib.face_utils import judge_side_face 11 | from VideoFace3D.face_track.lib.utils import Logger, mkdir 12 | from VideoFace3D.utils.Global import project_dir 13 | from VideoFace3D.face_track.src.sort import Sort 14 | from VideoFace3D.utils.video_utils import progressbar 15 | import copy 16 | 17 | import warnings 18 | warnings.filterwarnings("ignore") 19 | 20 | class FaceTracker(): 21 | def __init__(self, scale_rate=1.0, detect_interval=1, face_score_threshold=0.85, margin=15, echo=False): 22 | self.scale_rate = scale_rate 23 | self.detect_interval = detect_interval 24 | self.face_score_threshold = face_score_threshold 25 | self.margin = margin 26 | 27 | self.tracker = Sort() 28 | self.minsize = 40 # minimum size of face for mtcnn to detect 29 | self.threshold = [0.6, 0.7, 0.7] # three steps's threshold 30 | self.factor = 0.709 # scale factor 31 | 32 | self.echo = echo 33 | 34 | def start_track(self, video_path): 35 | with tf.Graph().as_default(): 36 | with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True), 37 | log_device_placement=False)) as sess: 38 | self.pnet, self.rnet, self.onet = detect_face.create_mtcnn(sess, os.path.join(project_dir, 39 | "../face_track/align")) 40 | 41 | cam = cv2.VideoCapture(video_path) 42 | frame_numbers = int(cam.get(cv2.CAP_PROP_FRAME_COUNT)) 43 | ccount = 0 44 | c = 0 45 | all_result = [] 46 | while True: 47 | ccount += 1 48 | final_faces = [] 49 | addtional_attribute_list = [] 50 | ret, frame = cam.read() 51 | if not ret: 52 | break 53 | 54 | frame = cv2.resize(frame, (0, 0), fx=self.scale_rate, fy=self.scale_rate) 55 | r_g_b_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 56 | if c % self.detect_interval == 0: 57 | img_size = np.asarray(frame.shape)[0:2] 58 | mtcnn_starttime = time() 59 | faces, points = detect_face.detect_face(r_g_b_frame, self.minsize, self.pnet, self.rnet, self.onet, 60 | self.threshold, 61 | self.factor) 62 | face_sums = faces.shape[0] 63 | if face_sums > 0: 64 | face_list = [] 65 | for i, item in enumerate(faces): 66 | score = round(faces[i, 4], 6) 67 | if score > self.face_score_threshold: 68 | det = np.squeeze(faces[i, 0:4]) 69 | 70 | # face rectangle 71 | det[0] = np.maximum(det[0] - self.margin, 0) 72 | det[1] = np.maximum(det[1] - self.margin, 0) 73 | det[2] = np.minimum(det[2] + self.margin, img_size[1]) 74 | det[3] = np.minimum(det[3] + self.margin, img_size[0]) 75 | face_list.append(item) 76 | 77 | # face cropped 78 | bb = np.array(det, dtype=np.int32) 79 | 80 | # use 5 face landmarks to judge the face is front or side 81 | squeeze_points = np.squeeze(points[:, i]) 82 | tolist = squeeze_points.tolist() 83 | facial_landmarks = [] 84 | for j in range(5): 85 | item = [tolist[j], tolist[(j + 5)]] 86 | facial_landmarks.append(item) 87 | 88 | cropped = frame[bb[1]:bb[3], bb[0]:bb[2], :].copy() 89 | 90 | dist_rate, high_ratio_variance, width_rate = judge_side_face( 91 | np.array(facial_landmarks)) 92 | 93 | # face addtional attribute(index 0:face score; index 1:0 represents front face and 1 for side face ) 94 | item_list = [cropped, score, dist_rate, high_ratio_variance, width_rate] 95 | addtional_attribute_list.append(item_list) 96 | 97 | final_faces = np.array(face_list) 98 | 99 | trackers = self.tracker.update(final_faces, img_size, None, addtional_attribute_list, self.detect_interval) 100 | 101 | people = [] 102 | for d in trackers: 103 | det = np.array([0,0,0,0]) 104 | d = d.astype(np.int32) 105 | det[0] = np.maximum(d[0] - self.margin, 0) 106 | det[1] = np.maximum(d[1] - self.margin, 0) 107 | det[2] = np.minimum(d[2] + self.margin, img_size[1]) 108 | det[3] = np.minimum(d[3] + self.margin, img_size[0]) 109 | bb = np.array(det, dtype=np.int32) 110 | people.append((bb, d[4])) 111 | 112 | all_result.append((frame, people)) 113 | 114 | if self.echo: 115 | progressbar(ccount, frame_numbers, prefix="tracking...") 116 | 117 | return all_result 118 | -------------------------------------------------------------------------------- /VideoFace3D/landmark_detect/ddfa_inference.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | __author__ = 'cleardusk' 4 | 5 | import numpy as np 6 | from math import sqrt 7 | import scipy.io as sio 8 | import matplotlib.pyplot as plt 9 | from VideoFace3D.landmark_detect.ddfa_ddfa import reconstruct_vertex 10 | 11 | 12 | def get_suffix(filename): 13 | """a.jpg -> jpg""" 14 | pos = filename.rfind('.') 15 | if pos == -1: 16 | return '' 17 | return filename[pos:] 18 | 19 | 20 | def crop_img(img, roi_box): 21 | h, w = img.shape[:2] 22 | 23 | sx, sy, ex, ey = [int(round(_)) for _ in roi_box] 24 | dh, dw = ey - sy, ex - sx 25 | if len(img.shape) == 3: 26 | res = np.zeros((dh, dw, 3), dtype=np.uint8) 27 | else: 28 | res = np.zeros((dh, dw), dtype=np.uint8) 29 | if sx < 0: 30 | sx, dsx = 0, -sx 31 | else: 32 | dsx = 0 33 | 34 | if ex > w: 35 | ex, dex = w, dw - (ex - w) 36 | else: 37 | dex = dw 38 | 39 | if sy < 0: 40 | sy, dsy = 0, -sy 41 | else: 42 | dsy = 0 43 | 44 | if ey > h: 45 | ey, dey = h, dh - (ey - h) 46 | else: 47 | dey = dh 48 | 49 | res[dsy:dey, dsx:dex] = img[sy:ey, sx:ex] 50 | return res 51 | 52 | 53 | def calc_hypotenuse(pts): 54 | bbox = [min(pts[0, :]), min(pts[1, :]), max(pts[0, :]), max(pts[1, :])] 55 | center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2] 56 | radius = max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 2 57 | bbox = [center[0] - radius, center[1] - radius, center[0] + radius, center[1] + radius] 58 | llength = sqrt((bbox[2] - bbox[0]) ** 2 + (bbox[3] - bbox[1]) ** 2) 59 | return llength / 3 60 | 61 | 62 | def parse_roi_box_from_landmark(pts): 63 | """calc roi box from landmark""" 64 | bbox = [min(pts[0, :]), min(pts[1, :]), max(pts[0, :]), max(pts[1, :])] 65 | center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2] 66 | radius = max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 2 67 | bbox = [center[0] - radius, center[1] - radius, center[0] + radius, center[1] + radius] 68 | 69 | llength = sqrt((bbox[2] - bbox[0]) ** 2 + (bbox[3] - bbox[1]) ** 2) 70 | center_x = (bbox[2] + bbox[0]) / 2 71 | center_y = (bbox[3] + bbox[1]) / 2 72 | 73 | roi_box = [0] * 4 74 | roi_box[0] = center_x - llength / 2 75 | roi_box[1] = center_y - llength / 2 76 | roi_box[2] = roi_box[0] + llength 77 | roi_box[3] = roi_box[1] + llength 78 | 79 | return roi_box 80 | 81 | 82 | def parse_roi_box_from_bbox(bbox): 83 | left, top, right, bottom = bbox 84 | old_size = (right - left + bottom - top) / 2 85 | center_x = right - (right - left) / 2.0 86 | center_y = bottom - (bottom - top) / 2.0 + old_size * 0.14 87 | size = int(old_size * 1.58) 88 | roi_box = [0] * 4 89 | roi_box[0] = center_x - size / 2 90 | roi_box[1] = center_y - size / 2 91 | roi_box[2] = roi_box[0] + size 92 | roi_box[3] = roi_box[1] + size 93 | return roi_box 94 | 95 | 96 | def dump_to_ply(vertex, tri, wfp): 97 | header = """ply 98 | format ascii 1.0 99 | element vertex {} 100 | property float x 101 | property float y 102 | property float z 103 | element face {} 104 | property list uchar int vertex_indices 105 | end_header""" 106 | 107 | n_vertex = vertex.shape[1] 108 | n_face = tri.shape[1] 109 | header = header.format(n_vertex, n_face) 110 | 111 | with open(wfp, 'w') as f: 112 | f.write(header + '\n') 113 | for i in range(n_vertex): 114 | x, y, z = vertex[:, i] 115 | f.write('{:.4f} {:.4f} {:.4f}\n'.format(x, y, z)) 116 | for i in range(n_face): 117 | idx1, idx2, idx3 = tri[:, i] 118 | f.write('3 {} {} {}\n'.format(idx1 - 1, idx2 - 1, idx3 - 1)) 119 | print('Dump tp {}'.format(wfp)) 120 | 121 | 122 | def dump_vertex(vertex, wfp): 123 | sio.savemat(wfp, {'vertex': vertex}) 124 | print('Dump to {}'.format(wfp)) 125 | 126 | 127 | def _predict_vertices(param, roi_bbox, dense, transform=True, rotate=True): 128 | vertex = reconstruct_vertex(param, dense=dense, rotate=rotate) 129 | sx, sy, ex, ey = roi_bbox 130 | scale_x = (ex - sx) / 120 131 | scale_y = (ey - sy) / 120 132 | vertex[0, :] = vertex[0, :] * scale_x + sx 133 | vertex[1, :] = vertex[1, :] * scale_y + sy 134 | 135 | s = (scale_x + scale_y) / 2 136 | vertex[2, :] *= s 137 | 138 | return vertex 139 | 140 | 141 | def predict_68pts(param, roi_box): 142 | return _predict_vertices(param, roi_box, dense=False) 143 | 144 | 145 | def predict_dense(param, roi_box, rotate=True): 146 | return _predict_vertices(param, roi_box, dense=True, rotate=rotate) 147 | 148 | 149 | def draw_landmarks(img, pts, style='fancy', wfp=None, show_flg=False, **kwargs): 150 | """Draw landmarks using matplotlib""" 151 | height, width = img.shape[:2] 152 | plt.figure(figsize=(12, height / width * 12)) 153 | plt.imshow(img[:, :, ::-1]) 154 | plt.subplots_adjust(left=0, right=1, top=1, bottom=0) 155 | plt.axis('off') 156 | 157 | if not type(pts) in [tuple, list]: 158 | pts = [pts] 159 | for i in range(len(pts)): 160 | if style == 'simple': 161 | plt.plot(pts[i][0, :], pts[i][1, :], 'o', markersize=4, color='g') 162 | 163 | elif style == 'fancy': 164 | alpha = 0.8 165 | markersize = 4 166 | lw = 1.5 167 | color = kwargs.get('color', 'w') 168 | markeredgecolor = kwargs.get('markeredgecolor', 'black') 169 | 170 | nums = [0, 17, 22, 27, 31, 36, 42, 48, 60, 68] 171 | 172 | # close eyes and mouths 173 | plot_close = lambda i1, i2: plt.plot([pts[i][0, i1], pts[i][0, i2]], [pts[i][1, i1], pts[i][1, i2]], 174 | color=color, lw=lw, alpha=alpha - 0.1) 175 | plot_close(41, 36) 176 | plot_close(47, 42) 177 | plot_close(59, 48) 178 | plot_close(67, 60) 179 | 180 | for ind in range(len(nums) - 1): 181 | l, r = nums[ind], nums[ind + 1] 182 | plt.plot(pts[i][0, l:r], pts[i][1, l:r], color=color, lw=lw, alpha=alpha - 0.1) 183 | 184 | plt.plot(pts[i][0, l:r], pts[i][1, l:r], marker='o', linestyle='None', markersize=markersize, 185 | color=color, 186 | markeredgecolor=markeredgecolor, alpha=alpha) 187 | 188 | if wfp is not None: 189 | plt.savefig(wfp, dpi=200) 190 | print('Save visualization result to {}'.format(wfp)) 191 | if show_flg: 192 | plt.show() 193 | 194 | 195 | def get_colors(image, vertices): 196 | [h, w, _] = image.shape 197 | vertices[0, :] = np.minimum(np.maximum(vertices[0, :], 0), w - 1) # x 198 | vertices[1, :] = np.minimum(np.maximum(vertices[1, :], 0), h - 1) # y 199 | ind = np.round(vertices).astype(np.int32) 200 | colors = image[ind[1, :], ind[0, :], :] # n x 3 201 | 202 | return colors 203 | 204 | 205 | def write_obj_with_colors(obj_name, vertices, triangles, colors): 206 | triangles = triangles.copy() # meshlab start with 1 207 | 208 | if obj_name.split('.')[-1] != 'obj': 209 | obj_name = obj_name + '.obj' 210 | 211 | # write obj 212 | with open(obj_name, 'w') as f: 213 | # write vertices & colors 214 | for i in range(vertices.shape[1]): 215 | s = 'v {:.4f} {:.4f} {:.4f} {} {} {}\n'.format(vertices[1, i], vertices[0, i], vertices[2, i], colors[i, 2], 216 | colors[i, 1], colors[i, 0]) 217 | f.write(s) 218 | 219 | # write f: ver ind/ uv ind 220 | for i in range(triangles.shape[1]): 221 | s = 'f {} {} {}\n'.format(triangles[0, i], triangles[1, i], triangles[2, i]) 222 | f.write(s) 223 | 224 | 225 | def main(): 226 | pass 227 | 228 | 229 | if __name__ == '__main__': 230 | main() 231 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/model.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from __future__ import absolute_import, division, print_function 3 | import torch 4 | import torchvision 5 | import pickle as pkl 6 | from torch import nn 7 | import torch.nn.functional as F 8 | 9 | 10 | class ResidualBlock(nn.Module): 11 | def __init__(self, in_channel, out_channel): 12 | super(ResidualBlock, self).__init__() 13 | # nbn1/nbn2/.../nbn5 abn1/abn2/.../abn5 14 | self.bn = nn.BatchNorm2d(in_channel) 15 | # nconv1/nconv2/.../nconv5 aconv1/aconv2/.../aconv5 16 | self.conv = nn.Conv2d(in_channel, out_channel, kernel_size=3, stride=1, padding=1) 17 | # nbn1r/nbn2r/.../nbn5r abn1r/abn2r/.../abn5r 18 | self.bnr = nn.BatchNorm2d(out_channel) 19 | # nconv1r/nconv2r/.../nconv5r aconv1r/aconv2r/.../anconv5r 20 | self.convr = nn.Conv2d(out_channel, out_channel, kernel_size=3, stride=1, padding=1) 21 | 22 | def forward(self, x): 23 | out = self.conv(F.relu(self.bn(x))) 24 | out = self.convr(F.relu(self.bnr(out))) 25 | out += x 26 | return out 27 | 28 | 29 | class SfSNet(nn.Module): # SfSNet = PS-Net in SfSNet_deploy.prototxt 30 | def __init__(self): 31 | # C64 32 | super(SfSNet, self).__init__() 33 | # TODO 初始化器 xavier 34 | self.conv1 = nn.Conv2d(3, 64, 7, 1, 3) 35 | self.bn1 = nn.BatchNorm2d(64) 36 | # C128 37 | self.conv2 = nn.Conv2d(64, 128, 3, 1, 1) 38 | self.bn2 = nn.BatchNorm2d(128) 39 | # C128 S2 40 | self.conv3 = nn.Conv2d(128, 128, 3, 2, 1) 41 | # ------------RESNET for normals------------ 42 | # RES1 43 | self.n_res1 = ResidualBlock(128, 128) 44 | # RES2 45 | self.n_res2 = ResidualBlock(128, 128) 46 | # RES3 47 | self.n_res3 = ResidualBlock(128, 128) 48 | # RES4 49 | self.n_res4 = ResidualBlock(128, 128) 50 | # RES5 51 | self.n_res5 = ResidualBlock(128, 128) 52 | # nbn6r 53 | self.nbn6r = nn.BatchNorm2d(128) 54 | # CD128 55 | # TODO 初始化器 bilinear 56 | self.nup6 = nn.ConvTranspose2d(128, 128, 4, 2, 1, groups=128, bias=False) 57 | # nconv6 58 | self.nconv6 = nn.Conv2d(128, 128, 1, 1, 0) 59 | # nbn6 60 | self.nbn6 = nn.BatchNorm2d(128) 61 | # CD 64 62 | self.nconv7 = nn.Conv2d(128, 64, 3, 1, 1) 63 | # nbn7 64 | self.nbn7 = nn.BatchNorm2d(64) 65 | # C*3 66 | self.Nconv0 = nn.Conv2d(64, 3, 1, 1, 0) 67 | 68 | # --------------------Albedo--------------- 69 | # RES1 70 | self.a_res1 = ResidualBlock(128, 128) 71 | # RES2 72 | self.a_res2 = ResidualBlock(128, 128) 73 | # RES3 74 | self.a_res3 = ResidualBlock(128, 128) 75 | # RES4 76 | self.a_res4 = ResidualBlock(128, 128) 77 | # RES5 78 | self.a_res5 = ResidualBlock(128, 128) 79 | # abn6r 80 | self.abn6r = nn.BatchNorm2d(128) 81 | # CD128 82 | self.aup6 = nn.ConvTranspose2d(128, 128, 4, 2, 1, groups=128, bias=False) 83 | # nconv6 84 | self.aconv6 = nn.Conv2d(128, 128, 1, 1, 0) 85 | # nbn6 86 | self.abn6 = nn.BatchNorm2d(128) 87 | # CD 64 88 | self.aconv7 = nn.Conv2d(128, 64, 3, 1, 1) 89 | # nbn7 90 | self.abn7 = nn.BatchNorm2d(64) 91 | # C*3 92 | self.Aconv0 = nn.Conv2d(64, 3, 1, 1, 0) 93 | 94 | # ---------------Light------------------ 95 | # lconv1 96 | self.lconv1 = nn.Conv2d(384, 128, 1, 1, 0) 97 | # lbn1 98 | self.lbn1 = nn.BatchNorm2d(128) 99 | # lpool2r 100 | self.lpool2r = nn.AvgPool2d(64) 101 | # fc_light 102 | self.fc_light = nn.Linear(128, 27) 103 | 104 | def forward(self, inputs): 105 | # C64 106 | x = F.relu(self.bn1(self.conv1(inputs))) 107 | # C128 108 | x = F.relu(self.bn2(self.conv2(x))) 109 | # C128 S2 110 | conv3 = self.conv3(x) 111 | # ------------RESNET for normals------------ 112 | # RES1 113 | x = self.n_res1(conv3) 114 | # RES2 115 | x = self.n_res2(x) 116 | # RES3 117 | x = self.n_res3(x) 118 | # RES4 119 | x = self.n_res4(x) 120 | # RES5 121 | nsum5 = self.n_res5(x) 122 | # nbn6r 123 | nrelu6r = F.relu(self.nbn6r(nsum5)) 124 | # CD128 125 | x = self.nup6(nrelu6r) 126 | # nconv6/nbn6/nrelu6 127 | x = F.relu(self.nbn6(self.nconv6(x))) 128 | # nconv7/nbn7/nrelu7 129 | x = F.relu(self.nbn7(self.nconv7(x))) 130 | # nconv0 131 | normal = self.Nconv0(x) 132 | # --------------------Albedo--------------- 133 | # RES1 134 | x = self.a_res1(conv3) 135 | # RES2 136 | x = self.a_res2(x) 137 | # RES3 138 | x = self.a_res3(x) 139 | # RES4 140 | x = self.a_res4(x) 141 | # RES5 142 | asum5 = self.a_res5(x) 143 | # nbn6r 144 | arelu6r = F.relu(self.abn6r(asum5)) 145 | # CD128 146 | x = self.aup6(arelu6r) 147 | # nconv6/nbn6/nrelu6 148 | x = F.relu(self.abn6(self.aconv6(x))) 149 | # nconv7/nbn7/nrelu7 150 | x = F.relu(self.abn7(self.aconv7(x))) 151 | # nconv0 152 | albedo = self.Aconv0(x) 153 | # ---------------Light------------------ 154 | # lconcat1, shape(1 256 64 64) 155 | x = torch.cat((nrelu6r, arelu6r), 1) 156 | # lconcat2, shape(1 384 64 64) 157 | x = torch.cat([x, conv3], 1) 158 | # lconv1/lbn1/lrelu1 shape(1 128 64 64) 159 | x = F.relu(self.lbn1(self.lconv1(x))) 160 | # lpool2r, shape(1 128 1 1) 161 | x = self.lpool2r(x) 162 | x = x.view(-1, 128) 163 | # fc_light 164 | light = self.fc_light(x) 165 | 166 | return normal, albedo, light 167 | 168 | def load_weights_from_pkl(self, weights_pkl): 169 | from torch import from_numpy 170 | with open(weights_pkl, 'rb') as wp: 171 | try: 172 | # for python3 173 | name_weights = pkl.load(wp, encoding='latin1') 174 | except TypeError as e: 175 | # for python2 176 | name_weights = pkl.load(wp) 177 | state_dict = {} 178 | 179 | def _set_deconv(layer, key): 180 | state_dict[layer+'.weight'] = from_numpy(name_weights[key]['weight']) 181 | 182 | def _set(layer, key): 183 | state_dict[layer + '.weight'] = from_numpy(name_weights[key]['weight']) 184 | state_dict[layer + '.bias'] = from_numpy(name_weights[key]['bias']) 185 | 186 | def _set_bn(layer, key): 187 | state_dict[layer + '.running_var'] = from_numpy(name_weights[key]['running_var']) 188 | state_dict[layer + '.running_mean'] = from_numpy(name_weights[key]['running_mean']) 189 | state_dict[layer + '.weight'] = torch.ones_like(state_dict[layer + '.running_var']) 190 | state_dict[layer + '.bias'] = torch.zeros_like(state_dict[layer + '.running_var']) 191 | 192 | def _set_res(layer, n_or_a, index): 193 | _set_bn(layer+'.bn', n_or_a + 'bn' + str(index)) 194 | _set(layer+'.conv', n_or_a + 'conv' + str(index)) 195 | _set_bn(layer+'.bnr', n_or_a + 'bn' + str(index) + 'r') 196 | _set(layer+'.convr', n_or_a + 'conv' + str(index) + 'r') 197 | 198 | _set('conv1', 'conv1') 199 | _set_bn('bn1', 'bn1') 200 | _set('conv2', 'conv2') 201 | _set_bn('bn2', 'bn2') 202 | _set('conv3', 'conv3') 203 | _set_res('n_res1', 'n', 1) 204 | _set_res('n_res2', 'n', 2) 205 | _set_res('n_res3', 'n', 3) 206 | _set_res('n_res4', 'n', 4) 207 | _set_res('n_res5', 'n', 5) 208 | _set_bn('nbn6r', 'nbn6r') 209 | _set_deconv('nup6', 'nup6') 210 | _set('nconv6', 'nconv6') 211 | _set_bn('nbn6', 'nbn6') 212 | _set('nconv7', 'nconv7') 213 | _set_bn('nbn7', 'nbn7') 214 | _set('Nconv0', 'Nconv0') 215 | _set_res('a_res1', 'a', 1) 216 | _set_res('a_res2', 'a', 2) 217 | _set_res('a_res3', 'a', 3) 218 | _set_res('a_res4', 'a', 4) 219 | _set_res('a_res5', 'a', 5) 220 | _set_bn('abn6r', 'abn6r') 221 | _set_deconv('aup6', 'aup6') 222 | _set('aconv6', 'aconv6') 223 | _set_bn('abn6', 'abn6') 224 | _set('aconv7', 'aconv7') 225 | _set_bn('abn7', 'abn7') 226 | _set('Aconv0', 'Aconv0') 227 | _set('lconv1', 'lconv1') 228 | _set_bn('lbn1', 'lbn1') 229 | _set('fc_light', 'fc_light') 230 | self.load_state_dict(state_dict) 231 | 232 | 233 | if __name__ == '__main__': 234 | net = SfSNet() 235 | net.eval() 236 | 237 | print(len(list(net.named_parameters()))) 238 | for name, param in list(net.named_parameters()): 239 | print(name, param.size()) 240 | -------------------------------------------------------------------------------- /VideoFace3D/utils/geometry.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from scipy.io import loadmat, savemat 4 | from PIL import Image 5 | from VideoFace3D.utils.Global import * 6 | import neural_renderer as nr 7 | import math 8 | 9 | 10 | def euler2rot(euler_angle): 11 | batch_size = euler_angle.shape[0] 12 | theta = -euler_angle[:, 0].reshape(-1, 1, 1) 13 | phi = -euler_angle[:, 1].reshape(-1, 1, 1) 14 | psi = euler_angle[:, 2].reshape(-1, 1, 1) 15 | one = torch.ones(batch_size, 1, 1).to(euler_angle.device) 16 | zero = torch.zeros(batch_size, 1, 1).to(euler_angle.device) 17 | rot_x = torch.cat(( 18 | torch.cat((one, zero, zero), 1), 19 | torch.cat((zero, theta.cos(), theta.sin()), 1), 20 | torch.cat((zero, -theta.sin(), theta.cos()), 1), 21 | ), 2) 22 | rot_y = torch.cat(( 23 | torch.cat((phi.cos(), zero, -phi.sin()), 1), 24 | torch.cat((zero, one, zero), 1), 25 | torch.cat((phi.sin(), zero, phi.cos()), 1), 26 | ), 2) 27 | rot_z = torch.cat(( 28 | torch.cat((psi.cos(), -psi.sin(), zero), 1), 29 | torch.cat((psi.sin(), psi.cos(), zero), 1), 30 | torch.cat((zero, zero, one), 1) 31 | ), 2) 32 | return torch.bmm(rot_x, torch.bmm(rot_y, rot_z)) 33 | 34 | 35 | def texture_from_point2faces(triangles, texutures): 36 | batch = len(texutures) 37 | tex = nr.vertices_to_faces(texutures, triangles) 38 | tex = torch.Tensor.mean(tex, dim=2) 39 | return tex.reshape((batch, tex.shape[1], 1, 1, 1, 3)) 40 | 41 | 42 | def POS(xp, x): 43 | npts = xp.shape[1] 44 | 45 | A = np.zeros([2 * npts, 8]) 46 | 47 | A[0:2 * npts - 1:2, 0:3] = x.transpose() 48 | A[0:2 * npts - 1:2, 3] = 1 49 | 50 | A[1:2 * npts:2, 4:7] = x.transpose() 51 | A[1:2 * npts:2, 7] = 1 52 | 53 | b = np.reshape(xp.transpose(), [2 * npts, 1]) 54 | 55 | k, _, _, _ = np.linalg.lstsq(A, b) 56 | 57 | R1 = k[0:3] 58 | R2 = k[4:7] 59 | sTx = k[3] 60 | sTy = k[7] 61 | s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2 62 | t = np.stack([sTx, sTy], axis=0) 63 | 64 | return t, s 65 | 66 | 67 | def alignment_and_crop(image_path, align_kp): 68 | Lm3D = loadmat(SIMILARITY_LM3D_ALL_MODEL_PATH) 69 | Lm3D = Lm3D['lm'] 70 | 71 | lm_idx = np.array([31, 37, 40, 43, 46, 49, 55]) - 1 72 | Lm3D = np.stack([Lm3D[lm_idx[0], :], np.mean(Lm3D[lm_idx[[1, 2]], :], 0), np.mean(Lm3D[lm_idx[[3, 4]], :], 0), 73 | Lm3D[lm_idx[5], :], Lm3D[lm_idx[6], :]], axis=0) 74 | Lm3D = Lm3D[[1, 2, 0, 3, 4], :] 75 | 76 | Lm2D = np.stack( 77 | [align_kp[lm_idx[0], :], np.mean(align_kp[lm_idx[[1, 2]], :], 0), np.mean(align_kp[lm_idx[[3, 4]], :], 0), 78 | align_kp[lm_idx[5], :], align_kp[lm_idx[6], :]], axis=0) 79 | Lm2D = Lm2D[[1, 2, 0, 3, 4], :] 80 | 81 | img = Image.open(image_path) 82 | w0, h0 = img.size 83 | 84 | Lm2D = np.stack([Lm2D[:, 0], h0 - 1 - Lm2D[:, 1]], axis=1) 85 | t, s = POS(Lm2D.transpose(), Lm3D.transpose()) 86 | 87 | img = img.transform(img.size, Image.AFFINE, (1, 0, t[0] - w0 / 2, 0, 1, h0 / 2 - t[1])) 88 | w = (w0 / s * 102).astype(np.int32) 89 | h = (h0 / s * 102).astype(np.int32) 90 | img = img.resize((w, h), resample=Image.BILINEAR) 91 | # lm = np.stack([lm[:, 0] - t[0] + w0 / 2, lm[:, 1] - t[1] + h0 / 2], axis=1) / s * 102 92 | lm68 = np.stack([align_kp[:, 0] - t[0] + w0 / 2, align_kp[:, 1] + t[1] - h0 / 2], axis=1) / s * 102 93 | # crop the image to 224*224 from image center 94 | left = (w / 2 - 112).astype(np.int32) 95 | right = left + 224 96 | up = (h / 2 - 112).astype(np.int32) 97 | below = up + 224 98 | 99 | img = img.crop((left, up, right, below)) 100 | img = np.array(img) 101 | img = img[:, :, ::-1] 102 | img = np.expand_dims(img, 0) 103 | lm68 = lm68 - np.reshape(np.array([(w / 2 - 112), (h / 2 - 112)]), [1, 2]) 104 | return img, np.expand_dims(lm68, 0) 105 | 106 | 107 | def estimate_affine_matrix_3d22d(X, x): 108 | ''' Using Golden Standard Algorithm for estimating an affine camera 109 | matrix P from world to image correspondences. 110 | See Alg.7.2. in MVGCV 111 | Code Ref: https://github.com/patrikhuber/eos/blob/master/include/eos/fitting/affine_camera_estimation.hpp 112 | x_homo = X_homo.dot(P_Affine) 113 | Args: 114 | X: [n, 3]. corresponding 3d points(fixed) 115 | x: [n, 2]. n>=4. 2d points(moving). x = PX 116 | Returns: 117 | P_Affine: [3, 4]. Affine camera matrix 118 | ''' 119 | X = X.T 120 | x = x.T 121 | assert (x.shape[1] == X.shape[1]) 122 | n = x.shape[1] 123 | assert (n >= 4) 124 | 125 | # --- 1. normalization 126 | # 2d points 127 | mean = np.mean(x, 1) # (2,) 128 | x = x - np.tile(mean[:, np.newaxis], [1, n]) 129 | average_norm = np.mean(np.sqrt(np.sum(x ** 2, 0))) 130 | scale = np.sqrt(2) / average_norm 131 | x = scale * x 132 | 133 | T = np.zeros((3, 3), dtype=np.float32) 134 | T[0, 0] = T[1, 1] = scale 135 | T[:2, 2] = -mean * scale 136 | T[2, 2] = 1 137 | 138 | # 3d points 139 | X_homo = np.vstack((X, np.ones((1, n)))) 140 | mean = np.mean(X, 1) # (3,) 141 | X = X - np.tile(mean[:, np.newaxis], [1, n]) 142 | m = X_homo[:3, :] - X 143 | average_norm = np.mean(np.sqrt(np.sum(X ** 2, 0))) 144 | scale = np.sqrt(3) / average_norm 145 | X = scale * X 146 | 147 | U = np.zeros((4, 4), dtype=np.float32) 148 | U[0, 0] = U[1, 1] = U[2, 2] = scale 149 | U[:3, 3] = -mean * scale 150 | U[3, 3] = 1 151 | 152 | # --- 2. equations 153 | A = np.zeros((n * 2, 8), dtype=np.float32); 154 | X_homo = np.vstack((X, np.ones((1, n)))).T 155 | A[:n, :4] = X_homo 156 | A[n:, 4:] = X_homo 157 | b = np.reshape(x, [-1, 1]) 158 | 159 | # --- 3. solution 160 | p_8 = np.linalg.pinv(A).dot(b) 161 | P = np.zeros((3, 4), dtype=np.float32) 162 | P[0, :] = p_8[:4, 0] 163 | P[1, :] = p_8[4:, 0] 164 | P[-1, -1] = 1 165 | 166 | # --- 4. denormalization 167 | P_Affine = np.linalg.inv(T).dot(P.dot(U)) 168 | return P_Affine 169 | 170 | 171 | def rad2degree(angles): 172 | ''' 173 | 174 | :param angles: [batch, 3] 175 | :return: 176 | ''' 177 | return angles * 180 / np.pi 178 | 179 | 180 | def isRotationMatrix(R): 181 | ''' checks if a matrix is a valid rotation matrix(whether orthogonal or not) 182 | ''' 183 | Rt = np.transpose(R) 184 | shouldBeIdentity = np.dot(Rt, R) 185 | I = np.identity(3, dtype=R.dtype) 186 | n = np.linalg.norm(I - shouldBeIdentity) 187 | return n < 1e-6 188 | 189 | 190 | def matrix2angle(R): 191 | ''' get three Euler angles from Rotation Matrix 192 | Args: 193 | R: (3,3). rotation matrix 194 | Returns: 195 | x: pitch 196 | y: yaw 197 | z: roll 198 | ''' 199 | assert (isRotationMatrix) 200 | sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0]) 201 | 202 | singular = sy < 1e-6 203 | 204 | if not singular: 205 | x = math.atan2(R[2, 1], R[2, 2]) 206 | y = math.atan2(-R[2, 0], sy) 207 | z = math.atan2(R[1, 0], R[0, 0]) 208 | else: 209 | x = math.atan2(-R[1, 2], R[1, 1]) 210 | y = math.atan2(-R[2, 0], sy) 211 | z = 0 212 | 213 | return x, y, z 214 | 215 | 216 | def P2sRt(P): 217 | ''' decompositing camera matrix P 218 | Args: 219 | P: (3, 4). Affine Camera Matrix. 220 | Returns: 221 | s: scale factor. 222 | R: (3, 3). rotation matrix. 223 | t: (3,). translation. 224 | ''' 225 | t = P[:, 3] 226 | R1 = P[0:1, :3] 227 | R2 = P[1:2, :3] 228 | s = (np.linalg.norm(R1) + np.linalg.norm(R2)) / 2.0 229 | r1 = R1 / np.linalg.norm(R1) 230 | r2 = R2 / np.linalg.norm(R2) 231 | r3 = np.cross(r1, r2) 232 | 233 | R = np.concatenate((r1, r2, r3), 0) 234 | return s, R, t 235 | 236 | 237 | def compute_face_norm(vertices, triangles): 238 | pt1_index, pt2_index, pt3_index = triangles[:, 0], triangles[:, 1], triangles[:, 2] 239 | pts1, pts2, pts3 = vertices[:, pt1_index, :], vertices[:, pt2_index, :], vertices[:, pt3_index, :] 240 | 241 | vec1 = pts1 - pts2 242 | vec2 = pts1 - pts3 243 | 244 | face_norm = torch.Tensor.cross(vec1, vec2) 245 | 246 | return face_norm 247 | 248 | 249 | def compute_point_norm(vertices, triangles, point_buf): 250 | batch = len(vertices) 251 | face_norm = compute_face_norm(vertices, triangles) 252 | 253 | face_norm = torch.cat([face_norm, torch.zeros((batch, 1, 3)).to(vertices.device)], dim=1) 254 | 255 | v_norm = torch.sum(face_norm[:, point_buf, :], dim=2) 256 | v_norm = v_norm / (torch.norm(v_norm, dim=2).unsqueeze(2)) 257 | return v_norm 258 | 259 | 260 | def texture_mapping(image, geo, s, R, t): 261 | pass 262 | 263 | 264 | from matplotlib.path import Path 265 | 266 | 267 | def inpolygon(xq, yq, xv, yv): 268 | """ 269 | reimplement inpolygon in matlab 270 | :type xq: np.ndarray 271 | :type yq: np.ndarray 272 | :type xv: np.ndarray 273 | :type yv: np.ndarray 274 | """ 275 | # 合并xv和yv为顶点数组 276 | vertices = np.vstack((xv, yv)).T 277 | # 定义Path对象 278 | path = Path(vertices) 279 | # 把xq和yq合并为test_points 280 | test_points = np.hstack([xq.reshape(xq.size, -1), yq.reshape(yq.size, -1)]) 281 | # 得到一个test_points是否严格在path内的mask，是bool值数组 282 | _in = path.contains_points(test_points) 283 | # 得到一个test_points是否在path内部或者在路径上的mask 284 | _in_on = path.contains_points(test_points, radius=-1e-10) 285 | # 得到一个test_points是否在path路径上的mask 286 | _on = _in ^ _in_on 287 | 288 | return _in_on, _on 289 | 290 | 291 | def create_mask_fiducial(fiducials, image): 292 | # fiducials: 2x68 293 | border_fid = fiducials[:, 0:17] 294 | face_fid = fiducials[:, 18:] 295 | 296 | c1 = np.array([[border_fid[0, 0]], [face_fid[1, 2]]]) 297 | c2 = np.array([[border_fid[0, 16]], [face_fid[1, 7]]]) 298 | 299 | eye = np.linalg.norm(face_fid[:, 22] - face_fid[:, 25]) 300 | c3, c4 = face_fid[:, 2], face_fid[:, 7] 301 | c3[1] = c3[1] - 0.3 * eye 302 | c4[1] = c4[1] - 0.3 * eye 303 | 304 | border = np.column_stack([c1, border_fid, c2, c4, c3]) 305 | 306 | h, w = image.shape[0:2] 307 | 308 | X, Y = np.meshgrid(np.arange(w), np.arange(h)) 309 | 310 | _in, _on = inpolygon(X.reshape(-1), Y.reshape(-1), border[0, :], border[1, :]) 311 | 312 | mask = np.round(np.reshape(_in + _on, (h, w))) 313 | return (mask * 255).astype(np.uint8) 314 | 315 | 316 | def save_obj(v,c,f,save_path): 317 | folder = os.path.split(save_path)[0] 318 | if not os.path.exists(folder): 319 | os.makedirs(folder) 320 | with open(save_path, 'w') as file: 321 | for i in range(len(c)): 322 | file.write('v %f %f %f %f %f %f\n' % (v[i, 0], v[i, 1], v[i, 2], c[i, 0], c[i, 1], c[i, 2])) 323 | 324 | file.write('\n') 325 | 326 | for i in range(len(f)): 327 | file.write('f %d %d %d\n' % (f[i, 0], f[i, 1], f[i, 2])) 328 | file.close() -------------------------------------------------------------------------------- /VideoFace3D/segmentation/faceparsing/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | 4 | 5 | import torch 6 | import torch.nn as nn 7 | import torch.nn.functional as F 8 | import torchvision 9 | 10 | from VideoFace3D.segmentation.faceparsing.resnet import Resnet18 11 | # from modules.bn import InPlaceABNSync as BatchNorm2d 12 | 13 | 14 | class ConvBNReLU(nn.Module): 15 | def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1, *args, **kwargs): 16 | super(ConvBNReLU, self).__init__() 17 | self.conv = nn.Conv2d(in_chan, 18 | out_chan, 19 | kernel_size = ks, 20 | stride = stride, 21 | padding = padding, 22 | bias = False) 23 | self.bn = nn.BatchNorm2d(out_chan) 24 | self.init_weight() 25 | 26 | def forward(self, x): 27 | x = self.conv(x) 28 | x = F.relu(self.bn(x)) 29 | return x 30 | 31 | def init_weight(self): 32 | for ly in self.children(): 33 | if isinstance(ly, nn.Conv2d): 34 | nn.init.kaiming_normal_(ly.weight, a=1) 35 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 36 | 37 | class BiSeNetOutput(nn.Module): 38 | def __init__(self, in_chan, mid_chan, n_classes, *args, **kwargs): 39 | super(BiSeNetOutput, self).__init__() 40 | self.conv = ConvBNReLU(in_chan, mid_chan, ks=3, stride=1, padding=1) 41 | self.conv_out = nn.Conv2d(mid_chan, n_classes, kernel_size=1, bias=False) 42 | self.init_weight() 43 | 44 | def forward(self, x): 45 | x = self.conv(x) 46 | x = self.conv_out(x) 47 | return x 48 | 49 | def init_weight(self): 50 | for ly in self.children(): 51 | if isinstance(ly, nn.Conv2d): 52 | nn.init.kaiming_normal_(ly.weight, a=1) 53 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 54 | 55 | def get_params(self): 56 | wd_params, nowd_params = [], [] 57 | for name, module in self.named_modules(): 58 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 59 | wd_params.append(module.weight) 60 | if not module.bias is None: 61 | nowd_params.append(module.bias) 62 | elif isinstance(module, nn.BatchNorm2d): 63 | nowd_params += list(module.parameters()) 64 | return wd_params, nowd_params 65 | 66 | 67 | class AttentionRefinementModule(nn.Module): 68 | def __init__(self, in_chan, out_chan, *args, **kwargs): 69 | super(AttentionRefinementModule, self).__init__() 70 | self.conv = ConvBNReLU(in_chan, out_chan, ks=3, stride=1, padding=1) 71 | self.conv_atten = nn.Conv2d(out_chan, out_chan, kernel_size= 1, bias=False) 72 | self.bn_atten = nn.BatchNorm2d(out_chan) 73 | self.sigmoid_atten = nn.Sigmoid() 74 | self.init_weight() 75 | 76 | def forward(self, x): 77 | feat = self.conv(x) 78 | atten = F.avg_pool2d(feat, feat.size()[2:]) 79 | atten = self.conv_atten(atten) 80 | atten = self.bn_atten(atten) 81 | atten = self.sigmoid_atten(atten) 82 | out = torch.mul(feat, atten) 83 | return out 84 | 85 | def init_weight(self): 86 | for ly in self.children(): 87 | if isinstance(ly, nn.Conv2d): 88 | nn.init.kaiming_normal_(ly.weight, a=1) 89 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 90 | 91 | 92 | class ContextPath(nn.Module): 93 | def __init__(self, *args, **kwargs): 94 | super(ContextPath, self).__init__() 95 | self.resnet = Resnet18() 96 | self.arm16 = AttentionRefinementModule(256, 128) 97 | self.arm32 = AttentionRefinementModule(512, 128) 98 | self.conv_head32 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1) 99 | self.conv_head16 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1) 100 | self.conv_avg = ConvBNReLU(512, 128, ks=1, stride=1, padding=0) 101 | 102 | self.init_weight() 103 | 104 | def forward(self, x): 105 | H0, W0 = x.size()[2:] 106 | feat8, feat16, feat32 = self.resnet(x) 107 | H8, W8 = feat8.size()[2:] 108 | H16, W16 = feat16.size()[2:] 109 | H32, W32 = feat32.size()[2:] 110 | 111 | avg = F.avg_pool2d(feat32, feat32.size()[2:]) 112 | avg = self.conv_avg(avg) 113 | avg_up = F.interpolate(avg, (H32, W32), mode='nearest') 114 | 115 | feat32_arm = self.arm32(feat32) 116 | feat32_sum = feat32_arm + avg_up 117 | feat32_up = F.interpolate(feat32_sum, (H16, W16), mode='nearest') 118 | feat32_up = self.conv_head32(feat32_up) 119 | 120 | feat16_arm = self.arm16(feat16) 121 | feat16_sum = feat16_arm + feat32_up 122 | feat16_up = F.interpolate(feat16_sum, (H8, W8), mode='nearest') 123 | feat16_up = self.conv_head16(feat16_up) 124 | 125 | return feat8, feat16_up, feat32_up # x8, x8, x16 126 | 127 | def init_weight(self): 128 | for ly in self.children(): 129 | if isinstance(ly, nn.Conv2d): 130 | nn.init.kaiming_normal_(ly.weight, a=1) 131 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 132 | 133 | def get_params(self): 134 | wd_params, nowd_params = [], [] 135 | for name, module in self.named_modules(): 136 | if isinstance(module, (nn.Linear, nn.Conv2d)): 137 | wd_params.append(module.weight) 138 | if not module.bias is None: 139 | nowd_params.append(module.bias) 140 | elif isinstance(module, nn.BatchNorm2d): 141 | nowd_params += list(module.parameters()) 142 | return wd_params, nowd_params 143 | 144 | 145 | ### This is not used, since I replace this with the resnet feature with the same size 146 | class SpatialPath(nn.Module): 147 | def __init__(self, *args, **kwargs): 148 | super(SpatialPath, self).__init__() 149 | self.conv1 = ConvBNReLU(3, 64, ks=7, stride=2, padding=3) 150 | self.conv2 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1) 151 | self.conv3 = ConvBNReLU(64, 64, ks=3, stride=2, padding=1) 152 | self.conv_out = ConvBNReLU(64, 128, ks=1, stride=1, padding=0) 153 | self.init_weight() 154 | 155 | def forward(self, x): 156 | feat = self.conv1(x) 157 | feat = self.conv2(feat) 158 | feat = self.conv3(feat) 159 | feat = self.conv_out(feat) 160 | return feat 161 | 162 | def init_weight(self): 163 | for ly in self.children(): 164 | if isinstance(ly, nn.Conv2d): 165 | nn.init.kaiming_normal_(ly.weight, a=1) 166 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 167 | 168 | def get_params(self): 169 | wd_params, nowd_params = [], [] 170 | for name, module in self.named_modules(): 171 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 172 | wd_params.append(module.weight) 173 | if not module.bias is None: 174 | nowd_params.append(module.bias) 175 | elif isinstance(module, nn.BatchNorm2d): 176 | nowd_params += list(module.parameters()) 177 | return wd_params, nowd_params 178 | 179 | 180 | class FeatureFusionModule(nn.Module): 181 | def __init__(self, in_chan, out_chan, *args, **kwargs): 182 | super(FeatureFusionModule, self).__init__() 183 | self.convblk = ConvBNReLU(in_chan, out_chan, ks=1, stride=1, padding=0) 184 | self.conv1 = nn.Conv2d(out_chan, 185 | out_chan//4, 186 | kernel_size = 1, 187 | stride = 1, 188 | padding = 0, 189 | bias = False) 190 | self.conv2 = nn.Conv2d(out_chan//4, 191 | out_chan, 192 | kernel_size = 1, 193 | stride = 1, 194 | padding = 0, 195 | bias = False) 196 | self.relu = nn.ReLU(inplace=True) 197 | self.sigmoid = nn.Sigmoid() 198 | self.init_weight() 199 | 200 | def forward(self, fsp, fcp): 201 | fcat = torch.cat([fsp, fcp], dim=1) 202 | feat = self.convblk(fcat) 203 | atten = F.avg_pool2d(feat, feat.size()[2:]) 204 | atten = self.conv1(atten) 205 | atten = self.relu(atten) 206 | atten = self.conv2(atten) 207 | atten = self.sigmoid(atten) 208 | feat_atten = torch.mul(feat, atten) 209 | feat_out = feat_atten + feat 210 | return feat_out 211 | 212 | def init_weight(self): 213 | for ly in self.children(): 214 | if isinstance(ly, nn.Conv2d): 215 | nn.init.kaiming_normal_(ly.weight, a=1) 216 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 217 | 218 | def get_params(self): 219 | wd_params, nowd_params = [], [] 220 | for name, module in self.named_modules(): 221 | if isinstance(module, nn.Linear) or isinstance(module, nn.Conv2d): 222 | wd_params.append(module.weight) 223 | if not module.bias is None: 224 | nowd_params.append(module.bias) 225 | elif isinstance(module, nn.BatchNorm2d): 226 | nowd_params += list(module.parameters()) 227 | return wd_params, nowd_params 228 | 229 | 230 | class BiSeNet(nn.Module): 231 | def __init__(self, n_classes, *args, **kwargs): 232 | super(BiSeNet, self).__init__() 233 | self.cp = ContextPath() 234 | ## here self.sp is deleted 235 | self.ffm = FeatureFusionModule(256, 256) 236 | self.conv_out = BiSeNetOutput(256, 256, n_classes) 237 | self.conv_out16 = BiSeNetOutput(128, 64, n_classes) 238 | self.conv_out32 = BiSeNetOutput(128, 64, n_classes) 239 | self.init_weight() 240 | 241 | def forward(self, x): 242 | H, W = x.size()[2:] 243 | feat_res8, feat_cp8, feat_cp16 = self.cp(x) # here return res3b1 feature 244 | feat_sp = feat_res8 # use res3b1 feature to replace spatial path feature 245 | feat_fuse = self.ffm(feat_sp, feat_cp8) 246 | 247 | feat_out = self.conv_out(feat_fuse) 248 | feat_out16 = self.conv_out16(feat_cp8) 249 | feat_out32 = self.conv_out32(feat_cp16) 250 | 251 | feat_out = F.interpolate(feat_out, (H, W), mode='bilinear', align_corners=True) 252 | feat_out16 = F.interpolate(feat_out16, (H, W), mode='bilinear', align_corners=True) 253 | feat_out32 = F.interpolate(feat_out32, (H, W), mode='bilinear', align_corners=True) 254 | return feat_out, feat_out16, feat_out32 255 | 256 | def init_weight(self): 257 | for ly in self.children(): 258 | if isinstance(ly, nn.Conv2d): 259 | nn.init.kaiming_normal_(ly.weight, a=1) 260 | if not ly.bias is None: nn.init.constant_(ly.bias, 0) 261 | 262 | def get_params(self): 263 | wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params = [], [], [], [] 264 | for name, child in self.named_children(): 265 | child_wd_params, child_nowd_params = child.get_params() 266 | if isinstance(child, FeatureFusionModule) or isinstance(child, BiSeNetOutput): 267 | lr_mul_wd_params += child_wd_params 268 | lr_mul_nowd_params += child_nowd_params 269 | else: 270 | wd_params += child_wd_params 271 | nowd_params += child_nowd_params 272 | return wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params 273 | 274 | 275 | if __name__ == "__main__": 276 | net = BiSeNet(19) 277 | net.cuda() 278 | net.eval() 279 | in_ten = torch.randn(16, 3, 640, 480).cuda() 280 | out, out16, out32 = net(in_ten) 281 | print(out.shape) 282 | 283 | net.get_params() 284 | -------------------------------------------------------------------------------- /VideoFace3D/SFS/SFSNet/mask.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | from __future__ import absolute_import, division, print_function 3 | import dlib 4 | import cv2 5 | import numpy as np 6 | import os 7 | import sys 8 | from matplotlib.path import Path 9 | 10 | 11 | class MaskGenerator: 12 | def __init__(self, landmarks_path): 13 | """ 14 | :param landmarks_path: the path of pretrained key points weight, 15 | it could be download from: 16 | http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2 17 | """ 18 | if not os.path.exists(landmarks_path): 19 | raise RuntimeError('face landmark file is not exist. please download if from: \n' 20 | 'http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2 ' 21 | 'and uncompress it.') 22 | self._detector = dlib.get_frontal_face_detector() 23 | self._predictor = dlib.shape_predictor(landmarks_path) 24 | 25 | def bounding_boxes(self, image): 26 | # convert to gray image 27 | gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 28 | # get rect contains face 29 | face_rects = self._detector(gray_image, 0) 30 | return face_rects 31 | 32 | def align(self, image, size=(240, 240), scale=1.8, warp=True, crop=True, resize=True, 33 | crop_function_version=0, align_multi=False, draw_landmarks=False): 34 | """ 35 | warp and crop image 36 | https://blog.csdn.net/qq_39438636/article/details/79304130 37 | 38 | :param image: a BGR format face image 39 | :type image: np.ndarray 40 | :param size: target size 41 | :param scale: 42 | :param warp: warp or not 43 | :param crop: crop or not 44 | :param resize: resize od not 45 | :param crop_function_version: crop function version 46 | :param align_multi: whther to detect multi face 47 | :param draw_landmarks: whether draw face landmarks 48 | :return: mask, image and whether successfully crop image 49 | """ 50 | # check option 51 | if crop_function_version == 1 and align_multi: 52 | raise RuntimeError("When align_multi is true, crop_function_version must be 0") 53 | # if image is too big, resize to a smaller image 54 | if np.min(image.shape[0:2]) > 1000: 55 | ratio = 1000 / np.min(image.shape[0:2]) 56 | image = cv2.resize(image, dsize=(0, 0), fx=ratio, fy=ratio) 57 | # make border for image 58 | border = int(np.min(image.shape[0:2]) * 0.3) 59 | image = cv2.copyMakeBorder(image, border, border, border, border, cv2.BORDER_CONSTANT) 60 | # backup image 61 | original_image = image.copy() 62 | # get rectangles which contains face 63 | face_rects = self.bounding_boxes(image) 64 | results = [] 65 | if len(face_rects) > 0: 66 | for i in range(len(face_rects)): 67 | # get 68 landmarks of face 68 | landmarks = np.array([[p.x, p.y] for p in self._predictor(original_image, face_rects[i]).parts()]) 69 | # draw landmarks 70 | if draw_landmarks: 71 | landmark_image = self.draw_landmarks(original_image, landmarks) 72 | # remove border 73 | _row, _col, _ = landmark_image.shape 74 | landmark_image = landmark_image[border:_row-border, border:_col-border, :] 75 | else: 76 | landmark_image = None 77 | # create mask using landmarks 78 | mask = create_mask_by_landmarks(landmarks.T, original_image) 79 | if warp: 80 | image, mask, r_mat = self._warp(original_image, mask, landmarks) 81 | landmarks = self._get_rotated_points(landmarks, r_mat) 82 | if crop: 83 | if crop_function_version == 0: 84 | image = self._crop_v0(image, landmarks, scale) 85 | mask = self._crop_v0(mask, landmarks, scale) 86 | elif crop_function_version == 1: 87 | image, mask, suc_ = self._crop_v1(image, mask, scale) 88 | if not suc_: 89 | sys.stderr.write('%s: Failed to crop image and mask\n' % __file__) 90 | else: 91 | raise RuntimeError("crop_function_version must be 0 or 1") 92 | 93 | if resize: 94 | results.append((True, cv2.resize(mask, size), cv2.resize(image, size), landmark_image)) 95 | else: 96 | results.append((True, mask, image, landmark_image)) 97 | 98 | if not align_multi: 99 | return results 100 | return results 101 | else: 102 | sys.stderr.write("%s: Can't detect face in image\n" % __file__) 103 | image = cv2.resize(image, size) 104 | return [(False, np.ones(image.shape, dtype=image.dtype) * 255, image, None)] 105 | 106 | @staticmethod 107 | def _get_rotated_points(points, rotate_mat): 108 | # Blog； https://www.cnblogs.com/zhoug2020/p/7842808.html 109 | # add 1 to every point 110 | __padding = np.ones((points.shape[0], 1), dtype=points.dtype) 111 | points = np.concatenate([points, __padding], axis=1) 112 | # add [0, 0, 1] to rotate matrix 113 | __padding = np.array([0, 0, 1], dtype=points.dtype).reshape(1, 3) 114 | rotate_mat = np.concatenate([rotate_mat, __padding], axis=0) 115 | # compute rotated landmarks 116 | rotate_landmarks = np.matmul(rotate_mat, points.T) 117 | # remove the padding and transpose landmarks 118 | rotate_landmarks = rotate_landmarks[0:2, :].T 119 | # return landmark as integer numpy array 120 | return rotate_landmarks.astype(points.dtype) 121 | 122 | @staticmethod 123 | def _warp(image, mask, landmarks): 124 | """ 125 | warp image and mask by landmarks 126 | :param image: 127 | :type image np.ndarray 128 | :param landmarks: 129 | :type landmarks np.ndarray 130 | :return: warped face and mask 131 | """ 132 | # landmarks.shape = (68, 2) 133 | landmarks = np.array(landmarks) 134 | # compute rotate angle, r_angle=arctan((y1-y2)/(x1-x2)) 135 | # landmarks[36]: corner of left eye 136 | # landmarks[42]: corner of right eye 137 | r_angle = np.arctan((landmarks[36][1] - landmarks[42][1]) / 138 | (landmarks[36][0] - landmarks[42][0])) 139 | r_angle = 180 * r_angle / np.pi 140 | # get rotation matrix 141 | rot_mat = cv2.getRotationMatrix2D(tuple(landmarks[2]), r_angle, scale=1) 142 | 143 | # rotate image and mask 144 | rotated_image = cv2.warpAffine(image, rot_mat, dsize=image.shape[0:2]) 145 | rotated_mask = cv2.warpAffine(mask, rot_mat, dsize=image.shape[0:2]) 146 | 147 | return rotated_image, rotated_mask, rot_mat 148 | 149 | def _crop_v0(self, image, landmarks, scale): 150 | """ 151 | crop image by face landmarks 152 | :param image: 153 | :param landmarks: 154 | :param scale: 155 | :return: 156 | """ 157 | # left eye: landmarks[36] 158 | # left mouth: landmarks[48] 159 | # nose: landmarks[29] 160 | # find the most left point and most right point 161 | landmarks_x = landmarks[:, 0] 162 | most_left_x = np.min(landmarks_x) 163 | most_right_x = np.max(landmarks_x) 164 | mid_x = (most_left_x + most_right_x) // 2 165 | # print(most_left_x, most_right_x, mid_x) 166 | # define new center point use mid_x and y from nose point 167 | center_point = [mid_x, landmarks[29][1]] 168 | # compute the distance between left eye(landmarks[36]) 169 | distance = most_right_x - mid_x 170 | size = distance * scale 171 | # print(center_point) 172 | # compute row_start, row_end, col_start, col_end 173 | row_start = int(center_point[1] - size) 174 | row_end = int(center_point[1] + size) 175 | col_start = int(center_point[0] - size) 176 | col_end = int(center_point[0] + size) 177 | # print('*' * 10) 178 | # print(row_start, row_end, col_start, col_end) 179 | # make range valid and compute padding 180 | if row_start < 0: 181 | padding_up = abs(row_start) 182 | row_start = 0 183 | else: 184 | padding_up = 0 185 | if col_start < 0: 186 | padding_left = abs(col_start) 187 | col_start = 0 188 | else: 189 | padding_left = 0 190 | if row_end > (image.shape[0] - 1): 191 | padding_down = row_end - (image.shape[0] - 1) 192 | row_end = image.shape[0] - 1 193 | else: 194 | padding_down = 0 195 | if col_end > (image.shape[1] - 1): 196 | padding_right = col_end - (image.shape[1] - 1) 197 | col_end = image.shape[1] - 1 198 | else: 199 | padding_right = 0 200 | # print(row_start, row_end, col_start, col_end) 201 | # print('*' * 10) 202 | # crop image 203 | cropped_image = self._crop_helper(image, row_start, row_end, col_start, col_end, 204 | padding_up, padding_down, padding_left, padding_right) 205 | return cropped_image 206 | 207 | def _crop_v1(self, image, mask, scale): 208 | face_rects = self.bounding_boxes(image) 209 | if len(face_rects) == 0: 210 | return image, mask, False 211 | # define crop size 212 | size = (face_rects[0].right() - face_rects[0].left()) / 2 213 | size *= scale 214 | # define new center point use mid_x and y from nose point 215 | _x = (face_rects[0].left() + face_rects[0].right()) // 2 216 | _y = (face_rects[0].top() + face_rects[0].bottom()) // 2 217 | center_point = [_x, _y] 218 | # compute the distance between left eye(landmarks[36]) 219 | # print(center_point) 220 | # compute row_start, row_end, col_start, col_end 221 | row_start = int(center_point[1] - size) 222 | row_end = int(center_point[1] + size) 223 | col_start = int(center_point[0] - size) 224 | col_end = int(center_point[0] + size) 225 | # print('*' * 10) 226 | # print(row_start, row_end, col_start, col_end) 227 | # make range valid and compute padding 228 | if row_start < 0: 229 | padding_up = abs(row_start) 230 | row_start = 0 231 | else: 232 | padding_up = 0 233 | if col_start < 0: 234 | padding_left = abs(col_start) 235 | col_start = 0 236 | else: 237 | padding_left = 0 238 | if row_end > (image.shape[0] - 1): 239 | padding_down = row_end - (image.shape[0] - 1) 240 | row_end = image.shape[0] - 1 241 | else: 242 | padding_down = 0 243 | if col_end > (image.shape[1] - 1): 244 | padding_right = col_end - (image.shape[1] - 1) 245 | col_end = image.shape[1] - 1 246 | else: 247 | padding_right = 0 248 | # print(row_start, row_end, col_start, col_end) 249 | # print('*' * 10) 250 | # crop image 251 | image = self._crop_helper(image, row_start, row_end, col_start, col_end, 252 | padding_up, padding_down, padding_left, padding_right) 253 | mask = self._crop_helper(mask, row_start, row_end, col_start, col_end, 254 | padding_up, padding_down, padding_left, padding_right) 255 | return image, mask, True 256 | 257 | @staticmethod 258 | def _crop_helper(image, row_start, row_end, col_start, col_end, 259 | padding_up, padding_down, padding_left, padding_right): 260 | cropped_image = image[row_start:row_end, col_start:col_end] 261 | 262 | # add padding to image 263 | rows, cols, _ = cropped_image.shape 264 | if padding_up > 0: 265 | padding = np.zeros(shape=(padding_up, cols, 3), dtype=cropped_image.dtype) 266 | cropped_image = np.vstack((padding, cropped_image)) 267 | if padding_down > 0: 268 | padding = np.zeros(shape=(padding_down, cols, 3), dtype=cropped_image.dtype) 269 | cropped_image = np.vstack((cropped_image, padding)) 270 | rows, cols, _ = cropped_image.shape 271 | if padding_left > 0: 272 | padding = np.zeros(shape=(rows, padding_left, 3), dtype=cropped_image.dtype) 273 | cropped_image = np.hstack((padding, cropped_image)) 274 | if padding_right > 0: 275 | padding = np.zeros(shape=(rows, padding_right, 3), dtype=cropped_image.dtype) 276 | cropped_image = np.hstack((cropped_image, padding)) 277 | return cropped_image 278 | 279 | @staticmethod 280 | def draw_landmarks(image, landmarks): 281 | landmark_im = image.copy() 282 | for i, landmark in enumerate(landmarks): 283 | cv2.circle(landmark_im, tuple(landmark), 3, (0, 0, 255)) 284 | cv2.putText(landmark_im, str(i), tuple(landmark), cv2.FONT_HERSHEY_SIMPLEX, 285 | 0.3, (0, 255, 0)) 286 | return landmark_im 287 | 288 | 289 | def create_mask_by_landmarks(landmarks, Image): 290 | """ 291 | create mask use fiducials of Image 292 | :param landmarks: the 68 landmarks detected using dlib 293 | :type landmarks np.ndarray 294 | :param Image: a 3-channel image 295 | :type Image np.ndarray 296 | :return: 297 | """ 298 | # fiducals is 2x68 299 | landmarks = np.float32(landmarks) 300 | border_fid = landmarks[:, 0:17] 301 | face_fid = landmarks[:, 17:] 302 | 303 | c1 = np.array([border_fid[0, 0], face_fid[1, 2]]) # left 304 | c2 = np.array([border_fid[0, 16], face_fid[1, 7]]) # right 305 | eye = np.linalg.norm(face_fid[:, 22] - face_fid[:, 25]) 306 | c3 = face_fid[:, 2] 307 | c3[1] = c3[1] - 0.3 * eye 308 | c4 = face_fid[:, 7] 309 | c4[1] = c4[1] - 0.3 * eye 310 | 311 | border = [c1, border_fid, c2, c4, c3] 312 | border = [item.reshape(2, -1) for item in border] 313 | border = np.hstack(border) 314 | 315 | M = Image.shape[0] # row -> y 316 | N = Image.shape[1] # col -> x 317 | 318 | y = np.arange(0, M, step=1, dtype=np.float32) 319 | x = np.arange(0, N, step=1, dtype=np.float32) 320 | X, Y = np.meshgrid(x, y) 321 | 322 | _in, _on = inpolygon(X, Y, border[0, :].T, border[1, :].T) 323 | 324 | mask = np.round(np.reshape(_in | _on, [M, N])) 325 | mask = 255 * np.uint8(mask) 326 | mask = np.repeat(np.expand_dims(mask, -1), 3, axis=-1) 327 | return mask 328 | 329 | 330 | def inpolygon(xq, yq, xv, yv): 331 | """ 332 | reimplement inpolygon in matlab 333 | :type xq: np.ndarray 334 | :type yq: np.ndarray 335 | :type xv: np.ndarray 336 | :type yv: np.ndarray 337 | """ 338 | # http://blog.sina.com.cn/s/blog_70012f010102xnel.html 339 | # merge xy and yv into vertices 340 | vertices = np.vstack((xv, yv)).T 341 | # define a Path object 342 | path = Path(vertices) 343 | # merge X and Y into test_points 344 | test_points = np.hstack([xq.reshape(xq.size, -1), yq.reshape(yq.size, -1)]) 345 | # get mask of test_points in path 346 | _in = path.contains_points(test_points) 347 | # get mask of test_points in path(include the points on path) 348 | _in_on = path.contains_points(test_points, radius=-1e-10) 349 | # get the points on path 350 | _on = _in ^ _in_on 351 | return _in_on, _on 352 | -------------------------------------------------------------------------------- /VideoFace3D/renderer/render.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | import math 3 | 4 | import torch 5 | import torch.nn as nn 6 | import numpy 7 | 8 | import neural_renderer as nr 9 | from VideoFace3D.renderer.weak_projection import weak_projection 10 | from VideoFace3D.utils.geometry import texture_from_point2faces, euler2rot, compute_face_norm, compute_point_norm 11 | from VideoFace3D.renderer.lightning import * 12 | from VideoFace3D.models.face_model import FaceModelBFM 13 | 14 | class Renderer(nn.Module): 15 | def __init__(self, image_size=256, anti_aliasing=True, background_color=[0, 0, 0], 16 | fill_back=True, camera_mode='weak_projection', 17 | K=None, R=None, t=None, dist_coeffs=None, orig_size=1024, 18 | perspective=True, viewing_angle=30, camera_direction=[0, 0, 1], 19 | near=0.1, far=100,light_mode="parallel",SH_Coeff=None, 20 | light_intensity_ambient=0.5, light_intensity_directional=0.5, 21 | light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1], 22 | light_direction=[0, 1, 0]): 23 | super(Renderer, self).__init__() 24 | # rendering 25 | self.image_size = image_size 26 | self.anti_aliasing = anti_aliasing 27 | self.background_color = background_color 28 | self.fill_back = fill_back 29 | 30 | self.light_mode= light_mode 31 | self.SH_coeff = SH_Coeff 32 | self.facemodel = FaceModelBFM() 33 | # camera 34 | self.camera_mode = camera_mode 35 | if self.camera_mode in ['projection', 'weak_projection']: 36 | self.K = K 37 | self.R = R 38 | self.t = t 39 | if isinstance(self.K, numpy.ndarray): 40 | self.K = torch.cuda.FloatTensor(self.K) 41 | if isinstance(self.R, numpy.ndarray): 42 | self.R = torch.cuda.FloatTensor(self.R) 43 | if isinstance(self.t, numpy.ndarray): 44 | self.t = torch.cuda.FloatTensor(self.t) 45 | self.dist_coeffs = dist_coeffs 46 | if dist_coeffs is None: 47 | self.dist_coeffs = torch.cuda.FloatTensor([[0., 0., 0., 0., 0.]]) 48 | self.orig_size = orig_size 49 | elif self.camera_mode in ['look', 'look_at']: 50 | self.perspective = perspective 51 | self.viewing_angle = viewing_angle 52 | self.eye = [0, 0, -(1. / math.tan(math.radians(self.viewing_angle)) + 1)] 53 | self.camera_direction = [0, 0, 1] 54 | else: 55 | raise ValueError('Camera mode has to be one of projection, look or look_at') 56 | 57 | self.near = near 58 | self.far = far 59 | 60 | # light 61 | self.light_intensity_ambient = light_intensity_ambient 62 | self.light_intensity_directional = light_intensity_directional 63 | self.light_color_ambient = light_color_ambient 64 | self.light_color_directional = light_color_directional 65 | self.light_direction = light_direction 66 | 67 | # rasterization 68 | self.rasterizer_eps = 1e-3 69 | 70 | def forward(self, vertices, faces=None, textures=None, mode=None, K=None, R=None, t=None, dist_coeffs=None, 71 | orig_size=224): 72 | ''' 73 | Implementation of forward rendering method 74 | The old API is preserved for back-compatibility with the Chainer implementation 75 | ''' 76 | 77 | if mode is None: 78 | return self.render(vertices, faces, textures, K, R, t, dist_coeffs, orig_size) 79 | elif mode is 'rgb': 80 | return self.render_rgb(vertices, faces, textures, K, R, t, dist_coeffs, orig_size) 81 | elif mode == 'silhouettes': 82 | return self.render_silhouettes(vertices, faces, K, R, t, dist_coeffs, orig_size) 83 | elif mode == 'depth': 84 | return self.render_depth(vertices, faces, K, R, t, dist_coeffs, orig_size) 85 | elif mode == 'points': 86 | return self.project_points(vertices, K, R, t, dist_coeffs, orig_size) 87 | else: 88 | raise ValueError("mode should be one of None, 'silhouettes' or 'depth'") 89 | 90 | def render_silhouettes(self, vertices, faces, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): 91 | 92 | # fill back 93 | if self.fill_back: 94 | faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1) 95 | 96 | # viewpoint transformation 97 | if self.camera_mode == 'look_at': 98 | vertices = nr.look_at(vertices, self.eye) 99 | # perspective transformation 100 | if self.perspective: 101 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 102 | elif self.camera_mode == 'look': 103 | vertices = nr.look(vertices, self.eye, self.camera_direction) 104 | # perspective transformation 105 | if self.perspective: 106 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 107 | elif self.camera_mode == 'projection': 108 | if K is None: 109 | K = self.K 110 | if R is None: 111 | R = self.R 112 | if t is None: 113 | t = self.t 114 | if dist_coeffs is None: 115 | dist_coeffs = self.dist_coeffs 116 | if orig_size is None: 117 | orig_size = self.orig_size 118 | vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) 119 | elif self.camera_mode == 'weak_projection': 120 | if K is None: 121 | K = self.K 122 | if R is None: 123 | R = self.R 124 | if t is None: 125 | t = self.t 126 | 127 | vertices = weak_projection(vertices, K, R, t, orig_size) 128 | 129 | # rasterization 130 | faces = nr.vertices_to_faces(vertices, faces) 131 | images = nr.rasterize_silhouettes(faces, self.image_size, self.anti_aliasing) 132 | return images 133 | 134 | def render_depth(self, vertices, faces, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): 135 | 136 | # fill back 137 | if self.fill_back: 138 | faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() 139 | 140 | # viewpoint transformation 141 | if self.camera_mode == 'look_at': 142 | vertices = nr.look_at(vertices, self.eye) 143 | # perspective transformation 144 | if self.perspective: 145 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 146 | elif self.camera_mode == 'look': 147 | vertices = nr.look(vertices, self.eye, self.camera_direction) 148 | # perspective transformation 149 | if self.perspective: 150 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 151 | elif self.camera_mode == 'projection': 152 | if K is None: 153 | K = self.K 154 | if R is None: 155 | R = self.R 156 | if t is None: 157 | t = self.t 158 | if dist_coeffs is None: 159 | dist_coeffs = self.dist_coeffs 160 | if orig_size is None: 161 | orig_size = self.orig_size 162 | vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) 163 | elif self.camera_mode == 'weak_projection': 164 | if K is None: 165 | K = self.K 166 | if R is None: 167 | R = self.R 168 | if t is None: 169 | t = self.t 170 | 171 | vertices = weak_projection(vertices, K, R, t, orig_size) 172 | # rasterization 173 | faces = nr.vertices_to_faces(vertices, faces) 174 | images = nr.rasterize_depth(faces, self.image_size, self.anti_aliasing) 175 | return images 176 | 177 | def render_rgb(self, vertices, faces, textures, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): 178 | # fill back 179 | if self.fill_back: 180 | faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() 181 | textures = torch.cat((textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) 182 | 183 | # lighting 184 | faces_lighting = nr.vertices_to_faces(vertices, faces) 185 | textures = nr.lighting( 186 | faces_lighting, 187 | textures, 188 | self.light_intensity_ambient, 189 | self.light_intensity_directional, 190 | self.light_color_ambient, 191 | self.light_color_directional, 192 | self.light_direction) 193 | 194 | # viewpoint transformation 195 | if self.camera_mode == 'look_at': 196 | vertices = nr.look_at(vertices, self.eye) 197 | # perspective transformation 198 | if self.perspective: 199 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 200 | elif self.camera_mode == 'look': 201 | vertices = nr.look(vertices, self.eye, self.camera_direction) 202 | # perspective transformation 203 | if self.perspective: 204 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 205 | elif self.camera_mode == 'projection': 206 | if K is None: 207 | K = self.K 208 | if R is None: 209 | R = self.R 210 | if t is None: 211 | t = self.t 212 | if dist_coeffs is None: 213 | dist_coeffs = self.dist_coeffs 214 | if orig_size is None: 215 | orig_size = self.orig_size 216 | vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) 217 | elif self.camera_mode == 'weak_projection': 218 | if K is None: 219 | K = self.K 220 | if R is None: 221 | R = self.R 222 | if t is None: 223 | t = self.t 224 | 225 | vertices = weak_projection(vertices, K, R, t, orig_size) 226 | 227 | # rasterization 228 | faces = nr.vertices_to_faces(vertices, faces) 229 | images = nr.rasterize( 230 | faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, 231 | self.background_color) 232 | return images 233 | 234 | def render(self, vertices, faces, textures, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): 235 | # fill back 236 | # viewpoint transformation 237 | if self.camera_mode == 'look_at': 238 | vertices = nr.look_at(vertices, self.eye) 239 | # perspective transformation 240 | if self.perspective: 241 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 242 | elif self.camera_mode == 'look': 243 | vertices = nr.look(vertices, self.eye, self.camera_direction) 244 | # perspective transformation 245 | if self.perspective: 246 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 247 | elif self.camera_mode == 'projection': 248 | if K is None: 249 | K = self.K 250 | if R is None: 251 | R = self.R 252 | if t is None: 253 | t = self.t 254 | if dist_coeffs is None: 255 | dist_coeffs = self.dist_coeffs 256 | if orig_size is None: 257 | orig_size = self.orig_size 258 | vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) 259 | elif self.camera_mode == 'weak_projection': 260 | if K is None: 261 | K = self.K 262 | if R is None: 263 | R = self.R 264 | if t is None: 265 | t = self.t 266 | 267 | vertices = weak_projection(vertices, K, R, t, orig_size) 268 | # 正则化一下，让脸颊到鼻尖的向量和z轴负方向夹角小于90度 (a,b,c)*(0,0,-1)>0 -> c<0 269 | # 另外将z值全部归到正轴上去 270 | #from MorphableModelFitting.models.face_model import FaceModelBFM 271 | 272 | 273 | front_idx, back_idx_1, back_idx_2 = self.facemodel.keypoints[30], self.facemodel.keypoints[0], self.facemodel.keypoints[16] 274 | for i in range(len(vertices)): 275 | back_z = (vertices[i, back_idx_1, 2] + vertices[i, back_idx_2, 2]) / 2 276 | if (vertices[i, front_idx, 2] - back_z) > 0: 277 | vertices[i, :, 2] = -vertices[i, :, 2] 278 | 279 | vertices[:, :, 2] = vertices[:, :, 2] - torch.min(vertices[:, :, 2], dim=1)[0].unsqueeze(1) + 1 280 | 281 | # lighting 282 | if self.light_mode == "parallel": 283 | textures = texture_from_point2faces(faces, textures) / 255 284 | if self.fill_back: 285 | faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() 286 | textures = torch.cat((textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) 287 | faces_lighting = nr.vertices_to_faces(vertices, faces) 288 | textures = nr.lighting( 289 | faces_lighting, 290 | textures, 291 | self.light_intensity_ambient, 292 | self.light_intensity_directional, 293 | self.light_color_ambient, 294 | self.light_color_directional, 295 | self.light_direction) 296 | elif self.light_mode == "SH": 297 | point_buf = torch.from_numpy(self.facemodel.point_buf).long() - 1 298 | point_norm = compute_point_norm(vertices, faces[0], point_buf) 299 | 300 | # texture = texture_from_point2faces(triangles, texture).reshape((batch, -1, 3)) 301 | textures, _ = Illumination_SH(textures, point_norm, self.SH_coeff) 302 | textures = texture_from_point2faces(faces, textures) / 255 303 | if self.fill_back: 304 | faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() 305 | textures = torch.cat((textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) 306 | else: 307 | return None 308 | 309 | # rasterization 310 | faces = nr.vertices_to_faces(vertices, faces) 311 | out = nr.rasterize_rgbad( 312 | faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, 313 | self.background_color) 314 | return out['rgb'], out['depth'], out['alpha'] 315 | 316 | def project_points(self, vertices, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): 317 | if self.camera_mode == 'look_at': 318 | vertices = nr.look_at(vertices, self.eye) 319 | # perspective transformation 320 | if self.perspective: 321 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 322 | elif self.camera_mode == 'look': 323 | vertices = nr.look(vertices, self.eye, self.camera_direction) 324 | # perspective transformation 325 | if self.perspective: 326 | vertices = nr.perspective(vertices, angle=self.viewing_angle) 327 | elif self.camera_mode == 'projection': 328 | if K is None: 329 | K = self.K 330 | if R is None: 331 | R = self.R 332 | if t is None: 333 | t = self.t 334 | if dist_coeffs is None: 335 | dist_coeffs = self.dist_coeffs 336 | if orig_size is None: 337 | orig_size = self.orig_size 338 | vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) 339 | elif self.camera_mode == 'weak_projection': 340 | if K is None: 341 | K = self.K 342 | if R is None: 343 | R = self.R 344 | if t is None: 345 | t = self.t 346 | 347 | vertices = weak_projection(vertices, K, R, t, orig_size) 348 | 349 | vertices[:, :, 0] = (orig_size / 2) * vertices[:, :, 0] + (orig_size / 2) 350 | vertices[:, :, 1] = orig_size - ((orig_size / 2) * vertices[:, :, 1] + (orig_size / 2)) 351 | return vertices 352 | --------------------------------------------------------------------------------