├── pyaam ├── __init__.py ├── tracker.py ├── draw.py ├── combined.py ├── perturbator.py ├── texture.py ├── texturemapper.py ├── utils.py ├── detector.py ├── shape.py ├── muct.py └── patches.py ├── .gitignore ├── calc_regmat.py ├── webcam.py ├── README.md ├── view_face.py ├── view_triangles.py ├── view_data.py ├── view_perturbations.py ├── matmul.py ├── train_model.py ├── do_perts.py ├── test_aam.py └── view_model.py /pyaam/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.npz 3 | *.h5 4 | data/muct 5 | -------------------------------------------------------------------------------- /calc_regmat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import division 4 | 5 | import numpy as np 6 | import tables as tb 7 | import matmul 8 | 9 | print 'opening file ...' 10 | h5 = tb.openFile('/media/disk2/perturbations.h5', mode='r') 11 | print 'get G ...' 12 | G = h5.root.residuals 13 | print 'get P copy ...' 14 | P = h5.root.perturbations[:] 15 | print 'P pseudoinverse ...' 16 | P_pinv = np.linalg.pinv(P) 17 | print 'alloc J' 18 | rows = G.shape[0] 19 | cols = P_pinv.shape[1] 20 | J = np.zeros((rows, cols)) 21 | print 'J = G * P_pinv' 22 | matmul.dot(G, P_pinv, out=J) 23 | print 'J pseudoinverse ...' 24 | R = np.linalg.pinv(J) 25 | print 'writing file ...' 26 | np.savez('data/regmat.npz', R=R) 27 | -------------------------------------------------------------------------------- /webcam.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import cv2 8 | import cv2.cv as cv 9 | 10 | cascade = cv2.CascadeClassifier('data/haarcascades/haarcascade_frontalface_default.xml') 11 | 12 | cv2.namedWindow('webcam') 13 | cam = cv2.VideoCapture(0) 14 | 15 | if not cam.isOpened(): 16 | sys.exit('no camera') 17 | 18 | while True: 19 | rval, img = cam.read() 20 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 21 | gray = cv2.equalizeHist(gray) 22 | rects = cascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=4, 23 | minSize=(30,30), flags=cv.CV_HAAR_SCALE_IMAGE) 24 | 25 | for x,y,w,h in rects: 26 | cv2.rectangle(img, (x,y), (x+w,y+h), cv.RGB(0,0,255), 2) 27 | 28 | cv2.imshow('webcam', img) 29 | 30 | if cv2.waitKey(5) == 27: 31 | break 32 | -------------------------------------------------------------------------------- /pyaam/tracker.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | from pyaam.shape import ShapeModel 6 | from pyaam.patches import PatchesModel 7 | from pyaam.detector import FaceDetector 8 | 9 | class FaceTracker(object): 10 | def __init__(self): 11 | self.shape = ShapeModel.load('data/shape.npz') 12 | self.patches = PatchesModel.load('data/patches.npz') 13 | self.detector = FaceDetector.load('data/detector.npz') 14 | self.points = None 15 | self.ssize = ((21,21), (11,11), (5,5)) 16 | 17 | def reset(self): 18 | self.points = None 19 | 20 | def track(self, img): 21 | if self.points is None: 22 | self.points = self.detector.detect(img) 23 | for ssize in self.ssize: 24 | self.fit(img, ssize) 25 | 26 | def fit(self, img, ssize): 27 | p = self.shape.calc_params(self.points) 28 | pts = self.shape.calc_shape(p) 29 | peaks = self.patches.calc_peaks(img, pts, ssize) 30 | 31 | p = self.shape.calc_params(peaks) 32 | self.points = self.shape.calc_shape(p) 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pyaam - active appearance model 2 | 3 | active appearance models implemented in python 4 | 5 | ## Instructions 6 | 7 | Download MUCT dataset: 8 | 9 | python -m pyaam.muct 10 | 11 | View MUCT dataset: 12 | 13 | ./view_data.py 14 | 15 | Train models: 16 | 17 | ./train_model.py shape 18 | ./train_model.py patches 19 | ./train_model.py detector 20 | ./train_model.py texture 21 | ./train_model.py combined 22 | 23 | View models: 24 | 25 | ./view_model.py shape 26 | ./view_model.py patches 27 | ./view_model.py texture 28 | ./view_model.py combined 29 | 30 | View face detector on webcam: 31 | 32 | ./view_face.py detector 33 | 34 | View face tracker (patches): 35 | 36 | ./view_face.py tracker 37 | 38 | Face tracker using AAMs coming soon! 39 | 40 | ## References 41 | 42 | - J. Saragih, "Non-rigid Face Tracking". In Mastering OpenCV with Practical Computer Vision Projects. PACKT, Oct 2012. 43 | - M.B. Stegmann, "Active appearance models: Theory, extensions and cases". Master Thesis. 2nd edition. Informatics and Mathematical Modelling, Technical University of Denmark. Aug 2000. 44 | - P. Martins, "Active Appearance Models for Facial Expression Recognition and Monocular Head Pose Estimation". MSc Thesis. Department of Electrical and Computer Engineering, University of Coimbra. June 2008. 45 | -------------------------------------------------------------------------------- /view_face.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import cv2 8 | import argparse 9 | from pyaam.draw import draw_muct_shape 10 | from pyaam.tracker import FaceTracker 11 | from pyaam.detector import FaceDetector 12 | 13 | 14 | 15 | def parse_args(): 16 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 17 | parser.add_argument('model', choices=['tracker', 'detector'], help='model name') 18 | parser.add_argument('--detector', default='data/detector.npz', help='face detector filename') 19 | return parser.parse_args() 20 | 21 | 22 | 23 | def view_face_tracker(): 24 | tracker = FaceTracker() 25 | cam = cv2.VideoCapture(0) 26 | if not cam.isOpened(): 27 | sys.exit('no camera') 28 | while True: 29 | val, img = cam.read() 30 | tracker.track(img) 31 | draw_muct_shape(img, tracker.points) 32 | cv2.imshow('face tracker', img) 33 | key = cv2.waitKey(10) 34 | if key == 27: 35 | break 36 | elif key == ord('r'): 37 | tracker.reset() 38 | 39 | 40 | 41 | def view_face_detector(detector_fn): 42 | detector = FaceDetector.load(detector_fn) 43 | cam = cv2.VideoCapture(0) 44 | if not cam.isOpened(): 45 | sys.exit('no camera') 46 | while True: 47 | val, img = cam.read() 48 | p = detector.detect(img) 49 | draw_muct_shape(img, p) 50 | cv2.imshow('face detector', img) 51 | if cv2.waitKey(10) == 27: 52 | break 53 | 54 | 55 | 56 | if __name__ == '__main__': 57 | args = parse_args() 58 | 59 | if args.model == 'detector': 60 | view_face_detector(args.detector) 61 | 62 | elif args.model == 'tracker': 63 | view_face_tracker() 64 | -------------------------------------------------------------------------------- /view_triangles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import cv2 8 | import argparse 9 | from pyaam.muct import MuctDataset 10 | from pyaam.shape import ShapeModel 11 | from pyaam.draw import draw_polygons, Color 12 | from pyaam.utils import get_aabb, normalize, warp_triangles, get_vertices 13 | 14 | 15 | 16 | def parse_args(): 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument('--no-tm', dest='use_texturemapper', action='store_false', 19 | help='disable TextureMapper') 20 | parser.add_argument('--no-norm', dest='normalize', action='store_false', 21 | help='disable roi normalization') 22 | parser.add_argument('--no-trigs', dest='show_triangles', action='store_false', 23 | help='hide triangles') 24 | return parser.parse_args() 25 | 26 | 27 | 28 | if __name__ == '__main__': 29 | args = parse_args() 30 | 31 | cv2.namedWindow('triangles') 32 | cv2.namedWindow('warp') 33 | 34 | smodel = ShapeModel.load('data/shape.npz') 35 | params = smodel.get_params(200, 150, 150) 36 | ref = smodel.calc_shape(params) 37 | ref = ref.reshape((len(ref)//2, 2)) 38 | 39 | verts = get_vertices(ref) 40 | 41 | muct = MuctDataset() 42 | muct.load(clean=True) 43 | 44 | if args.use_texturemapper: 45 | from pyaam.texturemapper import TextureMapper 46 | tm = TextureMapper(480, 640) 47 | warp_triangles = tm.warp_triangles 48 | 49 | for name, tag, lmks, flipped in muct.iterdata(): 50 | img = muct.image(name) 51 | pts = lmks.reshape((len(lmks)//2, 2)) 52 | aabb = get_aabb(pts) 53 | if args.normalize: 54 | img = normalize(img, aabb) 55 | orig = img.copy() 56 | if args.show_triangles: 57 | draw_polygons(img, pts[verts], Color.blue) 58 | cv2.imshow('triangles', img) 59 | warped = warp_triangles(orig, pts[verts], ref[verts]) 60 | if args.show_triangles: 61 | draw_polygons(warped, ref[verts], Color.blue) 62 | cv2.imshow('warp', warped[:300,:300]) 63 | key = cv2.waitKey() 64 | if key == 27: 65 | sys.exit() 66 | -------------------------------------------------------------------------------- /view_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import cv2 8 | import numpy as np 9 | from pyaam.draw import Color, draw_string, draw_points, draw_line, draw_pairs 10 | from pyaam.muct import MuctDataset 11 | 12 | 13 | 14 | class LandmarkDrawer(object): 15 | def __init__(self, name, img, img_flip, lmks, lmks_flip): 16 | self.name = name 17 | self.name_flip = name[0] + 'r' + name[1:] # using muct naming scheme 18 | self.img = img 19 | self.img_flip = img_flip 20 | self.lmks = lmks 21 | self.lmks_flip = lmks_flip 22 | 23 | def draw(self, flip=False, points=False, line=False, pairs=False): 24 | name = self.name_flip if flip else self.name 25 | img = self.img_flip if flip else self.img 26 | lmks = self.lmks_flip if flip else self.lmks 27 | # draw on image copy 28 | img = img.copy() 29 | # prepare points 30 | pts = np.column_stack((lmks[::2], lmks[1::2])) 31 | # draw 32 | draw_string(img, name) 33 | if line: draw_line(img, pts, Color.blue) 34 | if pairs: draw_pairs(img, pts, MuctDataset.PAIRS, Color.red) 35 | if points: draw_points(img, pts, Color.green) 36 | return img 37 | 38 | 39 | 40 | if __name__ == '__main__': 41 | muct = MuctDataset() 42 | muct.load(clean=True) 43 | 44 | cv2.namedWindow('muct') 45 | flip = points = line = pairs = False 46 | 47 | for name, tag, lmks, flipped in muct.iterdata(): 48 | image = muct.image(name) 49 | image_flip = muct.image(name, flip=True) 50 | drawer = LandmarkDrawer(name, image, image_flip, lmks, flipped) 51 | 52 | while True: 53 | img = drawer.draw(flip, points, line, pairs) 54 | cv2.imshow('muct', img) 55 | # handle keyboard events 56 | key = cv2.waitKey() 57 | if key == 27: 58 | sys.exit() 59 | elif key == ord(' '): 60 | break # next image 61 | elif key == ord('1'): 62 | flip = not flip 63 | elif key == ord('2'): 64 | points = not points 65 | elif key == ord('3'): 66 | line = not line 67 | elif key == ord('4'): 68 | pairs = not pairs 69 | -------------------------------------------------------------------------------- /pyaam/draw.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import cv2 6 | import cv2.cv as cv 7 | import numpy as np 8 | from pyaam.muct import MuctDataset 9 | from pyaam.utils import get_mask, get_vertices 10 | 11 | 12 | 13 | class Color: 14 | black = cv.RGB(0, 0, 0) 15 | white = cv.RGB(255, 255, 255) 16 | red = cv.RGB(255, 0, 0) 17 | green = cv.RGB(0, 255, 0) 18 | blue = cv.RGB(0, 0, 255) 19 | cyan = cv.RGB(0, 255, 255) 20 | magenta = cv.RGB(255, 0, 255) 21 | yellow = cv.RGB(255, 255, 0) 22 | 23 | 24 | 25 | def prepare(x): 26 | return x.round().astype('int32') 27 | 28 | def draw_string(img, text, font=cv2.FONT_HERSHEY_COMPLEX, scale=0.6, thickness=1): 29 | size, baseLine = cv2.getTextSize(text, font, scale, thickness) 30 | cv2.putText(img, text, (0, size[1]), font, scale, Color.black, thickness, cv2.CV_AA) 31 | cv2.putText(img, text, (1, size[1]+1), font, scale, Color.white, thickness, cv2.CV_AA) 32 | 33 | def draw_points(img, points, color, radius=2): 34 | points = prepare(points) 35 | for p in points: 36 | cv2.circle(img, tuple(p), radius, color) 37 | 38 | def draw_line(img, points, color): 39 | points = prepare(points) 40 | cv2.polylines(img, [points], False, color) 41 | 42 | def draw_pairs(img, points, pairs, color): 43 | points = prepare(points) 44 | for a,b in pairs: 45 | cv2.line(img, tuple(points[a]), tuple(points[b]), color) 46 | 47 | def draw_polygons(img, polygons, color): 48 | polygons = prepare(polygons) 49 | for p in polygons: 50 | cv2.polylines(img, [p], True, color) 51 | 52 | def draw_muct_shape(img, points): 53 | # convert vector of points into matrix of size (n_pts, 2) 54 | pts = points.reshape((len(points)//2, 2)) 55 | draw_pairs(img, pts, MuctDataset.PAIRS, Color.red) 56 | draw_points(img, pts, Color.green) 57 | 58 | def draw_texture(img, texture, points): 59 | texture = texture.round().reshape((texture.size//3, 3)) 60 | mask = get_mask(points, img.shape[:2]) 61 | img[mask] = texture 62 | return img 63 | 64 | def draw_face(img, points, texture, ref_shape, warp_triangles): 65 | verts = get_vertices(ref_shape) 66 | img_texture = np.zeros(img.shape, dtype='uint8') 67 | draw_texture(img_texture, texture, ref_shape) 68 | return warp_triangles(img_texture, ref_shape[verts], points[verts], img) 69 | -------------------------------------------------------------------------------- /pyaam/combined.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import numpy as np 6 | from pyaam.shape import ShapeModel 7 | from pyaam.texture import TextureModel 8 | from pyaam.texturemapper import TextureMapper 9 | from pyaam.utils import pca 10 | 11 | class CombinedModel(object): 12 | def __init__(self, model, variance, weights): 13 | self.model = model 14 | self.variance = variance 15 | self.weights = weights 16 | 17 | @classmethod 18 | def train(cls, lmks, imgs, ref, frac, kmax): 19 | smodel = ShapeModel.load('data/shape.npz') 20 | tmodel = TextureModel.load('data/texture.npz') 21 | 22 | # build diagonal matrix of weights that measures 23 | # the unit difference between shape and texture parameters 24 | r = tmodel.variance.sum() / smodel.variance.sum() 25 | Ws = r * np.eye(smodel.num_modes()) 26 | 27 | n_samples = lmks.shape[1] 28 | 29 | tm = TextureMapper(480, 640) 30 | 31 | # create empty matrix 32 | n_feats = smodel.num_modes() + tmodel.num_modes() 33 | A = np.empty((n_feats, n_samples)) 34 | 35 | # concatenate shape and texture parameters for each training example 36 | for i in xrange(n_samples): 37 | img = next(imgs) 38 | lmk = lmks[:,i] 39 | sparams = smodel.calc_params(lmk) 40 | tparams = tmodel.calc_params(img, lmk, ref, tm.warp_triangles) 41 | # ignore first 4 shape parameters 42 | A[:,i] = np.concatenate((Ws.dot(sparams[4:]), tparams)) 43 | 44 | D = pca(A, frac, kmax) 45 | 46 | # compute variance 47 | Q = pow(D.T.dot(A), 2) 48 | e = Q.sum(axis=1) / (n_samples-1) 49 | 50 | return cls(D, e, Ws) 51 | 52 | @classmethod 53 | def load(cls, filename): 54 | data = np.load(filename) 55 | return cls(data['model'], data['variance'], data['weights']) 56 | 57 | def save(self, filename): 58 | np.savez(filename, model=self.model, variance=self.variance, weights=self.weights) 59 | 60 | def num_modes(self): 61 | return self.model.shape[1] 62 | 63 | def calc_shp_tex_params(self, params, split): 64 | st_params = self.model.dot(params) 65 | sparams = np.linalg.inv(self.weights).dot(st_params[:split]) 66 | tparams = st_params[split:] 67 | return sparams, tparams 68 | -------------------------------------------------------------------------------- /pyaam/perturbator.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import numpy as np 6 | 7 | class Perturbator(object): 8 | def __init__(self, s_sigma, t_sigma): 9 | self.s_sigma = s_sigma 10 | self.t_sigma = t_sigma 11 | 12 | self.scale_perts = (0.95, 0.97, 0.99, 1.01, 1.03, 1.05) 13 | self.angle_perts = [np.radians(x) for x in (-5, -3, -1, 1, 3, 5)] 14 | self.trans_perts = (-6, -3, -1, 1, 3, 6) 15 | self.param_perts = (-0.5, -0.25, 0.25, 0.5) 16 | 17 | def num_perts(self): 18 | n_perts = 0 19 | n_perts += len(self.scale_perts) 20 | n_perts += len(self.angle_perts) 21 | n_perts += len(self.trans_perts) * 2 22 | n_params = len(self.s_sigma) + len(self.t_sigma) 23 | n_perts += len(self.param_perts) * n_params 24 | return n_perts 25 | 26 | def perturbations(self, shape, texture): 27 | pose = shape[:4] 28 | shape = shape[4:] 29 | 30 | a, b = pose[:2] 31 | scale = np.sqrt(a*a + b*b) 32 | angle = np.arctan2(b, a) 33 | 34 | for pert in self.scale_perts: 35 | pert_pose = pose.copy() 36 | pert_pose[0] = scale * pert * np.cos(angle) 37 | pert_pose[1] = scale * pert * np.sin(angle) 38 | yield np.concatenate((pert_pose, shape, texture)) 39 | 40 | for pert in self.angle_perts: 41 | pert_pose = pose.copy() 42 | pert_pose[0] = scale * np.cos(angle + pert) 43 | pert_pose[1] = scale * np.sin(angle + pert) 44 | yield np.concatenate((pert_pose, shape, texture)) 45 | 46 | for i in [2, 3]: # tx, ty 47 | for pert in self.trans_perts: 48 | pert_pose = pose.copy() 49 | pert_pose[i] += pert 50 | yield np.concatenate((pert_pose, shape, texture)) 51 | 52 | for i in xrange(len(shape)): 53 | for pert in self.param_perts: 54 | pert_shape = shape.copy() 55 | pert_shape[i] += pert * self.s_sigma[i] * scale 56 | yield np.concatenate((pose, pert_shape, texture)) 57 | 58 | for i in xrange(len(texture)): 59 | for pert in self.param_perts: 60 | pert_texture = texture.copy() 61 | pert_texture[i] += pert * self.t_sigma[i] 62 | yield np.concatenate((pose, shape, pert_texture)) 63 | -------------------------------------------------------------------------------- /pyaam/texture.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import cv2 6 | import numpy as np 7 | from pyaam.texturemapper import TextureMapper 8 | from pyaam.utils import get_mask, get_aabb, get_vertices, normalize, pca 9 | 10 | 11 | 12 | class TextureModel(object): 13 | def __init__(self, model, mean, variance): 14 | self.model = model 15 | self.mean = mean 16 | self.variance = variance 17 | 18 | @classmethod 19 | def train(cls, lmks, imgs, ref, frac, kmax): 20 | G = get_data_matrix(imgs, lmks, ref) 21 | Gm = G.mean(axis=1) 22 | G -= Gm[:,np.newaxis] 23 | N = lmks.shape[1] 24 | D = pca(G, frac, kmax) 25 | # normalize eigenvectors 26 | for i in xrange(D.shape[1]): 27 | D[:,i] /= np.linalg.norm(D[:,i]) 28 | # compute variance 29 | Q = D.T.dot(G) 30 | Q = pow(Q, 2) 31 | e = Q.sum(axis=1) / (N-1) 32 | return cls(D, Gm, e) 33 | 34 | @classmethod 35 | def load(cls, filename): 36 | arch = np.load(filename) 37 | return cls(arch['model'], arch['mean'], arch['variance']) 38 | 39 | def save(self, filename): 40 | np.savez(filename, model=self.model, mean=self.mean, variance=self.variance) 41 | 42 | def num_modes(self): 43 | return self.model.shape[1] 44 | 45 | def texture_vector_size(self): 46 | return self.model.shape[0] 47 | 48 | def calc_texture(self, params): 49 | t = self.mean + self.model.dot(params) 50 | return t.clip(0, 255) # clamp pixels intensities 51 | 52 | def calc_params(self, img, lmk, ref, warp_triangles): 53 | ref = ref.reshape((ref.size//2, 2)).astype('int32') 54 | src = lmk.reshape(ref.shape) 55 | img = normalize(img, get_aabb(src)) 56 | mask = get_mask(ref, img.shape[:2]) 57 | verts = get_vertices(ref) 58 | warp = warp_triangles(img, src[verts], ref[verts]) 59 | t = warp[mask].ravel() - self.mean 60 | p = self.model.T.dot(t) 61 | # clamp 62 | c = 3 63 | for i in xrange(len(self.variance)): 64 | v = c * np.sqrt(self.variance[i]) 65 | if abs(p[i]) > v: 66 | p[i] = v if p[i] > 0 else -v 67 | return p 68 | 69 | 70 | 71 | def get_data_matrix(imgs, lmks, ref): 72 | ref = ref.reshape((ref.size//2, 2)).astype('int32') 73 | mask = get_mask(ref, (640, 480)) # FIXME hardcoded image size 74 | verts = get_vertices(ref) 75 | tm = TextureMapper(480, 640) # ditto 76 | n_samples = lmks.shape[1] 77 | n_pixels = mask.sum() * 3 78 | G = np.empty((n_pixels, n_samples)) 79 | for i in xrange(n_samples): 80 | src = lmks[:,i].reshape(ref.shape) 81 | img = normalize(next(imgs), get_aabb(src)) 82 | warp = tm.warp_triangles(img, src[verts], ref[verts]) 83 | G[:,i] = warp[mask].ravel() 84 | return G 85 | -------------------------------------------------------------------------------- /pyaam/texturemapper.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import numpy as np 6 | from OpenGL.GL import * 7 | from OpenGL.GLU import * 8 | from OpenGL.GLUT import * 9 | from OpenGL.GL.framebufferobjects import * 10 | 11 | 12 | 13 | class TextureMapper(object): 14 | def __init__(self, width, height): 15 | self.width = width 16 | self.height = height 17 | 18 | # init glut and hide window 19 | glutInit() 20 | glutCreateWindow('TextureMapper') 21 | glutHideWindow() 22 | 23 | self.fbo = glGenFramebuffers(1) 24 | glBindFramebuffer(GL_FRAMEBUFFER, self.fbo) 25 | 26 | glViewport(0, 0, width, height) 27 | 28 | glClearColor(0, 0, 0, 0) 29 | glShadeModel(GL_SMOOTH) 30 | glEnable(GL_TEXTURE_2D) 31 | glPixelStorei(GL_UNPACK_ALIGNMENT, 1) 32 | 33 | def warp_triangles(self, image, src, dst, img_dst=None): 34 | width = self.width 35 | height = self.height 36 | 37 | dest_texture = glGenTextures(1) 38 | glBindTexture(GL_TEXTURE_2D, dest_texture) 39 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) 40 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) 41 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 42 | 0, GL_RGB, GL_UNSIGNED_BYTE, img_dst) 43 | glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, 44 | GL_TEXTURE_2D, dest_texture, 0) 45 | 46 | orig_texture = glGenTextures(1) 47 | glBindTexture(GL_TEXTURE_2D, orig_texture) 48 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR) 49 | glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR) 50 | glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 51 | 0, GL_RGB, GL_UNSIGNED_BYTE, image) 52 | 53 | if img_dst is None: 54 | glClear(GL_COLOR_BUFFER_BIT) 55 | 56 | glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL) 57 | 58 | glMatrixMode(GL_PROJECTION) 59 | glLoadIdentity() 60 | glOrtho(0, width, 0, height, -5, 5) 61 | glMatrixMode(GL_MODELVIEW) 62 | glLoadIdentity() 63 | 64 | # assign texture to each triangle 65 | glBegin(GL_TRIANGLES) 66 | for s,d in zip(src, dst): 67 | glTexCoord2f(s[0,0]/width, s[0,1]/height) 68 | glVertex3f(d[0,0], d[0,1], 0) 69 | glTexCoord2f(s[1,0]/width, s[1,1]/height) 70 | glVertex3f(d[1,0], d[1,1], 0) 71 | glTexCoord2f(s[2,0]/width, s[2,1]/height) 72 | glVertex3f(d[2,0], d[2,1], 0) 73 | glEnd() 74 | 75 | # read from fbo 76 | s = glReadPixels(0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE) 77 | warped = np.fromstring(s, dtype='uint8').reshape((height,width,3)) 78 | 79 | glDeleteTextures(orig_texture) 80 | glDeleteTextures(dest_texture) 81 | return warped 82 | 83 | def cleanup(self): 84 | glDeleteFramebuffers(1, self.fbo) 85 | -------------------------------------------------------------------------------- /pyaam/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import cv2 6 | import numpy as np 7 | from scipy.spatial import Delaunay 8 | 9 | 10 | 11 | def get_aabb(pts): 12 | """axis-aligned minimum bounding box""" 13 | x, y = np.floor(pts.min(axis=0)).astype(int) 14 | w, h = np.ceil(pts.ptp(axis=0)).astype(int) 15 | return x, y, w, h 16 | 17 | def get_vertices(pts): 18 | return Delaunay(pts).vertices 19 | 20 | def normalize(img, aabb): 21 | x, y, w, h = aabb 22 | # work on image copy 23 | img = img.copy() 24 | # .copy() required on linux 25 | img[y:y+h,x:x+w,0] = cv2.equalizeHist(img[y:y+h,x:x+w,0].copy()) 26 | img[y:y+h,x:x+w,1] = cv2.equalizeHist(img[y:y+h,x:x+w,1].copy()) 27 | img[y:y+h,x:x+w,2] = cv2.equalizeHist(img[y:y+h,x:x+w,2].copy()) 28 | return img 29 | 30 | def get_mask(pts, shape): 31 | pts = pts.astype('int32') 32 | mask = np.zeros(shape, dtype='uint8') 33 | hull = cv2.convexHull(pts[:,np.newaxis].copy()) # .copy() required on linux 34 | hull = hull.reshape((hull.shape[0], hull.shape[2])) 35 | cv2.fillConvexPoly(mask, hull, 255) 36 | return mask.astype(bool) 37 | 38 | # NOTE you should use pyaam.texturemapper.TextureMapper instead 39 | def warp_triangles(img, src, dst): 40 | result = np.zeros(img.shape, dtype='uint8') 41 | dsize = (img.shape[1], img.shape[0]) 42 | for s, d in zip(src, dst): 43 | mask = np.zeros(img.shape[:2], dtype='uint8') 44 | cv2.fillConvexPoly(mask, d.astype('int32'), 255) 45 | mask = mask.astype(bool) 46 | M = cv2.getAffineTransform(s.astype('float32'), d.astype('float32')) 47 | warp = cv2.warpAffine(img, M, dsize, flags=cv2.INTER_LINEAR) 48 | result[mask] = warp[mask] 49 | return result 50 | 51 | def pca(M, frac, kmax=None): 52 | """principal component analysis""" 53 | # see Stegmann's thesis section 5.6.1 for details 54 | enough_samples = M.shape[1] > M.shape[0] # each column is a sample 55 | # covariance matrix 56 | C = M.dot(M.T) if enough_samples else M.T.dot(M) 57 | C /= M.shape[1] 58 | u, s, v = np.linalg.svd(C) 59 | if not enough_samples: 60 | u = M.dot(u) 61 | if kmax is not None: 62 | s = s[:kmax] 63 | p = s.cumsum() / s.sum() 64 | k = p[p < frac].size 65 | return u[:,:k] 66 | 67 | def gram_schmid(V): 68 | """Gram-Schmid orthonormalization (in-place)""" 69 | n = V.shape[1] 70 | for i in xrange(n): 71 | for j in xrange(i): 72 | # subtract projection 73 | V[:,i] -= np.dot(V[:,i], V[:,j]) * V[:,j] 74 | # normalize 75 | V[:,i] /= np.linalg.norm(V[:,i]) 76 | # orthonormalization was performed in-place 77 | # but we return V for convenience 78 | return V 79 | 80 | def sample_texture(img, pts, ref, warp_triangles): 81 | """returns a texture vector""" 82 | aabb = get_aabb(pts) 83 | img = normalize(img, aabb) 84 | mask = get_mask(ref, img.shape[:2]) 85 | verts = get_vertices(ref) 86 | warp = warp_triangles(img, pts[verts], ref[verts]) 87 | return warp[mask].ravel() 88 | -------------------------------------------------------------------------------- /view_perturbations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import argparse 8 | import numpy as np 9 | import cv2 10 | 11 | from pyaam.muct import MuctDataset 12 | from pyaam.draw import draw_polygons, draw_texture, draw_face 13 | from pyaam.utils import get_vertices, sample_texture 14 | from pyaam.shape import ShapeModel 15 | from pyaam.texture import TextureModel 16 | from pyaam.texturemapper import TextureMapper 17 | from pyaam.perturbator import Perturbator 18 | 19 | 20 | 21 | def experiments(images, landmarks, smodel, tmodel, ref_shape): 22 | cv2.namedWindow('original') 23 | cv2.namedWindow('model') 24 | cv2.namedWindow('perturbed') 25 | tm = TextureMapper(480, 640) 26 | tri = get_vertices(ref_shape) 27 | split = smodel.num_modes() + 4 28 | perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance)) 29 | for i in xrange(len(landmarks)): 30 | # get image and corresponding landmarks 31 | img = next(images) 32 | lmks = landmarks[i] 33 | pts = lmks.reshape(ref_shape.shape) 34 | # get shape and texture model parameters for current example 35 | s_params = smodel.calc_params(lmks) 36 | t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles) 37 | params = np.concatenate((s_params, t_params)) 38 | 39 | cv2.imshow('original', img) 40 | 41 | shape = smodel.calc_shape(s_params) 42 | shape = shape.reshape((shape.size//2, 2)) 43 | texture = tmodel.calc_texture(t_params) 44 | warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) 45 | cv2.imshow('model', warped) 46 | 47 | for pert in perturbator.perturbations(s_params, t_params): 48 | s = pert[:split] 49 | t = pert[split:] 50 | x_image = smodel.calc_shape(s) 51 | x_image = x_image.reshape((x_image.size//2, 2)) 52 | g_image = sample_texture(img, x_image, ref_shape, tm.warp_triangles) 53 | g_model = tmodel.calc_texture(t) 54 | perturbation = pert - params 55 | residual = g_image - g_model 56 | 57 | warped = draw_face(img, x_image, g_model, ref_shape, tm.warp_triangles) 58 | cv2.imshow('perturbed', warped) 59 | 60 | key = cv2.waitKey() 61 | if key == ord('n'): 62 | break 63 | elif key == 27: 64 | sys.exit() 65 | 66 | 67 | 68 | def parse_args(): 69 | description = '' # FIXME write some description 70 | parser = argparse.ArgumentParser(description=description) 71 | parser.add_argument('--no-flip', action='store_false', dest='flipped', 72 | help='exclude flipped data') 73 | return parser.parse_args() 74 | 75 | if __name__ == '__main__': 76 | args = parse_args() 77 | 78 | muct = MuctDataset() 79 | muct.load(clean=True) 80 | data = muct.all_lmks() 81 | imgs = muct.iterimages(mirror=True) 82 | print 'training samples:', len(data) 83 | 84 | smodel = ShapeModel.load('data/shape.npz') 85 | tmodel = TextureModel.load('data/texture.npz') 86 | 87 | # get reference shape 88 | params = smodel.get_params(200, 150, 150) 89 | ref = smodel.calc_shape(params) 90 | ref = ref.reshape((ref.size//2, 2)) 91 | 92 | experiments(imgs, data, smodel, tmodel, ref) 93 | -------------------------------------------------------------------------------- /pyaam/detector.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import cv2 6 | import cv2.cv as cv 7 | import numpy as np 8 | 9 | 10 | 11 | CASCADE_FILENAME = 'data/haarcascades/haarcascade_frontalface_default.xml' 12 | 13 | 14 | 15 | class FaceDetector(object): 16 | def __init__(self, offset, ref): 17 | self.offset = offset 18 | self.ref_shape = ref 19 | self.cascade = cv2.CascadeClassifier(CASCADE_FILENAME) 20 | self.flags = cv.CV_HAAR_FIND_BIGGEST_OBJECT | cv.CV_HAAR_SCALE_IMAGE 21 | 22 | @classmethod 23 | def train(cls, lmks, imgs, ref, frac=0.8, scale_factor=1.1, min_neighbors=2, min_size=(30,30)): 24 | detector = cv2.CascadeClassifier(CASCADE_FILENAME) 25 | flags = cv.CV_HAAR_FIND_BIGGEST_OBJECT | cv.CV_HAAR_SCALE_IMAGE 26 | N = lmks.shape[1] 27 | xoffset = [] 28 | yoffset = [] 29 | zoffset = [] 30 | 31 | for i in xrange(N): 32 | pts = lmks[:,i] 33 | img = next(imgs) 34 | 35 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 36 | gray = cv2.equalizeHist(gray) 37 | rects = detector.detectMultiScale(gray, 38 | scaleFactor=scale_factor, 39 | minNeighbors=min_neighbors, 40 | minSize=min_size, 41 | flags=flags) 42 | 43 | if len(rects) == 0: 44 | continue 45 | 46 | x,y,w,h = rects[0] 47 | if enough_bounded_points(pts, (x,y,w,h), frac): 48 | center = pts.reshape((pts.size//2,2)).sum(axis=0) / (pts.size//2) 49 | xoffset.append((center[0] - (x + 0.5 * w)) / w) 50 | yoffset.append((center[1] - (y + 0.5 * h)) / w) 51 | zoffset.append(calc_scale(pts, ref) / w) 52 | 53 | xoffset.sort() 54 | yoffset.sort() 55 | zoffset.sort() 56 | 57 | detector_offset = (xoffset[len(xoffset)//2], 58 | yoffset[len(yoffset)//2], 59 | zoffset[len(zoffset)//2]) 60 | 61 | return cls(detector_offset, ref) 62 | 63 | @classmethod 64 | def load(cls, filename): 65 | arch = np.load(filename) 66 | return cls(arch['offset'], arch['ref_shape']) 67 | 68 | def save(self, filename): 69 | np.savez(filename, offset=self.offset, ref_shape=self.ref_shape) 70 | 71 | def detect(self, img, scale_factor=1.1, min_neighbors=2, min_size=(30,30)): 72 | gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 73 | gray = cv2.equalizeHist(gray) 74 | 75 | rects = self.cascade.detectMultiScale(gray, 76 | scaleFactor=scale_factor, 77 | minNeighbors=min_neighbors, 78 | minSize=min_size, 79 | flags=self.flags) 80 | 81 | if len(rects) == 0: 82 | return 83 | 84 | r = rects[0] 85 | scale = self.offset * r[2] 86 | p = self.ref_shape.copy() 87 | p[::2] = scale[2] * p[::2] + r[0] + 0.5 * r[2] + scale[0] 88 | p[1::2] = scale[2] * p[1::2] + r[1] + 0.5 * r[3] + scale[1] 89 | return p 90 | 91 | 92 | 93 | def calc_scale(pts, ref): 94 | pts = pts.reshape((pts.size//2,2)) 95 | c = pts.sum(axis=0) / pts.shape[0] 96 | p = pts - c 97 | p = p.reshape(p.size) 98 | ref = ref.reshape(ref.size) 99 | return ref.dot(p) / ref.dot(ref) 100 | 101 | def enough_bounded_points(pts, r, frac): 102 | n = len(pts) // 2 103 | bounded = pts[::2] >= r[0] 104 | bounded &= pts[::2] <= r[0]+r[2] 105 | bounded &= pts[1::2] >= r[1] 106 | bounded &= pts[1::2] <= r[1]+r[3] 107 | return bounded.sum() / n >= frac 108 | 109 | -------------------------------------------------------------------------------- /matmul.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # License: BSD 4 | # Created: October 11, 2013 5 | # Author: Francesc Alted 6 | # 7 | ######################################################################## 8 | 9 | """ 10 | Implementation of an out of core matrix-matrix multiplication for PyTables. 11 | """ 12 | 13 | import sys, math 14 | 15 | import numpy as np 16 | import tables as tb 17 | 18 | _MB = 2**20 19 | OOC_BUFFER_SIZE = 32*_MB 20 | """The buffer size for out-of-core operations. 21 | """ 22 | 23 | def dot(a, b, out=None): 24 | """ 25 | Matrix multiplication of two 2-D arrays. 26 | 27 | Parameters 28 | ---------- 29 | a : array_like 30 | First argument. 31 | b : array_like 32 | Second argument. 33 | out : array_like, optional 34 | Output argument. This must have the exact kind that would be 35 | returned if it was not used. 36 | 37 | Returns 38 | ------- 39 | output : CArray or scalar 40 | Returns the dot product of `a` and `b`. If `a` and `b` are 41 | both scalars or both 1-D arrays then a scalar is returned; 42 | otherwise a new CArray (in file dot.h5:/out) is returned. If 43 | `out` parameter is provided, then it is returned instead. 44 | 45 | Raises 46 | ------ 47 | ValueError 48 | If the last dimension of `a` is not the same size as the 49 | second-to-last dimension of `b`. 50 | """ 51 | 52 | if len(a.shape) != 2 or len(b.shape) != 2: 53 | raise (ValueError, "only 2-D matrices supported") 54 | 55 | if a.shape[1] != b.shape[0]: 56 | raise (ValueError, 57 | "last dimension of `a` does not match first dimension of `b`") 58 | 59 | l, m, n = a.shape[0], a.shape[1], b.shape[1] 60 | 61 | if out is not None: 62 | if out.shape != (l, n): 63 | raise (ValueError, "`out` array does not have the correct shape") 64 | else: 65 | f = tb.openFile('dot.h5', 'w') 66 | filters = tb.Filters(complevel=5, complib='blosc') 67 | out = f.createCArray(f.root, 'out', tb.Atom.from_dtype(a.dtype), 68 | shape=(l, n), filters=filters) 69 | 70 | # Compute a good block size 71 | buffersize = OOC_BUFFER_SIZE 72 | bl = math.sqrt(buffersize / out.dtype.itemsize) 73 | bl = 2**int(math.log(bl, 2)) 74 | for i in range(0, l, bl): 75 | for j in range(0, n, bl): 76 | for k in range(0, m, bl): 77 | a0 = a[i:min(i+bl, l), k:min(k+bl, m)] 78 | b0 = b[k:min(k+bl, m), j:min(j+bl, n)] 79 | out[i:i+bl, j:j+bl] += np.dot(a0, b0) 80 | 81 | return out 82 | 83 | if __name__ == "__main__": 84 | """Small benchmark for comparison against numpy.dot() speed""" 85 | from time import time 86 | 87 | # Matrix dimensions 88 | L, M, N = 1000, 100, 2000 89 | print "Multiplying (%d, %d) x (%d, %d) matrices" % (L, M, M, N) 90 | 91 | a = np.linspace(0, 1, L*M).reshape(L, M) 92 | b = np.linspace(0, 1, M*N).reshape(M, N) 93 | 94 | t0 = time() 95 | cdot = np.dot(a,b) 96 | print "Time for np.dot->", round(time()-t0, 3), cdot.shape 97 | 98 | f = tb.openFile('matrix-pt.h5', 'w') 99 | 100 | l, m, n = a.shape[0], a.shape[1], b.shape[1] 101 | 102 | filters = tb.Filters(complevel=5, complib='blosc') 103 | ad = f.createCArray(f.root, 'a', tb.Float64Atom(), (l,m), 104 | filters=filters) 105 | ad[:] = a 106 | bd = f.createCArray(f.root, 'b', tb.Float64Atom(), (m,n), 107 | filters=filters) 108 | bd[:] = b 109 | cd = f.createCArray(f.root, 'c', tb.Float64Atom(), (l,n), 110 | filters=filters) 111 | 112 | t0 = time() 113 | dot(a, b, out=cd) 114 | print "Time for ooc dot->", round(time()-t0, 3), cd.shape 115 | 116 | np.testing.assert_almost_equal(cd, cdot) 117 | 118 | f.close() 119 | 120 | -------------------------------------------------------------------------------- /train_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import argparse 5 | from pyaam.muct import MuctDataset 6 | from pyaam.shape import ShapeModel 7 | from pyaam.patches import PatchesModel 8 | from pyaam.texture import TextureModel 9 | from pyaam.combined import CombinedModel 10 | from pyaam.detector import FaceDetector 11 | 12 | 13 | 14 | def parse_args(): 15 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 16 | parser.add_argument('model', choices=['shape', 'patches', 'detector', 'texture', 'combined'], help='model name') 17 | parser.add_argument('--frac', type=float, default=0.99, help='fraction of variation') 18 | parser.add_argument('--kmax', type=int, default=20, help='maximum modes') 19 | parser.add_argument('--width', type=int, default=100, help='face width') 20 | parser.add_argument('--psize', type=int, default=11, help='patch size') 21 | parser.add_argument('--ssize', type=int, default=11, help='search window size') 22 | parser.add_argument('--var', type=float, default=1.0, help='variance of annotation error') 23 | parser.add_argument('-lmbda', type=float, default=1e-6, help='regularization weight') 24 | parser.add_argument('--mu', type=float, default=1e-3, help='initial stochastic gradient descent step size') 25 | parser.add_argument('--nsamples', type=int, default=1000, help='number of stochastic gradient descent samples') 26 | parser.add_argument('--face-width', type=int, default=100, help='face width') 27 | parser.add_argument('--shp-fn', default='data/shape.npz', help='shape model filename') 28 | parser.add_argument('--ptc-fn', default='data/patches.npz', help='patches model filename') 29 | parser.add_argument('--dtc-fn', default='data/detector.npz', help='face detector filename') 30 | parser.add_argument('--txt-fn', default='data/texture.npz', help='texture model filename') 31 | parser.add_argument('--cmb-fn', default='data/combined.npz', help='combined model filename') 32 | return parser.parse_args() 33 | 34 | 35 | 36 | if __name__ == '__main__': 37 | args = parse_args() 38 | 39 | muct = MuctDataset() 40 | muct.load(clean=True) 41 | data = muct.all_lmks() 42 | imgs = muct.iterimages(mirror=True) 43 | print 'training samples:', len(data) 44 | 45 | if args.model == 'shape': 46 | print 'training shape model ...' 47 | model = ShapeModel.train(data.T, args.frac, args.kmax) 48 | print 'retained:', model.num_modes(), 'modes' 49 | model.save(args.shp_fn) 50 | print 'wrote', args.shp_fn 51 | 52 | elif args.model == 'patches': 53 | print 'reading images ...' 54 | imgs = list(imgs) 55 | print 'training patches model ...' 56 | sm = ShapeModel.load(args.shp_fn) 57 | model = PatchesModel.train(data.T, imgs, sm.get_shape(args.face_width), args.psize, 58 | args.ssize, args.var, args.lmbda, args.mu, args.nsamples) 59 | model.save(args.ptc_fn) 60 | print 'wrote', args.ptc_fn 61 | 62 | elif args.model == 'detector': 63 | print 'training face detector ...' 64 | sm = ShapeModel.load(args.shp_fn) 65 | model = FaceDetector.train(data.T, imgs, sm.get_shape()) 66 | model.save(args.dtc_fn) 67 | print 'wrote', args.dtc_fn 68 | 69 | elif args.model == 'texture': 70 | print 'training texture model ...' 71 | sm = ShapeModel.load(args.shp_fn) 72 | ref = sm.get_shape(200, 150, 150) 73 | model = TextureModel.train(data.T, imgs, ref, args.frac, args.kmax) 74 | print 'retained:', model.num_modes(), 'modes' 75 | model.save(args.txt_fn) 76 | print 'wrote', args.txt_fn 77 | 78 | elif args.model == 'combined': 79 | print 'training combined model ...' 80 | sm = ShapeModel.load(args.shp_fn) 81 | ref = sm.get_shape(200, 150, 150) 82 | model = CombinedModel.train(data.T, imgs, ref, args.frac, args.kmax) 83 | print 'retained:', model.num_modes(), 'modes' 84 | model.save(args.cmb_fn) 85 | print 'wrote', args.cmb_fn 86 | -------------------------------------------------------------------------------- /do_perts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import argparse 8 | import numpy as np 9 | import tables as tb 10 | import cv2 11 | 12 | from pyaam.muct import MuctDataset 13 | from pyaam.draw import draw_polygons, draw_texture, draw_face 14 | from pyaam.utils import get_vertices, get_aabb, normalize, get_mask 15 | from pyaam.shape import ShapeModel 16 | from pyaam.texture import TextureModel 17 | from pyaam.texturemapper import TextureMapper 18 | from pyaam.perturbator import Perturbator 19 | 20 | 21 | 22 | def sample_texture(img, pts, ref, tm): 23 | """returns a texture vector""" 24 | img = normalize(img, get_aabb(pts)) 25 | mask = get_mask(ref, img.shape[:2]) 26 | verts = get_vertices(ref) 27 | warp = tm.warp_triangles(img, pts[verts], ref[verts]) 28 | return warp[mask].ravel() 29 | 30 | 31 | 32 | def experiments(images, landmarks, smodel, tmodel, ref_shape, fout): 33 | tm = TextureMapper(480, 640) 34 | tri = get_vertices(ref_shape) 35 | split = smodel.num_modes() + 4 36 | perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance)) 37 | 38 | n_samples = len(landmarks) 39 | n_perts = perturbator.num_perts() 40 | total_perts = n_perts * n_samples 41 | n_params = 4 + smodel.num_modes() + tmodel.num_modes() 42 | t_vec_sz = tmodel.texture_vector_size() 43 | 44 | h5 = tb.openFile(fout, mode='w', title='perturbations') 45 | filters = tb.Filters(complevel=5, complib='blosc') 46 | P = h5.createCArray(h5.root, 'perturbations', tb.Float64Atom(), 47 | shape=(n_params, total_perts), filters=filters) 48 | R = h5.createCArray(h5.root, 'residuals', tb.Float64Atom(), 49 | shape=(t_vec_sz, total_perts), filters=filters, 50 | chunkshape=(2048, 128)) 51 | 52 | for i in xrange(len(landmarks)): 53 | # get image and corresponding landmarks 54 | img = next(images) 55 | lmks = landmarks[i] 56 | pts = lmks.reshape(ref_shape.shape) 57 | # get shape and texture model parameters for current example 58 | s_params = smodel.calc_params(lmks) 59 | t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles) 60 | params = np.concatenate((s_params, t_params)) 61 | 62 | perturbations = perturbator.perturbations(s_params, t_params) 63 | for j,pert in enumerate(perturbations): 64 | col = n_perts * i + j 65 | print 'perturbation {:,} of {:,}'.format(col+1, total_perts) 66 | s = pert[:split] 67 | t = pert[split:] 68 | x_image = smodel.calc_shape(s) 69 | x_image = x_image.reshape((x_image.size//2, 2)) 70 | g_image = sample_texture(img, x_image, ref_shape, tm) 71 | g_model = tmodel.calc_texture(t) 72 | perturbation = pert - params 73 | residual = g_image - g_model 74 | 75 | P[:,col] = perturbation 76 | R[:,col] = residual 77 | 78 | h5.close() 79 | 80 | 81 | def parse_args(): 82 | description = '' # FIXME write some description 83 | parser = argparse.ArgumentParser(description=description) 84 | parser.add_argument('-o', dest='fout', default='data/perturbations.h5', 85 | help='output file') 86 | parser.add_argument('--no-flip', action='store_false', dest='flipped', 87 | help='exclude flipped data') 88 | return parser.parse_args() 89 | 90 | if __name__ == '__main__': 91 | args = parse_args() 92 | 93 | muct = MuctDataset() 94 | muct.load(clean=True) 95 | 96 | # If a face is too close to the image border and we perturbe 97 | # the scale then that face may grow beyond the image border. 98 | # Ignore problematic images. 99 | muct.ignore('i405wc-fn') 100 | 101 | data = muct.all_lmks() 102 | imgs = muct.iterimages(mirror=True) 103 | print 'training samples:', len(data) 104 | 105 | smodel = ShapeModel.load('data/shape.npz') 106 | tmodel = TextureModel.load('data/texture.npz') 107 | 108 | # get reference shape 109 | params = smodel.get_params(200, 150, 150) 110 | ref = smodel.calc_shape(params) 111 | ref = ref.reshape((ref.size//2, 2)) 112 | 113 | experiments(imgs, data, smodel, tmodel, ref, args.fout) 114 | print 'wrote', args.fout 115 | -------------------------------------------------------------------------------- /test_aam.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import division 4 | 5 | import sys 6 | import cv2 7 | import numpy as np 8 | 9 | from pyaam.muct import MuctDataset 10 | from pyaam.draw import draw_face, draw_muct_shape 11 | from pyaam.utils import get_vertices 12 | from pyaam.shape import ShapeModel 13 | from pyaam.texture import TextureModel 14 | from pyaam.detector import FaceDetector 15 | from pyaam.texturemapper import TextureMapper 16 | from pyaam.utils import sample_texture 17 | 18 | MAX_ITER = 10 19 | 20 | 21 | def get_instance(params, smodel, tmodel): 22 | split = smodel.num_params() 23 | s = params[:split] 24 | t = params[split:] 25 | shape = smodel.calc_shape(s) 26 | shape = shape.reshape((shape.size // 2, 2)) 27 | texture = tmodel.calc_texture(t) 28 | return shape, texture 29 | 30 | 31 | 32 | def test_aam(images, landmarks, smodel, tmodel, R, ref_shape): 33 | cv2.namedWindow('original') 34 | cv2.namedWindow('fitted') 35 | cv2.namedWindow('shape') 36 | tm = TextureMapper(480, 640) 37 | tri = get_vertices(ref_shape) 38 | for i in xrange(len(landmarks)): 39 | img = next(images) 40 | cv2.imshow('original', img) 41 | lmks = landmarks[i].reshape(ref_shape.shape) 42 | 43 | # detect face 44 | pts = detector.detect(img) 45 | # get params for detected face shape 46 | s_params = smodel.calc_params(pts) 47 | # mean texture 48 | t_params = np.zeros(tmodel.num_modes()) 49 | # concatenate parameters 50 | params = np.concatenate((s_params, t_params)) 51 | 52 | shape, texture = get_instance(params, smodel, tmodel) 53 | warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) 54 | cv2.imshow('fitted', warped) 55 | 56 | img2 = img.copy() 57 | draw_muct_shape(img2, shape.ravel()) 58 | cv2.imshow('shape', img2) 59 | 60 | key = cv2.waitKey() 61 | if key == ord(' '): 62 | pass 63 | elif key == ord('n'): 64 | continue 65 | elif key == 27: 66 | sys.exit() 67 | 68 | shape, texture = get_instance(params, smodel, tmodel) 69 | g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) 70 | # compute residual 71 | residual = g_image - texture 72 | # evaluate error 73 | E0 = np.dot(residual, residual) 74 | # predict model displacements 75 | pert = R.dot(residual) 76 | 77 | for i in xrange(MAX_ITER): 78 | shape, texture = get_instance(params, smodel, tmodel) 79 | g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) 80 | # compute residual 81 | residual = g_image - texture 82 | # predict model displacements 83 | pert = R.dot(residual) 84 | for alpha in (1.5, 1, 0.5, 0.25, 0.125): 85 | new_params = params - alpha * pert 86 | 87 | shape, texture = get_instance(new_params, smodel, tmodel) 88 | g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) 89 | residual = g_image - texture 90 | Ek = np.dot(residual, residual) 91 | 92 | if Ek < E0: 93 | params = new_params 94 | E0 = Ek 95 | break 96 | 97 | shape, texture = get_instance(params, smodel, tmodel) 98 | warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) 99 | cv2.imshow('fitted', warped) 100 | 101 | img2 = img.copy() 102 | draw_muct_shape(img2, shape.ravel()) 103 | cv2.imshow('shape', img2) 104 | 105 | key = cv2.waitKey() 106 | if key == ord(' '): 107 | continue 108 | elif key == ord('n'): 109 | break 110 | elif key == 27: 111 | sys.exit() 112 | 113 | 114 | 115 | if __name__ == '__main__': 116 | smodel = ShapeModel.load('data/shape.npz') 117 | tmodel = TextureModel.load('data/texture.npz') 118 | detector = FaceDetector.load('data/detector.npz') 119 | R = np.load('data/regmat.npz')['R'] 120 | 121 | muct = MuctDataset() 122 | muct.load(clean=True) 123 | data = muct.all_lmks() 124 | imgs = muct.iterimages(mirror=True) 125 | 126 | params = smodel.get_params(200, 150, 150) 127 | ref = smodel.calc_shape(params) 128 | ref = ref.reshape((ref.size//2, 2)) 129 | 130 | test_aam(imgs, data, smodel, tmodel, R, ref) 131 | -------------------------------------------------------------------------------- /pyaam/shape.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import numpy as np 6 | from pyaam.utils import pca, gram_schmid 7 | 8 | 9 | 10 | class ShapeModel(object): 11 | def __init__(self, model, variance): 12 | self.model = model 13 | self.variance = variance 14 | 15 | @classmethod 16 | def train(cls, X, frac, kmax): 17 | n_samples = X.shape[1] 18 | n_points = X.shape[0] // 2 19 | # align shapes 20 | Y = procrustes(X) 21 | # compute rigid transform 22 | R = calc_rigid_basis(Y) 23 | # project out rigidity 24 | P = R.T.dot(Y) 25 | dY = Y - R.dot(P) 26 | # compute non-rigid transformation 27 | D = pca(dY, frac, min(kmax, n_samples-1, n_points-1)) 28 | k = D.shape[1] 29 | # combine subspaces 30 | V = np.concatenate((R,D), axis=1) 31 | # project raw data onto subspace 32 | Q = V.T.dot(X) 33 | # normalize coordinates w.r.t. scale 34 | for i in xrange(n_samples): 35 | Q[:,i] /= Q[0,i] 36 | # compute variance 37 | e = np.empty(4+k, dtype=float) 38 | Q = pow(Q, 2) 39 | e[:4] = -1 # no clamping for rigid body coefficients 40 | e[4:] = Q[4:].sum(axis=1) / (n_samples-1) 41 | # return model 42 | return cls(V, e) 43 | 44 | @classmethod 45 | def load(cls, fname): 46 | arch = np.load(fname) 47 | return cls(arch['model'], arch['variance']) 48 | 49 | def save(self, fname): 50 | np.savez(fname, model=self.model, variance=self.variance) 51 | 52 | def num_params(self): 53 | return self.model.shape[1] 54 | 55 | def num_modes(self): 56 | return self.model.shape[1] - 4 57 | 58 | def calc_shape(self, params): 59 | return self.model.dot(params) 60 | 61 | def get_shape(self, scale=1, tranx=0, trany=0): 62 | params = self.get_params(scale, tranx, trany) 63 | return self.calc_shape(params) 64 | 65 | def calc_params(self, pts, c_factor=3.0): 66 | params = self.model.T.dot(pts) 67 | return self.clamp(params, c_factor) 68 | 69 | def get_params(self, scale=1, tranx=0, trany=0): 70 | # compute rigid parameters 71 | n = self.model.shape[0] // 2 72 | scale /= self.model[::2,0].ptp() 73 | tranx *= n / self.model[:,2].sum() 74 | trany *= n / self.model[:,3].sum() 75 | p = np.zeros(self.model.shape[1]) 76 | p[0] = scale 77 | p[2] = tranx 78 | p[3] = trany 79 | return p 80 | 81 | def clamp(self, p, c): 82 | scale = p[0] 83 | for i in xrange(len(p)): 84 | var = self.variance[i] 85 | if var < 0: 86 | # ignore rigid components 87 | continue 88 | v = c * np.sqrt(var) 89 | # preserve sign of coordinate 90 | if abs(p[i] / scale) > v: 91 | p[i] = v * scale if p[i] > 0 else -v * scale 92 | return p 93 | 94 | 95 | def procrustes(X, max_iters=100, tolerance=1e-6): 96 | """removes global rigid motion from a collection of shapes""" 97 | n_samples = X.shape[1] 98 | n_points = X.shape[0] // 2 99 | # copy of data to work on 100 | P = X.copy() 101 | 102 | # remove center of mass of each shape's instance 103 | P[::2,:] -= P[::2,:].sum(axis=0) / n_points 104 | P[1::2,:] -= P[1::2,:].sum(axis=0) / n_points 105 | 106 | # optimize scale and rotation 107 | C_old = None 108 | for _ in xrange(max_iters): 109 | # compute normalized canonical shape 110 | C = P.sum(axis=1) / n_samples 111 | C /= np.linalg.norm(C) 112 | 113 | # are we done? 114 | if C_old is not None and np.linalg.norm(C - C_old) < tolerance: 115 | break 116 | 117 | # keep copy of current estimate of canonical shape 118 | C_old = C.copy() 119 | 120 | # rotate and scale each shape to best match canonical shape 121 | for i in xrange(n_samples): 122 | R = rot_scale_align(P[:,i], C) 123 | pts = np.row_stack((P[::2,i], P[1::2,i])) 124 | P[:,i] = R.dot(pts).T.flatten() 125 | 126 | # return procrustes aligned shapes 127 | return P 128 | 129 | def rot_scale_align(src, dst): 130 | """computes the in-place rotation and scaling that best aligns 131 | shape instance `src` to shape instance `dst`""" 132 | # separate x and y 133 | srcx, srcy = src[::2], src[1::2] 134 | dstx, dsty = dst[::2], dst[1::2] 135 | # construct and solve linear system 136 | d = sum(pow(src, 2)) 137 | a = sum(srcx*dstx + srcy*dsty) / d 138 | b = sum(srcx*dsty - srcy*dstx) / d 139 | # return scale and rotation matrix 140 | return np.array([[a,-b],[b,a]]) 141 | 142 | def calc_rigid_basis(X): 143 | """model global transformation as linear subspace""" 144 | n_samples = X.shape[1] 145 | n_points = X.shape[0] // 2 146 | 147 | # compute canonical shape 148 | mean = X.mean(axis=1) 149 | 150 | # construct basis for similarity transform 151 | R = np.empty((2*n_points, 4), dtype=float) 152 | R[::2,0] = mean[::2] 153 | R[1::2,0] = mean[1::2] 154 | R[::2,1] = -mean[1::2] 155 | R[1::2,1] = mean[::2] 156 | R[::2,2] = 1 157 | R[1::2,2] = 0 158 | R[::2,3] = 0 159 | R[1::2,3] = 1 160 | 161 | return gram_schmid(R) 162 | -------------------------------------------------------------------------------- /pyaam/muct.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # The MUCT Face Database 4 | # http://www.milbo.org/muct/ 5 | 6 | from __future__ import division 7 | 8 | import os 9 | import sys 10 | import shutil 11 | import urllib2 12 | import tarfile 13 | import itertools 14 | import cv2 15 | import numpy as np 16 | 17 | 18 | 19 | # default dataset directory 20 | DEFAULT_DATADIR = 'data/muct' 21 | 22 | 23 | 24 | class MuctDataset(object): 25 | # landmark pair connections 26 | PAIRS = ( 27 | # jaw 28 | (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), 29 | (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), 30 | # right eyebrow 31 | (15, 16), (16, 17), (17, 18), (18, 19), (19, 20), (20, 15), 32 | # left eyebrow 33 | (21, 22), (22, 23), (23, 24), (24, 25), (25, 26), (26, 21), 34 | # left eye 35 | (27, 68), (68, 28), (28, 69), (69, 29), 36 | (29, 70), (70, 30), (30, 71), (71, 27), 37 | # right eye 38 | (32, 72), (72, 33), (33, 73), (73, 34), 39 | (34, 74), (74, 35), (35, 75), (75, 32), 40 | # nose 41 | (37, 38), (38, 39), (39, 40), (40, 41), 42 | (41, 42), (42, 43), (43, 44), (44, 45), 43 | # nose tip 44 | (41, 46), (46, 67), (67, 47), (47, 41), 45 | # upper lip 46 | (48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), 47 | (48, 65), (65, 64), (64, 63), (63, 54), 48 | # lower lip 49 | (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 48), 50 | (48, 60), (60, 61), (61, 62), (62, 54), 51 | ) 52 | 53 | # landmark flipping correspondences 54 | SYMMETRY = [14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 21, 55 | 22, 23, 24, 25, 26, 15, 16, 17, 18, 19, 20, 32, 33, 34, 56 | 35, 36, 27, 28, 29, 30, 31, 45, 44, 43, 42, 41, 40, 39, 57 | 38, 37, 47, 46, 54, 53, 52, 51, 50, 49, 48, 59, 58, 57, 58 | 56, 55, 62, 61, 60, 65, 64, 63, 66, 67, 72, 73, 74, 75, 59 | 68, 69, 70, 71] 60 | 61 | # dataset urls 62 | URLS = ( 63 | 'http://muct.googlecode.com/files/README.txt', 64 | 'http://muct.googlecode.com/files/muct-landmarks-v1.tar.gz', 65 | 'http://muct.googlecode.com/files/muct-a-jpg-v1.tar.gz', 66 | 'http://muct.googlecode.com/files/muct-b-jpg-v1.tar.gz', 67 | 'http://muct.googlecode.com/files/muct-c-jpg-v1.tar.gz', 68 | 'http://muct.googlecode.com/files/muct-d-jpg-v1.tar.gz', 69 | 'http://muct.googlecode.com/files/muct-e-jpg-v1.tar.gz', 70 | ) 71 | 72 | def __init__(self, datadir=DEFAULT_DATADIR): 73 | self._datadir = datadir 74 | self._img_fname = os.path.join(datadir, 'jpg/%s.jpg') 75 | 76 | def download(self): 77 | """downloads and unpacks the muct dataset""" 78 | # delete datadir if it already exists 79 | if os.path.exists(self._datadir): 80 | shutil.rmtree(self._datadir) 81 | # create datadir 82 | os.makedirs(self._datadir) 83 | # change directory to datadir but don't forget where you came from 84 | cwd = os.getcwd() 85 | os.chdir(self._datadir) 86 | # download all files 87 | for url in self.URLS: 88 | filename = url.split('/')[-1] 89 | download(url, filename) 90 | # unpack file if needed 91 | if filename.endswith('.tar.gz'): 92 | with tarfile.open(filename) as tar: 93 | tar.extractall() 94 | # return to original directory 95 | os.chdir(cwd) 96 | 97 | def load(self, clean=False): 98 | # read landmarks file 99 | fname = os.path.join(self._datadir, 'muct-landmarks/muct76-opencv.csv') 100 | data = np.loadtxt(fname, delimiter=',', skiprows=1, dtype=str) 101 | # separate data 102 | names = np.char.array(data[:,0]) 103 | tags = data[:,1] 104 | landmarks = data[:,2:].astype(float) 105 | # find flipped data 106 | flipped = names.startswith('ir') 107 | # keep data in self 108 | self.names = names[~flipped] 109 | self.tags = tags[~flipped] 110 | self.landmarks = landmarks[~flipped] 111 | self.landmarks_flip = landmarks[flipped] 112 | if clean: 113 | self.clean() 114 | 115 | def clean(self): 116 | """remove landmarks with unavailable points""" 117 | # unavailable points are marked with (0,0) 118 | is_complete = lambda x: all(x[::2] + x[1::2] != 0) 119 | keep = np.array(map(is_complete, self.landmarks)) 120 | self.names = self.names[keep] 121 | self.tags = self.tags[keep] 122 | self.landmarks = self.landmarks[keep] 123 | self.landmarks_flip = self.landmarks_flip[keep] 124 | 125 | def ignore(self, name): 126 | keep = self.names != name 127 | self.names = self.names[keep] 128 | self.tags = self.tags[keep] 129 | self.landmarks = self.landmarks[keep] 130 | self.landmarks_flip = self.landmarks_flip[keep] 131 | 132 | def image(self, name, flip=False): 133 | img = cv2.imread(self._img_fname % name) 134 | return cv2.flip(img, 1) if flip else img 135 | 136 | def iterimages(self, mirror=False): 137 | # iterate over all images 138 | for n in self.names: 139 | yield self.image(n) 140 | # iterate over all mirror images if required 141 | if mirror: 142 | for n in self.names: 143 | yield self.image(n, flip=True) 144 | 145 | def iterdata(self): 146 | return itertools.izip(self.names, self.tags, self.landmarks, self.landmarks_flip) 147 | 148 | def all_lmks(self): 149 | return np.concatenate((self.landmarks, self.landmarks_flip)) 150 | 151 | 152 | 153 | # http://stackoverflow.com/questions/22676/how-do-i-download-a-file-over-http-using-python/22776#22776 154 | def download(url, fname): 155 | """downloads file and shows progress""" 156 | fsize_dl = 0 157 | block_sz = 8192 158 | u = urllib2.urlopen(url) 159 | with open(fname, 'wb') as f: 160 | meta = u.info() 161 | fsize = int(meta.getheaders('Content-Length')[0]) 162 | sys.stdout.write('Downloading: %s Bytes: %s\n' % (fname, fsize)) 163 | while True: 164 | buffer = u.read(block_sz) 165 | if not buffer: 166 | break 167 | f.write(buffer) 168 | fsize_dl += len(buffer) 169 | status = '%10d [%3.2f%%]\r' % (fsize_dl, fsize_dl * 100 / fsize) 170 | sys.stdout.write(status) 171 | sys.stdout.flush() 172 | # overwrite progress message 173 | sys.stdout.write(' ' * 20 + '\r') 174 | sys.stdout.flush() 175 | 176 | 177 | 178 | # download dataset with command: 179 | # $ python -mpyaam.muct 180 | if __name__ == '__main__': 181 | muct = MuctDataset() 182 | muct.download() 183 | -------------------------------------------------------------------------------- /pyaam/patches.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import division 4 | 5 | import cv2 6 | import cv2.cv as cv 7 | import numpy as np 8 | 9 | 10 | 11 | class PatchesModel(object): 12 | def __init__(self, patches, ref_shape): 13 | self.patches = patches 14 | self.ref_shape = ref_shape 15 | 16 | @classmethod 17 | def train(cls, lmks, imgs, ref, psize, ssize, var, lmbda, mu_init, nsamples): 18 | patches = train_patches(lmks, imgs, ref, psize, ssize, var, lmbda, mu_init, nsamples) 19 | return cls(patches, ref) 20 | 21 | @classmethod 22 | def load(cls, filename): 23 | arch = np.load(filename) 24 | return cls(arch['patches'], arch['ref_shape']) 25 | 26 | def save(self, filename): 27 | np.savez(filename, patches=self.patches, ref_shape=self.ref_shape) 28 | 29 | def calc_peaks(self, img, points, ssize): 30 | points = points.reshape((len(points)//2, 2)) 31 | return calc_peaks(img, points, ssize, self.patches, self.ref_shape).flatten() 32 | 33 | 34 | 35 | def train_patch(images, psize, var=1.0, lmbda=1e-6, mu_init=1e-3, nsamples=1000): 36 | """ 37 | images - featured centered training images 38 | psize - desired patch model size 39 | var - variance of annotation error 40 | lmbda - regularization parameter 41 | mu_init - initial step size 42 | nsamples - number of stochastic samples 43 | """ 44 | 45 | h,w = psize 46 | N = len(images) 47 | n = w * h 48 | yimg, ximg = images[0].shape[:2] 49 | dx = ximg - w 50 | dy = yimg - h 51 | 52 | # ideal response map 53 | F = make_gaussian((dy,dx), var) 54 | 55 | # allocate memory 56 | dP = np.empty(psize) 57 | O = np.ones(psize) / n 58 | P = np.zeros(psize) 59 | 60 | # stochastic gradient descent 61 | mu = mu_init 62 | step = pow(1e-8/mu_init, 1/nsamples) 63 | for sample in xrange(nsamples): 64 | i = np.random.randint(N) 65 | I = convert_image(images[i]) 66 | dP[:] = 0 67 | # compute gradient direction 68 | for y in xrange(dy): 69 | for x in xrange(dx): 70 | Wi = I[y:y+h,x:x+w].copy() 71 | Wi -= Wi.dot(O) 72 | Wi = cv2.normalize(Wi) 73 | dP += (F[y,x] - P.dot(Wi)) * Wi 74 | # take a small step 75 | P += mu * (dP - lmbda*P) 76 | # reduce step size 77 | mu *= step 78 | 79 | return P 80 | 81 | def train_patches(lmks, imgs, ref, psize, ssize, var=1.0, lmbda=1e-6, mu_init=1e-3, nsamples=1000): 82 | """ 83 | ref - reference shape 84 | psize - desired patch size 85 | ssize - search window size 86 | var - variance of annotation error 87 | lmbda - regularization weight 88 | mu_init - initial stochastic gradient descent step size 89 | nsamples - number of stochastic gradient descent samples 90 | """ 91 | 92 | if isinstance(psize, int): 93 | psize = (psize, psize) 94 | if isinstance(ssize, int): 95 | ssize = (ssize, ssize) 96 | 97 | n = len(ref) // 2 98 | ximg = psize[1] + ssize[1] 99 | yimg = psize[0] + ssize[0] 100 | wsize = (yimg, ximg) 101 | 102 | patches = [] 103 | 104 | # train each patch model 105 | for i in xrange(n): 106 | print 'patch', i+1, 'of', n, '...' 107 | images = [] 108 | for j in xrange(lmks.shape[1]): 109 | im = imgs[j] 110 | pt = lmks[:,j] 111 | S = calc_simil(pt, ref) 112 | A = np.empty((2,3)) 113 | A[:2,:2] = S[:2,:2] 114 | A[0,2] = pt[2*i] - (A[0,0] * (ximg-1)/2 + A[0,1] * (yimg-1)/2) 115 | A[1,2] = pt[2*i+1] - (A[1,0] * (ximg-1)/2 + A[1,1] * (yimg-1)/2) 116 | I = cv2.warpAffine(im, A, wsize, flags=cv2.INTER_LINEAR+cv2.WARP_INVERSE_MAP) 117 | images.append(I) 118 | 119 | patch = train_patch(images, psize, var, lmbda, mu_init, nsamples) 120 | patches.append(patch) 121 | 122 | return np.array(patches) 123 | 124 | def calc_simil(pts, ref): 125 | # compute translation 126 | n = len(pts) // 2 127 | mx = pts[::2].sum() / n 128 | my = pts[1::2].sum() / n 129 | p = np.empty(pts.shape) 130 | p[::2] = pts[::2] - mx 131 | p[1::2] = pts[1::2] - my 132 | # compute rotation and scale 133 | a = np.sum(ref[::2] ** 2 + ref[1::2] ** 2) 134 | b = np.sum(ref[::2] * p[::2] + ref[1::2] * p[1::2]) 135 | c = np.sum(ref[::2] * p[1::2] + ref[1::2] * p[::2]) 136 | b /= a 137 | c /= a 138 | scale = np.sqrt(b ** 2 + c ** 2) 139 | theta = np.arctan2(c,b) 140 | sc = scale * np.cos(theta) 141 | ss = scale * np.sin(theta) 142 | return np.array([[sc,-ss,mx],[ss,sc,my]]) 143 | 144 | def make_gaussian(shape, var): 145 | """returns 2d gaussian of given shape and variance""" 146 | h,w = shape 147 | x = np.arange(w, dtype=float) 148 | y = np.arange(h, dtype=float)[:,np.newaxis] 149 | x0 = w // 2 150 | y0 = h // 2 151 | mat = np.exp(-0.5 * (pow(x-x0, 2) + pow(y-y0, 2)) / var) 152 | return cv2.normalize(mat, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX) 153 | 154 | def convert_image(img): 155 | gray = img.sum(axis=2) / img.shape[2] 156 | res = np.log(gray + 1) 157 | return res / res.max() 158 | 159 | def calc_response(img, patch, normalize=False): 160 | """calculates response map for image and patch""" 161 | img = convert_image(img).astype('float32') 162 | patch = patch.astype('float32') 163 | res = cv2.matchTemplate(img, patch, cv2.TM_CCOEFF_NORMED) 164 | if normalize: 165 | res = cv2.normalize(res, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX) 166 | res /= res.sum() 167 | return res 168 | 169 | def apply_simil(S, pts): 170 | p = np.empty(pts.shape) 171 | p[:,0] = S[0,0] * pts[:,0] + S[0,1] * pts[:,1] + S[0,2] 172 | p[:,1] = S[1,0] * pts[:,0] + S[1,1] * pts[:,1] + S[1,2] 173 | return p 174 | 175 | def inv_simil(S): 176 | Si = np.empty((2,3)) 177 | d = S[0,0] * S[1,1] - S[1,0] * S[0,1] 178 | Si[0,0] = S[1,1] / d 179 | Si[0,1] = -S[0,1] / d 180 | Si[1,0] = -S[1,0] / d 181 | Si[1,1] = S[0,0] / d 182 | Si[:,2] = -Si[:,:2].dot(S[:,2]) 183 | return Si 184 | 185 | def calc_peaks(img, points, ssize, patches, ref): 186 | assert len(points) == len(patches) 187 | pt = points.flatten() 188 | S = calc_simil(pt, ref) 189 | Si = inv_simil(S) 190 | pts = apply_simil(Si, points) 191 | for i in xrange(len(points)): 192 | patch = patches[i] 193 | psize = patch.shape 194 | wsize = (ssize[0]+psize[0],ssize[1]+psize[1]) 195 | A = np.empty((2,3)) 196 | A[:,:2] = S[:,:2] 197 | A[:,2] = points[i,:] - (A[:,0] * (wsize[1]-1)/2 + A[:,1] * (wsize[0]-1)/2) 198 | I = cv2.warpAffine(img, A, wsize, flags=cv2.INTER_LINEAR+cv2.WARP_INVERSE_MAP) 199 | R = calc_response(I, patch) 200 | maxloc = cv2.minMaxLoc(R)[-1] 201 | pts[i,0] = pts[i,0] + maxloc[0] - 0.5 * ssize[0] 202 | pts[i,1] = pts[i,1] + maxloc[1] - 0.5 * ssize[1] 203 | return apply_simil(S, pts) 204 | -------------------------------------------------------------------------------- /view_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import division 4 | 5 | import sys 6 | import cv2 7 | import cv2.cv as cv 8 | import argparse 9 | import numpy as np 10 | from pyaam.shape import ShapeModel 11 | from pyaam.patches import PatchesModel 12 | from pyaam.texture import TextureModel 13 | from pyaam.combined import CombinedModel 14 | from pyaam.draw import Color, draw_string, draw_muct_shape, draw_texture 15 | from pyaam.utils import get_vertices 16 | from pyaam.texturemapper import TextureMapper 17 | 18 | 19 | 20 | def parse_args(): 21 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 22 | parser.add_argument('model', choices=['shape', 'patches', 'texture', 'combined'], help='model name') 23 | parser.add_argument('--scale', type=float, default=200, help='scale') 24 | parser.add_argument('--tranx', type=int, default=150, help='translate x') 25 | parser.add_argument('--trany', type=int, default=150, help='translate y') 26 | parser.add_argument('--width', type=int, default=300, help='image width') 27 | parser.add_argument('--height', type=int, default=300, help='image height') 28 | parser.add_argument('--face-width', type=int, default=200, help='face width') 29 | parser.add_argument('--shp-fn', default='data/shape.npz', help='shape model filename') 30 | parser.add_argument('--ptc-fn', default='data/patches.npz', help='patches model filename') 31 | parser.add_argument('--txt-fn', default='data/texture.npz', help='texture model filename') 32 | parser.add_argument('--cmb-fn', default='data/combined.npz', help='combined model filename') 33 | return parser.parse_args() 34 | 35 | 36 | 37 | def genvals(): 38 | """generate trajectory of parameters""" 39 | vals = np.empty(200) 40 | vals[:50] = np.arange(50) / 50 41 | vals[50:100] = (50 - np.arange(50)) / 50 42 | vals[100:] = -vals[:100] 43 | return vals 44 | 45 | 46 | 47 | def view_shape_model(shp_fn, scale, tranx, trany, width, height): 48 | img = np.empty((height, width, 3), dtype='uint8') 49 | smodel = ShapeModel.load(shp_fn) 50 | vals = genvals() 51 | while True: 52 | for k in xrange(4, smodel.model.shape[1]): 53 | for v in vals: 54 | p = smodel.get_params(scale, tranx, trany) 55 | p[k] = p[0] * v * 3 * np.sqrt(smodel.variance[k]) 56 | img[:] = 0 # set to black 57 | s = 'mode: %d, val: %f sd' % (k-3, v*3) 58 | draw_string(img, s) 59 | q = smodel.calc_shape(p) 60 | draw_muct_shape(img, q) 61 | cv2.imshow('shape model', img) 62 | if cv2.waitKey(10) == 27: 63 | sys.exit() 64 | 65 | 66 | 67 | def view_texture_model(shp_fn, txt_fn, scale, tranx, trany, width, height): 68 | img = np.empty((height, width, 3), dtype='uint8') 69 | smodel = ShapeModel.load(shp_fn) 70 | tmodel = TextureModel.load(txt_fn) 71 | vals = genvals() 72 | # get reference shape 73 | ref = smodel.get_shape(scale, tranx, trany) 74 | ref = ref.reshape((ref.size//2, 2)) 75 | while True: 76 | for k in xrange(tmodel.num_modes()): 77 | for v in vals: 78 | p = np.zeros(tmodel.num_modes()) 79 | p[k] = v * 3 * np.sqrt(tmodel.variance[k]) 80 | img[:] = 0 81 | s = 'mode: %d, val: %f sd' % (k, v*3) 82 | draw_string(img, s) 83 | t = tmodel.calc_texture(p) 84 | draw_texture(img, t, ref) 85 | cv2.imshow('texture model', img) 86 | if cv2.waitKey(10) == 27: 87 | sys.exit() 88 | 89 | 90 | 91 | def view_combined_model(shp_fn, txt_fn, cmb_fn, scale, tranx, trany, width, height): 92 | img = np.empty((height, width, 3), dtype='uint8') 93 | cv2.namedWindow('combined model') 94 | tm = TextureMapper(img.shape[1], img.shape[0]) 95 | smodel = ShapeModel.load(shp_fn) 96 | tmodel = TextureModel.load(txt_fn) 97 | cmodel = CombinedModel.load(cmb_fn) 98 | vals = genvals() 99 | params = smodel.get_params(scale, tranx, trany) 100 | ref = smodel.calc_shape(params) 101 | ref = ref.reshape((ref.size//2, 2)) 102 | verts = get_vertices(ref) 103 | while True: 104 | for k in xrange(cmodel.num_modes()): 105 | for v in vals: 106 | p = np.zeros(cmodel.num_modes()) 107 | p[k] = v * 3 * np.sqrt(cmodel.variance[k]) 108 | sparams, tparams = cmodel.calc_shp_tex_params(p, smodel.num_modes()) 109 | params[4:] = sparams 110 | 111 | shp = smodel.calc_shape(params) 112 | shp = shp.reshape(ref.shape) 113 | t = tmodel.calc_texture(tparams) 114 | img[:] = 0 115 | draw_texture(img, t, ref) 116 | warped = tm.warp_triangles(img, ref[verts], shp[verts]) 117 | 118 | s = 'mode: %d, val: %f sd' % (k, v*3) 119 | draw_string(warped, s) 120 | cv2.imshow('combined model', warped) 121 | 122 | if cv2.waitKey(10) == 27: 123 | sys.exit() 124 | 125 | 126 | 127 | def view_patches_model(ptc_fn, shp_fn, width): 128 | pmodel = PatchesModel.load(ptc_fn) 129 | smodel = ShapeModel.load(shp_fn) 130 | ref = pmodel.ref_shape 131 | ref = np.column_stack((ref[::2], ref[1::2])) 132 | # compute scale factor 133 | scale = width / ref[:,0].ptp() 134 | height = int(scale * ref[:,1].ptp() + 0.5) 135 | # compute image width 136 | max_height = int(scale * pmodel.patches.shape[1]) 137 | max_width = int(scale * pmodel.patches.shape[2]) 138 | # create reference image 139 | image_size = (height+4*max_height, width+4*max_width, 3) 140 | image = np.empty(image_size, dtype='uint8') 141 | image[:] = 192 142 | patches = [] 143 | points = [] 144 | for i in xrange(len(pmodel.patches)): 145 | im = cv2.normalize(pmodel.patches[i], alpha=0, beta=255, norm_type=cv2.NORM_MINMAX) 146 | im = cv2.resize(im, (int(scale*im.shape[0]), int(scale*im.shape[1]))) 147 | im = im.astype('uint8') 148 | patches.append(cv2.cvtColor(im, cv.CV_GRAY2BGR)) 149 | h,w = patches[i].shape[:2] 150 | points.append((int(scale*ref[i,1] + image_size[0]/2 - h/2), 151 | int(scale*ref[i,0] + image_size[1]/2 - w/2))) 152 | y,x = points[i] 153 | image[y:y+h,x:x+w,:] = patches[i] 154 | cv2.namedWindow('patches model') 155 | i = 0 156 | while True: 157 | img = image.copy() 158 | y,x = points[i] 159 | h,w = patches[i].shape[:2] 160 | img[y:y+h,x:x+w,:] = patches[i] # draw current patch on top 161 | cv2.rectangle(img, (x,y), (x+w, y+h), Color.red, 2, cv2.CV_AA) 162 | text = 'patch %d' % (i+1) 163 | draw_string(img, text) 164 | cv2.imshow('patches model', img) 165 | c = cv2.waitKey(0) 166 | if c == 27: 167 | break 168 | elif c == ord('j'): 169 | i += 1 170 | elif c == ord('k'): 171 | i -= 1 172 | if i < 0: 173 | i = 0 174 | elif i >= len(pmodel.patches): 175 | i = len(pmodel.patches) - 1 176 | 177 | 178 | 179 | if __name__ == '__main__': 180 | args = parse_args() 181 | 182 | if args.model == 'shape': 183 | view_shape_model(args.shp_fn, args.scale, args.tranx, args.trany, 184 | args.width, args.height) 185 | 186 | elif args.model == 'patches': 187 | view_patches_model(args.ptc_fn, args.shp_fn, args.face_width) 188 | 189 | elif args.model == 'texture': 190 | view_texture_model(args.shp_fn, args.txt_fn, 200, 150, 150, args.width, args.height) 191 | 192 | elif args.model == 'combined': 193 | view_combined_model(args.shp_fn, args.txt_fn, args.cmb_fn, 200, 150, 150, args.width, args.height) 194 | --------------------------------------------------------------------------------