├── README.md ├── depth2mesh.p ├── imgs ├── 2327-3.jpg └── teaser.png ├── mesh2fine.p ├── raw2depth.m ├── run_net.lua ├── run_net.m ├── runme.m └── surface_show.m /README.md: -------------------------------------------------------------------------------- 1 | # Unrestricted Facial Geometry Reconstruction Using Image-to-Image Translation 2 | [[Arxiv]](https://arxiv.org/pdf/1703.10131.pdf) [[Video]](https://www.youtube.com/watch?v=6lUdSVcBB-k) 3 | 4 | Evaluation code for Unrestricted Facial Geometry Reconstruction Using Image-to-Image Translation. Given a single image, the code outputs the reconstructed mesh. 5 | 6 | 7 | 8 | ## Recent Updates 9 | **`2020.05.07`**: We have released a [PyTorch version](https://github.com/eladrich/pix2vertex.pytorch) of this repository and encourage you to give it a try! 10 | 11 | **`2018.07.20`**: Spiltted `postprocess` into the different steps composing the pipeline, changed models to `float` to save space. 12 | 13 | ## Setup & Usage 14 | The project was tested on Ubuntu 14.04 LTS with Matlab R2015b, to run it follow these instructions: 15 | - Make sure you have Torch installed on your machine. 16 | - Install the ```mattorch``` and ```nngraph``` packages. 17 | 18 | ```bash 19 | luarocks install mattorch 20 | luarocks install nngraph 21 | ``` 22 | - Download the model files and extract them into the ```models``` directroy. 23 | 24 | - Run the ```runme.m``` script in Matlab. 25 | 26 | ## Citation 27 | If you use this code for your research, please cite our paper Unrestricted Facial Geometry Reconstruction Using Image-to-Image Translation: 28 | 29 | ``` 30 | @article{sela2017unrestricted, 31 | title={Unrestricted Facial Geometry Reconstruction Using Image-to-Image Translation}, 32 | author={Sela, Matan and Richardson, Elad and Kimmel, Ron}, 33 | journal={arxiv}, 34 | year={2017} 35 | } 36 | ``` 37 | -------------------------------------------------------------------------------- /depth2mesh.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matansel/pix2vertex/8b1afa660dca757e50edbb3f4ae9d3251d6d60d7/depth2mesh.p -------------------------------------------------------------------------------- /imgs/2327-3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matansel/pix2vertex/8b1afa660dca757e50edbb3f4ae9d3251d6d60d7/imgs/2327-3.jpg -------------------------------------------------------------------------------- /imgs/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matansel/pix2vertex/8b1afa660dca757e50edbb3f4ae9d3251d6d60d7/imgs/teaser.png -------------------------------------------------------------------------------- /mesh2fine.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/matansel/pix2vertex/8b1afa660dca757e50edbb3f4ae9d3251d6d60d7/mesh2fine.p -------------------------------------------------------------------------------- /raw2depth.m: -------------------------------------------------------------------------------- 1 | function [ vis_Z, pipeline_args ] = raw2depth( img, im_pncc, im_depth ) 2 | %Apply preprocessing to the raw network result 3 | 4 | pipeline_args = {}; 5 | 6 | im_depth(im_depth<10 & im_depth>-10) = 0; 7 | im_pncc(im_pncc<10 & im_pncc>-10) = 0; 8 | 9 | load models/template; 10 | Sm = [template.X template.Y template.Z]; 11 | %Get points from network 12 | net_X = im_depth(:,:,1)*(max(Sm(:,1))-min(Sm(:,1)))/255+min(Sm(:,1)); 13 | net_Y = im_depth(:,:,2)*(max(Sm(:,2))-min(Sm(:,2)))/255+min(Sm(:,2)); 14 | net_Z = im_depth(:,:,3)*(max(Sm(:,3))-min(Sm(:,3)))/255+min(Sm(:,3)); 15 | 16 | mask = all(im_depth,3).*all(im_pncc,3); 17 | %mask = ~(all(im_depth<15 & im_depth>-15,3).*all(im_pncc<15 & im_pncc>-15,3)); 18 | inds = find(mask); %Look only on existing indices 19 | 20 | %Define uniform grid 21 | X = repmat(linspace(-1,1,size(im_depth,1)),size(im_depth,2),1); 22 | Y = repmat(linspace(1,-1,size(im_depth,2)),size(im_depth,1),1)'; 23 | 24 | %Normalize grid according to the network result 25 | X = (X-mean(X(inds)))/std(X(inds))*std(net_X(inds))+mean(net_X(inds)); 26 | Y = (Y-mean(Y(inds)))/std(Y(inds))*std(net_Y(inds))+mean(net_Y(inds)); 27 | Z = net_Z; 28 | 29 | f=1/(X(1,2)-X(1,1)); 30 | 31 | %Calculate mask 32 | se = strel('disk',3); 33 | mask = imerode(mask,se); 34 | BW = logical(mask); %%// Coins photo from MATLAB Library 35 | [L, num] = bwlabel(BW, 8); 36 | count_pixels_per_obj = sum(bsxfun(@eq,L(:),1:num)); 37 | [~,ind] = max(count_pixels_per_obj); 38 | mask = (L==ind); 39 | 40 | 41 | vis_Z = Z*f; 42 | vis_Z(~mask) = NaN; 43 | 44 | pipeline_args.template = template; 45 | pipeline_args.img = img; 46 | pipeline_args.im_pncc = im_pncc; 47 | pipeline_args.X = X; 48 | pipeline_args.Y = Y; 49 | pipeline_args.Z = Z; 50 | pipeline_args.mask = mask; 51 | 52 | end 53 | 54 | -------------------------------------------------------------------------------- /run_net.lua: -------------------------------------------------------------------------------- 1 | require 'nn' 2 | require 'nngraph' 3 | require 'mattorch' 4 | 5 | 6 | function parse_args() 7 | cmd = torch.CmdLine() 8 | cmd:text('Options') 9 | -- general options: 10 | cmd:option('-model', '', 'Model Path') 11 | cmd:option('-inputs', '', 'Images Path') 12 | cmd:option('-res', '', 'Res Path') 13 | cmd:text() 14 | params = cmd:parse(arg) 15 | return params 16 | end 17 | 18 | params = parse_args() 19 | 20 | 21 | model = torch.load(params.model) 22 | input = mattorch.load(params.inputs) 23 | in_im = input.img 24 | in_im = in_im:view(1,in_im:size(1),in_im:size(2),in_im:size(3)) 25 | in_im = in_im:permute(1,2,4,3) 26 | in_im:div(255.0):mul(2):add(-1) --Input Normalization 27 | output = model:forward(in_im:float()) 28 | output:add(1):div(2):mul(255) --Output Denormalization 29 | output = output:permute(1,2,4,3) 30 | mattorch.save(params.res, output:double()) 31 | -------------------------------------------------------------------------------- /run_net.m: -------------------------------------------------------------------------------- 1 | function [ res ] = run_net( img, net_path, save_path ) 2 | %run_net - Run the network in net_path 3 | % img - Input Image 4 | % net_path - The network path 5 | % save_path - Path for saving the network's output 6 | 7 | matname = [save_path '.mat']; 8 | img = double(img); 9 | save(matname, 'img'); 10 | resname = [save_path '-res.mat']; 11 | system(['OMP_NUM_THREADS=1 th run_net.lua -model ' net_path ' -inputs ' matname ... 12 | ' -res ' resname ]); 13 | net_res = load(resname); 14 | delete(matname); 15 | delete(resname); 16 | res = net_res.x; 17 | 18 | 19 | end 20 | 21 | -------------------------------------------------------------------------------- /runme.m: -------------------------------------------------------------------------------- 1 | clear; close all; clc; 2 | % This script runs an evaluation of the 3d face reconstruction method proposed in 3 | % Unrestricted Facial Geometry Reconstruction Using Image-to-Image Translation 4 | im_file = 'imgs/2327-3.jpg'; 5 | img = imread(im_file); 6 | img_size = 512; 7 | show_figs = true; 8 | 9 | % Crop input image 10 | FaceDetect = vision.CascadeObjectDetector; 11 | BB = step(FaceDetect,img); 12 | imsz = size(img); 13 | 14 | BB(1,1) = max(1,min(imsz(1),BB(1,1)-BB(1,3)*.1)); 15 | BB(1,2) = max(1,min(imsz(2),BB(1,2)-BB(1,4)*.1)); 16 | BB(1,3) = max(1,min(imsz(1),BB(1,3)*1.3)); 17 | BB(1,4) = max(1,min(imsz(1),BB(1,4)*1.3)); 18 | img= imcrop(img,BB(1,:)); 19 | img = imresize(img, [img_size img_size]); 20 | 21 | if show_figs 22 | figure(1); 23 | subplot(1,3,1); 24 | imshow(img); 25 | title('Cropped Input'); 26 | drawnow; 27 | end 28 | 29 | %% Run networks 30 | fprintf('Running Net...'); 31 | im_pncc = run_net( img, 'models/pncc_net_float.t7', im_file(1:end-4)); 32 | im_depth = run_net( img, 'models/depth_net_float.t7', im_file(1:end-4)); 33 | fprintf(' Done!\n'); 34 | 35 | % Visualize Network Result 36 | if show_figs 37 | figure(1); 38 | subplot(1,3,2); 39 | imshow(im_pncc/255); 40 | title('Correspondence'); 41 | subplot(1,3,3); 42 | imshow(im_depth(:,:,3)/255); 43 | title('Depth'); 44 | drawnow; 45 | end 46 | 47 | %% Process network result (scale and mask) 48 | 49 | [ Z, pipeline_args ] = raw2depth( img, im_pncc, im_depth ); 50 | 51 | if show_figs 52 | figure; 53 | set(gcf,'color','k'); 54 | sub_1 = subplot(1,2,1); 55 | h=surf(fliplr(Z),fliplr(img)); 56 | set(h,'LineStyle','none') 57 | axis equal 58 | grid off 59 | box on 60 | axis off 61 | view(-180,90) 62 | light('Position',[0 0 1]);lighting phong;material([.5 .6 .1]); 63 | title('Textured Network Result','Color', 'w'); 64 | 65 | sub_2 = subplot(1,2,2); 66 | h=surf(fliplr(Z)); 67 | set(h,'FaceColor',[.7 .8 1]); 68 | set(h,'LineStyle','none') 69 | axis equal 70 | grid off 71 | box on 72 | axis off 73 | view(-180,90) 74 | light('Position',[0 0 1]);lighting phong;material([.5 .6 .1]); 75 | title('Network Result','Color', 'w'); 76 | 77 | linkprop([sub_1 sub_2], 'View'); 78 | end 79 | 80 | %% Apply rigid deformation 81 | [mesh_result, pipeline_args] = depth2mesh( pipeline_args, show_figs ); 82 | 83 | if show_figs 84 | figure; 85 | sub_p(1) = subplot(1,2,1); 86 | surface_show(mesh_result.face,mesh_result.vertex,[.7 .8 1]);material([.5 .6 .3]) 87 | lighting phong;material([.5 .6 .1]); 88 | title('Mesh Result','Color', 'w'); 89 | sub_p(2) = subplot(1,2,2); 90 | surface_show(mesh_result.face,mesh_result.vertex,mesh_result.texture);material([.5 .6 .3]) 91 | lighting phong;material([.5 .6 .1]); 92 | title('Textured Mesh Result','Color', 'w'); 93 | linkprop([sub_p(1) sub_p(2)], 'View'); 94 | drawnow; 95 | end 96 | 97 | %% Apply detail extraction 98 | 99 | [ fine_result ] = mesh2fine(pipeline_args); 100 | 101 | if show_figs 102 | figure; 103 | sub_p(1) = subplot(1,2,1); 104 | surface_show(fine_result.face,fine_result.vertex,[.7 .8 1]);material([.5 .6 .3]) 105 | lighting phong;material([.5 .6 .1]); 106 | title('Final Result','Color', 'w'); 107 | sub_p(2) = subplot(1,2,2); 108 | surface_show(fine_result.face,fine_result.vertex,fine_result.texture);material([.5 .6 .3]) 109 | lighting phong;material([.5 .6 .1]); 110 | title('Textured Final Result','Color', 'w'); 111 | linkprop([sub_p(1) sub_p(2)], 'View'); 112 | drawnow; 113 | end -------------------------------------------------------------------------------- /surface_show.m: -------------------------------------------------------------------------------- 1 | function h = surface_show(varargin) 2 | if nargin ==1; 3 | tri = varargin{1}.tri; 4 | if isfield(varargin{1},'V') 5 | x2 = varargin{1}.V(:,1); 6 | y2 = varargin{1}.V(:,2); 7 | z2 = varargin{1}.V(:,3); 8 | else 9 | x2 = varargin{1}.X; 10 | y2 = varargin{1}.Y; 11 | z2 = varargin{1}.Z; 12 | end; 13 | if isfield(varargin{1},'I'); 14 | I = varargin{1}.I; 15 | end; 16 | elseif nargin==2 17 | tri = varargin{1}; 18 | x2 = varargin{2}(:,1); 19 | y2 = varargin{2}(:,2); 20 | z2 = varargin{2}(:,3); 21 | elseif nargin==3 22 | tri = varargin{1}; 23 | x2 = varargin{2}(:,1); 24 | y2 = varargin{2}(:,2); 25 | z2 = varargin{2}(:,3); 26 | I = varargin{3}; 27 | elseif nargin==4 28 | tri = varargin{1}; 29 | x2 = varargin{2}; 30 | y2 = varargin{3}; 31 | z2 = varargin{4}; 32 | elseif nargin==5 33 | tri = varargin{1}; 34 | x2 = varargin{2}; 35 | y2 = varargin{3}; 36 | z2 = varargin{4}; 37 | I = varargin{5}; 38 | end; 39 | 40 | if ~exist('I'); 41 | h = trisurf(tri,x2,y2,z2); 42 | shading interp; 43 | colormap gray; 44 | else 45 | if size(I,1)==1 46 | h=trisurf(tri,x2,y2,z2); 47 | shading interp; 48 | set(h,'FaceColor',I); 49 | else 50 | if size(I,2)==3; 51 | 52 | %I = I-min(min(I)); 53 | %I = (double(I)./double(max(max(I))))*255; 54 | I(I<0) = 0; 55 | I(I>255) = 255; 56 | C=double(I)/255; 57 | colormap(C); 58 | h = trisurf(tri,x2,y2,z2,1:size(x2,1),'edgecolor','none');%,'Visible','off'); 59 | %lighting phong; 60 | shading flat; 61 | else 62 | h = trisurf(tri,x2,y2,z2,I); 63 | shading interp; 64 | colormap jet; 65 | end; 66 | end 67 | end; 68 | axis equal; 69 | set(gcf,'color',[0 0 0]); 70 | axis off; 71 | grid off; 72 | %cameratoolbar; 73 | set(gca,'CameraViewAngleMode','manual'); 74 | camtarget([0 0 0]); 75 | %camup([0 1 0]); 76 | campos([0 0 14]); 77 | camproj('perspective'); 78 | light('Position',[0 0 1]);lighting phong;material dull 79 | 80 | end --------------------------------------------------------------------------------