├── FaceEditor.zip ├── README.md ├── cpu.theanorc ├── datagen.py ├── dutil.py ├── encoder.py ├── face_edit.py └── gpu.theanorc /FaceEditor.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HackerPoet/FaceEditor/69519606315925c237485a82223391cb3f511ec3/FaceEditor.zip -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Face Editor 2 | Unsupervised learning to find facial features. 3 | https://youtu.be/4VAkrUNLKSo 4 | -------------------------------------------------------------------------------- /cpu.theanorc: -------------------------------------------------------------------------------- 1 | [global] 2 | floatX=float32 3 | device=cpu 4 | -------------------------------------------------------------------------------- /datagen.py: -------------------------------------------------------------------------------- 1 | import os, random, sys 2 | import numpy as np 3 | import cv2 4 | from dutil import * 5 | 6 | NUM_IMAGES = 1769 7 | SAMPLES_PER_IMG = 10 8 | DOTS_PER_IMG = 60 9 | IMAGE_W = 144 10 | IMAGE_H = 192 11 | IMAGE_DIR = 'YB_PICTURES' 12 | NUM_SAMPLES = NUM_IMAGES * 2 * SAMPLES_PER_IMG 13 | 14 | def center_resize(img): 15 | assert(IMAGE_W == IMAGE_H) 16 | w, h = img.shape[0], img.shape[1] 17 | if w > h: 18 | x = (w-h)/2 19 | img = img[x:x+h,:] 20 | elif h > w: 21 | img = img[:,0:w] 22 | return cv2.resize(img, (IMAGE_W, IMAGE_H), interpolation = cv2.INTER_LINEAR) 23 | 24 | def yb_resize(img): 25 | assert(img.shape[1] == 151) 26 | assert(img.shape[0] == 197) 27 | return cv2.resize(img, (IMAGE_W, IMAGE_H), interpolation = cv2.INTER_LINEAR) 28 | 29 | def rand_dots(img, sample_ix): 30 | sample_ratio = float(sample_ix) / SAMPLES_PER_IMG 31 | return auto_canny(img, sample_ratio) 32 | 33 | x_data = np.empty((NUM_SAMPLES, NUM_CHANNELS, IMAGE_H, IMAGE_W), dtype=np.uint8) 34 | y_data = np.empty((NUM_SAMPLES, 3, IMAGE_H, IMAGE_W), dtype=np.uint8) 35 | ix = 0 36 | for root, subdirs, files in os.walk(IMAGE_DIR): 37 | for file in files: 38 | path = root + "\\" + file 39 | if not (path.endswith('.jpg') or path.endswith('.png')): 40 | continue 41 | img = cv2.imread(path) 42 | if img is None: 43 | assert(False) 44 | if len(img.shape) != 3 or img.shape[2] != 3: 45 | assert(False) 46 | if img.shape[0] < IMAGE_H or img.shape[1] < IMAGE_W: 47 | assert(False) 48 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 49 | img = yb_resize(img) 50 | for i in xrange(SAMPLES_PER_IMG): 51 | y_data[ix] = np.transpose(img, (2, 0, 1)) 52 | x_data[ix] = rand_dots(img, i) 53 | if ix < SAMPLES_PER_IMG*16: 54 | outimg = x_data[ix][0] 55 | cv2.imwrite('cargb' + str(ix) + '.png', outimg) 56 | print path 57 | ix += 1 58 | y_data[ix] = np.flip(y_data[ix - 1], axis=2) 59 | x_data[ix] = np.flip(x_data[ix - 1], axis=2) 60 | ix += 1 61 | 62 | sys.stdout.write('\r') 63 | progress = ix * 100 / NUM_SAMPLES 64 | sys.stdout.write(str(progress) + "%") 65 | sys.stdout.flush() 66 | assert(ix <= NUM_SAMPLES) 67 | 68 | assert(ix == NUM_SAMPLES) 69 | print "Saving..." 70 | np.save('x_data.npy', x_data) 71 | np.save('y_data.npy', y_data) -------------------------------------------------------------------------------- /dutil.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | 4 | def add_pos(arr): 5 | s = arr.shape 6 | result = np.empty((s[0], s[1] + 2, s[2], s[3]), dtype=np.float32) 7 | result[:,:s[1],:,:] = arr 8 | x = np.repeat(np.expand_dims(np.arange(s[3]) / float(s[3]), axis=0), s[2], axis=0) 9 | y = np.repeat(np.expand_dims(np.arange(s[2]) / float(s[2]), axis=0), s[3], axis=0) 10 | result[:,s[1] + 0,:,:] = x 11 | result[:,s[1] + 1,:,:] = np.transpose(y) 12 | return result 13 | 14 | def auto_canny(image, sigma=0.0): 15 | gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) 16 | grayed = np.where(gray < 20, 255, 0) 17 | 18 | lower = sigma*128 + 128 19 | upper = 255 20 | edged = cv2.Canny(image, lower, upper) 21 | 22 | return np.maximum(edged, grayed) 23 | 24 | def save_image(x, fname): 25 | img = np.transpose(x * 255, (1, 2, 0)) 26 | img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_RGB2BGR) 27 | cv2.imwrite(fname, img) 28 | -------------------------------------------------------------------------------- /encoder.py: -------------------------------------------------------------------------------- 1 | import sys, random 2 | import numpy as np 3 | from matplotlib import pyplot as plt 4 | from dutil import * 5 | import pydot 6 | 7 | SHIFT_AMOUNT = 9 8 | BATCH_SIZE = 8 9 | NUM_KERNELS = 20 10 | CONTINUE_TRAIN = False 11 | 12 | NUM_EPOCHS = 2000 13 | PARAM_SIZE = 80 14 | LR = 0.001 15 | NUM_RAND_FACES = BATCH_SIZE 16 | NUM_TEST_FACES = BATCH_SIZE 17 | 18 | def plotScores(scores, test_scores, fname, on_top=True): 19 | plt.clf() 20 | ax = plt.gca() 21 | ax.yaxis.tick_right() 22 | ax.yaxis.set_ticks_position('both') 23 | ax.yaxis.grid(True) 24 | plt.plot(scores) 25 | plt.plot(test_scores) 26 | plt.xlabel('Epoch') 27 | plt.ylim([0.0, 0.01]) 28 | loc = ('upper right' if on_top else 'lower right') 29 | plt.legend(['Train', 'Test'], loc=loc) 30 | plt.draw() 31 | plt.savefig(fname) 32 | 33 | #Load data set 34 | print "Loading Data..." 35 | y_train = np.load('y_data.npy').astype(np.float32) / 255.0 36 | y_train = y_train[:y_train.shape[0] - y_train.shape[0] % BATCH_SIZE] 37 | x_train = np.expand_dims(np.arange(y_train.shape[0]), axis=1) 38 | num_samples = y_train.shape[0] 39 | print "Loaded " + str(num_samples) + " Samples." 40 | 41 | ################################### 42 | # Create Model 43 | ################################### 44 | print "Loading Keras..." 45 | import os, math 46 | os.environ['THEANORC'] = "./gpu.theanorc" 47 | os.environ['KERAS_BACKEND'] = "theano" 48 | import theano 49 | print "Theano Version: " + theano.__version__ 50 | 51 | from keras.initializers import RandomUniform 52 | from keras.layers import Input, Dense, Activation, Dropout, Flatten, Reshape, SpatialDropout2D 53 | from keras.layers.convolutional import Conv2D, Conv2DTranspose, UpSampling2D 54 | from keras.layers.embeddings import Embedding 55 | from keras.layers.local import LocallyConnected2D 56 | from keras.layers.pooling import MaxPooling2D 57 | from keras.layers.noise import GaussianNoise 58 | from keras.models import Model, Sequential, load_model 59 | from keras.optimizers import Adam, RMSprop, SGD 60 | from keras.preprocessing.image import ImageDataGenerator 61 | from keras.regularizers import l1 62 | from keras.utils import plot_model 63 | from keras import backend as K 64 | K.set_image_data_format('channels_first') 65 | 66 | if CONTINUE_TRAIN: 67 | print "Loading Model..." 68 | model = load_model('Encoder.h5') 69 | else: 70 | print "Building Model..." 71 | model = Sequential() 72 | 73 | model.add(Embedding(num_samples, PARAM_SIZE, input_length=1)) 74 | model.add(Flatten(name='pre_encoder')) 75 | print model.output_shape 76 | assert(model.output_shape == (None, PARAM_SIZE)) 77 | 78 | model.add(Reshape((PARAM_SIZE, 1, 1), name='encoder')) 79 | print model.output_shape 80 | 81 | model.add(Conv2DTranspose(256, (4, 1))) #(4, 1) 82 | model.add(Activation("relu")) 83 | print model.output_shape 84 | 85 | model.add(Conv2DTranspose(256, 4)) #(7, 4) 86 | model.add(Activation("relu")) 87 | print model.output_shape 88 | 89 | model.add(Conv2DTranspose(256, 4)) #(10, 7) 90 | model.add(Activation("relu")) 91 | print model.output_shape 92 | 93 | model.add(Conv2DTranspose(256, 4, strides=2)) #(22, 16) 94 | model.add(Activation("relu")) 95 | print model.output_shape 96 | 97 | model.add(Conv2DTranspose(128, 4, strides=2)) #(46, 34) 98 | model.add(Activation("relu")) 99 | print model.output_shape 100 | 101 | model.add(Conv2DTranspose(128, 4, strides=2)) #(94, 70) 102 | model.add(Activation("relu")) 103 | print model.output_shape 104 | 105 | model.add(Conv2DTranspose(3, 6, strides=2)) #(192, 144) 106 | model.add(Activation("sigmoid")) 107 | print model.output_shape 108 | assert(model.output_shape[1:] == (3, 192, 144)) 109 | 110 | model.compile(optimizer=Adam(lr=LR), loss='mse') 111 | plot_model(model, to_file='model.png', show_shapes=True) 112 | 113 | ################################### 114 | # Encoder / Decoder 115 | ################################### 116 | print "Compiling SubModels..." 117 | func = K.function([model.get_layer('encoder').input, K.learning_phase()], 118 | [model.layers[-1].output]) 119 | enc_model = Model(inputs=model.input, 120 | outputs=model.get_layer('pre_encoder').output) 121 | 122 | rand_vecs = np.random.normal(0.0, 1.0, (NUM_RAND_FACES, PARAM_SIZE)) 123 | 124 | def make_rand_faces(rand_vecs, iters): 125 | x_enc = enc_model.predict(x_train, batch_size=BATCH_SIZE) 126 | 127 | x_mean = np.mean(x_enc, axis=0) 128 | x_stds = np.std(x_enc, axis=0) 129 | x_cov = np.cov((x_enc - x_mean).T) 130 | e, v = np.linalg.eig(x_cov) 131 | 132 | np.save('means.npy', x_mean) 133 | np.save('stds.npy', x_stds) 134 | np.save('evals.npy', e) 135 | np.save('evecs.npy', v) 136 | 137 | e_list = e.tolist() 138 | e_list.sort(reverse=True) 139 | plt.clf() 140 | plt.bar(np.arange(e.shape[0]), e_list, align='center') 141 | plt.draw() 142 | plt.savefig('evals.png') 143 | 144 | x_vecs = x_mean + np.dot(v, (rand_vecs * e).T).T 145 | y_faces = func([x_vecs, 0])[0] 146 | for i in xrange(y_faces.shape[0]): 147 | save_image(y_faces[i], 'rand' + str(i) + '.png') 148 | if i < 5 and (iters % 10) == 0: 149 | if not os.path.exists('morph' + str(i)): 150 | os.makedirs('morph' + str(i)) 151 | save_image(y_faces[i], 'morph' + str(i) + '/img' + str(iters) + '.png') 152 | 153 | make_rand_faces(rand_vecs, 0) 154 | 155 | ################################### 156 | # Train 157 | ################################### 158 | print "Training..." 159 | train_loss = [] 160 | 161 | for iters in xrange(NUM_EPOCHS): 162 | history = model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=1) 163 | 164 | loss = history.history['loss'][-1] 165 | train_loss.append(loss) 166 | print "Loss: " + str(loss) 167 | 168 | plotScores(train_loss, [], 'EncoderScores.png', True) 169 | 170 | if iters % 20 == 0: 171 | model.save('Encoder.h5') 172 | 173 | y_faces = model.predict(x_train[:NUM_TEST_FACES], batch_size=BATCH_SIZE) 174 | for i in xrange(y_faces.shape[0]): 175 | save_image(y_faces[i], 'gt' + str(i) + '.png') 176 | 177 | make_rand_faces(rand_vecs, iters) 178 | 179 | print "Saved" 180 | 181 | print "Done" 182 | -------------------------------------------------------------------------------- /face_edit.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | import random 3 | import numpy as np 4 | import cv2 5 | import h5py 6 | from dutil import * 7 | 8 | #User constants 9 | device = "cpu" 10 | enc_fname = 'Encoder.h5' 11 | background_color = (210, 210, 210) 12 | edge_color = (60, 60, 60) 13 | slider_color = (20, 20, 20) 14 | num_params = 80 15 | input_w = 144 16 | input_h = 192 17 | image_scale = 3 18 | image_padding = 10 19 | slider_w = 15 20 | slider_h = 100 21 | slider_px = 5 22 | slider_py = 10 23 | slider_cols = 20 24 | 25 | #Derived constants 26 | slider_w = slider_w + slider_px*2 27 | slider_h = slider_h + slider_py*2 28 | drawing_x = image_padding 29 | drawing_y = image_padding 30 | drawing_w = input_w * image_scale 31 | drawing_h = input_h * image_scale 32 | slider_rows = (num_params - 1) / slider_cols + 1 33 | sliders_x = drawing_x + drawing_w + image_padding 34 | sliders_y = image_padding 35 | sliders_w = slider_w * slider_cols 36 | sliders_h = slider_h * slider_rows 37 | window_w = drawing_w + image_padding*3 + sliders_w 38 | window_h = drawing_h + image_padding*2 39 | 40 | #Global variables 41 | prev_mouse_pos = None 42 | mouse_pressed = False 43 | cur_slider_ix = 0 44 | needs_update = True 45 | cur_params = np.zeros((num_params,), dtype=np.float32) 46 | cur_face = np.zeros((3, input_h, input_w), dtype=np.uint8) 47 | rgb_array = np.zeros((input_h, input_w, 3), dtype=np.uint8) 48 | 49 | #Keras 50 | print "Loading Keras..." 51 | import os 52 | os.environ['THEANORC'] = "./" + device + ".theanorc" 53 | os.environ['KERAS_BACKEND'] = "theano" 54 | import theano 55 | print "Theano Version: " + theano.__version__ 56 | from keras.models import Sequential, load_model, model_from_json 57 | from keras.layers import Dense, Activation, Dropout, Flatten, Reshape 58 | from keras.layers.convolutional import Conv2D, Conv2DTranspose, ZeroPadding2D 59 | from keras.layers.pooling import MaxPooling2D 60 | from keras.layers.noise import GaussianNoise 61 | from keras.layers.local import LocallyConnected2D 62 | from keras.optimizers import Adam, RMSprop, SGD 63 | from keras.regularizers import l2 64 | from keras.layers.advanced_activations import ELU 65 | from keras.preprocessing.image import ImageDataGenerator 66 | from keras.utils import plot_model 67 | from keras import backend as K 68 | K.set_image_data_format('channels_first') 69 | 70 | print "Loading Encoder..." 71 | enc_model = load_model(enc_fname) 72 | enc = K.function([enc_model.get_layer('encoder').input, K.learning_phase()], 73 | [enc_model.layers[-1].output]) 74 | 75 | print "Loading Statistics..." 76 | means = np.load('means.npy') 77 | stds = np.load('stds.npy') 78 | evals = np.sqrt(np.load('evals.npy')) 79 | evecs = np.load('evecs.npy') 80 | 81 | sort_inds = np.argsort(-evals) 82 | evals = evals[sort_inds] 83 | evecs = evecs[:,sort_inds] 84 | 85 | #Open a window 86 | pygame.init() 87 | pygame.font.init() 88 | screen = pygame.display.set_mode((window_w, window_h)) 89 | face_surface_mini = pygame.Surface((input_w, input_h)) 90 | face_surface = screen.subsurface((drawing_x, drawing_y, drawing_w, drawing_h)) 91 | pygame.display.set_caption('Face Editor - By ') 92 | font = pygame.font.SysFont("monospace", 15) 93 | 94 | def update_mouse_click(mouse_pos): 95 | global cur_slider_ix 96 | global mouse_pressed 97 | x = (mouse_pos[0] - sliders_x) 98 | y = (mouse_pos[1] - sliders_y) 99 | 100 | if x >= 0 and y >= 0 and x < sliders_w and y < sliders_h: 101 | slider_ix_w = x / slider_w 102 | slider_ix_h = y / slider_h 103 | 104 | cur_slider_ix = slider_ix_h * slider_cols + slider_ix_w 105 | mouse_pressed = True 106 | 107 | def update_mouse_move(mouse_pos): 108 | global needs_update 109 | y = (mouse_pos[1] - sliders_y) 110 | 111 | if y >= 0 and y < sliders_h: 112 | slider_row_ix = cur_slider_ix / slider_cols 113 | slider_val = y - slider_row_ix * slider_h 114 | 115 | slider_val = min(max(slider_val, slider_py), slider_h - slider_py) - slider_py 116 | val = (float(slider_val) / (slider_h - slider_py*2) - 0.5) * 6.0 117 | cur_params[cur_slider_ix] = val 118 | 119 | needs_update = True 120 | 121 | def draw_sliders(): 122 | for i in xrange(num_params): 123 | row = i / slider_cols 124 | col = i % slider_cols 125 | x = sliders_x + col * slider_w 126 | y = sliders_y + row * slider_h 127 | 128 | cx = x + slider_w / 2 129 | cy_1 = y + slider_py 130 | cy_2 = y + slider_h - slider_py 131 | pygame.draw.line(screen, slider_color, (cx, cy_1), (cx, cy_2)) 132 | 133 | py = y + int((cur_params[i] / 6.0 + 0.5) * (slider_h - slider_py*2)) + slider_py 134 | pygame.draw.circle(screen, slider_color, (cx, py), slider_w/2 - slider_px) 135 | 136 | cx_1 = x + slider_px 137 | cx_2 = x + slider_w - slider_px 138 | for j in xrange(7): 139 | ly = y + slider_h/2 + (j-3)*(slider_h/7) 140 | pygame.draw.line(screen, slider_color, (cx_1, ly), (cx_2, ly)) 141 | 142 | def draw_face(): 143 | pygame.surfarray.blit_array(face_surface_mini, np.transpose(cur_face, (2, 1, 0))) 144 | pygame.transform.scale(face_surface_mini, (drawing_w, drawing_h), face_surface) 145 | pygame.draw.rect(screen, (0,0,0), (drawing_x, drawing_y, drawing_w, drawing_h), 1) 146 | 147 | #Main loop 148 | running = True 149 | while running: 150 | #Process events 151 | for event in pygame.event.get(): 152 | if event.type == pygame.QUIT: 153 | running = False 154 | break 155 | elif event.type == pygame.MOUSEBUTTONDOWN: 156 | if pygame.mouse.get_pressed()[0]: 157 | prev_mouse_pos = pygame.mouse.get_pos() 158 | update_mouse_click(prev_mouse_pos) 159 | update_mouse_move(prev_mouse_pos) 160 | elif pygame.mouse.get_pressed()[2]: 161 | cur_params = np.zeros((num_params,), dtype=np.float32) 162 | needs_update = True 163 | elif event.type == pygame.MOUSEBUTTONUP: 164 | mouse_pressed = False 165 | prev_mouse_pos = None 166 | elif event.type == pygame.MOUSEMOTION and mouse_pressed: 167 | update_mouse_move(pygame.mouse.get_pos()) 168 | elif event.type == pygame.KEYDOWN: 169 | if event.key == pygame.K_r: 170 | cur_params = np.clip(np.random.normal(0.0, 1.0, (num_params,)), -3.0, 3.0) 171 | needs_update = True 172 | 173 | #Check if we need an update 174 | if needs_update: 175 | x = means + np.dot(evecs, (cur_params * evals).T).T 176 | #x = means + stds * cur_params 177 | x = np.expand_dims(x, axis=0) 178 | y = enc([x, 0])[0][0] 179 | cur_face = (y * 255.0).astype(np.uint8) 180 | needs_update = False 181 | 182 | #Draw to the screen 183 | screen.fill(background_color) 184 | draw_face() 185 | draw_sliders() 186 | 187 | #Flip the screen buffer 188 | pygame.display.flip() 189 | pygame.time.wait(10) 190 | -------------------------------------------------------------------------------- /gpu.theanorc: -------------------------------------------------------------------------------- 1 | [global] 2 | floatX=float32 3 | device=cuda 4 | 5 | [nvcc] 6 | compiler_bindir=C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin 7 | 8 | [dnn] 9 | enabled=True 10 | include_path=C:\CUDA\v8.0\include 11 | library_path=C:\CUDA\v8.0\lib\x64 12 | --------------------------------------------------------------------------------