├── README.md ├── conv_visualization.py ├── kfkd.py ├── kfkd_specialist.py ├── kfkd_top.py ├── kfkd_vgg_top.py └── model6_weights_5000.h5 /README.md: -------------------------------------------------------------------------------- 1 | # Kaggle facial keypoints detection by Keras 2 | 3 | This code is for the final model in this blog: https://elix-tech.github.io/ja/2016/06/02/kaggle-facial-keypoints-ja.html 4 | -------------------------------------------------------------------------------- /conv_visualization.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | from __future__ import print_function 3 | from scipy.misc import imsave 4 | import numpy as np 5 | import time 6 | import os 7 | import h5py 8 | 9 | from keras.models import Sequential 10 | from keras.layers import Convolution2D, Activation, MaxPooling2D, Dropout 11 | from keras import backend as K 12 | 13 | img_width = 96 14 | img_height = 96 15 | weights_path = '../examples/model6_weights_5000.h5' 16 | layer_name = 'conv3' 17 | 18 | 19 | model = Sequential() 20 | 21 | model.add(Convolution2D(32, 3, 3, input_shape=(1, 96, 96), name='conv1')) 22 | first_layer = model.layers[-1] 23 | input_img = first_layer.input 24 | 25 | model.add(Activation('relu')) 26 | model.add(MaxPooling2D(pool_size=(2, 2))) 27 | model.add(Dropout(0.1)) 28 | 29 | model.add(Convolution2D(64, 2, 2, name='conv2')) 30 | model.add(Activation('relu')) 31 | model.add(MaxPooling2D(pool_size=(2, 2))) 32 | model.add(Dropout(0.2)) 33 | 34 | model.add(Convolution2D(128, 2, 2, name='conv3')) 35 | model.add(Activation('relu')) 36 | model.add(MaxPooling2D(pool_size=(2, 2))) 37 | model.add(Dropout(0.3)) 38 | 39 | 40 | assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).' 41 | f = h5py.File(weights_path) 42 | 43 | layer_names = [n.decode('utf8') for n in f.attrs['layer_names']] 44 | weight_value_tuples = [] 45 | for k, name in enumerate(layer_names): 46 | if k >= len(model.layers): 47 | break 48 | g = f[name] 49 | weight_names = [n.decode('utf8') for n in g.attrs['weight_names']] 50 | if len(weight_names): 51 | weight_values = [g[weight_name] for weight_name in weight_names] 52 | layer = model.layers[k] 53 | symbolic_weights = layer.trainable_weights + layer.non_trainable_weights 54 | if len(weight_values) != len(symbolic_weights): 55 | raise Exception('Layer #' + str(k) + 56 | ' (named "' + layer.name + 57 | '" in the current model) was found to ' 58 | 'correspond to layer ' + name + 59 | ' in the save file. ' 60 | 'However the new layer ' + layer.name + 61 | ' expects ' + str(len(symbolic_weights)) + 62 | ' weights, but the saved weights have ' + 63 | str(len(weight_values)) + 64 | ' elements.') 65 | weight_value_tuples += zip(symbolic_weights, weight_values) 66 | K.batch_set_value(weight_value_tuples) 67 | f.close() 68 | 69 | print('Model loaded.') 70 | 71 | layer_dict = dict([(layer.name, layer) for layer in model.layers]) 72 | 73 | def normalize(x): 74 | return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) 75 | 76 | def deprocess_image(x): 77 | x -= x.mean() 78 | x /= (x.std() + 1e-5) 79 | x *= 0.1 80 | 81 | x += 0.5 82 | x = np.clip(x, 0, 1) 83 | 84 | x *= 255 85 | x = x.transpose((1, 2, 0)) 86 | x = np.clip(x, 0, 255).astype('uint8') 87 | return x 88 | 89 | kept_filters = [] 90 | for filter_index in range(0, 128): 91 | print('Processing filter %d' % filter_index) 92 | start_time = time.time() 93 | 94 | layer_output = layer_dict[layer_name].output 95 | loss = K.mean(layer_output[:, filter_index, :, :]) 96 | grads = K.gradients(loss, input_img)[0] 97 | grads = normalize(grads) 98 | iterate = K.function([input_img, K.learning_phase()], [loss, grads]) 99 | 100 | step = 5. 101 | 102 | input_img_data = np.random.random((1, 1, img_width, img_height)) * 20 + 128. 103 | 104 | for i in range(200): 105 | loss_value, grads_value = iterate([input_img_data, 0]) 106 | input_img_data += grads_value * step 107 | 108 | print('Current loss value:', loss_value) 109 | if loss_value <= 0.: 110 | break 111 | 112 | if loss_value > 0: 113 | img = deprocess_image(input_img_data[0]) 114 | kept_filters.append((img, loss_value)) 115 | end_time = time.time() 116 | print('Filter %d processed in %ds' % (filter_index, end_time - start_time)) 117 | 118 | 119 | nb_img_x = 6 120 | nb_img_y = 2 121 | 122 | kept_filters.sort(key=lambda x: x[1], reverse=True) 123 | kept_filters = kept_filters[:nb_img_x * nb_img_y] 124 | 125 | margin = 5 126 | width = nb_img_x * img_width + (nb_img_x - 1) * margin 127 | height = nb_img_y * img_height + (nb_img_y - 1) * margin 128 | stitched_filters = np.zeros((height, width, 3)) 129 | 130 | for i in range(nb_img_x): 131 | for j in range(nb_img_y): 132 | img, loss = kept_filters[j * nb_img_y + i] 133 | stitched_filters[(img_height + margin) * j: (img_height + margin) * j + img_height, 134 | (img_width + margin) * i: (img_width + margin) * i + img_width, :] = img 135 | 136 | imsave('stitched_filters_%s_%dx%d.png' % (layer_name, nb_img_x, nb_img_y), stitched_filters) 137 | 138 | 139 | -------------------------------------------------------------------------------- /kfkd.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import os 3 | import numpy as np 4 | from pandas.io.parsers import read_csv 5 | from sklearn.utils import shuffle 6 | from sklearn.cross_validation import train_test_split 7 | from collections import OrderedDict 8 | 9 | from keras.models import Sequential 10 | from keras.layers import Dense, Activation, Convolution2D, MaxPooling2D, Flatten, Dropout 11 | from keras.optimizers import SGD 12 | from keras.models import model_from_json 13 | from keras.preprocessing.image import ImageDataGenerator 14 | from keras.callbacks import LearningRateScheduler, EarlyStopping 15 | 16 | # Download from https://www.kaggle.com/c/facial-keypoints-detection/data 17 | FTRAIN = 'data/training.csv' 18 | FTEST = 'data/test.csv' 19 | 20 | SPECIALIST_SETTINGS = [ 21 | dict( 22 | columns=( 23 | 'left_eye_center_x', 'left_eye_center_y', 24 | 'right_eye_center_x', 'right_eye_center_y', 25 | ), 26 | flip_indices=((0, 2), (1, 3)), 27 | ), 28 | 29 | dict( 30 | columns=( 31 | 'nose_tip_x', 'nose_tip_y', 32 | ), 33 | flip_indices=(), 34 | ), 35 | 36 | dict( 37 | columns=( 38 | 'mouth_left_corner_x', 'mouth_left_corner_y', 39 | 'mouth_right_corner_x', 'mouth_right_corner_y', 40 | 'mouth_center_top_lip_x', 'mouth_center_top_lip_y', 41 | ), 42 | flip_indices=((0, 2), (1, 3)), 43 | ), 44 | 45 | dict( 46 | columns=( 47 | 'mouth_center_bottom_lip_x', 48 | 'mouth_center_bottom_lip_y', 49 | ), 50 | flip_indices=(), 51 | ), 52 | 53 | dict( 54 | columns=( 55 | 'left_eye_inner_corner_x', 'left_eye_inner_corner_y', 56 | 'right_eye_inner_corner_x', 'right_eye_inner_corner_y', 57 | 'left_eye_outer_corner_x', 'left_eye_outer_corner_y', 58 | 'right_eye_outer_corner_x', 'right_eye_outer_corner_y', 59 | ), 60 | flip_indices=((0, 2), (1, 3), (4, 6), (5, 7)), 61 | ), 62 | 63 | dict( 64 | columns=( 65 | 'left_eyebrow_inner_end_x', 'left_eyebrow_inner_end_y', 66 | 'right_eyebrow_inner_end_x', 'right_eyebrow_inner_end_y', 67 | 'left_eyebrow_outer_end_x', 'left_eyebrow_outer_end_y', 68 | 'right_eyebrow_outer_end_x', 'right_eyebrow_outer_end_y', 69 | ), 70 | flip_indices=((0, 2), (1, 3), (4, 6), (5, 7)), 71 | ), 72 | ] 73 | 74 | def load(test=False, cols=None): 75 | 76 | fname = FTEST if test else FTRAIN 77 | df = read_csv(os.path.expanduser(fname)) 78 | 79 | df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' ')) 80 | 81 | if cols: 82 | df = df[list(cols) + ['Image']] 83 | 84 | print(df.count()) 85 | df = df.dropna() 86 | 87 | X = np.vstack(df['Image'].values) / 255. 88 | X = X.astype(np.float32) 89 | 90 | if not test: 91 | y = df[df.columns[:-1]].values 92 | y = (y - 48) / 48 93 | X, y = shuffle(X, y, random_state=42) 94 | y = y.astype(np.float32) 95 | else: 96 | y = None 97 | 98 | return X, y 99 | 100 | def load2d(test=False, cols=None): 101 | X, y = load(test, cols) 102 | X = X.reshape(-1, 1, 96, 96) 103 | return X, y 104 | 105 | def plot_sample(x, y, axis): 106 | img = x.reshape(96, 96) 107 | axis.imshow(img, cmap='gray') 108 | axis.scatter(y[0::2]*48+48, y[1::2]*48+48, marker='x', s=10) 109 | 110 | class FlippedImageDataGenerator(ImageDataGenerator): 111 | flip_indices = [ 112 | (0, 2), (1, 3), 113 | (4, 8), (5, 9), (6, 10), (7, 11), 114 | (12, 16), (13, 17), (14, 18), (15, 19), 115 | (22, 24), (23, 25), 116 | ] 117 | 118 | def next(self): 119 | X_batch, y_batch = super(FlippedImageDataGenerator, self).next() 120 | batch_size = X_batch.shape[0] 121 | indices = np.random.choice(batch_size, batch_size/2, replace=False) 122 | X_batch[indices] = X_batch[indices, :, :, ::-1] 123 | 124 | if y_batch is not None: 125 | y_batch[indices, ::2] = y_batch[indices, ::2] * -1 126 | 127 | for a, b in self.flip_indices: 128 | y_batch[indices, a], y_batch[indices, b] = ( 129 | y_batch[indices, b], y_batch[indices, a] 130 | ) 131 | 132 | return X_batch, y_batch 133 | 134 | 135 | model = Sequential() 136 | 137 | model.add(Convolution2D(32, 3, 3, input_shape=(1, 96, 96))) 138 | model.add(Activation('relu')) 139 | model.add(MaxPooling2D(pool_size=(2, 2))) 140 | model.add(Dropout(0.1)) 141 | 142 | model.add(Convolution2D(64, 2, 2)) 143 | model.add(Activation('relu')) 144 | model.add(MaxPooling2D(pool_size=(2, 2))) 145 | model.add(Dropout(0.2)) 146 | 147 | model.add(Convolution2D(128, 2, 2)) 148 | model.add(Activation('relu')) 149 | model.add(MaxPooling2D(pool_size=(2, 2))) 150 | model.add(Dropout(0.3)) 151 | 152 | model.add(Flatten()) 153 | model.add(Dense(1000)) 154 | model.add(Activation('relu')) 155 | model.add(Dropout(0.5)) 156 | model.add(Dense(1000)) 157 | model.add(Activation('relu')) 158 | model.add(Dense(30)) 159 | 160 | 161 | def fit_specialists(fname_pretrain=None): 162 | specialists = OrderedDict() 163 | start = 0.03 164 | stop = 0.001 165 | nb_epoch = 10000 166 | 167 | for setting in SPECIALIST_SETTINGS: 168 | 169 | cols = setting['columns'] 170 | X, y = load2d(cols=cols) 171 | X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42) 172 | model_specialist = model_from_json(model.to_json()) 173 | 174 | if fname_pretrain: 175 | model_specialist.load_weights(fname_pretrain) 176 | 177 | model_specialist.layers.pop() 178 | model_specialist.outputs = [model_specialist.layers[-1].output] 179 | model_specialist.layers[-1].outbound_nodes = [] 180 | model_specialist.add(Dense(len(cols))) 181 | 182 | sgd = SGD(lr=start, momentum=0.9, nesterov=True) 183 | model_specialist.compile(loss='mean_squared_error', optimizer=sgd) 184 | 185 | # from keras.utils.visualize_util import plot 186 | # plot(model_specialist, to_file="model8_{}.png".format(cols[0]), show_shapes=True) 187 | 188 | flipgen = FlippedImageDataGenerator() 189 | flipgen.flip_indices = setting['flip_indices'] 190 | 191 | early_stop = EarlyStopping(patience=100) 192 | learning_rates = np.linspace(start, stop, nb_epoch) 193 | change_lr = LearningRateScheduler(lambda epoch: float(learning_rates[epoch])) 194 | 195 | print("Training model for columns {} for {} epochs".format(cols, nb_epoch)) 196 | 197 | hist = model_specialist.fit_generator(flipgen.flow(X_train, y_train), 198 | samples_per_epoch=X_train.shape[0], 199 | nb_epoch=nb_epoch, 200 | validation_data=(X_val, y_val), 201 | callbacks=[change_lr, early_stop]) 202 | 203 | specialists[cols] = model_specialist 204 | 205 | 206 | fit_specialists() 207 | -------------------------------------------------------------------------------- /kfkd_specialist.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import os 3 | import h5py 4 | import numpy as np 5 | from pandas.io.parsers import read_csv 6 | from sklearn.cross_validation import train_test_split 7 | from collections import OrderedDict 8 | from keras.models import Sequential 9 | from keras.layers import Convolution2D, Activation, MaxPooling2D, Dropout 10 | from keras.layers import Flatten, Dense, Dropout 11 | from keras.optimizers import SGD 12 | from keras.callbacks import LearningRateScheduler, EarlyStopping 13 | from keras import backend as K 14 | from sklearn.utils import shuffle 15 | 16 | # Download from https://www.kaggle.com/c/facial-keypoints-detection/data 17 | FTRAIN = 'data/training.csv' 18 | FTEST = 'data/test.csv' 19 | 20 | # Download from https://github.com/elix-tech/kaggle-facial-keypoints 21 | weights_path = '../examples/model6_weights_5000.h5' 22 | img_width = 96 23 | img_height = 96 24 | 25 | SPECIALIST_SETTINGS = [ 26 | dict( 27 | columns=( 28 | 'left_eye_center_x', 'left_eye_center_y', 29 | 'right_eye_center_x', 'right_eye_center_y', 30 | ), 31 | flip_indices=((0, 2), (1, 3)), 32 | ), 33 | 34 | dict( 35 | columns=( 36 | 'nose_tip_x', 'nose_tip_y', 37 | ), 38 | flip_indices=(), 39 | ), 40 | 41 | dict( 42 | columns=( 43 | 'mouth_left_corner_x', 'mouth_left_corner_y', 44 | 'mouth_right_corner_x', 'mouth_right_corner_y', 45 | 'mouth_center_top_lip_x', 'mouth_center_top_lip_y', 46 | ), 47 | flip_indices=((0, 2), (1, 3)), 48 | ), 49 | 50 | dict( 51 | columns=( 52 | 'mouth_center_bottom_lip_x', 53 | 'mouth_center_bottom_lip_y', 54 | ), 55 | flip_indices=(), 56 | ), 57 | 58 | dict( 59 | columns=( 60 | 'left_eye_inner_corner_x', 'left_eye_inner_corner_y', 61 | 'right_eye_inner_corner_x', 'right_eye_inner_corner_y', 62 | 'left_eye_outer_corner_x', 'left_eye_outer_corner_y', 63 | 'right_eye_outer_corner_x', 'right_eye_outer_corner_y', 64 | ), 65 | flip_indices=((0, 2), (1, 3), (4, 6), (5, 7)), 66 | ), 67 | 68 | dict( 69 | columns=( 70 | 'left_eyebrow_inner_end_x', 'left_eyebrow_inner_end_y', 71 | 'right_eyebrow_inner_end_x', 'right_eyebrow_inner_end_y', 72 | 'left_eyebrow_outer_end_x', 'left_eyebrow_outer_end_y', 73 | 'right_eyebrow_outer_end_x', 'right_eyebrow_outer_end_y', 74 | ), 75 | flip_indices=((0, 2), (1, 3), (4, 6), (5, 7)), 76 | ), 77 | ] 78 | 79 | def load(test=False, cols=None): 80 | fname = FTEST if test else FTRAIN 81 | df = read_csv(os.path.expanduser(fname)) 82 | 83 | df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' ')) 84 | 85 | if cols: 86 | df = df[list(cols) + ['Image']] 87 | 88 | print(df.count()) 89 | df = df.dropna() 90 | 91 | X = np.vstack(df['Image'].values) / 255. 92 | X = X.astype(np.float32) 93 | 94 | if not test: 95 | y = df[df.columns[:-1]].values 96 | y = (y - 48) / 48 97 | X, y = shuffle(X, y, random_state=42) 98 | y = y.astype(np.float32) 99 | else: 100 | y = None 101 | 102 | return X, y 103 | 104 | def load2d(test=False, cols=None): 105 | X, y = load(test, cols) 106 | X = X.reshape(-1, 1, 96, 96) 107 | return X, y 108 | 109 | def flip_image(X, y): 110 | flip_indices = [ 111 | (0, 2), (1, 3), 112 | (4, 8), (5, 9), (6, 10), (7, 11), 113 | (12, 16), (13, 17), (14, 18), (15, 19), 114 | (22, 24), (23, 25), 115 | ] 116 | 117 | X_flipped = np.array(X[:, :, :, ::-1]) 118 | y_flipped = np.array(y) 119 | y_flipped[:, ::2] = y_flipped[:, ::2] * -1 120 | 121 | for i in range(len(y)): 122 | for a, b in flip_indices: 123 | y_flipped[i, a], y_flipped[i, b] = (y_flipped[i, b], y_flipped[i, a]) 124 | return X_flipped, y_flipped 125 | 126 | def save_bottleneck_features(): 127 | model = Sequential() 128 | model.add(Convolution2D(32, 3, 3, input_shape=(1, img_width, img_height))) 129 | model.add(Activation('relu')) 130 | model.add(MaxPooling2D(pool_size=(2, 2))) 131 | model.add(Dropout(0.1)) 132 | 133 | model.add(Convolution2D(64, 2, 2)) 134 | model.add(Activation('relu')) 135 | model.add(MaxPooling2D(pool_size=(2, 2))) 136 | model.add(Dropout(0.2)) 137 | 138 | model.add(Convolution2D(128, 2, 2)) 139 | model.add(Activation('relu')) 140 | model.add(MaxPooling2D(pool_size=(2, 2))) 141 | model.add(Dropout(0.3)) 142 | 143 | assert os.path.exists(weights_path), 'Model weights not found (see "weights_path" variable in script).' 144 | f = h5py.File(weights_path) 145 | layer_names = [n.decode('utf8') for n in f.attrs['layer_names']] 146 | weight_value_tuples = [] 147 | for k, name in enumerate(layer_names): 148 | if k >= len(model.layers): 149 | break 150 | g = f[name] 151 | weight_names = [n.decode('utf8') for n in g.attrs['weight_names']] 152 | if len(weight_names): 153 | weight_values = [g[weight_name] for weight_name in weight_names] 154 | layer = model.layers[k] 155 | symbolic_weights = layer.trainable_weights + layer.non_trainable_weights 156 | if len(weight_values) != len(symbolic_weights): 157 | raise Exception('Layer #' + str(k) + 158 | ' (named "' + layer.name + 159 | '" in the current model) was found to ' 160 | 'correspond to layer ' + name + 161 | ' in the save file. ' 162 | 'However the new layer ' + layer.name + 163 | ' expects ' + str(len(symbolic_weights)) + 164 | ' weights, but the saved weights have ' + 165 | str(len(weight_values)) + 166 | ' elements.') 167 | weight_value_tuples += zip(symbolic_weights, weight_values) 168 | K.batch_set_value(weight_value_tuples) 169 | f.close() 170 | print('Model loaded.') 171 | 172 | X, y = load2d() 173 | X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42) 174 | X_flipped, y_flipped = flip_image(X_train, y_train) 175 | 176 | X_train = np.vstack((X_train, X_flipped)) 177 | y_train = np.vstack((y_train, y_flipped)) 178 | 179 | bottleneck_features_train = model.predict(X_train) 180 | np.save(open('bottleneck_features_train.npy', 'w'), bottleneck_features_train) 181 | np.save(open('label_train.npy', 'w'), y_train) 182 | 183 | bottleneck_features_validation = model.predict(X_val) 184 | np.save(open('bottleneck_features_validation.npy', 'w'), bottleneck_features_validation) 185 | np.save(open('label_validation.npy', 'w'), y_val) 186 | 187 | def fit_specialists(): 188 | specialists = OrderedDict() 189 | start = 0.01 190 | stop = 0.001 191 | nb_epoch = 300 192 | 193 | train_data = np.load(open('bottleneck_features_train.npy')) 194 | train_labels = np.load(open('label_train.npy')) 195 | 196 | validation_data = np.load(open('bottleneck_features_validation.npy')) 197 | validation_labels = np.load(open('label_validation.npy')) 198 | 199 | df = read_csv(os.path.expanduser(FTRAIN)) 200 | 201 | for setting in SPECIALIST_SETTINGS: 202 | 203 | cols = setting['columns'] 204 | indices = [index for index, column in enumerate(df.columns) if column in cols] 205 | train_labels_specialist = train_labels[:, indices] 206 | validation_labels_specialist = validation_labels[:, indices] 207 | 208 | model_specialist = Sequential() 209 | model_specialist.add(Flatten(input_shape=train_data.shape[1:])) 210 | model_specialist.add(Dense(1000)) 211 | model_specialist.add(Activation('relu')) 212 | model_specialist.add(Dropout(0.5)) 213 | model_specialist.add(Dense(1000)) 214 | model_specialist.add(Activation('relu')) 215 | model_specialist.add(Dense(len(cols))) 216 | 217 | sgd = SGD(lr=start, momentum=0.9, nesterov=True) 218 | model_specialist.compile(loss='mean_squared_error', optimizer=sgd) 219 | 220 | early_stop = EarlyStopping(patience=100) 221 | learning_rates = np.linspace(start, stop, nb_epoch) 222 | change_lr = LearningRateScheduler(lambda epoch: float(learning_rates[epoch])) 223 | 224 | print("Training model for columns {} for {} epochs".format(cols, nb_epoch)) 225 | 226 | hist = model_specialist.fit(train_data, train_labels_specialist, 227 | nb_epoch=nb_epoch, 228 | validation_data=(validation_data, validation_labels_specialist), 229 | callbacks=[change_lr, early_stop]) 230 | 231 | model_specialist.save_weights("model_{}.h5".format(cols[0])) 232 | np.savetxt("model_{}_loss.csv".format(cols[0]), hist.history['loss']) 233 | np.savetxt("model_{}_val_loss.csv".format(cols[0]), hist.history['val_loss']) 234 | 235 | specialists[cols] = model_specialist 236 | 237 | save_bottleneck_features() 238 | fit_specialists() -------------------------------------------------------------------------------- /kfkd_top.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import numpy as np 3 | from keras.models import Sequential 4 | from keras.layers import Activation, Flatten, Dense, Dropout 5 | from keras.optimizers import SGD 6 | from keras.callbacks import LearningRateScheduler, EarlyStopping 7 | 8 | start = 0.01 9 | stop = 0.001 10 | nb_epoch = 300 11 | 12 | train_data = np.load(open('bottleneck_features_train.npy')) 13 | train_labels = np.load(open('label_train.npy')) 14 | 15 | validation_data = np.load(open('bottleneck_features_validation.npy')) 16 | validation_labels = np.load(open('label_validation.npy')) 17 | 18 | model_top = Sequential() 19 | model_top.add(Flatten(input_shape=train_data.shape[1:])) 20 | model_top.add(Dense(1000)) 21 | model_top.add(Activation('relu')) 22 | model_top.add(Dropout(0.5)) 23 | model_top.add(Dense(1000)) 24 | model_top.add(Activation('relu')) 25 | model_top.add(Dense(30)) 26 | 27 | sgd = SGD(lr=start, momentum=0.9, nesterov=True) 28 | model_top.compile(loss='mean_squared_error', optimizer=sgd) 29 | 30 | early_stop = EarlyStopping(patience=100) 31 | learning_rates = np.linspace(start, stop, nb_epoch) 32 | change_lr = LearningRateScheduler(lambda epoch: float(learning_rates[epoch])) 33 | 34 | hist = model_top.fit(train_data, train_labels, 35 | nb_epoch=nb_epoch, 36 | validation_data=(validation_data, validation_labels), 37 | callbacks=[change_lr, early_stop]) 38 | 39 | model_top.save_weights("model_top.h5") 40 | np.savetxt("model_top_loss.csv", hist.history['loss']) 41 | np.savetxt("model_top_val_loss.csv", hist.history['val_loss']) -------------------------------------------------------------------------------- /kfkd_vgg_top.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import os 3 | import h5py 4 | import numpy as np 5 | from pandas.io.parsers import read_csv 6 | from sklearn.cross_validation import train_test_split 7 | from keras.models import Sequential 8 | from keras.layers import ZeroPadding2D, Convolution2D, MaxPooling2D 9 | from keras.layers import Flatten, Dense, Dropout 10 | from keras.optimizers import SGD 11 | from keras.callbacks import LearningRateScheduler 12 | from sklearn.utils import shuffle 13 | 14 | # Download from https://www.kaggle.com/c/facial-keypoints-detection/data 15 | FTRAIN = 'data/training.csv' 16 | FTEST = 'data/test.csv' 17 | 18 | # Download from https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3 19 | weight_path = '../examples/vgg16_weights.h5' 20 | img_width = 96 21 | img_height = 96 22 | 23 | def load(test=False, cols=None): 24 | 25 | fname = FTEST if test else FTRAIN 26 | df = read_csv(os.path.expanduser(fname)) 27 | 28 | df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' ')) 29 | 30 | if cols: 31 | df = df[list(cols) + ['Image']] 32 | 33 | print(df.count()) 34 | df = df.dropna() 35 | 36 | X = np.vstack(df['Image'].values) / 255. 37 | X = X.astype(np.float32) 38 | 39 | if not test: 40 | y = df[df.columns[:-1]].values 41 | y = (y - 48) / 48 42 | X, y = shuffle(X, y, random_state=42) 43 | y = y.astype(np.float32) 44 | else: 45 | y = None 46 | 47 | return X, y 48 | 49 | def load2d(test=False, cols=None): 50 | X, y = load(test, cols) 51 | X = X.reshape(-1, 1, 96, 96) 52 | return X, y 53 | 54 | def flip_image(X, y): 55 | flip_indices = [ 56 | (0, 2), (1, 3), 57 | (4, 8), (5, 9), (6, 10), (7, 11), 58 | (12, 16), (13, 17), (14, 18), (15, 19), 59 | (22, 24), (23, 25), 60 | ] 61 | 62 | X_flipped = np.array(X[:, :, :, ::-1]) 63 | y_flipped = np.array(y) 64 | y_flipped[:, ::2] = y_flipped[:, ::2] * -1 65 | 66 | for i in range(len(y)): 67 | for a, b in flip_indices: 68 | y_flipped[i, a], y_flipped[i, b] = (y_flipped[i, b], y_flipped[i, a]) 69 | return X_flipped, y_flipped 70 | 71 | def gray_to_rgb(X): 72 | X_transpose = np.array(X.transpose(0, 2, 3, 1)) 73 | ret = np.empty((X.shape[0], img_width, img_height, 3), dtype=np.float32) 74 | ret[:, :, :, 0] = X_transpose[:, :, :, 0] 75 | ret[:, :, :, 1] = X_transpose[:, :, :, 0] 76 | ret[:, :, :, 2] = X_transpose[:, :, :, 0] 77 | return ret.transpose(0, 3, 1, 2) 78 | 79 | def save_bottleneck_features(): 80 | model = Sequential() 81 | model.add(ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height))) 82 | 83 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1')) 84 | model.add(ZeroPadding2D((1, 1))) 85 | model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_2')) 86 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 87 | 88 | model.add(ZeroPadding2D((1, 1))) 89 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1')) 90 | model.add(ZeroPadding2D((1, 1))) 91 | model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_2')) 92 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 93 | 94 | model.add(ZeroPadding2D((1, 1))) 95 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1')) 96 | model.add(ZeroPadding2D((1, 1))) 97 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_2')) 98 | model.add(ZeroPadding2D((1, 1))) 99 | model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_3')) 100 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 101 | 102 | model.add(ZeroPadding2D((1, 1))) 103 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1')) 104 | model.add(ZeroPadding2D((1, 1))) 105 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2')) 106 | model.add(ZeroPadding2D((1, 1))) 107 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_3')) 108 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 109 | 110 | model.add(ZeroPadding2D((1, 1))) 111 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1')) 112 | model.add(ZeroPadding2D((1, 1))) 113 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_2')) 114 | model.add(ZeroPadding2D((1, 1))) 115 | model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_3')) 116 | model.add(MaxPooling2D((2, 2), strides=(2, 2))) 117 | 118 | 119 | assert os.path.exists(weight_path), 'Model weights not found (see "weights_path" variable in script).' 120 | f = h5py.File(weight_path) 121 | for k in range(f.attrs['nb_layers']): 122 | if k >= len(model.layers): 123 | break 124 | 125 | g = f['layer_{}'.format(k)] 126 | weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])] 127 | model.layers[k].set_weights(weights) 128 | f.close() 129 | print('Model loaded.') 130 | 131 | X, y = load2d() 132 | X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42) 133 | X_flipped, y_flipped = flip_image(X_train, y_train) 134 | 135 | X_train = np.vstack((X_train, X_flipped)) 136 | y_train = np.vstack((y_train, y_flipped)) 137 | X_train = gray_to_rgb(X_train) 138 | X_val = gray_to_rgb(X_val) 139 | 140 | bottleneck_features_train = model.predict(X_train) 141 | np.save(open('bottleneck_features_train.npy', 'w'), bottleneck_features_train) 142 | np.save(open('label_train.npy', 'w'), y_train) 143 | 144 | bottleneck_features_validation = model.predict(X_val) 145 | np.save(open('bottleneck_features_validation.npy', 'w'), bottleneck_features_validation) 146 | np.save(open('label_validation.npy', 'w'), y_val) 147 | 148 | 149 | def train_top_model(): 150 | start = 0.03 151 | stop = 0.001 152 | nb_epoch = 300 153 | 154 | train_data = np.load(open('bottleneck_features_train.npy')) 155 | train_labels = np.load(open('label_train.npy')) 156 | 157 | validation_data = np.load(open('bottleneck_features_validation.npy')) 158 | validation_labels = np.load(open('label_validation.npy')) 159 | 160 | model = Sequential() 161 | model.add(Flatten(input_shape=train_data.shape[1:])) 162 | model.add(Dense(1000, activation='relu')) 163 | model.add(Dropout(0.5)) 164 | model.add(Dense(1000, activation='relu')) 165 | model.add(Dense(30)) 166 | 167 | sgd = SGD(lr=start, momentum=0.9, nesterov=True) 168 | model.compile(loss='mean_squared_error', optimizer=sgd) 169 | learning_rates = np.linspace(start, stop, nb_epoch) 170 | change_lr = LearningRateScheduler(lambda epoch: float(learning_rates[epoch])) 171 | hist = model.fit(train_data, train_labels, 172 | nb_epoch=nb_epoch, 173 | validation_data=(validation_data, validation_labels), 174 | callbacks=[change_lr]) 175 | 176 | model.save_weights('model_top_vgg.h5') 177 | np.savetxt('model_top_vgg_flip_loss.csv', hist.history['loss']) 178 | np.savetxt('model_top_vgg_flip_val_loss.csv', hist.history['val_loss']) 179 | 180 | 181 | save_bottleneck_features() 182 | train_top_model() -------------------------------------------------------------------------------- /model6_weights_5000.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elix-tech/kaggle-facial-keypoints/8c4c9688ab4a374822d3d8b19b8484b209915b46/model6_weights_5000.h5 --------------------------------------------------------------------------------