├── README.md ├── config.py ├── crop_images.py ├── demo.py ├── mae_callback.py ├── main.py ├── model.py ├── requirements.txt ├── train.py └── train_generator.py /README.md: -------------------------------------------------------------------------------- 1 | This repo accompanies the blog post [Estimating Body Mass Index from Face Images Using Keras and Transfer Learning](https://medium.com/@leosimmons/making-a-bmi-classifier-with-keras-and-transfer-learning-de25e1bc0212). 2 | 3 | Instructions for running the demo 4 | 5 | 1. Clone this repo 6 | 2. Make sure requirements in `requirements.txt` are installed 7 | 3. Download [the trained model weights](https://drive.google.com/open?id=1cOkj2Ivrpo_siGDtsNSZGDnh0NRcQUzw) and put file in the repo directory 8 | 4. `python demo.py` 9 | 10 | If you have more than one face in the frame of your video camera, run the script with the `--multiple` flag, or else the results will be off. 11 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | RESNET50_DEFAULT_IMG_WIDTH = 224 2 | MARGIN = .1 3 | TRAIN_BATCH_SIZE = 16 4 | VALIDATION_SIZE = 100 5 | 6 | ORIGINAL_IMGS_DIR = 'images' 7 | ORIGINAL_IMGS_INFO_FILE = 'data.csv' 8 | AGE_TRAINED_WEIGHTS_FILE = 'age_only_resnet50_weights.061-3.300-4.410.hdf5' 9 | CROPPED_IMGS_DIR = 'normalized_images' 10 | CROPPED_IMGS_INFO_FILE = 'normalized_data.csv' 11 | TOP_LAYER_LOG_DIR = 'logs/top_layer' 12 | ALL_LAYERS_LOG_DIR = 'logs/all_layers' 13 | -------------------------------------------------------------------------------- /crop_images.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import cv2 4 | import dlib 5 | from matplotlib import pyplot as plt 6 | import numpy as np 7 | import config 8 | 9 | detector = dlib.get_frontal_face_detector() 10 | 11 | 12 | def crop_faces(plot_images=False, max_images_to_plot=5): 13 | bad_crop_count = 0 14 | if not os.path.exists(config.CROPPED_IMGS_DIR): 15 | os.makedirs(config.CROPPED_IMGS_DIR) 16 | print 'Cropping faces and saving to %s' % config.CROPPED_IMGS_DIR 17 | good_cropped_images = [] 18 | good_cropped_img_file_names = [] 19 | detected_cropped_images = [] 20 | original_images_detected = [] 21 | for file_name in sorted(os.listdir(config.ORIGINAL_IMGS_DIR)): 22 | np_img = cv2.imread(os.path.join(config.ORIGINAL_IMGS_DIR,file_name)) 23 | detected = detector(np_img, 1) 24 | img_h, img_w, _ = np.shape(np_img) 25 | original_images_detected.append(np_img) 26 | 27 | if len(detected) != 1: 28 | bad_crop_count += 1 29 | continue 30 | 31 | d = detected[0] 32 | x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height() 33 | xw1 = int(x1 - config.MARGIN * w) 34 | yw1 = int(y1 - config.MARGIN * h) 35 | xw2 = int(x2 + config.MARGIN * w) 36 | yw2 = int(y2 + config.MARGIN * h) 37 | cropped_img = crop_image(np_img, xw1, yw1, xw2, yw2) 38 | norm_file_path = '%s/%s' % (config.CROPPED_IMGS_DIR, file_name) 39 | cv2.imwrite(norm_file_path, cropped_img) 40 | 41 | good_cropped_img_file_names.append(file_name) 42 | 43 | # save info of good cropped images 44 | with open(config.ORIGINAL_IMGS_INFO_FILE, 'r') as f: 45 | column_headers = f.read().splitlines()[0] 46 | all_imgs_info = f.read().splitlines()[1:] 47 | cropped_imgs_info = [l for l in all_imgs_info if l.split(',')[-1] in good_cropped_img_file_names] 48 | 49 | with open(config.CROPPED_IMGS_INFO_FILE, 'w') as f: 50 | f.write('%s\n' % column_headers) 51 | for l in cropped_imgs_info: 52 | f.write('%s\n' % l) 53 | 54 | print 'Cropped %d images and saved in %s - info in %s' % (len(original_images_detected), config.CROPPED_IMGS_DIR, config.CROPPED_IMGS_INFO_FILE) 55 | print 'Error detecting face in %d images - info in Data/unnormalized.txt' % bad_crop_count 56 | 57 | if plot_images: 58 | print 'Plotting images...' 59 | img_index = 0 60 | plot_index = 1 61 | plot_n_cols = 3 62 | plot_n_rows = len(original_images_detected) if len(original_images_detected) < max_images_to_plot else max_images_to_plot 63 | for row in range(plot_n_rows): 64 | plt.subplot(plot_n_rows,plot_n_cols,plot_index) 65 | plt.imshow(original_images_detected[img_index].astype('uint8')) 66 | plot_index += 1 67 | 68 | plt.subplot(plot_n_rows,plot_n_cols,plot_index) 69 | plt.imshow(detected_cropped_images[img_index]) 70 | plot_index += 1 71 | 72 | plt.subplot(plot_n_rows,plot_n_cols,plot_index) 73 | plt.imshow(good_cropped_images[img_index]) 74 | plot_index += 1 75 | 76 | img_index += 1 77 | plt.show() 78 | return good_cropped_images 79 | 80 | 81 | 82 | # image cropping method taken from: 83 | # https://stackoverflow.com/questions/15589517/how-to-crop-an-image-in-opencv-using-python 84 | def crop_image(img, x1, y1, x2, y2): 85 | if x1 < 0 or y1 < 0 or x2 > img.shape[1] or y2 > img.shape[0]: 86 | img, x1, x2, y1, y2 = pad_img_to_fit_bbox(img, x1, x2, y1, y2) 87 | return img[y1:y2, x1:x2, :] 88 | 89 | def pad_img_to_fit_bbox(img, x1, x2, y1, y2): 90 | img = cv2.copyMakeBorder(img, - min(0, y1), max(y2 - img.shape[0], 0), 91 | -min(0, x1), max(x2 - img.shape[1], 0), cv2.BORDER_REPLICATE) 92 | y2 += -min(0, y1) 93 | y1 += -min(0, y1) 94 | x2 += -min(0, x1) 95 | x1 += -min(0, x1) 96 | return img, x1, x2, y1, y2 97 | 98 | if __name__ == '__main__': 99 | crop_faces() 100 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | 2 | import cv2 3 | import sys 4 | import dlib 5 | import numpy as np 6 | from contextlib import contextmanager 7 | import urllib2 8 | from model import get_model 9 | import config 10 | 11 | 12 | 13 | 14 | def get_trained_model(): 15 | weights_file = 'bmi_model_weights.h5' 16 | model = get_model(ignore_age_weights=True) 17 | model.load_weights(weights_file) 18 | return model 19 | 20 | 21 | def draw_label(image, point, label, font=cv2.FONT_HERSHEY_SIMPLEX, font_scale=1, thickness=2): 22 | size = cv2.getTextSize(label, font, font_scale, thickness)[0] 23 | x, y = point 24 | cv2.rectangle(image, (x, y - size[1]), (x + size[0], y), (255, 0, 0), cv2.FILLED) 25 | cv2.putText(image, label, point, font, font_scale, (255, 255, 255), thickness) 26 | 27 | 28 | @contextmanager 29 | def video_capture(*args, **kwargs): 30 | cap = cv2.VideoCapture(*args, **kwargs) 31 | try: 32 | yield cap 33 | finally: 34 | cap.release() 35 | 36 | 37 | def yield_images_from_camera(): 38 | with video_capture(0) as cap: 39 | cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) 40 | cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) 41 | 42 | while True: 43 | ret, img = cap.read() 44 | if not ret: 45 | raise RuntimeError("Failed to capture image") 46 | yield img 47 | 48 | 49 | def run_demo(): 50 | args = sys.argv[1:] 51 | multiple_targets = '--multiple' in args 52 | single_or_multiple = 'multiple faces' if multiple_targets else 'single face' 53 | model = get_trained_model() 54 | print 'Loading model to detect BMI of %s...' % single_or_multiple 55 | 56 | NUMBER_OF_FRAMES_IN_AVG = 20 57 | last_seen_bmis = [] 58 | detector = dlib.get_frontal_face_detector() 59 | 60 | for img in yield_images_from_camera(): 61 | input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 62 | img_h, img_w, _ = np.shape(input_img) 63 | 64 | detected = detector(input_img, 1) 65 | faces = np.empty((len(detected), config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH, 3)) 66 | 67 | if len(detected) > 0: 68 | for i, d in enumerate(detected): 69 | x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, d.bottom() + 1, d.width(), d.height() 70 | xw1 = max(int(x1 - config.MARGIN * w), 0) 71 | yw1 = max(int(y1 - config.MARGIN * h), 0) 72 | xw2 = min(int(x2 + config.MARGIN * w), img_w - 1) 73 | yw2 = min(int(y2 + config.MARGIN * h), img_h - 1) 74 | cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2) 75 | faces[i, :, :, :] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1, :], (config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH)) / 255.00 76 | 77 | predictions = model.predict(faces) 78 | 79 | if multiple_targets: 80 | for i, d in enumerate(detected): 81 | label = str(predictions[i][0]) 82 | draw_label(img, (d.left(), d.top()), label) 83 | else: 84 | last_seen_bmis.append(predictions[0]) 85 | if len(last_seen_bmis) > NUMBER_OF_FRAMES_IN_AVG: 86 | last_seen_bmis.pop(0) 87 | elif len(last_seen_bmis) < NUMBER_OF_FRAMES_IN_AVG: 88 | continue 89 | avg_bmi = sum(last_seen_bmis) / float(NUMBER_OF_FRAMES_IN_AVG) 90 | label = str(avg_bmi) 91 | draw_label(img, (d.left(), d.top()), label) 92 | 93 | cv2.imshow('result', img) 94 | key = cv2.waitKey(30) 95 | 96 | if key == 27: # ESC 97 | break 98 | 99 | 100 | if __name__ == '__main__': 101 | run_demo() 102 | -------------------------------------------------------------------------------- /mae_callback.py: -------------------------------------------------------------------------------- 1 | 2 | from keras.callbacks import Callback 3 | import numpy as np 4 | import config 5 | import cv2 6 | 7 | 8 | def get_mae(actual, predicted): 9 | n_samples = predicted.shape[0] 10 | diff_sum = 0.00 11 | for i in range(n_samples): 12 | p = predicted[i][0] 13 | a = actual[i] 14 | d = abs(p - a) 15 | diff_sum += d 16 | return diff_sum / n_samples 17 | 18 | 19 | class MAECallback(Callback): 20 | 21 | 22 | def on_train_begin(self, logs={}): 23 | self._data = [] 24 | 25 | 26 | def on_epoch_end(self, batch, logs={}): 27 | with open(config.CROPPED_IMGS_INFO_FILE, 'r') as f: 28 | test_images_info = f.read().splitlines()[-config.VALIDATION_SIZE:] 29 | test_x = [] 30 | test_y = [] 31 | for info in test_images_info: 32 | bmi = float(info.split(',')[1]) 33 | test_y.append(bmi) 34 | file_name = info.split(',')[4] 35 | file_path = '%s/%s' % (config.CROPPED_IMGS_DIR, file_name) 36 | print 37 | print 38 | print 39 | print file_path 40 | img = cv2.imread(file_path) 41 | img = cv2.resize(img, (config.RESNET50_DEFAULT_IMG_WIDTH,config.RESNET50_DEFAULT_IMG_WIDTH)) 42 | test_x.append(img/255.00) 43 | X_val = np.array(test_x) 44 | y_val = np.array(test_y) 45 | y_predict = np.asarray(self.model.predict(X_val)) 46 | val_mae = get_mae(y_val, y_predict) 47 | logs['val_mae'] = val_mae 48 | self._data.append({ 49 | 'val_mae': val_mae, 50 | }) 51 | return 52 | 53 | 54 | def get_data(self): 55 | return self._data 56 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | 2 | from model import get_model 3 | from train import train_top_layer, train_all_layers 4 | 5 | 6 | if __name__ == '__main__': 7 | model = get_model() 8 | train_top_layer(model) 9 | # train_all_layers(model) 10 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | 2 | from tensorflow.python.keras.models import Model 3 | from tensorflow.python.keras.applications import ResNet50 4 | from tensorflow.python.keras.layers import Dense 5 | import config 6 | 7 | def get_age_model(): 8 | 9 | age_model = ResNet50( 10 | include_top=False, 11 | weights='imagenet', 12 | input_shape=(config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH, 3), 13 | pooling='avg' 14 | ) 15 | 16 | prediction = Dense(units=101, 17 | kernel_initializer='he_normal', 18 | use_bias=False, 19 | activation='softmax', 20 | name='pred_age')(age_model.output) 21 | 22 | age_model = Model(inputs=age_model.input, outputs=prediction) 23 | return age_model 24 | 25 | 26 | def get_model(ignore_age_weights=False): 27 | 28 | base_model = get_age_model() 29 | if not ignore_age_weights: 30 | base_model.load_weights(config.AGE_TRAINED_WEIGHTS_FILE) 31 | print 'Loaded weights from age classifier' 32 | last_hidden_layer = base_model.get_layer(index=-2) 33 | 34 | base_model = Model( 35 | inputs=base_model.input, 36 | outputs=last_hidden_layer.output) 37 | prediction = Dense(1, kernel_initializer='normal')(base_model.output) 38 | 39 | model = Model(inputs=base_model.input, outputs=prediction) 40 | return model 41 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.0 2 | astor==0.7.1 3 | Augmentor==0.2.3 4 | backports.functools-lru-cache==1.5 5 | backports.weakref==1.0.post1 6 | cycler==0.10.0 7 | dlib==19.16.0 8 | enum34==1.1.6 9 | funcsigs==1.0.2 10 | future==0.17.1 11 | futures==3.2.0 12 | gast==0.2.2 13 | grpcio==1.18.0 14 | h5py==2.9.0 15 | Keras==2.2.4 16 | Keras-Applications==1.0.7 17 | Keras-Preprocessing==1.0.9 18 | kiwisolver==1.0.1 19 | Markdown==3.0.1 20 | matplotlib==2.2.3 21 | mock==2.0.0 22 | numpy==1.16.1 23 | opencv-python==4.0.0.21 24 | pandas==0.24.1 25 | pbr==5.1.2 26 | Pillow==5.4.1 27 | protobuf==3.6.1 28 | pyparsing==2.3.1 29 | python-dateutil==2.8.0 30 | pytz==2018.9 31 | PyYAML==3.13 32 | scipy==1.2.0 33 | six==1.12.0 34 | subprocess32==3.5.3 35 | tensorboard==1.12.2 36 | tensorflow==1.12.0 37 | termcolor==1.1.0 38 | tqdm==4.30.0 39 | Werkzeug==0.14.1 40 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | 2 | import cv2 3 | import numpy as np 4 | from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard 5 | from train_generator import train_generator, plot_imgs_from_generator 6 | from mae_callback import MAECallback 7 | import config 8 | 9 | 10 | 11 | batches_per_epoch=train_generator.n //train_generator.batch_size 12 | 13 | 14 | def train_top_layer(model): 15 | 16 | print 'Training top layer...' 17 | 18 | for l in model.layers[:-1]: 19 | l.trainable = False 20 | 21 | model.compile( 22 | loss='mean_squared_error', 23 | optimizer='adam' 24 | ) 25 | 26 | mae_callback = MAECallback() 27 | 28 | early_stopping_callback = EarlyStopping( 29 | monitor='val_mae', 30 | mode='min', 31 | verbose=1, 32 | patience=1) 33 | 34 | model_checkpoint_callback = ModelCheckpoint( 35 | 'saved_models/top_layer_trained_weights.{epoch:02d}-{val_mae:.2f}.h5', 36 | monitor='val_mae', 37 | mode='min', 38 | verbose=1, 39 | save_best_only=True 40 | ) 41 | 42 | tensorboard_callback = TensorBoard( 43 | log_dir=config.TOP_LAYER_LOG_DIR, 44 | batch_size=train_generator.batch_size 45 | ) 46 | 47 | model.fit_generator( 48 | generator=train_generator, 49 | steps_per_epoch=batches_per_epoch, 50 | epochs=20, 51 | callbacks=[ 52 | mae_callback, 53 | early_stopping_callback, 54 | model_checkpoint_callback, 55 | tensorboard_callback 56 | ] 57 | ) 58 | 59 | 60 | def train_all_layers(model): 61 | 62 | print 'Training all layers...' 63 | 64 | for l in model.layers: 65 | l.trainable = True 66 | 67 | mae_callback = MAECallback() 68 | 69 | early_stopping_callback = EarlyStopping( 70 | monitor='val_mae', 71 | mode='min', 72 | verbose=1, 73 | patience=10) 74 | 75 | model_checkpoint_callback = ModelCheckpoint( 76 | 'saved_models/all_layers_trained_weights.{epoch:02d}-{val_mae:.2f}.h5', 77 | monitor='val_mae', 78 | mode='min', 79 | verbose=1, 80 | save_best_only=True) 81 | 82 | tensorboard_callback = TensorBoard( 83 | log_dir=config.ALL_LAYERS_LOG_DIR, 84 | batch_size=train_generator.batch_size 85 | ) 86 | 87 | model.compile( 88 | loss='mean_squared_error', 89 | optimizer='adam' 90 | ) 91 | 92 | model.fit_generator( 93 | generator=train_generator, 94 | steps_per_epoch=batches_per_epoch, 95 | epochs=100, 96 | callbacks=[ 97 | mae_callback, 98 | early_stopping_callback, 99 | model_checkpoint_callback, 100 | tensorboard_callback 101 | ] 102 | ) 103 | 104 | 105 | def test_model(model): 106 | 107 | with open(config.CROPPED_IMGS_INFO_FILE, 'r') as f: 108 | test_images_info = f.read().splitlines()[-config.VALIDATION_SIZE:] 109 | 110 | test_X = [] 111 | test_y = [] 112 | for info in test_images_info: 113 | bmi = float(info.split(',')[1]) 114 | test_y.append(bmi) 115 | file_name = info.split(',')[4] 116 | file_path = '%s/%s' % (config.CROPPED_IMGS_DIR, file_name) 117 | img = cv2.imread(file_path) 118 | img = cv2.resize(img, (config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH)) 119 | test_X.append(img) 120 | 121 | test_X = np.array(test_X) 122 | test_y = np.array(test_y) 123 | 124 | mae = get_mae(test_y, model.predict(test_X)) 125 | print '\nMAE:', mae 126 | -------------------------------------------------------------------------------- /train_generator.py: -------------------------------------------------------------------------------- 1 | 2 | from keras.preprocessing.image import ImageDataGenerator 3 | import pandas as pd 4 | import Augmentor 5 | from PIL import Image 6 | import random 7 | import numpy as np 8 | import matplotlib.pyplot as plt 9 | import math 10 | import config 11 | 12 | 13 | def plot_imgs_from_generator(generator, number_imgs_to_show=9): 14 | print ('Plotting images...') 15 | n_rows_cols = int(math.ceil(math.sqrt(number_imgs_to_show))) 16 | plot_index = 1 17 | x_batch, _ = next(generator) 18 | while plot_index <= number_imgs_to_show: 19 | plt.subplot(n_rows_cols, n_rows_cols, plot_index) 20 | plt.imshow(x_batch[plot_index-1]) 21 | plot_index += 1 22 | plt.show() 23 | 24 | 25 | def augment_image(np_img): 26 | p = Augmentor.Pipeline() 27 | p.rotate(probability=1, max_left_rotation=5, max_right_rotation=5) 28 | p.flip_left_right(probability=0.5) 29 | # p.zoom_random(probability=0.5, percentage_area=0.95) 30 | p.random_distortion(probability=0.25, grid_width=2, grid_height=2, magnitude=8) 31 | p.random_color(probability=1, min_factor=0.8, max_factor=1.2) 32 | p.random_contrast(probability=.5, min_factor=0.8, max_factor=1.2) 33 | p.random_brightness(probability=1, min_factor=0.5, max_factor=1.5) 34 | 35 | image = [Image.fromarray(np_img.astype('uint8'))] 36 | for operation in p.operations: 37 | r = round(random.uniform(0, 1), 1) 38 | if r <= operation.probability: 39 | image = operation.perform_operation(image) 40 | image = [np.array(i).astype('float64') for i in image] 41 | return image[0] 42 | 43 | image_processor = ImageDataGenerator( 44 | rescale=1./255, 45 | preprocessing_function=augment_image 46 | ) 47 | 48 | # subtract validation size from training data 49 | with open(config.CROPPED_IMGS_INFO_FILE) as f: 50 | for i, _ in enumerate(f): 51 | pass 52 | training_n = i - config.VALIDATION_SIZE 53 | 54 | train_df=pd.read_csv(config.CROPPED_IMGS_INFO_FILE, nrows=training_n) 55 | 56 | train_generator=image_processor.flow_from_dataframe( 57 | dataframe=train_df, 58 | directory=config.CROPPED_IMGS_DIR, 59 | x_col='name', 60 | y_col='bmi', 61 | class_mode='other', 62 | color_mode='rgb', 63 | target_size=(config.RESNET50_DEFAULT_IMG_WIDTH,config.RESNET50_DEFAULT_IMG_WIDTH), 64 | batch_size=config.TRAIN_BATCH_SIZE) 65 | --------------------------------------------------------------------------------