├── .gitignore ├── README.md ├── _config.yml ├── camera.py ├── face_model.h5 ├── face_model.json ├── filter.py ├── filters ├── scary_eye.png └── sharingan.png ├── haarcascade_frontalface_default.xml └── model.py /.gitignore: -------------------------------------------------------------------------------- 1 | dataset 2 | __pycache__ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Snapchat-Filter-Using-Facial-Keypoints 2 | This Repository contains my project to apply snapchat filters to a camera feed. **Not the best filters in the world - Just an Experiment** 3 | 4 | The model is trained using code in [this repository](https://github.com/piyush2896/Facial-Keypoints-Detection). 5 | 6 | Dataset on which the model is trained is avilable at kaggle over [here](https://www.kaggle.com/c/facial-keypoints-detection/data) 7 | 8 | 9 | ## Dependencies 10 | 1. OpenCV 11 | 2. Matplotlib 12 | 3. Numpy 13 | 4. Keras 14 | 5. Tensorflow 15 | 16 | ## Usage 17 | Run [camera.py](https://github.com/piyush2896/Snapchat-Filter-Using-Facial-Keypoints/blob/master/camera.py) to see the results -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /camera.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | from filter import apply_filter 4 | from model import FaceKeypointsCaptureModel 5 | 6 | rgb = cv2.VideoCapture(0) 7 | facec = cv2.CascadeClassifier('haarcascade_frontalface_default.xml') 8 | labels = {} 9 | 10 | def __get_data__(): 11 | """ 12 | __get_data__: Gets data from the VideoCapture object and classifies them 13 | to a face or no face. 14 | 15 | returns: tuple (faces in image, frame read, grayscale frame) 16 | """ 17 | _, fr = rgb.read() 18 | gray = cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY) 19 | faces = facec.detectMultiScale(gray, 1.3, 5) 20 | 21 | return faces, fr, gray 22 | 23 | def start_app(cnn): 24 | skip_frame = 10 25 | data = [] 26 | flag = False 27 | ix = 0 28 | while True: 29 | ix += 1 30 | 31 | faces, fr, gray_fr = __get_data__() 32 | for (x, y, w, h) in faces: 33 | fc = gray_fr[y:y+h, x:x+w] 34 | 35 | roi = cv2.resize(fc, (96, 96)) 36 | pred, pred_dict = cnn.predict_points(roi[np.newaxis, :, :, np.newaxis]) 37 | pred, pred_dict = cnn.scale_prediction((x, fc.shape[1]+x), (y, fc.shape[0]+y)) 38 | 39 | fr = apply_filter(fr, pred_dict) 40 | if cv2.waitKey(1) == 27: 41 | break 42 | cv2.imshow('Filter', fr) 43 | cv2.destroyAllWindows() 44 | 45 | 46 | if __name__ == '__main__': 47 | model = FaceKeypointsCaptureModel("face_model.json", "face_model.h5") 48 | start_app(model) -------------------------------------------------------------------------------- /face_model.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piyush2896/Snapchat-Filter-Using-Facial-Keypoints/62f5c109cd1e022ac3a9262107b1b9a20f19b11b/face_model.h5 -------------------------------------------------------------------------------- /face_model.json: -------------------------------------------------------------------------------- 1 | {"config": [{"config": {"gamma_regularizer": null, "beta_regularizer": null, "dtype": "float32", "epsilon": 0.001, "name": "batch_normalization_1", "axis": -1, "scale": true, "moving_mean_initializer": {"config": {}, "class_name": "Zeros"}, "gamma_initializer": {"config": {}, "class_name": "Ones"}, "beta_initializer": {"config": {}, "class_name": "Zeros"}, "center": true, "batch_input_shape": [null, 96, 96, 1], "beta_constraint": null, "gamma_constraint": null, "momentum": 0.99, "moving_variance_initializer": {"config": {}, "class_name": "Ones"}, "trainable": true}, "class_name": "BatchNormalization"}, {"config": {"bias_regularizer": null, "dilation_rate": [1, 1], "bias_constraint": null, "dtype": "float32", "filters": 24, "name": "conv2d_1", "strides": [1, 1], "bias_initializer": {"config": {}, "class_name": "Zeros"}, "data_format": "channels_last", "kernel_initializer": {"config": {"mode": "fan_in", "scale": 2.0, "seed": null, "distribution": "normal"}, "class_name": "VarianceScaling"}, "use_bias": true, "padding": "same", "kernel_size": [5, 5], "kernel_constraint": null, "activity_regularizer": null, "activation": "linear", "kernel_regularizer": null, "batch_input_shape": [null, 96, 96, 1], "trainable": true}, "class_name": "Conv2D"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_1"}, "class_name": "Activation"}, {"config": {"padding": "valid", "pool_size": [2, 2], "trainable": true, "name": "max_pooling2d_1", "strides": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D"}, {"config": {"bias_regularizer": null, "dilation_rate": [1, 1], "bias_constraint": null, "trainable": true, "filters": 36, "name": "conv2d_2", "strides": [1, 1], "kernel_regularizer": null, "data_format": "channels_last", "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "use_bias": true, "padding": "valid", "bias_initializer": {"config": {}, "class_name": "Zeros"}, "kernel_size": [5, 5], "activity_regularizer": null, "activation": "linear", "kernel_constraint": null}, "class_name": "Conv2D"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_2"}, "class_name": "Activation"}, {"config": {"padding": "valid", "pool_size": [2, 2], "trainable": true, "name": "max_pooling2d_2", "strides": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D"}, {"config": {"bias_regularizer": null, "dilation_rate": [1, 1], "bias_constraint": null, "trainable": true, "filters": 48, "name": "conv2d_3", "strides": [1, 1], "kernel_regularizer": null, "data_format": "channels_last", "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "use_bias": true, "padding": "valid", "bias_initializer": {"config": {}, "class_name": "Zeros"}, "kernel_size": [5, 5], "activity_regularizer": null, "activation": "linear", "kernel_constraint": null}, "class_name": "Conv2D"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_3"}, "class_name": "Activation"}, {"config": {"padding": "valid", "pool_size": [2, 2], "trainable": true, "name": "max_pooling2d_3", "strides": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D"}, {"config": {"bias_regularizer": null, "dilation_rate": [1, 1], "bias_constraint": null, "trainable": true, "filters": 64, "name": "conv2d_4", "strides": [1, 1], "kernel_regularizer": null, "data_format": "channels_last", "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "use_bias": true, "padding": "valid", "bias_initializer": {"config": {}, "class_name": "Zeros"}, "kernel_size": [3, 3], "activity_regularizer": null, "activation": "linear", "kernel_constraint": null}, "class_name": "Conv2D"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_4"}, "class_name": "Activation"}, {"config": {"padding": "valid", "pool_size": [2, 2], "trainable": true, "name": "max_pooling2d_4", "strides": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D"}, {"config": {"trainable": true, "name": "flatten_1"}, "class_name": "Flatten"}, {"config": {"units": 500, "bias_constraint": null, "trainable": true, "name": "dense_1", "kernel_constraint": null, "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_regularizer": null, "use_bias": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "activity_regularizer": null, "activation": "linear", "kernel_regularizer": null}, "class_name": "Dense"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_5"}, "class_name": "Activation"}, {"config": {"units": 90, "bias_constraint": null, "trainable": true, "name": "dense_2", "kernel_constraint": null, "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_regularizer": null, "use_bias": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "activity_regularizer": null, "activation": "linear", "kernel_regularizer": null}, "class_name": "Dense"}, {"config": {"activation": "relu", "trainable": true, "name": "activation_6"}, "class_name": "Activation"}, {"config": {"units": 30, "bias_constraint": null, "trainable": true, "name": "dense_3", "kernel_constraint": null, "kernel_initializer": {"config": {"mode": "fan_avg", "scale": 1.0, "seed": null, "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_regularizer": null, "use_bias": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "activity_regularizer": null, "activation": "linear", "kernel_regularizer": null}, "class_name": "Dense"}], "keras_version": "2.0.4", "class_name": "Sequential", "backend": "tensorflow"} -------------------------------------------------------------------------------- /filter.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from model import FaceKeypointsCaptureModel 4 | 5 | 6 | def apply_filter(frame, pts_dict): 7 | left_eye_center_x = pts_dict["left_eye_center_x"] 8 | left_eye_center_y = pts_dict["left_eye_center_y"] 9 | left_eye_inner_corner_x = pts_dict["left_eye_inner_corner_x"] 10 | left_eye_inner_corner_y = pts_dict["left_eye_inner_corner_y"] 11 | radius_left = distance((left_eye_center_x, left_eye_center_y), 12 | (left_eye_inner_corner_x, left_eye_inner_corner_y)) 13 | 14 | right_eye_center_x = pts_dict["right_eye_center_x"] 15 | right_eye_center_y = pts_dict["right_eye_center_y"] 16 | right_eye_inner_corner_x = pts_dict["right_eye_inner_corner_x"] 17 | right_eye_inner_corner_y = pts_dict["right_eye_inner_corner_y"] 18 | radius_right = distance((right_eye_center_x, right_eye_center_y), 19 | (right_eye_inner_corner_x, right_eye_inner_corner_y)) 20 | 21 | frame = apply_filter_eye_helper(frame, int(left_eye_center_x), 22 | int(left_eye_center_y), int(radius_left)) 23 | frame = apply_filter_eye_helper(frame, int(right_eye_center_x), 24 | int(right_eye_center_y), int(radius_right)) 25 | return frame 26 | 27 | 28 | def apply_filter_eye_helper(frame, x, y, radius): 29 | adjust_rad = radius - 3 30 | filter_img = cv2.resize(cv2.imread("filters/sharingan.png"), 31 | (2*adjust_rad, 2*adjust_rad)) 32 | 33 | slice = frame[y-adjust_rad:y+adjust_rad, x-adjust_rad:x+adjust_rad, :] 34 | for i in range(slice.shape[2]): 35 | for j in range(slice.shape[1]): 36 | slice[filter_img[:, j, i] != 0, j, i] = filter_img[filter_img[:, j, i]!=0, j, i] 37 | frame[y-adjust_rad:y+adjust_rad, x-adjust_rad:x+adjust_rad, :] = slice 38 | return frame 39 | 40 | def distance(pt1, pt2): 41 | pt1 = np.array(pt1) 42 | pt2 = np.array(pt2) 43 | return np.sqrt(sum((pt1-pt2)**2)) 44 | 45 | if __name__ == '__main__': 46 | model = FaceKeypointsCaptureModel("face_model.json", "face_model.h5") 47 | 48 | import matplotlib.pyplot as plt 49 | import cv2 50 | img = cv2.cvtColor(cv2.imread('dataset/trial1.jpg'), cv2.COLOR_BGR2GRAY) 51 | img1 = cv2.resize(img, (96, 96)) 52 | img1 = img1[np.newaxis, :, :, np.newaxis] 53 | 54 | print(img1.shape) 55 | 56 | pts, pts_dict = model.predict_points(img1) 57 | pts1, pred_dict1 = model.scale_prediction((0, 200), (0, 200)) 58 | 59 | fr = apply_filter(img, pred_dict1) 60 | 61 | plt.figure(0) 62 | plt.imshow(fr, cmap='gray') 63 | plt.show() -------------------------------------------------------------------------------- /filters/scary_eye.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piyush2896/Snapchat-Filter-Using-Facial-Keypoints/62f5c109cd1e022ac3a9262107b1b9a20f19b11b/filters/scary_eye.png -------------------------------------------------------------------------------- /filters/sharingan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piyush2896/Snapchat-Filter-Using-Facial-Keypoints/62f5c109cd1e022ac3a9262107b1b9a20f19b11b/filters/sharingan.png -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | from keras.models import model_from_json 2 | import numpy as np 3 | 4 | 5 | class FaceKeypointsCaptureModel(object): 6 | 7 | COLUMNS = ['left_eye_center_x', 'left_eye_center_y', 8 | 'right_eye_center_x', 'right_eye_center_y', 9 | 'left_eye_inner_corner_x', 'left_eye_inner_corner_y', 10 | 'left_eye_outer_corner_x', 'left_eye_outer_corner_y', 11 | 'right_eye_inner_corner_x', 'right_eye_inner_corner_y', 12 | 'right_eye_outer_corner_x', 'right_eye_outer_corner_y', 13 | 'left_eyebrow_inner_end_x', 'left_eyebrow_inner_end_y', 14 | 'left_eyebrow_outer_end_x', 'left_eyebrow_outer_end_y', 15 | 'right_eyebrow_inner_end_x', 'right_eyebrow_inner_end_y', 16 | 'right_eyebrow_outer_end_x', 'right_eyebrow_outer_end_y', 17 | 'nose_tip_x', 'nose_tip_y', 18 | 'mouth_left_corner_x', 'mouth_left_corner_y', 19 | 'mouth_right_corner_x', 'mouth_right_corner_y', 20 | 'mouth_center_top_lip_x', 'mouth_center_top_lip_y', 21 | 'mouth_center_bottom_lip_x', 'mouth_center_bottom_lip_y'] 22 | 23 | def __init__(self, model_json_file, model_weights_file): 24 | # load model from JSON file 25 | with open(model_json_file, "r") as json_file: 26 | loaded_model_json = json_file.read() 27 | self.loaded_model = model_from_json(loaded_model_json) 28 | 29 | # load weights into the new model 30 | self.loaded_model.load_weights(model_weights_file) 31 | print("Model loaded from disk") 32 | self.loaded_model.summary() 33 | 34 | def predict_points(self, img): 35 | self.preds = self.loaded_model.predict(img) % 96 36 | 37 | self.pred_dict = dict([(point, val) for point, val in zip(FaceKeypointsCaptureModel.COLUMNS, self.preds[0])]) 38 | 39 | return self.preds, self.pred_dict 40 | 41 | def scale_prediction(self, out_range_x=(-1, 1), out_range_y=(-1, 1)): 42 | range_ = [0, 96] 43 | self.preds = ((self.preds - range_[0]) / (range_[1] - range_[0])) 44 | self.preds[:, range(0, 30, 2)] = ((self.preds[:, range(0, 30, 2)] * 45 | (out_range_x[1] - out_range_x[0])) + out_range_x[0]) 46 | self.preds[:, range(1, 30, 2)] = ((self.preds[:, range(1, 30, 2)] * 47 | (out_range_y[1] - out_range_y[0])) + out_range_y[0]) 48 | 49 | self.pred_dict = dict([(point, val) for point, val in zip(FaceKeypointsCaptureModel.COLUMNS, self.preds[0])]) 50 | return self.preds, self.pred_dict 51 | 52 | 53 | if __name__ == '__main__': 54 | model = FaceKeypointsCaptureModel("face_model.json", "face_model.h5") 55 | 56 | import matplotlib.pyplot as plt 57 | import cv2 58 | img = cv2.cvtColor(cv2.imread('dataset/trial1.jpg'), cv2.COLOR_BGR2GRAY) 59 | img1 = cv2.resize(img, (96, 96)) 60 | img1 = img1[np.newaxis, :, :, np.newaxis] 61 | 62 | print(img1.shape) 63 | 64 | pts, pts_dict = model.predict_points(img1) 65 | pts1, pred_dict1 = model.scale_prediction((0, 200)) 66 | 67 | plt.figure(0) 68 | plt.subplot(1, 2, 1) 69 | plt.imshow(img, cmap='gray', interpolation=None) 70 | plt.scatter(pts1[range(0, 30, 2)], pts1[range(1, 30, 2)], marker='x') 71 | 72 | plt.subplot(1, 2, 2) 73 | plt.imshow(img1[0, :, :, 0], cmap='gray', interpolation=None) 74 | plt.scatter(pts[0, range(0, 30, 2)], pts[0, range(1, 30, 2)], marker='x') 75 | plt.show() --------------------------------------------------------------------------------