├── dependencies.txt ├── final report.pdf ├── requirements.txt ├── _mini_XCEPTION.102-0.66.hdf5 ├── REAL TIME STRESS DETECTION AND PROPOSED SOLUTION.pptx ├── preprocess.py ├── load_and_process.py ├── license.txt ├── emotion_recognition.py ├── README.md ├── train_emotion_classifier.py ├── blink_detection.py ├── real_time_video.py ├── eyebrow_detection.py ├── cnn1.py └── cnn.py /dependencies.txt: -------------------------------------------------------------------------------- 1 | keras 2 | tensorflow 3 | dlib 4 | imutils 5 | sklearn 6 | cv2 7 | matplotlib 8 | numpy 9 | scipy 10 | -------------------------------------------------------------------------------- /final report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nancypareta/Stress-detection-Techniques-and-Chat-Bot-Depression-Therapy/HEAD/final report.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | opencv_python==4.1.1.26 2 | Keras==2.3.1 3 | pandas==0.25.3 4 | numpy==1.17.4 5 | imutils==0.5.3 6 | scikit_learn==0.22.1 7 | -------------------------------------------------------------------------------- /_mini_XCEPTION.102-0.66.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nancypareta/Stress-detection-Techniques-and-Chat-Bot-Depression-Therapy/HEAD/_mini_XCEPTION.102-0.66.hdf5 -------------------------------------------------------------------------------- /REAL TIME STRESS DETECTION AND PROPOSED SOLUTION.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nancypareta/Stress-detection-Techniques-and-Chat-Bot-Depression-Therapy/HEAD/REAL TIME STRESS DETECTION AND PROPOSED SOLUTION.pptx -------------------------------------------------------------------------------- /preprocess.py: -------------------------------------------------------------------------------- 1 | def preprocess_input(x): 2 | x = x.astype('float32') 3 | x = x / 255.0 4 | x = x - 0.5 5 | x = x * 2 6 | return x 7 | 8 | '''def preprocess_input(x, v2=True): 9 | x = x.astype('float32') 10 | x = x / 255.0 11 | if v2: 12 | # x = x - 0.5 13 | # x = x * 2.0 14 | return x''' 15 | -------------------------------------------------------------------------------- /load_and_process.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import cv2 3 | import numpy as np 4 | 5 | 6 | dataset_path = r'D:\Stress-Detection-master\Code\fer2013\fer2013\fer2013.csv' 7 | image_size=(48,48) 8 | 9 | def load_fer2013(): 10 | data = pd.read_csv(dataset_path) 11 | pixels = data['pixels'].tolist() 12 | width, height = 48, 48 13 | faces = [] 14 | for pixel_sequence in pixels: 15 | face = [int(pixel) for pixel in pixel_sequence.split(' ')] 16 | face = np.asarray(face).reshape(width, height) 17 | face = cv2.resize(face.astype('uint8'),image_size) 18 | faces.append(face.astype('float32')) 19 | faces = np.asarray(faces) 20 | faces = np.expand_dims(faces, -1) 21 | emotions = pd.get_dummies(data['emotion']) 22 | return faces, emotions 23 | 24 | def preprocess_input(x, v2=True): 25 | x = x.astype('float32') 26 | x = x / 255.0 27 | if v2: 28 | x = x - 0.5 29 | x = x * 2.0 30 | return x -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2018] [Omar Ayman] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /emotion_recognition.py: -------------------------------------------------------------------------------- 1 | from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping 2 | from keras.callbacks import ReduceLROnPlateau 3 | from keras.preprocessing.image import ImageDataGenerator 4 | from load_and_process import load_fer2013 5 | from load_and_process import preprocess_input 6 | from models.cnn import mini_XCEPTION 7 | from sklearn.model_selection import train_test_split 8 | 9 | # parameters 10 | batch_size = 32 11 | num_epochs = 100 12 | input_shape = (48, 48, 1) 13 | validation_split = .2 14 | verbose = 1 15 | num_classes = 7 16 | patience = 50 17 | base_path = 'models/' 18 | 19 | # data generator 20 | data_generator = ImageDataGenerator( 21 | featurewise_center=False, 22 | featurewise_std_normalization=False, 23 | rotation_range=10, 24 | width_shift_range=0.1, 25 | height_shift_range=0.1, 26 | zoom_range=.1, 27 | horizontal_flip=True) 28 | 29 | # model parameters/compilation 30 | model = mini_XCEPTION(input_shape, num_classes) 31 | model.compile(optimizer='adam', loss='categorical_crossentropy', 32 | metrics=['accuracy']) 33 | model.summary() 34 | 35 | 36 | 37 | 38 | 39 | # callbacks 40 | log_file_path = base_path + '_emotion_training.log' 41 | csv_logger = CSVLogger(log_file_path, append=False) 42 | early_stop = EarlyStopping('val_loss', patience=patience) 43 | reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, 44 | patience=int(patience/4), verbose=1) 45 | trained_models_path = base_path + '_mini_XCEPTION' 46 | model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5' 47 | model_checkpoint = ModelCheckpoint(model_names, 'val_loss', verbose=1, 48 | save_best_only=True) 49 | callbacks = [model_checkpoint, csv_logger, early_stop, reduce_lr] 50 | 51 | # loading dataset 52 | faces, emotions = load_fer2013() 53 | faces = preprocess_input(faces) 54 | num_samples, num_classes = emotions.shape 55 | xtrain, xtest,ytrain,ytest = train_test_split(faces, emotions,test_size=0.2,shuffle=True) 56 | model.fit_generator(data_generator.flow(xtrain, ytrain, 57 | batch_size), 58 | steps_per_epoch=len(xtrain) / batch_size, 59 | epochs=num_epochs, verbose=1, callbacks=callbacks, 60 | validation_data=(xtest,ytest)) 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ABSTRACT 2 | REAL TIME STRESS DETECTION AND PROPOSED SOLUTIONS 3 | 4 | 5 | Stress has become part and parcel in every individual’s day to day lifestyle. From youngsters to sandwich generation, stress has become a major concern for the wellness economy of country India. In spite of, a lot of progress has been done in recognizing stress physiologically, there are still barriers in recognizing stress in real lifetime in cost effective way and providing hand to hand solutions. Human psychological stress and human emotion are very much interconnected. In computational psychology study, the relationship between stress and emotions is the key to understanding the human behavior. Research has been done for detecting facial emotions from images using deep learning but has not been explicitly taken up yet to find psychological stress. We have reviewed various stress recognition techniques and Proposed a hybrid system in which a convolutional neural network (CNN) is used to detect the stress levels of human being through face musculature detected from image .A CNN is trained to detect and recognize facial expressions and classify human faces into discrete emotion categories (Anger, Disgust, Neutral, Fear, Sad, Happy and Surprise). The Real time face stress detection model is image processing-based model which is having two parts: Emotion Recognition and Stress level calculation. 6 | 7 | The Real time emotion recognition model will return the emotion predicted in real time. The model classifies face as stressed and not stressed. A model is trained on the fer2013 dataset (https://www.kaggle.com/deadskull7/fer2013) .The stress level is calculated with the help of eyebrows contraction and displacement from the mean position. The distance between the left and right eyebrow is being calculated and then the stress level is calculated using exponential function and normalized between 1 to 100. 8 | Chatbot-Depression Therapy to provide real time therapeutic solutions to alleviate depression. 9 | Chatbot System is implemented using deep learning for detection and management of stress and depression and provide suggestions accordingly based on user’s mental condition. Technologies: Keras, genism python libraries, anaconda environment, the dataset being used is obtained from Kaggle. 10 | Keywords: Cognitive behavior therapy, Deep learning, Convolution Neural Network (CNN), Facial Expression Recognition. 11 | 12 | 13 | -------------------------------------------------------------------------------- /train_emotion_classifier.py: -------------------------------------------------------------------------------- 1 | """ 2 | Description: Train emotion classification model 3 | """ 4 | 5 | from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping 6 | from keras.callbacks import ReduceLROnPlateau 7 | from keras.preprocessing.image import ImageDataGenerator 8 | from load_and_process import load_fer2013 9 | from load_and_process import preprocess_input 10 | from models.cnn import mini_XCEPTION 11 | from sklearn.model_selection import train_test_split 12 | 13 | # parameters 14 | batch_size = 32 15 | num_epochs = 10000 16 | input_shape = (48, 48, 1) 17 | validation_split = .2 18 | verbose = 1 19 | num_classes = 7 20 | patience = 50 21 | base_path = 'models/' 22 | 23 | # data generator 24 | data_generator = ImageDataGenerator( 25 | featurewise_center=False, 26 | featurewise_std_normalization=False, 27 | rotation_range=10, 28 | width_shift_range=0.1, 29 | height_shift_range=0.1, 30 | zoom_range=.1, 31 | horizontal_flip=True) 32 | 33 | # model parameters/compilation 34 | model = mini_XCEPTION(input_shape, num_classes) 35 | model.compile(optimizer='adam', loss='categorical_crossentropy', 36 | metrics=['accuracy']) 37 | model.summary() 38 | 39 | 40 | 41 | 42 | 43 | # callbacks 44 | log_file_path = base_path + '_emotion_training.log' 45 | csv_logger = CSVLogger(log_file_path, append=False) 46 | early_stop = EarlyStopping('val_loss', patience=patience) 47 | reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, 48 | patience=int(patience/4), verbose=1) 49 | trained_models_path = base_path + '_mini_XCEPTION' 50 | model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5' 51 | model_checkpoint = ModelCheckpoint(model_names, 'val_loss', verbose=1, 52 | save_best_only=True) 53 | callbacks = [model_checkpoint, csv_logger, early_stop, reduce_lr] 54 | 55 | # loading dataset 56 | faces, emotions = load_fer2013() 57 | faces = preprocess_input(faces) 58 | num_samples, num_classes = emotions.shape 59 | xtrain, xtest,ytrain,ytest = train_test_split(faces, emotions,test_size=0.2,shuffle=True) 60 | model.fit_generator(data_generator.flow(xtrain, ytrain, 61 | batch_size), 62 | steps_per_epoch=len(xtrain) / batch_size, 63 | epochs=num_epochs, verbose=1, callbacks=callbacks, 64 | validation_data=(xtest,ytest)) 65 | -------------------------------------------------------------------------------- /blink_detection.py: -------------------------------------------------------------------------------- 1 | from scipy.spatial import distance as dist 2 | from imutils.video import VideoStream 3 | from imutils import face_utils 4 | import numpy as np 5 | import imutils 6 | import time 7 | import dlib 8 | import cv2 9 | 10 | #get the location of the eyes 11 | def eye_aspect_ratio(eye): 12 | # compute the euclidean distances between the vertical landamrks 13 | A = dist.euclidean(eye[1], eye[5]) 14 | B = dist.euclidean(eye[2], eye[4]) 15 | 16 | # compute the euclidean distance between the horizontal 17 | C = dist.euclidean(eye[0], eye[3]) 18 | 19 | # compute the eye aspect ratio 20 | eye_opening_ratio = (A + B) / (2.0 * C) 21 | 22 | # return the eye aspect ratio 23 | return eye_opening_ratio 24 | 25 | 26 | # the consecuting frame factor tells us to consider this amount of farme. 27 | ar_thresh = 0.3 28 | eye_ar_consec_frame = 5 29 | counter = 0 30 | total = 0 31 | 32 | # get the frontal face detector and shape predictor 33 | detector = dlib.get_frontal_face_detector() 34 | predictor = dlib.shape_predictor(r"C:\Users\Lenovo\Downloads/shape_predictor_68_face_landmarks.dat") 35 | cap = cv2.VideoCapture(0) 36 | while(True): 37 | _,frame = cap.read() 38 | frame = cv2.flip(frame,1) 39 | frame = imutils.resize(frame, width=500,height=500) 40 | 41 | (lBegin, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] 42 | (rBegin, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"] 43 | #preprocessing the image 44 | gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) 45 | clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8)) 46 | clahe_image = clahe.apply(gray) 47 | detections = detector(clahe_image,0) 48 | for detection in detections: 49 | shape = predictor(gray,detection) 50 | shape = face_utils.shape_to_np(shape) 51 | left_eye = shape[lBegin:lEnd] 52 | right_eye = shape[rBegin:rEnd] 53 | 54 | leftEyeHull = cv2.convexHull(left_eye) 55 | rightEyeHull= cv2.convexHull(right_eye) 56 | cv2.drawContours(clahe_image, [leftEyeHull], -1, (0, 255, 0), 1) 57 | cv2.drawContours(clahe_image, [rightEyeHull], -1, (0, 255, 0), 1) 58 | #calculating the EAR 59 | left_eye_Ear = eye_aspect_ratio(left_eye) 60 | right_eye_Ear = eye_aspect_ratio(right_eye) 61 | 62 | avg_Ear = (left_eye_Ear + right_eye_Ear)/2.0 63 | 64 | if avg_Eareye_ar_consec_frame: 68 | total+= 1 69 | counter = 0 70 | cv2.putText(clahe_image, "Blinks: {}".format(total), (10, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 71 | cv2.putText(clahe_image, "EAR: {:.2f}".format(avg_Ear), (300, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 72 | cv2.imshow("Frame", clahe_image) 73 | 74 | key = cv2.waitKey(1) & 0xFF 75 | if key == ord('q'): 76 | break 77 | cv2.destroyAllWindows() 78 | cap.release() 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /real_time_video.py: -------------------------------------------------------------------------------- 1 | from keras.preprocessing.image import img_to_array 2 | import imutils 3 | import cv2 4 | from keras.models import load_model 5 | import numpy as np 6 | 7 | # parameters for loading data and images 8 | detection_model_path = 'haarcascade_files/haarcascade_frontalface_default.xml' 9 | emotion_model_path = 'models/_mini_XCEPTION.102-0.66.hdf5' 10 | 11 | # hyper-parameters for bounding boxes shape 12 | # loading models 13 | face_detection = cv2.CascadeClassifier(detection_model_path) 14 | emotion_classifier = load_model(emotion_model_path, compile=False) 15 | EMOTIONS = ["angry" ,"disgust","scared", "happy", "sad", "surprised", 16 | "neutral"] 17 | 18 | 19 | #feelings_faces = [] 20 | #for index, emotion in enumerate(EMOTIONS): 21 | # feelings_faces.append(cv2.imread('emojis/' + emotion + '.png', -1)) 22 | 23 | # starting video streaming 24 | cv2.namedWindow('your_face') 25 | camera = cv2.VideoCapture(0) 26 | while True: 27 | frame = camera.read()[1] 28 | #reading the frame 29 | frame = imutils.resize(frame,width=300) 30 | gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 31 | faces = face_detection.detectMultiScale(gray,scaleFactor=1.1,minNeighbors=5,minSize=(30,30),flags=cv2.CASCADE_SCALE_IMAGE) 32 | 33 | canvas = np.zeros((250, 300, 3), dtype="uint8") 34 | frameClone = frame.copy() 35 | if len(faces) > 0: 36 | faces = sorted(faces, reverse=True, 37 | key=lambda x: (x[2] - x[0]) * (x[3] - x[1]))[0] 38 | (fX, fY, fW, fH) = faces 39 | # Extract the ROI of the face from the grayscale image, resize it to a fixed 28x28 pixels, and then prepare 40 | # the ROI for classification via the CNN 41 | roi = gray[fY:fY + fH, fX:fX + fW] 42 | roi = cv2.resize(roi, (64, 64)) 43 | roi = roi.astype("float") / 255.0 44 | roi = img_to_array(roi) 45 | roi = np.expand_dims(roi, axis=0) 46 | 47 | 48 | preds = emotion_classifier.predict(roi)[0] 49 | emotion_probability = np.max(preds) 50 | label = EMOTIONS[preds.argmax()] 51 | else: continue 52 | 53 | 54 | for (i, (emotion, prob)) in enumerate(zip(EMOTIONS, preds)): 55 | # construct the label text 56 | text = "{}: {:.2f}%".format(emotion, prob * 100) 57 | 58 | # draw the label + probability bar on the canvas 59 | # emoji_face = feelings_faces[np.argmax(preds)] 60 | 61 | 62 | w = int(prob * 300) 63 | cv2.rectangle(canvas, (7, (i * 35) + 5), 64 | (w, (i * 35) + 35), (0, 0, 255), -1) 65 | cv2.putText(canvas, text, (10, (i * 35) + 23), 66 | cv2.FONT_HERSHEY_SIMPLEX, 0.45, 67 | (255, 255, 255), 2) 68 | cv2.putText(frameClone, label, (fX, fY - 10), 69 | cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2) 70 | cv2.rectangle(frameClone, (fX, fY), (fX + fW, fY + fH), 71 | (0, 0, 255), 2) 72 | # for c in range(0, 3): 73 | # frame[200:320, 10:130, c] = emoji_face[:, :, c] * \ 74 | # (emoji_face[:, :, 3] / 255.0) + frame[200:320, 75 | # 10:130, c] * (1.0 - emoji_face[:, :, 3] / 255.0) 76 | 77 | 78 | cv2.imshow('your_face', frameClone) 79 | cv2.imshow("Probabilities", canvas) 80 | if cv2.waitKey(1) & 0xFF == ord('q'): 81 | break 82 | 83 | camera.release() 84 | cv2.destroyAllWindows() 85 | -------------------------------------------------------------------------------- /eyebrow_detection.py: -------------------------------------------------------------------------------- 1 | from scipy.spatial import distance as dist 2 | from imutils.video import VideoStream 3 | from imutils import face_utils 4 | import numpy as np 5 | import imutils 6 | import time 7 | import dlib 8 | import cv2 9 | import matplotlib.pyplot as plt 10 | from keras.preprocessing.image import img_to_array 11 | from keras.models import load_model 12 | 13 | def eye_brow_distance(leye,reye): 14 | global points 15 | distq = dist.euclidean(leye,reye) 16 | points.append(int(distq)) 17 | return distq 18 | 19 | def emotion_finder(faces,frame): 20 | global emotion_classifier 21 | EMOTIONS = ["angry" ,"disgust","scared", "happy", "sad", "surprised","neutral"] 22 | x,y,w,h = face_utils.rect_to_bb(faces) 23 | frame = frame[y:y+h,x:x+w] 24 | roi = cv2.resize(frame,(64,64)) 25 | roi = roi.astype("float") / 255.0 26 | roi = img_to_array(roi) 27 | roi = np.expand_dims(roi,axis=0) 28 | preds = emotion_classifier.predict(roi)[0] 29 | emotion_probability = np.max(preds) 30 | label = EMOTIONS[preds.argmax()] 31 | if label in ['scared','sad']: 32 | label = 'stressed' 33 | else: 34 | label = 'not stressed' 35 | return label 36 | 37 | def normalize_values(points,disp): 38 | normalized_value = abs(disp - np.min(points))/abs(np.max(points) - np.min(points)) 39 | stress_value = np.exp(-(normalized_value)) 40 | print(stress_value) 41 | if stress_value>=75: 42 | return stress_value,"High Stress" 43 | else: 44 | return stress_value,"low_stress" 45 | 46 | detector = dlib.get_frontal_face_detector() 47 | predictor = dlib.shape_predictor(r"C:\Users\Lenovo\Downloads/shape_predictor_68_face_landmarks.dat") 48 | emotion_classifier = load_model(r"D:\Stress-Detection-master/_mini_XCEPTION.102-0.66.hdf5", compile=False) 49 | cap = cv2.VideoCapture(0) 50 | points = [] 51 | while(True): 52 | _,frame = cap.read() 53 | frame = cv2.flip(frame,1) 54 | frame = imutils.resize(frame, width=500,height=500) 55 | 56 | 57 | (lBegin, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eyebrow"] 58 | (rBegin, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eyebrow"] 59 | 60 | #preprocessing the image 61 | gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) 62 | 63 | detections = detector(gray,0) 64 | for detection in detections: 65 | emotion = emotion_finder(detection,gray) 66 | cv2.putText(frame, emotion, (10,10),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) 67 | shape = predictor(frame,detection) 68 | shape = face_utils.shape_to_np(shape) 69 | 70 | leyebrow = shape[lBegin:lEnd] 71 | reyebrow = shape[rBegin:rEnd] 72 | 73 | reyebrowhull = cv2.convexHull(reyebrow) 74 | leyebrowhull = cv2.convexHull(leyebrow) 75 | 76 | cv2.drawContours(frame, [reyebrowhull], -1, (0, 255, 0), 1) 77 | cv2.drawContours(frame, [leyebrowhull], -1, (0, 255, 0), 1) 78 | 79 | distq = eye_brow_distance(leyebrow[-1],reyebrow[0]) 80 | stress_value,stress_label = normalize_values(points,distq) 81 | cv2.putText(frame,"stress level:{}".format(str(int(stress_value*100))),(20,40),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) 82 | 83 | cv2.imshow("Frame", frame) 84 | 85 | key = cv2.waitKey(1) & 0xFF 86 | if key == ord('q'): 87 | break 88 | cv2.destroyAllWindows() 89 | cap.release() 90 | plt.plot(range(len(points)),points,'ro') 91 | plt.title("Stress Levels") 92 | plt.show() 93 | -------------------------------------------------------------------------------- /cnn1.py: -------------------------------------------------------------------------------- 1 | from keras.layers import Activation, Convolution2D, Dropout, Conv2D 2 | from keras.layers import AveragePooling2D, BatchNormalization 3 | from keras.layers import GlobalAveragePooling2D 4 | from keras.models import Sequential 5 | from keras.layers import Flatten 6 | from keras.models import Model 7 | from keras.layers import Input 8 | from keras.layers import MaxPooling2D 9 | from keras.layers import SeparableConv2D 10 | from keras import layers 11 | from keras.regularizers import l2 12 | 13 | def mini_XCEPTION(input_shape, num_classes, l2_regularization=0.01): 14 | regularization = l2(l2_regularization) 15 | 16 | # base 17 | img_input = Input(input_shape) 18 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 19 | use_bias=False)(img_input) 20 | x = BatchNormalization()(x) 21 | x = Activation('relu')(x) 22 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 23 | use_bias=False)(x) 24 | x = BatchNormalization()(x) 25 | x = Activation('relu')(x) 26 | 27 | # module 1 28 | residual = Conv2D(16, (1, 1), strides=(2, 2), 29 | padding='same', use_bias=False)(x) 30 | residual = BatchNormalization()(residual) 31 | 32 | x = SeparableConv2D(16, (3, 3), padding='same', 33 | kernel_regularizer=regularization, 34 | use_bias=False)(x) 35 | x = BatchNormalization()(x) 36 | x = Activation('relu')(x) 37 | x = SeparableConv2D(16, (3, 3), padding='same', 38 | kernel_regularizer=regularization, 39 | use_bias=False)(x) 40 | x = BatchNormalization()(x) 41 | 42 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 43 | x = layers.add([x, residual]) 44 | 45 | # module 2 46 | 47 | residual = Conv2D(32, (1, 1), strides=(2, 2), 48 | padding='same', use_bias=False)(x) 49 | residual = BatchNormalization()(residual) 50 | 51 | x = SeparableConv2D(32, (3, 3), padding='same', 52 | kernel_regularizer=regularization, 53 | use_bias=False)(x) 54 | x = BatchNormalization()(x) 55 | x = Activation('relu')(x) 56 | x = SeparableConv2D(32, (3, 3), padding='same', 57 | kernel_regularizer=regularization, 58 | use_bias=False)(x) 59 | x = BatchNormalization()(x) 60 | 61 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 62 | x = layers.add([x, residual]) 63 | 64 | # module 3 65 | residual = Conv2D(64, (1, 1), strides=(2, 2), 66 | padding='same', use_bias=False)(x) 67 | residual = BatchNormalization()(residual) 68 | 69 | x = SeparableConv2D(64, (3, 3), padding='same', 70 | kernel_regularizer=regularization, 71 | use_bias=False)(x) 72 | x = BatchNormalization()(x) 73 | x = Activation('relu')(x) 74 | x = SeparableConv2D(64, (3, 3), padding='same', 75 | kernel_regularizer=regularization, 76 | use_bias=False)(x) 77 | x = BatchNormalization()(x) 78 | 79 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 80 | x = layers.add([x, residual]) 81 | 82 | # module 4 83 | residual = Conv2D(128, (1, 1), strides=(2, 2), 84 | padding='same', use_bias=False)(x) 85 | residual = BatchNormalization()(residual) 86 | 87 | x = SeparableConv2D(128, (3, 3), padding='same', 88 | kernel_regularizer=regularization, 89 | use_bias=False)(x) 90 | x = BatchNormalization()(x) 91 | x = Activation('relu')(x) 92 | x = SeparableConv2D(128, (3, 3), padding='same', 93 | kernel_regularizer=regularization, 94 | use_bias=False)(x) 95 | x = BatchNormalization()(x) 96 | 97 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 98 | x = layers.add([x, residual]) 99 | 100 | x = Conv2D(num_classes, (3, 3), 101 | #kernel_regularizer=regularization, 102 | padding='same')(x) 103 | x = GlobalAveragePooling2D()(x) 104 | output = Activation('softmax',name='predictions')(x) 105 | 106 | model = Model(img_input, output) 107 | return model 108 | 109 | 110 | -------------------------------------------------------------------------------- /cnn.py: -------------------------------------------------------------------------------- 1 | from keras.layers import Activation, Convolution2D, Dropout, Conv2D 2 | from keras.layers import AveragePooling2D, BatchNormalization 3 | from keras.layers import GlobalAveragePooling2D 4 | from keras.models import Sequential 5 | from keras.layers import Flatten 6 | from keras.models import Model 7 | from keras.layers import Input 8 | from keras.layers import MaxPooling2D 9 | from keras.layers import SeparableConv2D 10 | from keras import layers 11 | from keras.regularizers import l2 12 | 13 | def simple_CNN(input_shape, num_classes): 14 | 15 | model = Sequential() 16 | model.add(Convolution2D(filters=16, kernel_size=(7, 7), padding='same', 17 | name='image_array', input_shape=input_shape)) 18 | model.add(BatchNormalization()) 19 | model.add(Convolution2D(filters=16, kernel_size=(7, 7), padding='same')) 20 | model.add(BatchNormalization()) 21 | model.add(Activation('relu')) 22 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 23 | model.add(Dropout(.5)) 24 | 25 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 26 | model.add(BatchNormalization()) 27 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 28 | model.add(BatchNormalization()) 29 | model.add(Activation('relu')) 30 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 31 | model.add(Dropout(.5)) 32 | 33 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 34 | model.add(BatchNormalization()) 35 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 36 | model.add(BatchNormalization()) 37 | model.add(Activation('relu')) 38 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 39 | model.add(Dropout(.5)) 40 | 41 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same')) 42 | model.add(BatchNormalization()) 43 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same')) 44 | model.add(BatchNormalization()) 45 | model.add(Activation('relu')) 46 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 47 | model.add(Dropout(.5)) 48 | 49 | model.add(Convolution2D(filters=256, kernel_size=(3, 3), padding='same')) 50 | model.add(BatchNormalization()) 51 | model.add(Convolution2D(filters=num_classes, kernel_size=(3, 3), padding='same')) 52 | model.add(GlobalAveragePooling2D()) 53 | model.add(Activation('softmax',name='predictions')) 54 | return model 55 | 56 | def simpler_CNN(input_shape, num_classes): 57 | 58 | model = Sequential() 59 | model.add(Convolution2D(filters=16, kernel_size=(5, 5), padding='same', 60 | name='image_array', input_shape=input_shape)) 61 | model.add(BatchNormalization()) 62 | model.add(Convolution2D(filters=16, kernel_size=(5, 5), 63 | strides=(2, 2), padding='same')) 64 | model.add(BatchNormalization()) 65 | model.add(Activation('relu')) 66 | model.add(Dropout(.25)) 67 | 68 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 69 | model.add(BatchNormalization()) 70 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), 71 | strides=(2, 2), padding='same')) 72 | model.add(BatchNormalization()) 73 | model.add(Activation('relu')) 74 | model.add(Dropout(.25)) 75 | 76 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 77 | model.add(BatchNormalization()) 78 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), 79 | strides=(2, 2), padding='same')) 80 | model.add(BatchNormalization()) 81 | model.add(Activation('relu')) 82 | model.add(Dropout(.25)) 83 | 84 | model.add(Convolution2D(filters=64, kernel_size=(1, 1), padding='same')) 85 | model.add(BatchNormalization()) 86 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), 87 | strides=(2, 2), padding='same')) 88 | model.add(BatchNormalization()) 89 | model.add(Activation('relu')) 90 | model.add(Dropout(.25)) 91 | 92 | model.add(Convolution2D(filters=256, kernel_size=(1, 1), padding='same')) 93 | model.add(BatchNormalization()) 94 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), 95 | strides=(2, 2), padding='same')) 96 | 97 | model.add(Convolution2D(filters=256, kernel_size=(1, 1), padding='same')) 98 | model.add(BatchNormalization()) 99 | model.add(Convolution2D(filters=num_classes, kernel_size=(3, 3), 100 | strides=(2, 2), padding='same')) 101 | 102 | model.add(Flatten()) 103 | #model.add(GlobalAveragePooling2D()) 104 | model.add(Activation('softmax',name='predictions')) 105 | return model 106 | 107 | def tiny_XCEPTION(input_shape, num_classes, l2_regularization=0.01): 108 | regularization = l2(l2_regularization) 109 | 110 | # base 111 | img_input = Input(input_shape) 112 | x = Conv2D(5, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 113 | use_bias=False)(img_input) 114 | x = BatchNormalization()(x) 115 | x = Activation('relu')(x) 116 | x = Conv2D(5, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 117 | use_bias=False)(x) 118 | x = BatchNormalization()(x) 119 | x = Activation('relu')(x) 120 | 121 | # module 1 122 | residual = Conv2D(8, (1, 1), strides=(2, 2), 123 | padding='same', use_bias=False)(x) 124 | residual = BatchNormalization()(residual) 125 | 126 | x = SeparableConv2D(8, (3, 3), padding='same', 127 | kernel_regularizer=regularization, 128 | use_bias=False)(x) 129 | x = BatchNormalization()(x) 130 | x = Activation('relu')(x) 131 | x = SeparableConv2D(8, (3, 3), padding='same', 132 | kernel_regularizer=regularization, 133 | use_bias=False)(x) 134 | x = BatchNormalization()(x) 135 | 136 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 137 | x = layers.add([x, residual]) 138 | 139 | # module 2 140 | residual = Conv2D(16, (1, 1), strides=(2, 2), 141 | padding='same', use_bias=False)(x) 142 | residual = BatchNormalization()(residual) 143 | 144 | x = SeparableConv2D(16, (3, 3), padding='same', 145 | kernel_regularizer=regularization, 146 | use_bias=False)(x) 147 | x = BatchNormalization()(x) 148 | x = Activation('relu')(x) 149 | x = SeparableConv2D(16, (3, 3), padding='same', 150 | kernel_regularizer=regularization, 151 | use_bias=False)(x) 152 | x = BatchNormalization()(x) 153 | 154 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 155 | x = layers.add([x, residual]) 156 | 157 | # module 3 158 | residual = Conv2D(32, (1, 1), strides=(2, 2), 159 | padding='same', use_bias=False)(x) 160 | residual = BatchNormalization()(residual) 161 | 162 | x = SeparableConv2D(32, (3, 3), padding='same', 163 | kernel_regularizer=regularization, 164 | use_bias=False)(x) 165 | x = BatchNormalization()(x) 166 | x = Activation('relu')(x) 167 | x = SeparableConv2D(32, (3, 3), padding='same', 168 | kernel_regularizer=regularization, 169 | use_bias=False)(x) 170 | x = BatchNormalization()(x) 171 | 172 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 173 | x = layers.add([x, residual]) 174 | 175 | # module 4 176 | residual = Conv2D(64, (1, 1), strides=(2, 2), 177 | padding='same', use_bias=False)(x) 178 | residual = BatchNormalization()(residual) 179 | 180 | x = SeparableConv2D(64, (3, 3), padding='same', 181 | kernel_regularizer=regularization, 182 | use_bias=False)(x) 183 | x = BatchNormalization()(x) 184 | x = Activation('relu')(x) 185 | x = SeparableConv2D(64, (3, 3), padding='same', 186 | kernel_regularizer=regularization, 187 | use_bias=False)(x) 188 | x = BatchNormalization()(x) 189 | 190 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 191 | x = layers.add([x, residual]) 192 | 193 | x = Conv2D(num_classes, (3, 3), 194 | #kernel_regularizer=regularization, 195 | padding='same')(x) 196 | x = GlobalAveragePooling2D()(x) 197 | output = Activation('softmax',name='predictions')(x) 198 | 199 | model = Model(img_input, output) 200 | return model 201 | 202 | 203 | def mini_XCEPTION(input_shape, num_classes, l2_regularization=0.01): 204 | regularization = l2(l2_regularization) 205 | 206 | # base 207 | img_input = Input(input_shape) 208 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 209 | use_bias=False)(img_input) 210 | x = BatchNormalization()(x) 211 | x = Activation('relu')(x) 212 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 213 | use_bias=False)(x) 214 | x = BatchNormalization()(x) 215 | x = Activation('relu')(x) 216 | 217 | # module 1 218 | residual = Conv2D(16, (1, 1), strides=(2, 2), 219 | padding='same', use_bias=False)(x) 220 | residual = BatchNormalization()(residual) 221 | 222 | x = SeparableConv2D(16, (3, 3), padding='same', 223 | kernel_regularizer=regularization, 224 | use_bias=False)(x) 225 | x = BatchNormalization()(x) 226 | x = Activation('relu')(x) 227 | x = SeparableConv2D(16, (3, 3), padding='same', 228 | kernel_regularizer=regularization, 229 | use_bias=False)(x) 230 | x = BatchNormalization()(x) 231 | 232 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 233 | x = layers.add([x, residual]) 234 | 235 | # module 2 236 | residual = Conv2D(32, (1, 1), strides=(2, 2), 237 | padding='same', use_bias=False)(x) 238 | residual = BatchNormalization()(residual) 239 | 240 | x = SeparableConv2D(32, (3, 3), padding='same', 241 | kernel_regularizer=regularization, 242 | use_bias=False)(x) 243 | x = BatchNormalization()(x) 244 | x = Activation('relu')(x) 245 | x = SeparableConv2D(32, (3, 3), padding='same', 246 | kernel_regularizer=regularization, 247 | use_bias=False)(x) 248 | x = BatchNormalization()(x) 249 | 250 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 251 | x = layers.add([x, residual]) 252 | 253 | # module 3 254 | residual = Conv2D(64, (1, 1), strides=(2, 2), 255 | padding='same', use_bias=False)(x) 256 | residual = BatchNormalization()(residual) 257 | 258 | x = SeparableConv2D(64, (3, 3), padding='same', 259 | kernel_regularizer=regularization, 260 | use_bias=False)(x) 261 | x = BatchNormalization()(x) 262 | x = Activation('relu')(x) 263 | x = SeparableConv2D(64, (3, 3), padding='same', 264 | kernel_regularizer=regularization, 265 | use_bias=False)(x) 266 | x = BatchNormalization()(x) 267 | 268 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 269 | x = layers.add([x, residual]) 270 | 271 | # module 4 272 | residual = Conv2D(128, (1, 1), strides=(2, 2), 273 | padding='same', use_bias=False)(x) 274 | residual = BatchNormalization()(residual) 275 | 276 | x = SeparableConv2D(128, (3, 3), padding='same', 277 | kernel_regularizer=regularization, 278 | use_bias=False)(x) 279 | x = BatchNormalization()(x) 280 | x = Activation('relu')(x) 281 | x = SeparableConv2D(128, (3, 3), padding='same', 282 | kernel_regularizer=regularization, 283 | use_bias=False)(x) 284 | x = BatchNormalization()(x) 285 | 286 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 287 | x = layers.add([x, residual]) 288 | 289 | x = Conv2D(num_classes, (3, 3), 290 | #kernel_regularizer=regularization, 291 | padding='same')(x) 292 | x = GlobalAveragePooling2D()(x) 293 | output = Activation('softmax',name='predictions')(x) 294 | 295 | model = Model(img_input, output) 296 | return model 297 | 298 | def big_XCEPTION(input_shape, num_classes): 299 | img_input = Input(input_shape) 300 | x = Conv2D(32, (3, 3), strides=(2, 2), use_bias=False)(img_input) 301 | x = BatchNormalization(name='block1_conv1_bn')(x) 302 | x = Activation('relu', name='block1_conv1_act')(x) 303 | x = Conv2D(64, (3, 3), use_bias=False)(x) 304 | x = BatchNormalization(name='block1_conv2_bn')(x) 305 | x = Activation('relu', name='block1_conv2_act')(x) 306 | 307 | residual = Conv2D(128, (1, 1), strides=(2, 2), 308 | padding='same', use_bias=False)(x) 309 | residual = BatchNormalization()(residual) 310 | 311 | x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x) 312 | x = BatchNormalization(name='block2_sepconv1_bn')(x) 313 | x = Activation('relu', name='block2_sepconv2_act')(x) 314 | x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x) 315 | x = BatchNormalization(name='block2_sepconv2_bn')(x) 316 | 317 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 318 | x = layers.add([x, residual]) 319 | 320 | residual = Conv2D(256, (1, 1), strides=(2, 2), 321 | padding='same', use_bias=False)(x) 322 | residual = BatchNormalization()(residual) 323 | 324 | x = Activation('relu', name='block3_sepconv1_act')(x) 325 | x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x) 326 | x = BatchNormalization(name='block3_sepconv1_bn')(x) 327 | x = Activation('relu', name='block3_sepconv2_act')(x) 328 | x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x) 329 | x = BatchNormalization(name='block3_sepconv2_bn')(x) 330 | 331 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 332 | x = layers.add([x, residual]) 333 | x = Conv2D(num_classes, (3, 3), 334 | #kernel_regularizer=regularization, 335 | padding='same')(x) 336 | x = GlobalAveragePooling2D()(x) 337 | output = Activation('softmax',name='predictions')(x) 338 | 339 | model = Model(img_input, output) 340 | return model 341 | 342 | 343 | if __name__ == "__main__": 344 | input_shape = (64, 64, 1) 345 | num_classes = 7 346 | #model = tiny_XCEPTION(input_shape, num_classes) 347 | #model.summary() 348 | #model = mini_XCEPTION(input_shape, num_classes) 349 | #model.summary() 350 | #model = big_XCEPTION(input_shape, num_classes) 351 | #model.summary() 352 | model = simple_CNN((48, 48, 1), num_classes) 353 | model.summary() 354 | --------------------------------------------------------------------------------