├── binary.png
├── landmarks.png
├── README.md
├── cpp
    └── aspect_ratio.cpp
└── binary_thresholding
    └── detector_with_alarm.py


/binary.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sahnimanas/Fatigue-Detection/HEAD/binary.png


--------------------------------------------------------------------------------
/landmarks.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sahnimanas/Fatigue-Detection/HEAD/landmarks.png


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Fatigue-Detection
 2 | Eye state classification using OpenCV and DLib to estimate Percentage Eye Closure (PERCLOS) and alert a drowsy person (such as a driver).
 3 | 
 4 | ##### Dependencies:
 5 | ... 1. OpenCV (3.0 or later)
 6 | ... 2. Dlib (19.0 or later, for facial landmarking)
 7 | 
 8 | #### Using aspect ratio of major/minor axes (cpp)
 9 | Uses DLib facial landmark detector to find the major and minor axes of eyes, as well as mouth. The aspect ratio of major to minor axes is used to determine whether eye/mouth is open; which allows for eye-state classification and yawning detection.
10 | Requires a pre-trained DLib facial landmark detector model in a .dat file. 
11 | 
12 | ![alt text][landmarks]
13 | 
14 | [landmarks]: https://raw.githubusercontent.com/sahnimanas/Fatigue-Detection/master/landmarks.png "Facial landmarks"
15 | 
16 | #### Using binary thresholding
17 | Uses OpenCV Haar Cascade classifiers to detect face, and then for eyes within a rough area defined inside the face bounding-box. Eye state classification is done by thresholding the image on skin color and counting the number of black pixels, with the threshold normalized for skin color via HSV histogram
18 | 
19 | ![alt text][binary]
20 | 
21 | [binary]: https://raw.githubusercontent.com/sahnimanas/Fatigue-Detection/master/binary.png "Binary thresholding"
22 | 


--------------------------------------------------------------------------------
/cpp/aspect_ratio.cpp:
--------------------------------------------------------------------------------
  1 | #include<vector>
  2 | #include<Windows.h>
  3 | #include <dlib/opencv.h>
  4 | #include <opencv2/highgui/highgui.hpp>
  5 | #include <dlib/image_processing/frontal_face_detector.h>
  6 | #include <dlib/image_processing/render_face_detections.h>
  7 | #include <dlib/image_processing.h>
  8 | #include <dlib/gui_widgets.h>
  9 | 
 10 | #define CHECK_PERIOD 10
 11 | 
 12 | using namespace dlib;
 13 | using namespace std;
 14 | using namespace cv;
 15 | 
 16 | double computeavg(std::vector<double> &window)
 17 | {
 18 | 	double avg = 0.0;
 19 | 	for (std::vector<double>::iterator i = window.begin(); i != window.end(); i++)
 20 | 	{
 21 | 		avg += *i;
 22 | 	}
 23 | 
 24 | 	avg /= window.size();
 25 | 	return avg;
 26 | }
 27 | 
 28 | void disp(std::vector<double> &window)
 29 | {
 30 | 	cout << computeavg(window) << "-------";
 31 | 	for (std::vector<double>::iterator i = window.begin(); i != window.end(); i++)
 32 | 		cout << *i << "\t";
 33 | 	cout << endl;
 34 | }
 35 | 
 36 | void calibrate(std::vector<double> &pastRatios, VideoCapture &cap, frontal_face_detector &detector, shape_predictor &pose_model, image_window &win)
 37 | {
 38 | 	cout << "Collecting calibration info\n";
 39 | 	Mat temp;
 40 | 	double ratio = 0;
 41 | 	for (int i = 0; i < CHECK_PERIOD * 3; i++)
 42 | 	{
 43 | 		cap >> temp;
 44 | 		cv_image<bgr_pixel> cimg(temp);
 45 | 
 46 | 		// Detect faces 
 47 | 		std::vector<rectangle> faces = detector(cimg);
 48 | 
 49 | 		// Find the pose of each face.
 50 | 		std::vector<full_object_detection> shapes;
 51 | 		for (unsigned long i = 0; i < faces.size(); ++i)
 52 | 			shapes.push_back(pose_model(cimg, faces[i]));
 53 | 
 54 | 		win.clear_overlay();
 55 | 		if (shapes.empty())
 56 | 		{
 57 | 			win.set_image(cimg);
 58 | 			win.add_overlay(render_face_detections(shapes));
 59 | 			i--;
 60 | 			continue;
 61 | 		}
 62 | 
 63 | 		for (int i = 36; i <= 47; i++)
 64 | 			draw_solid_circle(cimg, shapes[0].part(i), 2.0, bgr_pixel(0, 0, 255));
 65 | 		win.set_image(cimg);
 66 | 		win.add_overlay(render_face_detections(shapes));
 67 | 		
 68 | 		if (i < CHECK_PERIOD)
 69 | 			continue;
 70 | 		
 71 | 		ratio = (shapes[0].part(36) - shapes[0].part(39)).length_squared();
 72 | 		ratio /= (shapes[0].part(37) - shapes[0].part(41)).length_squared();
 73 | 		
 74 | 		pastRatios.push_back(ratio);
 75 | 		
 76 | 		disp(pastRatios);
 77 | 	}
 78 | 	cout << "------------------DONE CALIBRATING------------------\n";
 79 | }
 80 | 
 81 | int main(int argc, char* argv[])
 82 | {
 83 |     if (argc < 2) {
 84 |         cout << "Usage: " << argv[0] << " <POSE_MODEL_PATH" << endl
 85 |              << "Download the DLib pose model from  "
 86 | 	     << "http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2"
 87 | 	     << " and provide the path to the extracted *.dat file as POSE_MODEL_PATH"
 88 | 	     << endl;
 89 | 	return 0;
 90 |     }
 91 | 	
 92 |     try
 93 |     {
 94 |         cv::VideoCapture cap(0);
 95 |         if (!cap.isOpened())
 96 |         {
 97 |             cerr << "Unable to connect to camera" << endl;
 98 |             return 1;
 99 |         }
100 | 
101 |         image_window win;
102 | 
103 |         frontal_face_detector detector = get_frontal_face_detector();
104 |         shape_predictor pose_model;
105 |         deserialize(argv[1]) >> pose_model;
106 | 		double ratio;
107 | 		Mat temp;
108 |         // Grab and process frames until the main window is closed by the user.
109 | 		std::vector<double> window;
110 | 		calibrate(window, cap, detector, pose_model, win);
111 | 		double runningAvg = computeavg(window);
112 |         while(!win.is_closed())
113 |         {
114 |             // Grab a frame
115 |             cap >> temp;
116 |             cv_image<bgr_pixel> cimg(temp);
117 | 			
118 |             // Detect faces 
119 |             std::vector<rectangle> faces = detector(cimg);
120 | 
121 | 			// Find the pose of each face.
122 |             std::vector<full_object_detection> shapes;
123 |             for (unsigned long i = 0; i < faces.size(); ++i)
124 |                 shapes.push_back(pose_model(cimg, faces[i]));
125 | 			
126 | 			win.clear_overlay();
127 | 			if (shapes.empty())
128 | 			{
129 | 				win.set_image(cimg);
130 | 				win.add_overlay(render_face_detections(shapes));
131 | 				continue;
132 | 			}
133 | 			
134 | 			for (int i = 36; i <= 47; i++)
135 | 				draw_solid_circle(cimg, shapes[0].part(i), 2.0, bgr_pixel(0, 0, 255));
136 |             win.set_image(cimg);
137 |             win.add_overlay(render_face_detections(shapes));
138 | 			
139 | 			ratio = (shapes[0].part(36) - shapes[0].part(39)).length_squared();
140 | 			ratio /= (shapes[0].part(37) - shapes[0].part(41)).length_squared();
141 | 
142 | 			runningAvg += (ratio - window.front()) / CHECK_PERIOD;
143 | 			window.push_back(ratio);
144 | 			window.erase(window.begin());
145 | 			
146 | 			disp(window);
147 |         }
148 |     }
149 |     catch(serialization_error& e)
150 |     {
151 |         cout << "Need landmarks.dat file first" << endl;
152 |         cout << endl << e.what() << endl;
153 |     }
154 |     catch(exception& e)
155 |     {
156 |         cout << e.what() << endl;
157 |     }
158 | }
159 | 


--------------------------------------------------------------------------------
/binary_thresholding/detector_with_alarm.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import cv2
  3 | import matplotlib
  4 | from matplotlib import pyplot as plt
  5 | import time
  6 | import winsound
  7 | 
  8 | face_cascade = cv2.CascadeClassifier('C:/opencv/build/etc/haarcascades/haarcascade_frontalface_alt.xml')
  9 | Leye_cascade = cv2.CascadeClassifier('C:/opencv/build/etc/haarcascades/haarcascade_lefteye_2splits.xml')
 10 | Reye_cascade = cv2.CascadeClassifier('C:/opencv/build/etc/haarcascades/haarcascade_righteye_2splits.xml')
 11 | eye_cascade  = cv2.CascadeClassifier('C:/opencv/build/etc/haarcascades/haarcascade_eye.xml')
 12 | 
 13 | simulate_real_time = "true"
 14 | 
 15 | 
 16 | 
 17 | process_eye = 0
 18 | eyeq_len = 5
 19 | eyeq = []
 20 | 
 21 | def push_val(val):
 22 |     if(val < 800):
 23 |         if len(eyeq) <= eyeq_len:
 24 |             eyeq.append(val)
 25 |         else:
 26 |             eyeq.append(val)
 27 |             eyeq.pop(0)
 28 |     return avg_eyeq()
 29 | 
 30 | def avg_eyeq():
 31 |     #calculate average
 32 |     avg = 0
 33 |     for i in eyeq:
 34 |         avg = avg + i
 35 |     avg = avg / len(eyeq)
 36 |     
 37 |     return avg
 38 | 
 39 | def detect_and_draw(img, gray):
 40 |     faces = face_cascade.detectMultiScale(gray, 1.3, 5)
 41 |     for (x,y,w,h) in faces:
 42 |         img = cv2.rectangle(img, (x,y),(x+w,y+h),(255,0,0),2)
 43 |         roi_gray = gray[(y+h/4):(y+0.55*h), (x+0.13*w):(x+w-0.13*w)]
 44 |         roi_color = img[(y+h/4):(y+0.55*h), (x+0.13*w):(x+w-0.13*w)]
 45 |         eyes = eye_cascade.detectMultiScale(roi_gray)
 46 |         
 47 |         max_eyes = 2
 48 |         cnt_eye = 0
 49 |         for (ex,ey,ew,eh) in eyes:
 50 |             if(cnt_eye == max_eyes):
 51 |                 break;
 52 |             
 53 |             image_name = 'Eye_' + str(cnt_eye)
 54 |             #print image_name
 55 |             
 56 |             #change dimentionas
 57 |             #ex = ex + (ew/6)
 58 |             #ew = ew - (ew/6)
 59 |             #ey = ey + (eh/3)
 60 |             #eh = eh/3
 61 |             
 62 |             cv2.rectangle(roi_color,(ex,ey),(ex+ew,ey+eh),(0,255,0),1)
 63 |             
 64 |             roi_eye_gray = roi_gray[ey:ey+eh, ex:ex+ew]
 65 |             roi_eye_color = roi_color[ey:ey+eh, ex:ex+ew]
 66 | 
 67 |             # create & normalize histogram ---------
 68 |             hist = cv2.calcHist([roi_eye_gray], [0],None, [256], [0,256])
 69 |             histn = []
 70 |             max_val = 0
 71 |             for i in hist:
 72 |                 value = int(i[0])
 73 |                 histn.append(value)
 74 |                 if(value > max_val):
 75 |                     max_val = value
 76 |             for index, value in enumerate(histn):
 77 |                 histn[index] = ((value * 256) / max_val)
 78 |             
 79 |             threshold = np.argmax(histn)
 80 |             #print histn
 81 |             # normalize histogram ends ---------
 82 | 
 83 |             
 84 | 
 85 |             # Slice
 86 |             roi_eye_gray2 = roi_eye_gray.copy()
 87 |             #roi_eye_gray2 = cv2.threshold(roi_eye_gray2, 50, 255, cv2.THRESH_TOZERO)
 88 |             #roi_eye_gray2 = cv2.adaptiveThreshold(roi_eye_gray2, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 11, 10)
 89 | 
 90 |             total_white = 0
 91 |             total_black = 0
 92 |             for i in range(0, roi_eye_gray2.shape[0]):
 93 |                 for j in range(0, roi_eye_gray2.shape[1]):
 94 |                     pixel_value = roi_eye_gray2[i, j]
 95 |                     if(pixel_value >= threshold):
 96 |                         roi_eye_gray2[i, j] = 255
 97 |                         total_white = total_white + 1
 98 |                     else:
 99 |                         roi_eye_gray2[i, j] = 0
100 |                         total_black = total_black + 1
101 |             
102 |             binary=cv2.resize(roi_eye_gray2,None,fx=2,fy=2)            
103 |             cv2.imshow('binary',binary)
104 |             if image_name == "Eye_0":
105 |                 ag = push_val(total_white)
106 |                 #print image_name, " : ", total_white, " : ", ag
107 | 
108 |             #print "Black ", total_black
109 |             #print "White ", total_white
110 |             
111 |             if(simulate_real_time == "true"):
112 |                 pass
113 |                 # put number on image
114 |                 if(cnt_eye == 0):
115 |                     cv2.putText(img, ""+str(total_white), (10, 40), cv2.FONT_HERSHEY_PLAIN, 2, (0,255,0))
116 |                 else:
117 |                     cv2.putText(img, ""+str(total_white), (520, 40), cv2.FONT_HERSHEY_PLAIN, 2, (0,255,0))
118 |                 cv2.putText(img, ""+str(threshold), (10, 240), cv2.FONT_HERSHEY_PLAIN, 2, (0,255,0))
119 |             else:
120 |                 # Plot Histogram
121 |                 plt.subplot(2,3,((cnt_eye*3)+1)),plt.hist(roi_eye_gray.ravel(), 256, [0,256])
122 |                 plt.title(image_name+' Hist')
123 |                 # Plot Eye Images
124 |                 plt.subplot(2,3,((cnt_eye*3)+2)),plt.imshow(roi_eye_color, 'gray')
125 |                 plt.title(image_name+' Image Threshold')
126 | 
127 |                 # Plot Eye Images after threshold
128 |                 plt.subplot(2,3,((cnt_eye*3)+3)),plt.imshow(roi_eye_gray2, 'gray')
129 |                 plt.title(image_name+' Image')
130 |             
131 |             cnt_eye = cnt_eye + 1
132 |         if len(eyes) == 0:
133 |             ag = push_val(0)
134 |         
135 |         # Decision Making
136 |         average = avg_eyeq()
137 |         if average > 30:
138 | 
139 |             print "Eye_X: ", average
140 |             #player.pause()
141 |         else:
142 |             print "---------------------", average
143 |             winsound.Beep(1000,100)
144 | #            pygame.mixer.music.play()
145 |             #if player.playing == False:
146 |             #    print "Play music"
147 |             #    player.queue(music)
148 |             #    player.play()
149 | 
150 |         #time.sleep(0.5)
151 |     cv2.imshow('frame', img)
152 |     if(simulate_real_time == "false"):
153 |         plt.show()
154 |     #img.releaseImage()
155 |     
156 | 
157 | if __name__ == '__main__':
158 |     if(simulate_real_time == "true"):
159 |         cap = cv2.VideoCapture(0)
160 |         countQuit = 10
161 |         k=0;
162 |         while(True):
163 |             #time.sleep(1)
164 |             ret, frame = cap.read()
165 |             if frame != None:
166 |                 #frame = cv2.resize(frame, (600, 350))
167 |                 cv2.imshow('frame',frame)
168 |                 gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
169 |                 detect_and_draw(frame, gray)
170 |                 if cv2.waitKey(1) & 0xFF == ord('q'):
171 |                     break
172 |             else:
173 |                 print "Frame Grabbed Problem";
174 |                 countQuit = countQuit - 1
175 |                 if countQuit <= 0:
176 |                     break
177 |                 else:
178 |                     continue
179 |         cap.release()
180 |     else:
181 |         # Capture frame-by-frame
182 |         frame = cv2.imread('face_img.jpg')
183 |         
184 |         # Resize Image
185 |         frame = cv2.resize(frame, (600, 350))
186 | 
187 |         # Our operations on the frame come here
188 |         gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
189 | 
190 |         cv2.waitKey(1);
191 |         detect_and_draw(frame, gray)
192 |         #cv2.waitKey(0)
193 | 
194 |         # Display the resulting frame
195 |         #cv2.imshow('frame',gray)
196 |     
197 |     cv2.destroyAllWindows()
198 | 


--------------------------------------------------------------------------------