├── README.md ├── baojin.py ├── check.py ├── convert.py ├── data_provider.py ├── detect_class.py ├── evaluate.py ├── extract_face.py ├── haarcascade_files ├── haarcascade_eye.xml └── haarcascade_frontalface_default.xml ├── load_and_process.py ├── models ├── __pycache__ │ └── cnn.cpython-36.pyc ├── _mini_XCEPTION.102-0.66.hdf5 └── cnn.py ├── requirements.txt ├── split_train_test.py ├── tkinter_UI.exe ├── tkinter_UI.py ├── 系统说明.txt └── 运行说明.txt /README.md: -------------------------------------------------------------------------------- 1 | 开发环境： Pycharm + Python3.6 + 卷积神经网络算法 2 | 3 | 基于人脸表面特征的疲劳检测，主要分为三个部分，打哈欠、眨眼、点头。本实验从人脸朝向、位置、瞳孔朝向、眼睛开合度、眨眼频率、瞳孔收缩率等数据入手，并通过这些数据，实时地计算出驾驶员的注意力集中程度，分析驾驶员是否疲劳驾驶和及时作出安全提示。 4 | 5 | 视觉疲劳检测原理：因为人在疲倦时大概会产生两种状态： 眨眼：正常人的眼睛每分钟大约要眨动10-15次，每次眨眼大概0.2-0.4秒，如果疲倦时眨眼次数会增多，速度也会变慢。打哈欠：此时嘴会长大而且会保持一定的状态。因此检测人是否疲劳可以从眼睛的开合度，眨眼频率，以及嘴巴张合程度来判断一个人是否疲劳。 6 | 7 | 检测工具 8 | dlib ：一个很经典的用于图像处理的开源库，shape_predictor_68_face_landmarks.dat是一个用于人脸68个关键点检测的dat模型库，使用这个模型库可以很方便地进行人脸检测，并进行简单的应用。 9 | 10 | 眨眼计算原理： 11 | （1） 计算眼睛的宽高比 12 | 基本原理：计算 眼睛长宽比 Eye Aspect Ratio，EAR.当人眼睁开时，EAR在某个值上下波动，当人眼闭合时，EAR迅速下降，理论上会接近于零，当时人脸检测模型还没有这么精确。所以我们认为当EAR低于某个阈值时，眼睛处于闭合状态。为检测眨眼次数，需要设置同一次眨眼的连续帧数。眨眼速度比较快，一般1~3帧就完成了眨眼动作。两个阈值都要根据实际情况设置。 13 | （2）当前帧两双眼睛宽高比与前一帧的差值的绝对值（EAR）大于0.2，则认为是疲劳 14 | 15 | 代码思路 16 | 第一步：使用dlib.get_frontal_face_detector() 获得脸部位置检测器 17 | 第二步：使用dlib.shape_predictor获得脸部特征位置检测器 18 | 第三步：分别获取左右眼面部标志的索引 19 | 第四步：打开cv2 本地摄像头 20 | 第五步：从视频流进行循环，读取图片，并对图片做维度扩大，并进灰度化 21 | 第六步：使用detector(gray, 0) 进行脸部位置检测 22 | 第七步：循环脸部位置信息，使用predictor(gray, rect)获得脸部特征位置的信息 23 | 第八步：将脸部特征信息转换为数组array的格式 24 | 第九步：提取左眼和右眼坐标 25 | 第十步：构造函数计算左右眼的EAR值，使用平均值作为最终的EAR 26 | 第十一步：使用cv2.convexHull获得凸包位置，使用drawContours画出轮廓位置进行画图操作 27 | 第十二步：进行画图操作，用矩形框标注人脸 28 | 第十三步：分别计算左眼和右眼的评分求平均作为最终的评分，如果小于阈值，则加1，如果连续3次都小于阈值，则表示进行了一次眨眼活动 29 | 第十四步：进行画图操作，68个特征点标识 30 | 第十五步：进行画图操作，同时使用cv2.putText将眨眼次数进行显示 31 | 第十六步：统计总眨眼次数大于50次屏幕显示睡着。 32 | 33 | -------------------------------------------------------------------------------- /baojin.py: -------------------------------------------------------------------------------- 1 | from pygame import mixer 2 | import time 3 | 4 | a=mixer.init() 5 | mixer.music.load('d:/pljs.mp3') 6 | mixer.music.play(3,0.0) 7 | time.sleep(30) 8 | #mixer.music.stop() 9 | 10 | -------------------------------------------------------------------------------- /check.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import model 3 | import numpy as np 4 | from PIL import Image 5 | 6 | x = tf.placeholder(tf.float32, [None, 64*64]) 7 | 8 | keep_prob = tf.placeholder(tf.float32) #dropout (keep probability) 9 | 10 | num_classes = 2 11 | 12 | checkpoint_dir = "path/to/model/" 13 | 14 | sess = tf.Session() 15 | 16 | logit = model.create_model(x, num_classes, keep_prob) 17 | check_tag = tf.argmax(logit, 1) 18 | 19 | saver = tf.train.Saver() 20 | 21 | coord = tf.train.Coordinator() 22 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 23 | ckpt = tf.train.get_checkpoint_state(checkpoint_dir) 24 | 25 | if ckpt and ckpt.model_checkpoint_path: 26 | saver.restore(sess, ckpt.model_checkpoint_path) 27 | 28 | def pre_process(img_path): 29 | img = Image.open(img_path).convert('L') 30 | img.save(img_path) 31 | 32 | def check_state(img_path): 33 | global logit 34 | pre_process(img_path) 35 | image_raw_data = tf.gfile.FastGFile(img_path, 'rb').read() 36 | split_name = img_path.split(".")[1] 37 | if split_name == "jpg": 38 | img_data = tf.image.decode_jpeg(image_raw_data) 39 | elif split_name == "png": 40 | img_data = tf.image.decode_png(image_raw_data) 41 | image = tf.image.convert_image_dtype(img_data, tf.float32) 42 | image = tf.image.resize_images(image, (64, 64), method=0) 43 | #image = img.astype(np.float32) 44 | image -= 0.5 45 | image *= 2 46 | image = tf.reshape(image, shape=(1,64*64)) 47 | image = sess.run(image) 48 | output = sess.run(check_tag, {x:image, keep_prob:1.}) 49 | print (output) 50 | if (output[0] == 1): 51 | print ("smile") 52 | else: 53 | print ("no smile") 54 | check_state("3.png") 55 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | 5 | import os,sys 6 | import tensorflow as tf 7 | import random 8 | 9 | tf.app.flags.DEFINE_string( 10 | 'dataset_dir', 'data', 11 | 'The directory where the output TFRecords and temporary files are saved.') 12 | 13 | FLAGS = tf.app.flags.FLAGS 14 | 15 | _IMAGE_SIZE = None #Note!-not use,just show 16 | _NUM_CHANNELS = None #Note!-not use,just show 17 | 18 | # The number of images in the training set. Note!-not use,just show 19 | _NUM_TRAIN_SAMPLES = None 20 | 21 | # The number of images to be kept from the training set for the validation set. 22 | _NUM_VALIDATION = 0 23 | 24 | # The number of images in the test set. Note!-not use,just show 25 | _NUM_TEST_SAMPLES = None 26 | 27 | # Seed for repeatability. 28 | _RANDOM_SEED = 0 29 | 30 | class ImageReader(object): 31 | 32 | """Helper class that provides TensorFlow image coding utilities.""" 33 | 34 | def __init__(self): 35 | # Initializes function that decodes none-RGB jpeg data. 36 | self._decode_jpeg_data = tf.placeholder(dtype=tf.string) 37 | # You can use "tf.image.decode_jpeg" instead. 38 | self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=1) 39 | 40 | def read_image_dims(self, sess, image_data): 41 | image = self.decode_jpeg(sess, image_data) 42 | return image.shape[0], image.shape[1] 43 | 44 | def decode_jpeg(self, sess, image_data): 45 | image = sess.run( 46 | self._decode_jpeg, feed_dict={self._decode_jpeg_data: image_data}) 47 | assert len(image.shape) == 3 48 | assert image.shape[2] == 1 49 | return image 50 | 51 | def _get_output_filename(dataset_dir, split_name): 52 | """Creates the output filename. 53 | 54 | Args: 55 | dataset_dir: The directory where the temporary files are stored. 56 | split_name: The name of the train/test split. 57 | 58 | Returns: 59 | An absolute file path. 60 | """ 61 | return '%s/FACE_%s.tfrecord' % (dataset_dir, split_name) 62 | 63 | def _get_filenames(dataset_dir): 64 | """Returns a list of filenames and inferred class names. 65 | 66 | Args: 67 | dataset_dir: A directory containing a set jpeg encoded images. 68 | 69 | Returns: 70 | A list of image file paths, relative to `dataset_dir`. 71 | """ 72 | photo_filenames = [] 73 | for filename in os.listdir(dataset_dir): 74 | photo_filenames.append(filename) 75 | return photo_filenames 76 | 77 | def _extract_labels(label_filename): 78 | 79 | """Extract the labels into a dict of filenames to int labels. 80 | 81 | Args: 82 | labels_filename: The filename of the labels. 83 | 84 | Returns: 85 | A dictionary of filenames to int labels. 86 | """ 87 | print('Extracting labels from: ', label_filename) 88 | label_file = tf.gfile.FastGFile(label_filename, 'r').readlines() 89 | label_lines = [line.rstrip('\n').split() for line in label_file] 90 | labels = {} 91 | for line in label_lines: 92 | assert len(line) == 2 93 | labels[line[0]] = int(line[1]) 94 | return labels 95 | 96 | def _int64_feature(values): 97 | """Returns a TF-Feature of int64s. 98 | 99 | Args: 100 | values: A scalar or list of values. 101 | 102 | Returns: 103 | a TF-Feature. 104 | """ 105 | if not isinstance(values, (tuple, list)): 106 | values = [values] 107 | return tf.train.Feature(int64_list=tf.train.Int64List(value=values)) 108 | 109 | def _bytes_feature(values): 110 | """Returns a TF-Feature of bytes. 111 | 112 | Args: 113 | values: A string. 114 | 115 | Returns: 116 | a TF-Feature. 117 | """ 118 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values])) 119 | 120 | def image_to_tfexample(image_data, image_format, height, width, class_id): 121 | return tf.train.Example(features=tf.train.Features(feature={ 122 | 'image/encoded': _bytes_feature(image_data), 123 | 'image/format': _bytes_feature(image_format), 124 | 'image/class/label': _int64_feature(class_id), 125 | 'image/height': _int64_feature(height), 126 | 'image/width': _int64_feature(width), 127 | })) 128 | 129 | def _convert_dataset(split_name, filenames, filename_to_class_id, dataset_dir): 130 | """Converts the given filenames to a TFRecord dataset. 131 | 132 | Args: 133 | split_name: The name of the dataset, either 'train' or 'valid'. 134 | filenames: A list of absolute paths to jpeg images. 135 | filename_to_class_id: A dictionary from filenames (strings) to class ids 136 | (integers). 137 | dataset_dir: The directory where the converted datasets are stored. 138 | """ 139 | print('Converting the {} split.'.format(split_name)) 140 | # Train and validation splits are both in the train directory. 141 | if split_name in ['train', 'valid']: 142 | jpeg_directory = os.path.join(dataset_dir,'train') 143 | elif split_name == 'test': 144 | jpeg_directory = os.path.join(dataset_dir, 'test') 145 | 146 | with tf.Graph().as_default(): 147 | image_reader = ImageReader() 148 | 149 | with tf.Session('') as sess: 150 | output_filename = _get_output_filename(dataset_dir, split_name) 151 | with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer: 152 | for filename in filenames: 153 | # Read the filename: 154 | image_data = tf.gfile.FastGFile( 155 | os.path.join(jpeg_directory, filename), 'rb').read() 156 | #print (len(image_data)) 157 | height, width = image_reader.read_image_dims(sess, image_data) 158 | 159 | class_id = filename_to_class_id[filename] 160 | 161 | example = image_to_tfexample(image_data, 'jpeg'.encode(), height, 162 | width, class_id) 163 | tfrecord_writer.write(example.SerializeToString()) 164 | sys.stdout.write('\n') 165 | sys.stdout.flush() 166 | 167 | 168 | def run(dataset_dir): 169 | """Runs conversion operation. 170 | 171 | Args: 172 | dataset_dir: The dataset directory where the dataset is stored. 173 | """ 174 | if not tf.gfile.Exists(dataset_dir): 175 | tf.gfile.MakeDirs(dataset_dir) 176 | 177 | train_filename = _get_output_filename(dataset_dir, 'train') 178 | testing_filename = _get_output_filename(dataset_dir, 'test') 179 | 180 | if tf.gfile.Exists(train_filename) and tf.gfile.Exists(testing_filename): 181 | print('Dataset files already exist. Exiting without re-creating them.') 182 | return 183 | 184 | train_validation_filenames = _get_filenames( 185 | os.path.join(dataset_dir, 'train')) 186 | test_filenames = _get_filenames( 187 | os.path.join(dataset_dir, 'test')) 188 | 189 | # Divide into train and validation: 190 | random.seed(_RANDOM_SEED) 191 | random.shuffle(train_validation_filenames) 192 | train_filenames = train_validation_filenames[_NUM_VALIDATION:] 193 | validation_filenames = train_validation_filenames[:_NUM_VALIDATION] 194 | 195 | train_validation_filenames_to_class_ids = _extract_labels( 196 | os.path.join(dataset_dir, 'train_labels.txt')) 197 | test_filenames_to_class_ids = _extract_labels( 198 | os.path.join(dataset_dir, 'test_labels.txt')) 199 | 200 | # Convert the train, validation, and test sets. 201 | _convert_dataset('train', train_filenames, 202 | train_validation_filenames_to_class_ids, dataset_dir) 203 | _convert_dataset('valid', validation_filenames, 204 | train_validation_filenames_to_class_ids, dataset_dir) 205 | _convert_dataset('test', test_filenames, test_filenames_to_class_ids, 206 | dataset_dir) 207 | 208 | print('\nFinished converting the dataset!') 209 | 210 | 211 | def main(_): 212 | assert FLAGS.dataset_dir 213 | run(FLAGS.dataset_dir) 214 | 215 | 216 | -------------------------------------------------------------------------------- /data_provider.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | 4 | _FILE_PATTERN = 'FACE_%s.tfrecord' 5 | dataset_dir = 'data' 6 | reader = tf.TFRecordReader() 7 | 8 | 9 | keys_to_features = { 10 | 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 11 | 'image/format': tf.FixedLenFeature((), tf.string, default_value='raw'), 12 | 'image/class/label': tf.FixedLenFeature([1], tf.int64), 13 | } 14 | 15 | num_classes = 2 16 | 17 | def get_data(split_name): 18 | file_pattern = os.path.join(dataset_dir, _FILE_PATTERN % split_name) 19 | filename_queue = tf.train.string_input_producer([file_pattern]) 20 | _, serialized_example = reader.read(filename_queue) 21 | features = tf.parse_single_example(serialized_example,features = keys_to_features ) 22 | #image = tf.decode_raw(features['image/encoded'], tf.uint8) 23 | image = tf.image.decode_png(features['image/encoded']) 24 | #label = tf.cast(features['image/class/label'],tf.float32) 25 | label = tf.one_hot(features['image/class/label'], num_classes) 26 | label = tf.reshape(label, shape=(num_classes,)) 27 | print ("label:", label) 28 | image = tf.image.convert_image_dtype(image, tf.float32) 29 | image -= 0.5 30 | image *= 2 31 | image = tf.reshape(image, shape=(64*64,)) 32 | print (image, label) 33 | return (image, label) 34 | 35 | 36 | #test_image, test_label = get_data("test") 37 | #print (test_image) 38 | #print (test_label) 39 | -------------------------------------------------------------------------------- /detect_class.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import dlib 4 | import numpy as np 5 | import cv2 6 | import wx 7 | import wx.xrc 8 | import wx.adv 9 | # import the necessary packages 10 | from scipy.spatial import distance as dist 11 | from imutils.video import FileVideoStream 12 | from imutils.video import VideoStream 13 | from imutils import face_utils 14 | import numpy as np 15 | import argparse 16 | import imutils 17 | import datetime,time 18 | import math 19 | import os 20 | 21 | 22 | 23 | COVER = '' 24 | 25 | class Fatigue_detecting(wx.Frame): 26 | 27 | def __init__( self, parent, title ): 28 | wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = title, pos = wx.DefaultPosition, size = wx.Size( 873,535 ), style = wx.DEFAULT_FRAME_STYLE|wx.TAB_TRAVERSAL ) 29 | 30 | self.SetSizeHints( wx.DefaultSize, wx.DefaultSize ) 31 | self.SetBackgroundColour( wx.SystemSettings.GetColour( wx.SYS_COLOUR_MENU ) ) 32 | 33 | bSizer1 = wx.BoxSizer( wx.VERTICAL ) 34 | bSizer2 = wx.BoxSizer( wx.HORIZONTAL ) 35 | bSizer3 = wx.BoxSizer( wx.VERTICAL ) 36 | 37 | self.m_animCtrl1 = wx.adv.AnimationCtrl( self, wx.ID_ANY, wx.adv.NullAnimation, wx.DefaultPosition, wx.DefaultSize, wx.adv.AC_DEFAULT_STYLE ) 38 | bSizer3.Add( self.m_animCtrl1, 1, wx.ALL|wx.EXPAND, 5 ) 39 | bSizer2.Add( bSizer3, 9, wx.EXPAND, 5 ) 40 | bSizer4 = wx.BoxSizer( wx.VERTICAL ) 41 | sbSizer1 = wx.StaticBoxSizer( wx.StaticBox( self, wx.ID_ANY, u"参数设置" ), wx.VERTICAL ) 42 | sbSizer2 = wx.StaticBoxSizer( wx.StaticBox( sbSizer1.GetStaticBox(), wx.ID_ANY, u"视频源" ), wx.VERTICAL ) 43 | gSizer1 = wx.GridSizer( 0, 2, 0, 8 ) 44 | m_choice1Choices = [ u"摄像头ID_0", u"摄像头ID_1", u"摄像头ID_2" ] 45 | self.m_choice1 = wx.Choice( sbSizer2.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.Size( 90,25 ), m_choice1Choices, 0 ) 46 | self.m_choice1.SetSelection( 0 ) 47 | gSizer1.Add( self.m_choice1, 0, wx.ALL, 5 ) 48 | self.camera_button1 = wx.Button( sbSizer2.GetStaticBox(), wx.ID_ANY, u"开始检测", wx.DefaultPosition, wx.Size( 90,25 ), 0 ) 49 | gSizer1.Add( self.camera_button1, 0, wx.ALL, 5 ) 50 | self.vedio_button2 = wx.Button( sbSizer2.GetStaticBox(), wx.ID_ANY, u"打开视频文件", wx.DefaultPosition, wx.Size( 90,25 ), 0 ) 51 | gSizer1.Add( self.vedio_button2, 0, wx.ALL, 5 ) 52 | 53 | self.off_button3 = wx.Button( sbSizer2.GetStaticBox(), wx.ID_ANY, u"暂停", wx.DefaultPosition, wx.Size( 90,25 ), 0 ) 54 | gSizer1.Add( self.off_button3, 0, wx.ALL, 5 ) 55 | sbSizer2.Add( gSizer1, 1, wx.EXPAND, 5 ) 56 | sbSizer1.Add( sbSizer2, 2, wx.EXPAND, 5 ) 57 | sbSizer3 = wx.StaticBoxSizer( wx.StaticBox( sbSizer1.GetStaticBox(), wx.ID_ANY, u"疲劳检测" ), wx.VERTICAL ) 58 | bSizer5 = wx.BoxSizer( wx.HORIZONTAL ) 59 | self.yawn_checkBox1 = wx.CheckBox( sbSizer3.GetStaticBox(), wx.ID_ANY, u"打哈欠检测", wx.Point( -1,-1 ), wx.Size( -1,15 ), 0 ) 60 | self.yawn_checkBox1.SetValue(True) 61 | bSizer5.Add( self.yawn_checkBox1, 0, wx.ALL, 5 ) 62 | self.blink_checkBox2 = wx.CheckBox( sbSizer3.GetStaticBox(), wx.ID_ANY, u"闭眼检测", wx.Point( -1,-1 ), wx.Size( -1,15 ), 0 ) 63 | self.blink_checkBox2.SetValue(True) 64 | bSizer5.Add( self.blink_checkBox2, 0, wx.ALL, 5 ) 65 | sbSizer3.Add( bSizer5, 1, wx.EXPAND, 5 ) 66 | bSizer6 = wx.BoxSizer( wx.HORIZONTAL ) 67 | self.nod_checkBox7 = wx.CheckBox( sbSizer3.GetStaticBox(), wx.ID_ANY, u"点头检测", wx.Point( -1,-1 ), wx.Size( -1,15 ), 0 ) 68 | self.nod_checkBox7.SetValue(True) 69 | bSizer6.Add( self.nod_checkBox7, 0, wx.ALL, 5 ) 70 | self.m_staticText1 = wx.StaticText( sbSizer3.GetStaticBox(), wx.ID_ANY, u"疲劳时间(秒):", wx.DefaultPosition, wx.Size( -1,15 ), 0 ) 71 | self.m_staticText1.Wrap( -1 ) 72 | bSizer6.Add( self.m_staticText1, 0, wx.ALL, 5 ) 73 | m_listBox2Choices = [ u"3", u"4", u"5", u"6", u"7", u"8" ] 74 | self.m_listBox2 = wx.ListBox( sbSizer3.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.Size( 50,24 ), m_listBox2Choices, 0 ) 75 | bSizer6.Add( self.m_listBox2, 0, 0, 5 ) 76 | sbSizer3.Add( bSizer6, 1, wx.EXPAND, 5 ) 77 | sbSizer1.Add( sbSizer3, 2, 0, 5 ) 78 | sbSizer4 = wx.StaticBoxSizer( wx.StaticBox( sbSizer1.GetStaticBox(), wx.ID_ANY, u"脱岗检测" ), wx.VERTICAL ) 79 | bSizer8 = wx.BoxSizer( wx.HORIZONTAL ) 80 | self.m_checkBox4 = wx.CheckBox( sbSizer4.GetStaticBox(), wx.ID_ANY, u"脱岗检测", wx.DefaultPosition, wx.Size( -1,15 ), 0 ) 81 | self.m_checkBox4.SetValue(True) 82 | bSizer8.Add( self.m_checkBox4, 0, wx.ALL, 5 ) 83 | self.m_staticText2 = wx.StaticText( sbSizer4.GetStaticBox(), wx.ID_ANY, u"脱岗时间(秒):", wx.DefaultPosition, wx.Size( -1,15 ), 0 ) 84 | self.m_staticText2.Wrap( -1 ) 85 | bSizer8.Add( self.m_staticText2, 0, wx.ALL, 5 ) 86 | m_listBox21Choices = [ u"5", u"10", u"15", u"20", u"25", u"30" ] 87 | self.m_listBox21 = wx.ListBox( sbSizer4.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.Size( 50,24 ), m_listBox21Choices, 0 ) 88 | bSizer8.Add( self.m_listBox21, 0, 0, 5 ) 89 | sbSizer4.Add( bSizer8, 1, 0, 5 ) 90 | sbSizer1.Add( sbSizer4, 1, 0, 5 ) 91 | sbSizer5 = wx.StaticBoxSizer( wx.StaticBox( sbSizer1.GetStaticBox(), wx.ID_ANY, u"分析区域" ), wx.VERTICAL ) 92 | bSizer9 = wx.BoxSizer( wx.HORIZONTAL ) 93 | self.m_staticText3 = wx.StaticText( sbSizer5.GetStaticBox(), wx.ID_ANY, u"检测区域： ", wx.DefaultPosition, wx.DefaultSize, 0 ) 94 | self.m_staticText3.Wrap( -1 ) 95 | bSizer9.Add( self.m_staticText3, 0, wx.ALL, 5 ) 96 | m_choice2Choices = [ u"全视频检测", u"部分区域选取" ] 97 | self.m_choice2 = wx.Choice( sbSizer5.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize, m_choice2Choices, 0 ) 98 | self.m_choice2.SetSelection( 0 ) 99 | bSizer9.Add( self.m_choice2, 0, wx.ALL, 5 ) 100 | sbSizer5.Add( bSizer9, 1, wx.EXPAND, 5 ) 101 | sbSizer1.Add( sbSizer5, 1, 0, 5 ) 102 | sbSizer6 = wx.StaticBoxSizer( wx.StaticBox( sbSizer1.GetStaticBox(), wx.ID_ANY, u"状态输出" ), wx.VERTICAL ) 103 | self.m_textCtrl3 = wx.TextCtrl( sbSizer6.GetStaticBox(), wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.TE_MULTILINE|wx.TE_READONLY ) 104 | sbSizer6.Add( self.m_textCtrl3, 1, wx.ALL|wx.EXPAND, 5 ) 105 | sbSizer1.Add( sbSizer6, 5, wx.EXPAND, 5 ) 106 | bSizer4.Add( sbSizer1, 1, wx.EXPAND, 5 ) 107 | bSizer2.Add( bSizer4, 3, wx.EXPAND, 5 ) 108 | bSizer1.Add( bSizer2, 1, wx.EXPAND, 5 ) 109 | 110 | self.SetSizer( bSizer1 ) 111 | self.Layout() 112 | self.Centre( wx.BOTH ) 113 | 114 | # Connect Events 115 | self.m_choice1.Bind( wx.EVT_CHOICE, self.cameraid_choice )#绑定事件 116 | self.camera_button1.Bind( wx.EVT_BUTTON, self.camera_on )#开 117 | self.vedio_button2.Bind( wx.EVT_BUTTON, self.vedio_on ) 118 | self.off_button3.Bind( wx.EVT_BUTTON, self.off )#关 119 | 120 | self.m_listBox2.Bind( wx.EVT_LISTBOX, self.AR_CONSEC_FRAMES )# 闪烁阈值设置 121 | self.m_listBox21.Bind( wx.EVT_LISTBOX, self.OUT_AR_CONSEC_FRAMES )# 脱岗时间设置 122 | 123 | # 封面图片 124 | self.image_cover = wx.Image(COVER, wx.BITMAP_TYPE_ANY) 125 | # 显示图片在m_animCtrl1上 126 | self.bmp = wx.StaticBitmap(self.m_animCtrl1, -1, wx.Bitmap(self.image_cover)) 127 | 128 | # 设置窗口标题的图标 129 | self.icon = wx.Icon('./images/123.ico', wx.BITMAP_TYPE_ICO) 130 | self.SetIcon(self.icon) 131 | # 系统事件 132 | self.Bind(wx.EVT_CLOSE, self.OnClose) 133 | 134 | print("wxpython界面初始化加载完成！") 135 | 136 | """参数""" 137 | # 默认为摄像头0 138 | self.VIDEO_STREAM = 0 139 | self.CAMERA_STYLE = False # False未打开摄像头，True摄像头已打开 140 | # 闪烁阈值（秒） 141 | self.AR_CONSEC_FRAMES_check = 3 142 | self.OUT_AR_CONSEC_FRAMES_check = 5 143 | # 眼睛长宽比 144 | self.EYE_AR_THRESH = 0.2 145 | self.EYE_AR_CONSEC_FRAMES = self.AR_CONSEC_FRAMES_check 146 | # 打哈欠长宽比 147 | self.MAR_THRESH = 0.5 148 | self.MOUTH_AR_CONSEC_FRAMES = self.AR_CONSEC_FRAMES_check 149 | # 瞌睡点头 150 | self.HAR_THRESH = 0.3 151 | self.NOD_AR_CONSEC_FRAMES = self.AR_CONSEC_FRAMES_check 152 | 153 | """计数""" 154 | # 初始化帧计数器和眨眼总数 155 | self.COUNTER = 0 156 | self.TOTAL = 0 157 | # 初始化帧计数器和打哈欠总数 158 | self.mCOUNTER = 0 159 | self.mTOTAL = 0 160 | # 初始化帧计数器和点头总数 161 | self.hCOUNTER = 0 162 | self.hTOTAL = 0 163 | # 离职时间长度 164 | self.oCOUNTER = 0 165 | 166 | """姿态""" 167 | # 世界坐标系(UVW)：填写3D参考点，该模型参考http://aifi.isr.uc.pt/Downloads/OpenGL/glAnthropometric3DModel.cpp 168 | self.object_pts = np.float32([[6.825897, 6.760612, 4.402142], #33左眉左上角 169 | [1.330353, 7.122144, 6.903745], #29左眉右角 170 | [-1.330353, 7.122144, 6.903745], #34右眉左角 171 | [-6.825897, 6.760612, 4.402142], #38右眉右上角 172 | [5.311432, 5.485328, 3.987654], #13左眼左上角 173 | [1.789930, 5.393625, 4.413414], #17左眼右上角 174 | [-1.789930, 5.393625, 4.413414], #25右眼左上角 175 | [-5.311432, 5.485328, 3.987654], #21右眼右上角 176 | [2.005628, 1.409845, 6.165652], #55鼻子左上角 177 | [-2.005628, 1.409845, 6.165652], #49鼻子右上角 178 | [2.774015, -2.080775, 5.048531], #43嘴左上角 179 | [-2.774015, -2.080775, 5.048531],#39嘴右上角 180 | [0.000000, -3.116408, 6.097667], #45嘴中央下角 181 | [0.000000, -7.415691, 4.070434]])#6下巴角 182 | 183 | # 相机坐标系(XYZ)：添加相机内参 184 | self.K = [6.5308391993466671e+002, 0.0, 3.1950000000000000e+002, 185 | 0.0, 6.5308391993466671e+002, 2.3950000000000000e+002, 186 | 0.0, 0.0, 1.0]# 等价于矩阵[fx, 0, cx; 0, fy, cy; 0, 0, 1] 187 | # 图像中心坐标系(uv)：相机畸变参数[k1, k2, p1, p2, k3] 188 | self.D = [7.0834633684407095e-002, 6.9140193737175351e-002, 0.0, 0.0, -1.3073460323689292e+000] 189 | 190 | # 像素坐标系(xy)：填写凸轮的本征和畸变系数 191 | self.cam_matrix = np.array(self.K).reshape(3, 3).astype(np.float32) 192 | self.dist_coeffs = np.array(self.D).reshape(5, 1).astype(np.float32) 193 | 194 | # 重新投影3D点的世界坐标轴以验证结果姿势 195 | self.reprojectsrc = np.float32([[10.0, 10.0, 10.0], 196 | [10.0, 10.0, -10.0], 197 | [10.0, -10.0, -10.0], 198 | [10.0, -10.0, 10.0], 199 | [-10.0, 10.0, 10.0], 200 | [-10.0, 10.0, -10.0], 201 | [-10.0, -10.0, -10.0], 202 | [-10.0, -10.0, 10.0]]) 203 | # 绘制正方体12轴 204 | self.line_pairs = [[0, 1], [1, 2], [2, 3], [3, 0], 205 | [4, 5], [5, 6], [6, 7], [7, 4], 206 | [0, 4], [1, 5], [2, 6], [3, 7]] 207 | 208 | 209 | def __del__( self ): 210 | pass 211 | 212 | def get_head_pose(self,shape):# 头部姿态估计 213 | # （像素坐标集合）填写2D参考点，注释遵循https://ibug.doc.ic.ac.uk/resources/300-W/ 214 | # 17左眉左上角/21左眉右角/22右眉左上角/26右眉右上角/36左眼左上角/39左眼右上角/42右眼左上角/ 215 | # 45右眼右上角/31鼻子左上角/35鼻子右上角/48左上角/54嘴右上角/57嘴中央下角/8下巴角 216 | image_pts = np.float32([shape[17], shape[21], shape[22], shape[26], shape[36], 217 | shape[39], shape[42], shape[45], shape[31], shape[35], 218 | shape[48], shape[54], shape[57], shape[8]]) 219 | # solvePnP计算姿势——求解旋转和平移矩阵： 220 | # rotation_vec表示旋转矩阵，translation_vec表示平移矩阵，cam_matrix与K矩阵对应，dist_coeffs与D矩阵对应。 221 | _, rotation_vec, translation_vec = cv2.solvePnP(self.object_pts, image_pts, self.cam_matrix, self.dist_coeffs) 222 | # projectPoints重新投影误差：原2d点和重投影2d点的距离（输入3d点、相机内参、相机畸变、r、t，输出重投影2d点） 223 | reprojectdst, _ = cv2.projectPoints(self.reprojectsrc, rotation_vec, translation_vec, self.cam_matrix,self.dist_coeffs) 224 | reprojectdst = tuple(map(tuple, reprojectdst.reshape(8, 2)))# 以8行2列显示 225 | 226 | # 计算欧拉角calc euler angle 227 | # 参考https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#decomposeprojectionmatrix 228 | rotation_mat, _ = cv2.Rodrigues(rotation_vec)#罗德里格斯公式（将旋转矩阵转换为旋转向量） 229 | pose_mat = cv2.hconcat((rotation_mat, translation_vec))# 水平拼接，vconcat垂直拼接 230 | # decomposeProjectionMatrix将投影矩阵分解为旋转矩阵和相机矩阵 231 | _, _, _, _, _, _, euler_angle = cv2.decomposeProjectionMatrix(pose_mat) 232 | 233 | pitch, yaw, roll = [math.radians(_) for _ in euler_angle] 234 | 235 | pitch = math.degrees(math.asin(math.sin(pitch))) 236 | roll = -math.degrees(math.asin(math.sin(roll))) 237 | yaw = math.degrees(math.asin(math.sin(yaw))) 238 | #print('pitch:{}, yaw:{}, roll:{}'.format(pitch, yaw, roll)) 239 | 240 | return reprojectdst, euler_angle# 投影误差，欧拉角 241 | 242 | def eye_aspect_ratio(self,eye): 243 | # 垂直眼标志（X，Y）坐标 244 | A = dist.euclidean(eye[1], eye[5])# 计算两个集合之间的欧式距离 245 | B = dist.euclidean(eye[2], eye[4]) 246 | # 计算水平之间的欧几里得距离 247 | # 水平眼标志（X，Y）坐标 248 | C = dist.euclidean(eye[0], eye[3]) 249 | # 眼睛长宽比的计算 250 | ear = (A + B) / (2.0 * C) 251 | # 返回眼睛的长宽比 252 | return ear 253 | 254 | def mouth_aspect_ratio(self,mouth):# 嘴部 255 | A = np.linalg.norm(mouth[2] - mouth[9]) # 51, 59 256 | B = np.linalg.norm(mouth[4] - mouth[7]) # 53, 57 257 | C = np.linalg.norm(mouth[0] - mouth[6]) # 49, 55 258 | mar = (A + B) / (2.0 * C) 259 | return mar 260 | 261 | 262 | def _learning_face(self,event): 263 | """dlib的初始化调用""" 264 | # 使用人脸检测器get_frontal_face_detector 265 | self.detector = dlib.get_frontal_face_detector() 266 | # dlib的68点模型，使用作者训练好的特征预测器 267 | self.predictor = dlib.shape_predictor("./model/shape_predictor_68_face_landmarks.dat") 268 | self.m_textCtrl3.AppendText(u"加载模型成功!!!\n") 269 | # 分别获取左右眼面部标志的索引 270 | (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] 271 | (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"] 272 | (mStart, mEnd) = face_utils.FACIAL_LANDMARKS_IDXS["mouth"] 273 | 274 | #建cv2摄像头对象，这里使用电脑自带摄像头，如果接了外部摄像头，则自动切换到外部摄像头 275 | self.cap = cv2.VideoCapture(self.VIDEO_STREAM) 276 | 277 | if self.cap.isOpened()==True:# 返回true/false 检查初始化是否成功 278 | self.CAMERA_STYLE = True 279 | self.m_textCtrl3.AppendText(u"打开摄像头成功!!!\n") 280 | else: 281 | self.m_textCtrl3.AppendText(u"摄像头打开失败!!!\n") 282 | #显示封面图 283 | self.bmp.SetBitmap(wx.Bitmap(self.image_cover)) 284 | # 成功打开视频，循环读取视频流 285 | while(self.cap.isOpened()): 286 | # cap.read() 287 | # 返回两个值： 288 | # 一个布尔值true/false，用来判断读取视频是否成功/是否到视频末尾 289 | # 图像对象，图像的三维矩阵 290 | flag, im_rd = self.cap.read() 291 | # 取灰度 292 | img_gray = cv2.cvtColor(im_rd, cv2.COLOR_RGB2GRAY) 293 | 294 | # 使用人脸检测器检测每一帧图像中的人脸。并返回人脸数faces 295 | faces = self.detector(img_gray, 0) 296 | # 如果检测到人脸 297 | if(len(faces)!=0): 298 | # enumerate方法同时返回数据对象的索引和数据，k为索引，d为faces中的对象 299 | for k, d in enumerate(faces): 300 | # 用红色矩形框出人脸 301 | cv2.rectangle(im_rd, (d.left(), d.top()), (d.right(), d.bottom()), (0, 0, 255),1) 302 | # 使用预测器得到68点数据的坐标 303 | shape = self.predictor(im_rd, d) 304 | # 圆圈显示每个特征点 305 | for i in range(68): 306 | cv2.circle(im_rd, (shape.part(i).x, shape.part(i).y), 2, (0, 255, 0), -1, 8) 307 | # 将脸部特征信息转换为数组array的格式 308 | shape = face_utils.shape_to_np(shape) 309 | """ 310 | 打哈欠 311 | """ 312 | if self.yawn_checkBox1.GetValue()== True: 313 | # 嘴巴坐标 314 | mouth = shape[mStart:mEnd] 315 | # 打哈欠 316 | mar = self.mouth_aspect_ratio(mouth) 317 | # 使用cv2.convexHull获得凸包位置，使用drawContours画出轮廓位置进行画图操作 318 | mouthHull = cv2.convexHull(mouth) 319 | cv2.drawContours(im_rd, [mouthHull], -1, (0, 255, 0), 1) 320 | # 同理，判断是否打哈欠 321 | if mar > self.MAR_THRESH:# 张嘴阈值0.5 322 | self.mCOUNTER += 1 323 | else: 324 | # 如果连续3次都小于阈值，则表示打了一次哈欠 325 | if self.mCOUNTER >= self.MOUTH_AR_CONSEC_FRAMES:# 阈值：3 326 | self.mTOTAL += 1 327 | #显示 328 | cv2.putText(im_rd, "Yawning!", (10, 60),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 329 | self.m_textCtrl3.AppendText(time.strftime('%Y-%m-%d %H:%M ', time.localtime())+u"打哈欠\n") 330 | # 重置嘴帧计数器 331 | self.mCOUNTER = 0 332 | cv2.putText(im_rd, "COUNTER: {}".format(self.mCOUNTER), (150, 60),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 333 | cv2.putText(im_rd, "MAR: {:.2f}".format(mar), (300, 60),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 334 | cv2.putText(im_rd, "Yawning: {}".format(self.mTOTAL), (450, 60),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,0), 2) 335 | else: 336 | pass 337 | """ 338 | 眨眼 339 | """ 340 | if self.blink_checkBox2.GetValue()== True: 341 | # 提取左眼和右眼坐标 342 | leftEye = shape[lStart:lEnd] 343 | rightEye = shape[rStart:rEnd] 344 | # 构造函数计算左右眼的EAR值，使用平均值作为最终的EAR 345 | leftEAR = self.eye_aspect_ratio(leftEye) 346 | rightEAR = self.eye_aspect_ratio(rightEye) 347 | ear = (leftEAR + rightEAR) / 2.0 348 | leftEyeHull = cv2.convexHull(leftEye) 349 | rightEyeHull = cv2.convexHull(rightEye) 350 | # 使用cv2.convexHull获得凸包位置，使用drawContours画出轮廓位置进行画图操作 351 | cv2.drawContours(im_rd, [leftEyeHull], -1, (0, 255, 0), 1) 352 | cv2.drawContours(im_rd, [rightEyeHull], -1, (0, 255, 0), 1) 353 | # 循环，满足条件的，眨眼次数+1 354 | if ear < self.EYE_AR_THRESH:# 眼睛长宽比：0.2 355 | self.COUNTER += 1 356 | 357 | else: 358 | # 如果连续3次都小于阈值，则表示进行了一次眨眼活动 359 | if self.COUNTER >= self.EYE_AR_CONSEC_FRAMES:# 阈值：3 360 | self.TOTAL += 1 361 | self.m_textCtrl3.AppendText(time.strftime('%Y-%m-%d %H:%M ', time.localtime())+u"眨眼\n") 362 | # 重置眼帧计数器 363 | self.COUNTER = 0 364 | # 第十四步：进行画图操作，同时使用cv2.putText将眨眼次数进行显示 365 | cv2.putText(im_rd, "Faces: {}".format(len(faces)), (10, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 366 | cv2.putText(im_rd, "COUNTER: {}".format(self.COUNTER), (150, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 367 | cv2.putText(im_rd, "EAR: {:.2f}".format(ear), (300, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 368 | cv2.putText(im_rd, "Blinks: {}".format(self.TOTAL), (450, 30),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,0), 2) 369 | else: 370 | pass 371 | """ 372 | 瞌睡点头 373 | """ 374 | if self.nod_checkBox7.GetValue()== True: 375 | # 获取头部姿态 376 | reprojectdst, euler_angle = self.get_head_pose(shape) 377 | har = euler_angle[0, 0]# 取pitch旋转角度 378 | if har > self.HAR_THRESH:# 点头阈值0.3 379 | self.hCOUNTER += 1 380 | else: 381 | # 如果连续3次都小于阈值，则表示瞌睡点头一次 382 | if self.hCOUNTER >= self.NOD_AR_CONSEC_FRAMES:# 阈值：3 383 | self.hTOTAL += 1 384 | self.m_textCtrl3.AppendText(time.strftime('%Y-%m-%d %H:%M ', time.localtime())+u"瞌睡点头\n") 385 | # 重置点头帧计数器 386 | self.hCOUNTER = 0 387 | # 绘制正方体12轴(视频流尺寸过大时，reprojectdst会超出int范围，建议压缩检测视频尺寸) 388 | # for start, end in self.line_pairs: 389 | # cv2.line(im_rd, reprojectdst[start], reprojectdst[end], (0, 0, 255)) 390 | # 显示角度结果 391 | cv2.putText(im_rd, "X: " + "{:7.2f}".format(euler_angle[0, 0]), (10, 90), cv2.FONT_HERSHEY_SIMPLEX,0.75, (0, 255, 0), thickness=2)# GREEN 392 | cv2.putText(im_rd, "Y: " + "{:7.2f}".format(euler_angle[1, 0]), (150, 90), cv2.FONT_HERSHEY_SIMPLEX,0.75, (255, 0, 0), thickness=2)# BLUE 393 | cv2.putText(im_rd, "Z: " + "{:7.2f}".format(euler_angle[2, 0]), (300, 90), cv2.FONT_HERSHEY_SIMPLEX,0.75, (0, 0, 255), thickness=2)# RED 394 | cv2.putText(im_rd, "Nod: {}".format(self.hTOTAL), (450, 90),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,0), 2) 395 | else: 396 | pass 397 | 398 | #print('嘴巴实时长宽比:{:.2f} '.format(mar)+"\t是否张嘴："+str([False,True][mar > self.MAR_THRESH])) 399 | #print('眼睛实时长宽比:{:.2f} '.format(ear)+"\t是否眨眼："+str([False,True][self.COUNTER>=1])) 400 | else: 401 | # 没有检测到人脸 402 | self.oCOUNTER+=1 403 | cv2.putText(im_rd, "No Face", (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255),3, cv2.LINE_AA) 404 | if self.oCOUNTER >= self.OUT_AR_CONSEC_FRAMES_check: 405 | self.m_textCtrl3.AppendText(time.strftime('%Y-%m-%d %H:%M ', time.localtime())+u"员工脱岗!!!\n") 406 | self.oCOUNTER = 0 407 | 408 | # 确定疲劳提示:眨眼50次，打哈欠15次，瞌睡点头30次 409 | if self.TOTAL >= 50 or self.mTOTAL>=15 or self.hTOTAL>=30: 410 | cv2.putText(im_rd, "SLEEP!!!", (100, 200),cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 3) 411 | #self.m_textCtrl3.AppendText(u"疲劳") 412 | 413 | # opencv中imread的图片内部是BGR排序，wxPython的StaticBitmap需要的图片是RGB排序，不转换会出现颜色变换 414 | height,width = im_rd.shape[:2] 415 | image1 = cv2.cvtColor(im_rd, cv2.COLOR_BGR2RGB) 416 | pic = wx.Bitmap.FromBuffer(width,height,image1) 417 | # 显示图片在panel上： 418 | self.bmp.SetBitmap(pic) 419 | 420 | # 释放摄像头 421 | self.cap.release() 422 | 423 | def camera_on(self,event): 424 | """使用多线程，子线程运行后台的程序，主线程更新前台的UI，这样不会互相影响""" 425 | import _thread 426 | # 创建子线程，按钮调用这个方法， 427 | _thread.start_new_thread(self._learning_face, (event,)) 428 | 429 | def cameraid_choice( self, event ): 430 | # 摄像头编号 431 | cameraid = int(event.GetString()[-1])# 截取最后一个字符 432 | if cameraid == 0: 433 | self.m_textCtrl3.AppendText(u"准备打开本地摄像头!!!\n") 434 | if cameraid == 1 or cameraid == 2: 435 | self.m_textCtrl3.AppendText(u"准备打开外置摄像头!!!\n") 436 | self.VIDEO_STREAM = cameraid 437 | 438 | def vedio_on( self, event ): 439 | if self.CAMERA_STYLE == True :# 释放摄像头资源 440 | # 弹出关闭摄像头提示窗口 441 | dlg = wx.MessageDialog(None, u'确定要关闭摄像头？', u'操作提示', wx.YES_NO | wx.ICON_QUESTION) 442 | if(dlg.ShowModal() == wx.ID_YES): 443 | self.cap.release()#释放摄像头 444 | self.bmp.SetBitmap(wx.Bitmap(self.image_cover))#封面 445 | dlg.Destroy()#取消弹窗 446 | # 选择文件夹对话框窗口 447 | dialog = wx.FileDialog(self,u"选择视频检测",os.getcwd(),'',wildcard="(*.mp4)|*.mp4",style=wx.FD_OPEN | wx.FD_CHANGE_DIR) 448 | if dialog.ShowModal() == wx.ID_OK: 449 | #如果确定了选择的文件夹，将文件夹路径写到m_textCtrl3控件 450 | self.m_textCtrl3.SetValue(u"文件路径:"+dialog.GetPath()+"\n") 451 | self.VIDEO_STREAM = str(dialog.GetPath())# 更新全局变量路径 452 | dialog.Destroy 453 | """使用多线程，子线程运行后台的程序，主线程更新前台的UI，这样不会互相影响""" 454 | import _thread 455 | # 创建子线程，按钮调用这个方法， 456 | _thread.start_new_thread(self._learning_face, (event,)) 457 | 458 | def AR_CONSEC_FRAMES( self, event ): 459 | self.m_textCtrl3.AppendText(u"设置疲劳间隔为:\t"+event.GetString()+"秒\n") 460 | self.AR_CONSEC_FRAMES_check = int(event.GetString()) 461 | 462 | def OUT_AR_CONSEC_FRAMES( self, event ): 463 | self.m_textCtrl3.AppendText(u"设置脱岗间隔为:\t"+event.GetString()+"秒\n") 464 | self.OUT_AR_CONSEC_FRAMES_check = int(event.GetString()) 465 | 466 | def off(self,event): 467 | """关闭摄像头，显示封面页""" 468 | self.cap.release() 469 | self.bmp.SetBitmap(wx.Bitmap(self.image_cover)) 470 | 471 | def OnClose(self, evt): 472 | """关闭窗口事件函数""" 473 | dlg = wx.MessageDialog(None, u'确定要关闭本窗口？', u'操作提示', wx.YES_NO | wx.ICON_QUESTION) 474 | if(dlg.ShowModal() == wx.ID_YES): 475 | self.Destroy() 476 | print("检测结束，成功退出程序!!!") 477 | 478 | 479 | -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | import model 4 | import data_provider 5 | 6 | TEXT_EXAMPLES = 800 7 | num_classes = 2 8 | 9 | global_step = tf.Variable(0, trainable=False) 10 | 11 | x = tf.placeholder(tf.float32, [None, 64*64]) 12 | y = tf.placeholder(tf.float32, [None, num_classes]) 13 | keep_prob = tf.placeholder(tf.float32) #dropout (keep probability) 14 | 15 | image, label = data_provider.get_data("test") 16 | image = tf.reshape(image, shape=(1,64*64)) 17 | label = tf.reshape(label, shape=(1,2)) 18 | 19 | logit = model.create_model(x, num_classes, keep_prob) 20 | correct_pred = tf.equal(tf.argmax(logit, 1), tf.argmax(y, 1)) 21 | 22 | correct_num = 0 23 | 24 | saver = tf.train.Saver() 25 | 26 | checkpoint_dir = "path/to/model/" 27 | 28 | with tf.Session() as sess: 29 | 30 | coord = tf.train.Coordinator() 31 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 32 | 33 | ckpt = tf.train.get_checkpoint_state(checkpoint_dir) 34 | if ckpt and ckpt.model_checkpoint_path: 35 | saver.restore(sess, ckpt.model_checkpoint_path) 36 | 37 | for i in range(TEXT_EXAMPLES): 38 | test_image, test_label = sess.run([image, label]) 39 | correct = sess.run([correct_pred], feed_dict= 40 | {x:test_image, y:test_label, keep_prob:1.}) 41 | 42 | if correct[0] == True: 43 | #print ("*************", correct) 44 | correct_num += 1 45 | 46 | acc = correct_num / TEXT_EXAMPLES 47 | print ("After %s iterator(s) the accuracy is %f" %(global_step, acc)) 48 | 49 | -------------------------------------------------------------------------------- /extract_face.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | from functools import partial 4 | from scipy.misc import imresize 5 | from PIL import Image 6 | 7 | 8 | data_set = "genki4k/files/" 9 | output_dir = "genki4k/files_crip/" 10 | 11 | 12 | harr_model_path = 'face_det/data/haarcascades' 13 | frontal_model= os.path.join(harr_model_path, 'haarcascade_frontalface_default.xml') 14 | profile_model = os.path.join(harr_model_path, 'haarcascade_profileface.xml') 15 | 16 | # 正脸检测的模型 17 | frontal_dector = partial(cv2.CascadeClassifier(frontal_model).detectMultiScale, 18 | scaleFactor=1.1, 19 | minNeighbors=5, 20 | minSize=(100, 100)) 21 | 22 | # 侧脸检测的模型 23 | profile_dector = partial(cv2.CascadeClassifier(profile_model).detectMultiScale, 24 | scaleFactor=1.1, 25 | minNeighbors=5, 26 | minSize=(100, 100)) 27 | 28 | if not os.path.exists(output_dir): 29 | os.makedirs(output_dir) 30 | 31 | def extract_face(image, save_path): 32 | image = cv2.imread(image) 33 | faces = frontal_dector(image) 34 | for (x, y, z, w) in faces: 35 | cv2.rectangle(image, (x,y), (x+z, y+w), (0, 255, 0), 1) 36 | save_img = image[y:y+w,x:x+z] 37 | 38 | cv2.imwrite(save_path, save_img) 39 | save_img = Image.open(save_path) 40 | save_img = save_img.resize((64,64)) 41 | save_img = save_img.convert('L') 42 | save_img.save(save_path) 43 | 44 | imgs = os.listdir(data_set) 45 | for img in imgs: 46 | extract_face(data_set+img, output_dir+img) 47 | -------------------------------------------------------------------------------- /load_and_process.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import cv2 3 | import numpy as np 4 | 5 | 6 | dataset_path = 'fer2013/fer2013/fer2013.csv' 7 | image_size=(48,48) 8 | 9 | def load_fer2013(): 10 | data = pd.read_csv(dataset_path) 11 | pixels = data['pixels'].tolist() 12 | width, height = 48, 48 13 | faces = [] 14 | for pixel_sequence in pixels: 15 | face = [int(pixel) for pixel in pixel_sequence.split(' ')] 16 | face = np.asarray(face).reshape(width, height) 17 | face = cv2.resize(face.astype('uint8'),image_size) 18 | faces.append(face.astype('float32')) 19 | faces = np.asarray(faces) 20 | faces = np.expand_dims(faces, -1) 21 | emotions = pd.get_dummies(data['emotion']).as_matrix() 22 | return faces, emotions 23 | 24 | def preprocess_input(x, v2=True): 25 | x = x.astype('float32') 26 | x = x / 255.0 27 | if v2: 28 | x = x - 0.5 29 | x = x * 2.0 30 | return x -------------------------------------------------------------------------------- /models/__pycache__/cnn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangjianlin1985/Python_FatigueDrivingDetection/654f1ecd4c168976b7ec2ed3f0a7e7c24b017ecf/models/__pycache__/cnn.cpython-36.pyc -------------------------------------------------------------------------------- /models/_mini_XCEPTION.102-0.66.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangjianlin1985/Python_FatigueDrivingDetection/654f1ecd4c168976b7ec2ed3f0a7e7c24b017ecf/models/_mini_XCEPTION.102-0.66.hdf5 -------------------------------------------------------------------------------- /models/cnn.py: -------------------------------------------------------------------------------- 1 | from keras.layers import Activation, Convolution2D, Dropout, Conv2D 2 | from keras.layers import AveragePooling2D, BatchNormalization 3 | from keras.layers import GlobalAveragePooling2D 4 | from keras.models import Sequential 5 | from keras.layers import Flatten 6 | from keras.models import Model 7 | from keras.layers import Input 8 | from keras.layers import MaxPooling2D 9 | from keras.layers import SeparableConv2D 10 | from keras import layers 11 | from keras.regularizers import l2 12 | 13 | def simple_CNN(input_shape, num_classes): 14 | 15 | model = Sequential() 16 | model.add(Convolution2D(filters=16, kernel_size=(7, 7), padding='same', 17 | name='image_array', input_shape=input_shape)) 18 | model.add(BatchNormalization()) 19 | model.add(Convolution2D(filters=16, kernel_size=(7, 7), padding='same')) 20 | model.add(BatchNormalization()) 21 | model.add(Activation('relu')) 22 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 23 | model.add(Dropout(.5)) 24 | 25 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 26 | model.add(BatchNormalization()) 27 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 28 | model.add(BatchNormalization()) 29 | model.add(Activation('relu')) 30 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 31 | model.add(Dropout(.5)) 32 | 33 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 34 | model.add(BatchNormalization()) 35 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 36 | model.add(BatchNormalization()) 37 | model.add(Activation('relu')) 38 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 39 | model.add(Dropout(.5)) 40 | 41 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same')) 42 | model.add(BatchNormalization()) 43 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), padding='same')) 44 | model.add(BatchNormalization()) 45 | model.add(Activation('relu')) 46 | model.add(AveragePooling2D(pool_size=(2, 2), padding='same')) 47 | model.add(Dropout(.5)) 48 | 49 | model.add(Convolution2D(filters=256, kernel_size=(3, 3), padding='same')) 50 | model.add(BatchNormalization()) 51 | model.add(Convolution2D(filters=num_classes, kernel_size=(3, 3), padding='same')) 52 | model.add(GlobalAveragePooling2D()) 53 | model.add(Activation('softmax',name='predictions')) 54 | return model 55 | 56 | def simpler_CNN(input_shape, num_classes): 57 | 58 | model = Sequential() 59 | model.add(Convolution2D(filters=16, kernel_size=(5, 5), padding='same', 60 | name='image_array', input_shape=input_shape)) 61 | model.add(BatchNormalization()) 62 | model.add(Convolution2D(filters=16, kernel_size=(5, 5), 63 | strides=(2, 2), padding='same')) 64 | model.add(BatchNormalization()) 65 | model.add(Activation('relu')) 66 | model.add(Dropout(.25)) 67 | 68 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same')) 69 | model.add(BatchNormalization()) 70 | model.add(Convolution2D(filters=32, kernel_size=(5, 5), 71 | strides=(2, 2), padding='same')) 72 | model.add(BatchNormalization()) 73 | model.add(Activation('relu')) 74 | model.add(Dropout(.25)) 75 | 76 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), padding='same')) 77 | model.add(BatchNormalization()) 78 | model.add(Convolution2D(filters=64, kernel_size=(3, 3), 79 | strides=(2, 2), padding='same')) 80 | model.add(BatchNormalization()) 81 | model.add(Activation('relu')) 82 | model.add(Dropout(.25)) 83 | 84 | model.add(Convolution2D(filters=64, kernel_size=(1, 1), padding='same')) 85 | model.add(BatchNormalization()) 86 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), 87 | strides=(2, 2), padding='same')) 88 | model.add(BatchNormalization()) 89 | model.add(Activation('relu')) 90 | model.add(Dropout(.25)) 91 | 92 | model.add(Convolution2D(filters=256, kernel_size=(1, 1), padding='same')) 93 | model.add(BatchNormalization()) 94 | model.add(Convolution2D(filters=128, kernel_size=(3, 3), 95 | strides=(2, 2), padding='same')) 96 | 97 | model.add(Convolution2D(filters=256, kernel_size=(1, 1), padding='same')) 98 | model.add(BatchNormalization()) 99 | model.add(Convolution2D(filters=num_classes, kernel_size=(3, 3), 100 | strides=(2, 2), padding='same')) 101 | 102 | model.add(Flatten()) 103 | #model.add(GlobalAveragePooling2D()) 104 | model.add(Activation('softmax',name='predictions')) 105 | return model 106 | 107 | def tiny_XCEPTION(input_shape, num_classes, l2_regularization=0.01): 108 | regularization = l2(l2_regularization) 109 | 110 | # base 111 | img_input = Input(input_shape) 112 | x = Conv2D(5, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 113 | use_bias=False)(img_input) 114 | x = BatchNormalization()(x) 115 | x = Activation('relu')(x) 116 | x = Conv2D(5, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 117 | use_bias=False)(x) 118 | x = BatchNormalization()(x) 119 | x = Activation('relu')(x) 120 | 121 | # module 1 122 | residual = Conv2D(8, (1, 1), strides=(2, 2), 123 | padding='same', use_bias=False)(x) 124 | residual = BatchNormalization()(residual) 125 | 126 | x = SeparableConv2D(8, (3, 3), padding='same', 127 | kernel_regularizer=regularization, 128 | use_bias=False)(x) 129 | x = BatchNormalization()(x) 130 | x = Activation('relu')(x) 131 | x = SeparableConv2D(8, (3, 3), padding='same', 132 | kernel_regularizer=regularization, 133 | use_bias=False)(x) 134 | x = BatchNormalization()(x) 135 | 136 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 137 | x = layers.add([x, residual]) 138 | 139 | # module 2 140 | residual = Conv2D(16, (1, 1), strides=(2, 2), 141 | padding='same', use_bias=False)(x) 142 | residual = BatchNormalization()(residual) 143 | 144 | x = SeparableConv2D(16, (3, 3), padding='same', 145 | kernel_regularizer=regularization, 146 | use_bias=False)(x) 147 | x = BatchNormalization()(x) 148 | x = Activation('relu')(x) 149 | x = SeparableConv2D(16, (3, 3), padding='same', 150 | kernel_regularizer=regularization, 151 | use_bias=False)(x) 152 | x = BatchNormalization()(x) 153 | 154 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 155 | x = layers.add([x, residual]) 156 | 157 | # module 3 158 | residual = Conv2D(32, (1, 1), strides=(2, 2), 159 | padding='same', use_bias=False)(x) 160 | residual = BatchNormalization()(residual) 161 | 162 | x = SeparableConv2D(32, (3, 3), padding='same', 163 | kernel_regularizer=regularization, 164 | use_bias=False)(x) 165 | x = BatchNormalization()(x) 166 | x = Activation('relu')(x) 167 | x = SeparableConv2D(32, (3, 3), padding='same', 168 | kernel_regularizer=regularization, 169 | use_bias=False)(x) 170 | x = BatchNormalization()(x) 171 | 172 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 173 | x = layers.add([x, residual]) 174 | 175 | # module 4 176 | residual = Conv2D(64, (1, 1), strides=(2, 2), 177 | padding='same', use_bias=False)(x) 178 | residual = BatchNormalization()(residual) 179 | 180 | x = SeparableConv2D(64, (3, 3), padding='same', 181 | kernel_regularizer=regularization, 182 | use_bias=False)(x) 183 | x = BatchNormalization()(x) 184 | x = Activation('relu')(x) 185 | x = SeparableConv2D(64, (3, 3), padding='same', 186 | kernel_regularizer=regularization, 187 | use_bias=False)(x) 188 | x = BatchNormalization()(x) 189 | 190 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 191 | x = layers.add([x, residual]) 192 | 193 | x = Conv2D(num_classes, (3, 3), 194 | #kernel_regularizer=regularization, 195 | padding='same')(x) 196 | x = GlobalAveragePooling2D()(x) 197 | output = Activation('softmax',name='predictions')(x) 198 | 199 | model = Model(img_input, output) 200 | return model 201 | 202 | 203 | def mini_XCEPTION(input_shape, num_classes, l2_regularization=0.01): 204 | regularization = l2(l2_regularization) 205 | 206 | # base 207 | img_input = Input(input_shape) 208 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 209 | use_bias=False)(img_input) 210 | x = BatchNormalization()(x) 211 | x = Activation('relu')(x) 212 | x = Conv2D(8, (3, 3), strides=(1, 1), kernel_regularizer=regularization, 213 | use_bias=False)(x) 214 | x = BatchNormalization()(x) 215 | x = Activation('relu')(x) 216 | 217 | # module 1 218 | residual = Conv2D(16, (1, 1), strides=(2, 2), 219 | padding='same', use_bias=False)(x) 220 | residual = BatchNormalization()(residual) 221 | 222 | x = SeparableConv2D(16, (3, 3), padding='same', 223 | kernel_regularizer=regularization, 224 | use_bias=False)(x) 225 | x = BatchNormalization()(x) 226 | x = Activation('relu')(x) 227 | x = SeparableConv2D(16, (3, 3), padding='same', 228 | kernel_regularizer=regularization, 229 | use_bias=False)(x) 230 | x = BatchNormalization()(x) 231 | 232 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 233 | x = layers.add([x, residual]) 234 | 235 | # module 2 236 | residual = Conv2D(32, (1, 1), strides=(2, 2), 237 | padding='same', use_bias=False)(x) 238 | residual = BatchNormalization()(residual) 239 | 240 | x = SeparableConv2D(32, (3, 3), padding='same', 241 | kernel_regularizer=regularization, 242 | use_bias=False)(x) 243 | x = BatchNormalization()(x) 244 | x = Activation('relu')(x) 245 | x = SeparableConv2D(32, (3, 3), padding='same', 246 | kernel_regularizer=regularization, 247 | use_bias=False)(x) 248 | x = BatchNormalization()(x) 249 | 250 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 251 | x = layers.add([x, residual]) 252 | 253 | # module 3 254 | residual = Conv2D(64, (1, 1), strides=(2, 2), 255 | padding='same', use_bias=False)(x) 256 | residual = BatchNormalization()(residual) 257 | 258 | x = SeparableConv2D(64, (3, 3), padding='same', 259 | kernel_regularizer=regularization, 260 | use_bias=False)(x) 261 | x = BatchNormalization()(x) 262 | x = Activation('relu')(x) 263 | x = SeparableConv2D(64, (3, 3), padding='same', 264 | kernel_regularizer=regularization, 265 | use_bias=False)(x) 266 | x = BatchNormalization()(x) 267 | 268 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 269 | x = layers.add([x, residual]) 270 | 271 | # module 4 272 | residual = Conv2D(128, (1, 1), strides=(2, 2), 273 | padding='same', use_bias=False)(x) 274 | residual = BatchNormalization()(residual) 275 | 276 | x = SeparableConv2D(128, (3, 3), padding='same', 277 | kernel_regularizer=regularization, 278 | use_bias=False)(x) 279 | x = BatchNormalization()(x) 280 | x = Activation('relu')(x) 281 | x = SeparableConv2D(128, (3, 3), padding='same', 282 | kernel_regularizer=regularization, 283 | use_bias=False)(x) 284 | x = BatchNormalization()(x) 285 | 286 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 287 | x = layers.add([x, residual]) 288 | 289 | x = Conv2D(num_classes, (3, 3), 290 | #kernel_regularizer=regularization, 291 | padding='same')(x) 292 | x = GlobalAveragePooling2D()(x) 293 | output = Activation('softmax',name='predictions')(x) 294 | 295 | model = Model(img_input, output) 296 | return model 297 | 298 | def big_XCEPTION(input_shape, num_classes): 299 | img_input = Input(input_shape) 300 | x = Conv2D(32, (3, 3), strides=(2, 2), use_bias=False)(img_input) 301 | x = BatchNormalization(name='block1_conv1_bn')(x) 302 | x = Activation('relu', name='block1_conv1_act')(x) 303 | x = Conv2D(64, (3, 3), use_bias=False)(x) 304 | x = BatchNormalization(name='block1_conv2_bn')(x) 305 | x = Activation('relu', name='block1_conv2_act')(x) 306 | 307 | residual = Conv2D(128, (1, 1), strides=(2, 2), 308 | padding='same', use_bias=False)(x) 309 | residual = BatchNormalization()(residual) 310 | 311 | x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x) 312 | x = BatchNormalization(name='block2_sepconv1_bn')(x) 313 | x = Activation('relu', name='block2_sepconv2_act')(x) 314 | x = SeparableConv2D(128, (3, 3), padding='same', use_bias=False)(x) 315 | x = BatchNormalization(name='block2_sepconv2_bn')(x) 316 | 317 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 318 | x = layers.add([x, residual]) 319 | 320 | residual = Conv2D(256, (1, 1), strides=(2, 2), 321 | padding='same', use_bias=False)(x) 322 | residual = BatchNormalization()(residual) 323 | 324 | x = Activation('relu', name='block3_sepconv1_act')(x) 325 | x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x) 326 | x = BatchNormalization(name='block3_sepconv1_bn')(x) 327 | x = Activation('relu', name='block3_sepconv2_act')(x) 328 | x = SeparableConv2D(256, (3, 3), padding='same', use_bias=False)(x) 329 | x = BatchNormalization(name='block3_sepconv2_bn')(x) 330 | 331 | x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) 332 | x = layers.add([x, residual]) 333 | x = Conv2D(num_classes, (3, 3), 334 | #kernel_regularizer=regularization, 335 | padding='same')(x) 336 | x = GlobalAveragePooling2D()(x) 337 | output = Activation('softmax',name='predictions')(x) 338 | 339 | model = Model(img_input, output) 340 | return model 341 | 342 | 343 | if __name__ == "__main__": 344 | input_shape = (64, 64, 1) 345 | num_classes = 7 346 | #model = tiny_XCEPTION(input_shape, num_classes) 347 | #model.summary() 348 | #model = mini_XCEPTION(input_shape, num_classes) 349 | #model.summary() 350 | #model = big_XCEPTION(input_shape, num_classes) 351 | #model.summary() 352 | model = simple_CNN((48, 48, 1), num_classes) 353 | model.summary() 354 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | decorator==4.4.2 2 | dlib==19.7.0 3 | imageio==2.16.0 4 | imutils==0.5.4 5 | networkx==2.5.1 6 | numpy==1.19.4 7 | opencv-python==4.4.0.46 8 | pandas==1.1.4 9 | Pillow==8.0.1 10 | pygame==2.1.2 11 | python-dateutil==2.8.2 12 | pytz==2023.3 13 | PyWavelets==1.1.1 14 | scikit-image==0.15.0 15 | scipy==1.5.4 16 | six==1.16.0 17 | -------------------------------------------------------------------------------- /split_train_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import random 4 | 5 | data_set = "genki4k/files_crip/" 6 | 7 | subjects = os.listdir(data_set) 8 | 9 | random.shuffle(subjects) 10 | 11 | test_subjects = subjects[:800] 12 | train_subjects = subjects[800:] 13 | 14 | def generate(subjects, target_set): 15 | for img in subjects: 16 | src = data_set + img 17 | tar = target_set + img 18 | shutil.copyfile(src, tar) 19 | 20 | 21 | train_set = "data/train/" 22 | test_set = "data/test/" 23 | 24 | if not os.path.exists(train_set): 25 | os.makedirs(train_set) 26 | 27 | if not os.path.exists(test_set): 28 | os.makedirs(test_set) 29 | 30 | generate(train_subjects, train_set) 31 | generate(test_subjects, test_set) 32 | -------------------------------------------------------------------------------- /tkinter_UI.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangjianlin1985/Python_FatigueDrivingDetection/654f1ecd4c168976b7ec2ed3f0a7e7c24b017ecf/tkinter_UI.exe -------------------------------------------------------------------------------- /tkinter_UI.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | from tkinter import * 3 | import cv2 4 | from PIL import Image,ImageTk 5 | 6 | from scipy.spatial import distance as dist 7 | from imutils.video import FileVideoStream 8 | from imutils.video import VideoStream 9 | from imutils import face_utils 10 | import numpy as np # 数据处理的库 numpy 11 | import argparse 12 | import imutils 13 | import time 14 | import dlib 15 | import cv2 16 | import math 17 | import time 18 | from threading import Thread 19 | from pygame import mixer 20 | import time 21 | 22 | 23 | 24 | # 世界坐标系(UVW)：填写3D参考点 25 | object_pts = np.float32([[6.825897, 6.760612, 4.402142], # 33左眉左上角 26 | [1.330353, 7.122144, 6.903745], # 29左眉右角 27 | [-1.330353, 7.122144, 6.903745], # 34右眉左角 28 | [-6.825897, 6.760612, 4.402142], # 38右眉右上角 29 | [5.311432, 5.485328, 3.987654], # 13左眼左上角 30 | [1.789930, 5.393625, 4.413414], # 17左眼右上角 31 | [-1.789930, 5.393625, 4.413414], # 25右眼左上角 32 | [-5.311432, 5.485328, 3.987654], # 21右眼右上角 33 | [2.005628, 1.409845, 6.165652], # 55鼻子左上角 34 | [-2.005628, 1.409845, 6.165652], # 49鼻子右上角 35 | [2.774015, -2.080775, 5.048531], # 43嘴左上角 36 | [-2.774015, -2.080775, 5.048531], # 39嘴右上角 37 | [0.000000, -3.116408, 6.097667], # 45嘴中央下角 38 | [0.000000, -7.415691, 4.070434]]) # 6下巴角 39 | 40 | # 相机坐标系(XYZ)：添加相机内参 41 | K = [6.5308391993466671e+002, 0.0, 3.1950000000000000e+002, 42 | 0.0, 6.5308391993466671e+002, 2.3950000000000000e+002, 43 | 0.0, 0.0, 1.0] # 等价于矩阵[fx, 0, cx; 0, fy, cy; 0, 0, 1] 44 | 45 | # 图像中心坐标系(uv)：相机畸变参数[k1, k2, p1, p2, k3] 46 | D = [7.0834633684407095e-002, 6.9140193737175351e-002, 0.0, 0.0, -1.3073460323689292e+000] 47 | 48 | # 像素坐标系(xy)：填写凸轮的本征和畸变系数 49 | cam_matrix = np.array(K).reshape(3, 3).astype(np.float32) 50 | dist_coeffs = np.array(D).reshape(5, 1).astype(np.float32) 51 | 52 | # 重新投影3D点的世界坐标轴以验证结果姿势 53 | reprojectsrc = np.float32([[10.0, 10.0, 10.0], 54 | [10.0, 10.0, -10.0], 55 | [10.0, -10.0, -10.0], 56 | [10.0, -10.0, 10.0], 57 | [-10.0, 10.0, 10.0], 58 | [-10.0, 10.0, -10.0], 59 | [-10.0, -10.0, -10.0], 60 | [-10.0, -10.0, 10.0]]) 61 | # 绘制正方体12轴 62 | line_pairs = [[0, 1], [1, 2], [2, 3], [3, 0], 63 | [4, 5], [5, 6], [6, 7], [7, 4], 64 | [0, 4], [1, 5], [2, 6], [3, 7]] 65 | 66 | 67 | 68 | EYE_AR_THRESH = 0.25 69 | EYE_AR_CONSEC_FRAMES = 3 70 | 71 | 72 | MAR_THRESH = 0.5 73 | MOUTH_AR_CONSEC_FRAMES = 3 74 | 75 | 76 | HAR_THRESH = 0.3 77 | NOD_AR_CONSEC_FRAMES = 3 78 | 79 | 80 | COUNTER = 0 81 | TOTAL = 0 82 | 83 | mCOUNTER = 0 84 | mTOTAL = 0 85 | 86 | hCOUNTER = 0 87 | hTOTAL = 0 88 | 89 | 90 | detector = dlib.get_frontal_face_detector() 91 | predictor = dlib.shape_predictor( 92 | 'dancheng_model/resnet_dancheng_model_train_fk.h5') 93 | 94 | # 分别获取左右眼面部标志的索引 95 | (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] 96 | (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"] 97 | (mStart, mEnd) = face_utils.FACIAL_LANDMARKS_IDXS["mouth"] 98 | 99 | 100 | global_pitch = 0 101 | global_yaw = 0 102 | global_roll = 0 103 | 104 | EAR = '' 105 | MAR = '' 106 | 107 | def get_head_pose(shape): # 头部姿态估计 108 | # （像素坐标集合）填写2D参考点，注释遵循https://ibug.doc.ic.ac.uk/resources/300-W/ 109 | image_pts = np.float32([shape[17], shape[21], shape[22], shape[26], shape[36], 110 | shape[39], shape[42], shape[45], shape[31], shape[35], 111 | shape[48], shape[54], shape[57], shape[8]]) 112 | # solvePnP计算姿势——求解旋转和平移矩阵： 113 | # rotation_vec表示旋转矩阵，translation_vec表示平移矩阵，cam_matrix与K矩阵对应，dist_coeffs与D矩阵对应。 114 | _, rotation_vec, translation_vec = cv2.solvePnP(object_pts, image_pts, cam_matrix, dist_coeffs) 115 | # projectPoints重新投影误差：原2d点和重投影2d点的距离（输入3d点、相机内参、相机畸变、r、t，输出重投影2d点） 116 | reprojectdst, _ = cv2.projectPoints(reprojectsrc, rotation_vec, translation_vec, cam_matrix, dist_coeffs) 117 | reprojectdst = tuple(map(tuple, reprojectdst.reshape(8, 2))) # 以8行2列显示 118 | 119 | # 计算欧拉角calc euler angle 120 | rotation_mat, _ = cv2.Rodrigues(rotation_vec) # 罗德里格斯公式（将旋转矩阵转换为旋转向量） 121 | pose_mat = cv2.hconcat((rotation_mat, translation_vec)) # 水平拼接，vconcat垂直拼接 122 | # decomposeProjectionMatrix将投影矩阵分解为旋转矩阵和相机矩阵 123 | _, _, _, _, _, _, euler_angle = cv2.decomposeProjectionMatrix(pose_mat) 124 | 125 | pitch, yaw, roll = [math.radians(_) for _ in euler_angle] 126 | 127 | pitch = math.degrees(math.asin(math.sin(pitch))) 128 | roll = -math.degrees(math.asin(math.sin(roll))) 129 | yaw = math.degrees(math.asin(math.sin(yaw))) 130 | 131 | global global_pitch, global_yaw, global_roll 132 | global_pitch, global_yaw, global_roll = pitch, yaw, roll 133 | print('pitch:{}, yaw:{}, roll:{}'.format(pitch, yaw, roll)) 134 | 135 | return reprojectdst, euler_angle # 投影误差，欧拉角 136 | 137 | 138 | 139 | def construct_dict(file_path): 140 | word_freq = {} 141 | with open(file_path, "rb") as f: 142 | for line in f: 143 | info = line.split() 144 | word = info[0] 145 | frequency = info[1] 146 | word_freq[word] = frequency 147 | 148 | return word_freq 149 | 150 | def computeTextureWeights(fin, sigma, sharpness): 151 | # print(fin) 152 | # fin = fin / 255.0 153 | 154 | dt0_v = np.diff(fin, 1, 0) # 垂直差分 155 | dt0_v = np.concatenate((dt0_v, fin[:1, :] - fin[-1:, :]), axis=0) # 第0行减去最后一行 156 | 157 | dt0_h = np.diff(fin, 1, 1) # 水平差分 158 | dt0_h = np.concatenate((dt0_h, fin[:, :1] - fin[:, -1:]), axis=1) # 第0列减去最后一列 159 | 160 | gauker_h = cv2.filter2D(dt0_h, -1, np.ones((1, sigma)), borderType=cv2.BORDER_CONSTANT) 161 | gauker_v = cv2.filter2D(dt0_v, -1, np.ones((sigma, 1)), borderType=cv2.BORDER_CONSTANT) 162 | # cv2这个filter2D（镜像翻转）与MATLAB的filter2（补0）不同 163 | 164 | W_h = 1.0 / (abs(gauker_h) * abs(dt0_h) + sharpness) 165 | W_v = 1.0 / (abs(gauker_v) * abs(dt0_v) + sharpness) 166 | 167 | return W_h, W_v 168 | 169 | 170 | def convertCol(tmp): # 按照列转成列。[[1, 2, 3], [4, 5, 6], [7, 8, 9]] # 转成[147258369].T(竖着) 171 | return np.reshape(tmp.T, (tmp.shape[0] * tmp.shape[1], 1)) 172 | 173 | 174 | def solveLinearEquation(IN, wx, wy, lambd): 175 | print('IN', IN.shape) 176 | r, c, ch = IN.shape[0], IN.shape[1], 1 177 | k = r * c 178 | dx = -lambd * convertCol(wx) # 按列转成一列 179 | dy = -lambd * convertCol(wy) 180 | tempx = np.concatenate((wx[:, -1:], wx[:, 0:-1]), 1) # 最后一列插入到第一列前面 181 | tempy = np.concatenate((wy[-1:, :], wy[0:-1, :]), 0) # 最后一行插入到第一行前面 182 | dxa = -lambd * convertCol(tempx) 183 | dya = -lambd * convertCol(tempy) 184 | tempx = np.concatenate((wx[:, -1:], np.zeros((r, c - 1))), 1) # 取wx最后一列放在第一列，其他为0 185 | tempy = np.concatenate((wy[-1:, :], np.zeros((r - 1, c))), 0) # 取wy最后一行放在第一行，其他为0 186 | dxd1 = -lambd * convertCol(tempx) 187 | dyd1 = -lambd * convertCol(tempy) 188 | wx[:, -1:] = 0 # 最后一列置为0 189 | wy[-1:, :] = 0 # 最后一行置为0 190 | dxd2 = -lambd * convertCol(wx) 191 | dyd2 = -lambd * convertCol(wy) 192 | 193 | Ax = spdiags(np.concatenate((dxd1, dxd2), 1).T, np.array([-k + r, -r]), k, k) 194 | Ay = spdiags(np.concatenate((dyd1, dyd2), 1).T, np.array([-r + 1, -1]), k, k) 195 | # spdiags，与MATLAB不同，scipy是根据行来构造sp，而matlab是根据列来构造sp 196 | 197 | D = 1 - (dx + dy + dxa + dya) 198 | A = (Ax + Ay) + (Ax + Ay).T + spdiags(D.T, np.array([0]), k, k) 199 | 200 | A = A / 1000.0 # 需修改 201 | 202 | matCol = convertCol(IN) 203 | print('spsolve开始', str(datetime.now())) 204 | OUT = spsolve(A, matCol, permc_spec="MMD_AT_PLUS_A") 205 | print('spsolve结束', str(datetime.now())) 206 | OUT = OUT / 1000 207 | OUT = np.reshape(OUT, (c, r)).T 208 | return OUT 209 | 210 | 211 | def tsmooth(I, lambd=0.5, sigma=5, sharpness=0.001): 212 | # print(I.shape) 213 | wx, wy = computeTextureWeights(I, sigma, sharpness) 214 | S = solveLinearEquation(I, wx, wy, lambd) 215 | return S 216 | 217 | 218 | def rgb2gm(I): 219 | print('I', I.shape) 220 | # I = np.maximum(I, 0.0) 221 | if I.shape[2] and I.shape[2] == 3: 222 | I = np.power(np.multiply(np.multiply(I[:, :, 0], I[:, :, 1]), I[:, :, 2]), (1.0 / 3)) 223 | return I 224 | 225 | 226 | def YisBad(Y, isBad): # 此处需要修改得更高效 227 | return Y[isBad >= 1] 228 | # Z = [] 229 | # [rows, cols] = Y.shape 230 | # for i in range(rows): 231 | # for j in range(cols): 232 | # if isBad[i, j] >= 122: 233 | # Z.append(Y[i, j]) 234 | # return np.array([Z]).T 235 | 236 | 237 | def applyK(I, k, a=-0.3293, b=1.1258): 238 | # print(type(I)) 239 | if not type(I) == 'numpy.ndarray': 240 | I = np.array(I) 241 | # print(type(I)) 242 | beta = np.exp((1 - (k ** a)) * b) 243 | gamma = (k ** a) 244 | BTF = np.power(I, gamma) * beta 245 | # try: 246 | # BTF = (I ** gamma) * beta 247 | # except: 248 | # print('gamma:', gamma, '---beta:', beta) 249 | # BTF = I 250 | return BTF 251 | 252 | 253 | def maxEntropyEnhance(I, isBad, mink=1, maxk=10): 254 | # Y = rgb2gm(np.real(np.maximum(imresize(I, (50, 50), interp='bicubic') / 255.0, 0))) 255 | Y = imresize(I, (50, 50), interp='bicubic') / 255.0 256 | Y = rgb2gm(Y) 257 | # bicubic较为接近 258 | # Y = rgb2gm(np.real(np.maximum(cv2.resize(I, (50, 50), interpolation=cv2.INTER_LANCZOS4 ), 0))) 259 | # INTER_AREA 较为接近 260 | # import matplotlib.pyplot as plt 261 | # plt.imshow(Y, cmap='gray');plt.show() 262 | 263 | print('isBad', isBad.shape) 264 | isBad = imresize(isBad.astype(int), (50, 50), interp='nearest') 265 | print('isBad', isBad.shape) 266 | 267 | # plt.imshow(isBad, cmap='gray');plt.show() 268 | 269 | # 取出isBad为真的Y的值，形成一个列向量Y 270 | # Y = YisBad(Y, isBad) # 此处需要修改得更高效 271 | Y = Y[isBad >= 1] 272 | 273 | # Y = sorted(Y) 274 | 275 | print('-entropy(Y)', -entropy(Y)) 276 | 277 | def f(k): 278 | return -entropy(applyK(Y, k)) 279 | 280 | # opt_k = mink 281 | # k = mink 282 | # minF = f(k) 283 | # while k<= maxk: 284 | # k+=0.0001 285 | # if f(k) 0.8 302 | J3 = maxEntropyEnhance(I, isBad, 0.1, 0.5) # 求k和曝光图 303 | J3 = J3 * (1 - W) # 曝光图*权重 304 | fused = I3 + J3 # 增强图 305 | return I 306 | 307 | 308 | def oneHDR(I, mu=0.5, a=-0.3293, b=1.1258): 309 | # mu照度图T的指数，数值越大，增强程度越大 310 | I = I / 255.0 311 | t_b = I[:, :, 0] # t_b表示三通道图转成灰度图（灰度值为RGB中的最大值）,亮度矩阵L 312 | for i in range(I.shape[2] - 1): # 防止输入图片非三通道 313 | t_b = np.maximum(t_b, I[:, :, i + 1]) 314 | # t_b2 = cv2.resize(t_b, (0, 0), fx=0.5, fy=0.5) 315 | print('t_b', t_b.shape) 316 | # t_b2 = misc.imresize(t_b, (ceil(t_b.shape[0] / 2), ceil(t_b.shape[1] / 2)),interp='bicubic') 317 | # print('t_b2', t_b2.shape) 318 | # t_b2 = t_b / 255.0 319 | 320 | t_b2 = imresize(t_b, (256, 256), interp='bicubic', mode='F') # / 255 321 | t_our = tsmooth(t_b2) # 求解照度图T（灰度图） 322 | print('t_our前', t_our.shape) 323 | t_our = imresize(t_our, t_b.shape, interp='bicubic', mode='F') # / 255 324 | print('t_our后', t_our.shape) 325 | 326 | # W: Weight Matrix 与 I2 327 | # 照度图L（灰度图） -> 照度图L（RGB图）：灰度值重复3次赋给RGB 328 | # size为(I, 3) ， 防止与原图尺寸有偏差 329 | t = np.ndarray(I.shape) 330 | for ii in range(I.shape[2]): 331 | t[:, :, ii] = t_our 332 | print('t', t.shape) 333 | 334 | W = t ** mu # 原图的权重。三维矩阵 335 | 336 | cv2.imwrite(filepath + 'W.jpg', W * 255) 337 | cv2.imwrite(filepath + '1-W.jpg', (1 - W) * 255) 338 | cv2.imwrite(filepath + 't.jpg', t * 255) 339 | cv2.imwrite(filepath + '1-t.jpg', (1 - t) * 255) 340 | 341 | print('W', W.shape) 342 | # 变暗 343 | # isBad = t_our > 0.8 # 是高光照的像素点 344 | # I = maxEntropyEnhance(I, isBad) # 求k和曝光图 345 | # 变暗 346 | I2 = I * W # 原图*权重 347 | 348 | # 曝光率->k ->J 349 | isBad = t_our < 0.5 # 是低光照的像素点 350 | J = maxEntropyEnhance(I, isBad) # 求k和曝光图 351 | J2 = J * (1 - W) # 曝光图*权重 352 | fused = I2 + J2 # 增强图 353 | 354 | # 存储中间结果 355 | cv2.imwrite(filepath + 'I2.jpg', I2 * 255.0) 356 | cv2.imwrite(filepath + 'J2.jpg', J2 * 255.0) 357 | 358 | # 变暗 359 | # fused = HDR2dark(fused, t_our, W) 360 | 361 | return fused 362 | # return res 363 | 364 | # 图像增强，效果不错，人脸光照充足情况下没必要使用 365 | def test(): 366 | inputImg = cv2.imread(filepath + 'input.jpg') 367 | outputImg = oneHDR(inputImg) 368 | # outputImg = outputImg * 255.0 369 | cv2.imwrite(filepath + 'out.jpg', outputImg * 255) 370 | print("HDR完成，已保存到本地") 371 | 372 | print('程序结束', str(datetime.now())) 373 | 374 | cv2.imshow('inputImg', inputImg) 375 | cv2.imshow('outputImg', outputImg) 376 | # print(inputImg.dtype,outputImg.dtype) 377 | # outputImg = outputImg.astype(int) 378 | # print(inputImg.dtype, outputImg.dtype) 379 | # compare = np.concatenate((inputImg,outputImg),axis=1) 380 | # cv2.imshow('compare', compare) 381 | cv2.waitKey(0) 382 | cv2.destroyAllWindows() 383 | 384 | def load_cn_words_dict(file_path): 385 | cn_words_dict = "" 386 | with open(file_path, "rb") as f: 387 | for word in f: 388 | cn_words_dict += word.strip().decode("utf-8") 389 | return cn_words_dict 390 | 391 | 392 | def edits1(phrase, cn_words_dict): 393 | splits = [(phrase[:i], phrase[i:]) for i in range(len(phrase) + 1)] 394 | deletes = [L + R[1:] for L, R in splits if R] 395 | transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1] 396 | replaces = [L + c + R[1:] for L, R in splits if R for c in cn_words_dict] 397 | inserts = [L + c + R for L, R in splits for c in cn_words_dict] 398 | return set(deletes + transposes + replaces + inserts) 399 | 400 | 401 | def known(phrases): return set(phrase for phrase in phrases if phrase.encode("utf-8") in phrase_freq) 402 | 403 | 404 | def get_candidates(error_phrase): 405 | candidates_1st_order = [] 406 | candidates_2nd_order = [] 407 | candidates_3nd_order = [] 408 | 409 | p = Pinyin() 410 | error_pinyin = p.get_pinyin(error_phrase) 411 | re.sub("-", "/", error_pinyin) 412 | cn_words_dict = load_cn_words_dict("HW10/Autochecker4Chinese-master/cn_dict.txt") 413 | candidate_phrases = list(known(edits1(error_phrase, cn_words_dict))) 414 | 415 | for candidate_phrase in candidate_phrases: 416 | # candidate_pinyin = pinyin.get(candidate_phrase, format="strip", delimiter="/").encode("utf-8") 417 | candidate_pinyin = p.get_pinyin(candidate_phrase) 418 | re.sub("-", "/", candidate_pinyin) 419 | if candidate_pinyin == error_pinyin: 420 | candidates_1st_order.append(candidate_phrase) 421 | elif candidate_pinyin.split("/")[0] == error_pinyin.split("/")[0]: 422 | candidates_2nd_order.append(candidate_phrase) 423 | else: 424 | candidates_3nd_order.append(candidate_phrase) 425 | 426 | return candidates_1st_order, candidates_2nd_order, candidates_3nd_order 427 | 428 | 429 | def find_max(c1_order): 430 | maxo = '' 431 | maxi = 0 432 | for i in range(0, len(c1_order)): 433 | if c1_order[i].encode("utf-8") in phrase_freq: 434 | freq = int(phrase_freq.get(c1_order[i].encode('utf-8'))) 435 | if freq > maxi: 436 | maxi = freq 437 | maxo = c1_order[i] 438 | return maxo 439 | 440 | 441 | def auto_correct(error_phrase): 442 | c1_order, c2_order, c3_order = get_candidates(error_phrase) 443 | if c1_order: 444 | return find_max(c1_order) 445 | elif c2_order: 446 | return find_max(c2_order) 447 | else: 448 | return find_max(c3_order) 449 | 450 | 451 | def auto_correct_sentence(error_sentence, verbose=True): 452 | jieba_cut = jieba.cut(error_sentence, cut_all=False) 453 | seg_list = "\t".join(jieba_cut).split("\t") 454 | 455 | correct_sentence = "" 456 | 457 | for phrase in seg_list: 458 | 459 | correct_phrase = phrase 460 | # check if item is a punctuation 461 | if phrase not in PUNCTUATION_LIST: 462 | # check if the phrase in our dict, if not then it is a misspelled phrase 463 | if phrase.encode('utf-8') not in phrase_freq.keys(): 464 | correct_phrase = auto_correct(phrase) 465 | if verbose: 466 | print(phrase, correct_phrase) 467 | 468 | correct_sentence += correct_phrase 469 | 470 | return correct_sentence 471 | 472 | 473 | 474 | def test_case(err_sent_1): 475 | print('===============') 476 | correct_sent = auto_correct_sentence(err_sent_1) 477 | t1 = "original sentence:" + err_sent_1 + "\n==>\n" + "corrected sentence:" + correct_sent 478 | print(t1) 479 | 480 | 481 | def create_inverted_index(fenci_txt, param): 482 | pass 483 | 484 | 485 | def charge_spimi(): 486 | fenci_txt = "fenci.txt" 487 | with codecs.open(fenci_txt, 'w', encoding="UTF-8-SIG") as f: 488 | path = "data\\page" 489 | files = os.listdir(path) 490 | s = [] 491 | for file in files: 492 | if not os.path.isdir(file): 493 | print('>>>处理', file) 494 | for line in open(path + "\\" + file, encoding='utf-8', errors='ignore').readlines(): 495 | # 去标点 496 | line = re.sub(r"[0-9\s+\.\!\/_,$%^*()?;；:-【】+\"\']+|[+——！，;:。？、~@#￥%……&*（）]+", " ", line) 497 | # 分词 498 | seg_list = jieba.cut(line, cut_all=True) 499 | # 写入199801_new.txt 500 | f.write(" ".join(seg_list) + "\n") 501 | # 建立倒排索引 502 | create_inverted_index(fenci_txt, re.sub(r'\D', "", file)) 503 | 504 | with codecs.open("my_index.txt", 'w', encoding="UTF-8-SIG") as i: 505 | for key in index.keys(): 506 | i.write(key + str(index[key]) + "\n") 507 | 508 | 509 | 510 | 511 | def eye_aspect_ratio(eye): 512 | # 垂直眼标志（X，Y）坐标 513 | A = dist.euclidean(eye[1], eye[5]) 514 | B = dist.euclidean(eye[2], eye[4]) 515 | # 计算水平之间的欧几里得距离 516 | C = dist.euclidean(eye[0], eye[3]) 517 | ear = (A + B) / (2.0 * C) 518 | return ear 519 | 520 | 521 | def mouth_aspect_ratio(mouth): # 嘴部 522 | A = np.linalg.norm(mouth[2] - mouth[9]) # 51, 59 523 | B = np.linalg.norm(mouth[4] - mouth[7]) # 53, 57 524 | C = np.linalg.norm(mouth[0] - mouth[6]) # 49, 55 525 | mar = (A + B) / (2.0 * C) 526 | return mar 527 | 528 | 529 | def dete_tired(frame): 530 | frame = imutils.resize(frame, width=660) 531 | gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 532 | rects = detector(gray, 0) 533 | 534 | mar = '' 535 | ear = '' 536 | global COUNTER, TOTAL, mCOUNTER, mTOTAL, hCOUNTER, hTOTAL 537 | 538 | # 循环脸部位置信息，使用predictor(gray, rect)获得脸部特征位置的信息 539 | for rect in rects: 540 | shape = predictor(gray, rect) 541 | 542 | # 将脸部特征信息转换为数组array的格式 543 | shape = face_utils.shape_to_np(shape) 544 | 545 | # 提取左眼和右眼坐标 546 | leftEye = shape[lStart:lEnd] 547 | rightEye = shape[rStart:rEnd] 548 | # 嘴巴坐标 549 | mouth = shape[mStart:mEnd] 550 | 551 | # 构造函数计算左右眼的EAR值，使用平均值作为最终的EAR 552 | leftEAR = eye_aspect_ratio(leftEye) 553 | rightEAR = eye_aspect_ratio(rightEye) 554 | ear = (leftEAR + rightEAR) / 2.0 555 | mar = mouth_aspect_ratio(mouth) 556 | 557 | leftEyeHull = cv2.convexHull(leftEye) 558 | rightEyeHull = cv2.convexHull(rightEye) 559 | cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1) 560 | cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1) 561 | mouthHull = cv2.convexHull(mouth) 562 | cv2.drawContours(frame, [mouthHull], -1, (0, 255, 0), 1) 563 | 564 | left = rect.left() 565 | top = rect.top() 566 | right = rect.right() 567 | bottom = rect.bottom() 568 | cv2.rectangle(frame, (left, top), (right, bottom), (0, 255, 0), 1) 569 | 570 | ''' 571 | 分别计算左眼和右眼的评分求平均作为最终的评分，如果小于阈值，则加1，如果连续3次都小于阈值，则表示进行了一次眨眼活动 572 | ''' 573 | # 循环，满足条件的，眨眼次数+1 574 | if ear < EYE_AR_THRESH: # 眼睛长宽比：0.2 575 | COUNTER += 1 576 | 577 | else: 578 | # 如果连续3次都小于阈值，则表示进行了一次眨眼活动 579 | if COUNTER >= EYE_AR_CONSEC_FRAMES: # 阈值：3 580 | TOTAL += 1 581 | # 重置眼帧计数器 582 | COUNTER = 0 583 | 584 | global EAR, MAR 585 | EAR = "{:.2f}".format(ear) 586 | MAR = "{:.2f}".format(mar) 587 | 588 | # 进行画图操作，同时使用cv2.putText将眨眼次数进行显示 589 | cv2.putText(frame, "Faces: {}".format(len(rects)), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 590 | cv2.putText(frame, "COUNTER: {}".format(COUNTER), (150, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 591 | cv2.putText(frame, "EAR: {:.2f}".format(ear), (300, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 592 | 593 | ''' 594 | 计算张嘴评分，如果小于阈值，则加1，如果连续3次都小于阈值，则表示打了一次哈欠，同一次哈欠大约在3帧 595 | ''' 596 | if mar > MAR_THRESH: # 张嘴阈值0.5 597 | mCOUNTER += 1 598 | cv2.putText(frame, "Yawning!", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 599 | else: 600 | # 如果连续3次都小于阈值，则表示打了一次哈欠 601 | if mCOUNTER >= MOUTH_AR_CONSEC_FRAMES: # 阈值：3 602 | mTOTAL += 1 603 | # 重置嘴帧计数器 604 | mCOUNTER = 0 605 | cv2.putText(frame, "COUNTER: {}".format(mCOUNTER), (150, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 606 | cv2.putText(frame, "MAR: {:.2f}".format(mar), (300, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) 607 | 608 | """ 609 | 瞌睡点头 610 | """ 611 | # 获取头部姿态 612 | reprojectdst, euler_angle = get_head_pose(shape) 613 | 614 | har = euler_angle[0, 0] # 取pitch旋转角度 615 | if har > HAR_THRESH: # 点头阈值0.3 616 | hCOUNTER += 1 617 | else: 618 | # 如果连续3次都小于阈值，则表示瞌睡点头一次 619 | if hCOUNTER >= NOD_AR_CONSEC_FRAMES: # 阈值：3 620 | hTOTAL += 1 621 | # 重置点头帧计数器 622 | hCOUNTER = 0 623 | 624 | # 绘制正方体12轴 625 | # for start, end in line_pairs: 626 | # cv2.line(frame, reprojectdst[start], reprojectdst[end], (0, 0, 255)) 627 | # 显示角度结果 628 | cv2.putText(frame, "X: " + "{:7.2f}".format(euler_angle[0, 0]), (10, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 629 | (0, 255, 0), thickness=2) # GREEN 630 | cv2.putText(frame, "Y: " + "{:7.2f}".format(euler_angle[1, 0]), (150, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 631 | (255, 0, 0), thickness=2) # BLUE 632 | cv2.putText(frame, "Z: " + "{:7.2f}".format(euler_angle[2, 0]), (300, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.75, 633 | (0, 0, 255), thickness=2) # RED 634 | 635 | for (x, y) in shape: 636 | cv2.circle(frame, (x, y), 1, (0, 0, 255), -1) 637 | 638 | # print('嘴巴实时长宽比:{:.2f} '.format(mar) + "\t是否张嘴：" + str([False, True][mar > MAR_THRESH])) 639 | # print('眼睛实时长宽比:{:.2f} '.format(ear) + "\t是否眨眼：" + str([False, True][COUNTER >= 1])) 640 | 641 | # 确定疲劳提示:眨眼50次，打哈欠15次，瞌睡点头15次 642 | 643 | #if TOTAL >= 20 or mTOTAL >= 10 or hTOTAL >= 15: 644 | if TOTAL >= 4 or mTOTAL >= 4 or hTOTAL >= 4: 645 | cv2.putText(frame, "SLEEP!!!", (100, 200), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 3) 646 | mixer.init() 647 | mixer.music.load('d:/pljs.mp3') 648 | mixer.music.play(5, 0.0) 649 | time.sleep(2) 650 | mixer.music.stop() 651 | return frame 652 | 653 | 654 | def take_snapshot(): 655 | # video_loop() 656 | global TOTAL, mTOTAL, hTOTAL 657 | TOTAL = 0 658 | mTOTAL = 0 659 | hTOTAL = 0 660 | 661 | def video_loop(): 662 | success, img = camera.read() # 从摄像头读取照片 663 | if success: 664 | detect_result = dete_tired(img) 665 | cv2image = cv2.cvtColor(detect_result, cv2.COLOR_BGR2RGBA)#转换颜色从BGR到RGBA 666 | current_image = Image.fromarray(cv2image)#将图像转换成Image对象 667 | imgtk = ImageTk.PhotoImage(image=current_image) 668 | panel.imgtk = imgtk 669 | panel.config(image=imgtk) 670 | 671 | ####################################updata######################## 672 | global hCOUNTER, mCOUNTER, TOTAL, mTOTAL, hTOTAL 673 | global global_pitch, global_yaw, global_roll 674 | global EAR, MAR 675 | 676 | root.update() # 不断更新 677 | root.after(10) 678 | Label(root, text='打盹时间:'+str(hCOUNTER), font=("黑体", 14), fg="red", width=12, height=2).place(x=10, y=570, 679 | anchor='nw') 680 | Label(root, text='打哈欠时间:' + str(mCOUNTER), font=("黑体", 14), fg="red", width=12, height=2).place(x=140, y=570, 681 | anchor='nw') 682 | 683 | Label(root, text='眨眼次数:' + str(TOTAL), font=("黑体", 14), fg="red", width=12, height=2).place(x=270, y=570, 684 | anchor='nw') 685 | Label(root, text='哈欠次数:' + str(mTOTAL), font=("黑体", 14), fg="red", width=12, height=2).place(x=400, y=570, 686 | anchor='nw') 687 | Label(root, text='钓鱼次数:' + str(hTOTAL), font=("黑体", 14), fg="red", width=12, height=2).place(x=530, y=570, 688 | anchor='nw') 689 | 690 | Label(root, text='嘴部面积 : ' + str(MAR), font=("黑体", 14), fg="red", width=20, height=2).place(x=130, y=610, anchor='nw') 691 | Label(root, text='眼部面积 : ' + str(EAR), font=("黑体", 14), fg="red", width=20, height=2).place(x=320, y=610, anchor='nw') 692 | # Label(root, text='头部yaw:' + str(global_yaw), font=("黑体", 14), fg="red", width=12, height=2).place(x=140, y=600, anchor='nw') 693 | # Label(root, text='头部横滚角:' + str(global_roll), font=("黑体", 14), fg="red", width=12, height=2).place(x=270, y=600, anchor='nw') 694 | 695 | root.after(1, video_loop) 696 | 697 | 698 | 699 | if __name__=='__main__': 700 | camera = cv2.VideoCapture(0) # 摄像头 701 | 702 | root = Tk() 703 | root.title("opencv + tkinter") 704 | # root.protocol('WM_DELETE_WINDOW', detector) 705 | 706 | panel = Label(root) # initialize image panel 707 | panel.pack(padx=10, pady=10) 708 | root.config(cursor="arrow") 709 | 710 | btn = Button(root, text="疲劳提醒解锁!", command=take_snapshot) 711 | btn.pack(fill="both", expand=True, padx=10, pady=10) 712 | 713 | # strs = '测试文字' 714 | Label(root, text=' ', font=("黑体", 14), fg="red", width=12, height=2).pack(fill="both", expand=True, padx=10, pady=20) 715 | 716 | 717 | 718 | video_loop() 719 | 720 | root.mainloop() 721 | # 当一切都完成后，关闭摄像头并释放所占资源 722 | camera.release() 723 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /系统说明.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangjianlin1985/Python_FatigueDrivingDetection/654f1ecd4c168976b7ec2ed3f0a7e7c24b017ecf/系统说明.txt -------------------------------------------------------------------------------- /运行说明.txt: -------------------------------------------------------------------------------- 1 | 2 | ### 配置环境 3 | python 3.6.8 4 | opencv-python == 4.4.0.46 5 | numpy == 1.19.4 6 | pillow == 8.0.1 7 | scikit-image==0.15.0 8 | pandas==1.1.4 9 | 10 | dlib == 19.7.0 (dlib无法使用pip安装，见dlib安装说明) 11 | 12 | 13 | ### 代码运行 14 | 15 | tkinter_UI.py tkinter界面，直接运行即可 16 | 17 | 18 | 19 | 20 | --------------------------------------------------------------------------------