├── .gitignore ├── 20200407_173126.mp4 ├── EAMNet.py ├── LICENSE ├── Readme.md ├── SimpleVGGNet.py ├── Train.py ├── __init__.py ├── best0428ep150.h5 ├── layer_factory.py ├── mtcnn.py ├── network.py └── run.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /20200407_173126.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aristochi/MTCNN_CNN_DangerDrivingDetection/750659b7fa368e29c8da367f5ba14f50d36b40eb/20200407_173126.mp4 -------------------------------------------------------------------------------- /EAMNet.py: -------------------------------------------------------------------------------- 1 | # 导入所需模块 2 | from keras.models import Sequential 3 | from keras.layers.normalization import BatchNormalization 4 | from keras.layers.convolutional import Conv2D 5 | from keras.layers.convolutional import MaxPooling2D 6 | from keras.layers.convolutional import AveragePooling2D 7 | from keras.initializers import TruncatedNormal 8 | from keras.layers.core import Activation 9 | from keras.layers.core import Flatten 10 | from keras.layers.core import Dropout 11 | from keras.layers.core import Dense 12 | from keras import backend as K 13 | 14 | class EAMNET: 15 | @staticmethod 16 | def build(width, height, depth, classes): 17 | model = Sequential() 18 | inputShape = (height, width, depth) 19 | chanDim = -1 20 | 21 | if K.image_data_format() == "channels_first": 22 | inputShape = (depth, height, width) 23 | chanDim = 1 24 | 25 | # CONV => RELU => POOL 26 | model.add(Conv2D(32, (5, 5), padding="same", 27 | input_shape=inputShape,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 28 | model.add(Activation("relu")) 29 | model.add(BatchNormalization(axis=chanDim)) 30 | model.add(MaxPooling2D(pool_size=(2, 2))) 31 | model.add(Dropout(0.25)) 32 | 33 | # (CONV => RELU) * 2 => POOL 34 | model.add(Conv2D(32, (5, 5), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 35 | model.add(Activation("relu")) 36 | model.add(BatchNormalization(axis=chanDim)) 37 | model.add(AveragePooling2D(pool_size=(2, 2))) 38 | model.add(Dropout(0.25)) 39 | 40 | # (CONV => RELU) * 3 => POOL 41 | model.add(Conv2D(64, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 42 | model.add(Activation("relu")) 43 | model.add(BatchNormalization(axis=chanDim)) 44 | model.add(AveragePooling2D(pool_size=(2, 2))) 45 | model.add(Dropout(0.25)) 46 | 47 | # FC层 48 | model.add(Flatten()) 49 | model.add(Dense(64,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 50 | model.add(Activation("relu")) 51 | model.add(BatchNormalization()) 52 | model.add(Dropout(0.6)) 53 | 54 | # softmax 分类 55 | model.add(Dense(classes,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 56 | model.add(Activation("softmax")) 57 | 58 | return model 59 | model=EAMNET.build(32,32,1,6) 60 | model.summary() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 本项目是用于判断是否闭眼或者张开嘴哈欠和吸烟打电话等手势行为， 2 | 功能涵盖7类情绪识别，眨眼判断，哈欠判断，吸烟，打电话等， 3 | 达到危险驾驶检测的功能。 4 | 项目描述可以看https://blog.csdn.net/Aristochi/article/details/105655810 5 | 有问题可以私信 6 | ``` 7 | The project was designed to determine gestures such as closing eyes or yawning and smoking or talking on the phone. 8 | The functions cover 7 categories of emotion recognition, blinking judgment, yawning judgment, smoking, telephone, etc. to achieve the function of dangerous driving detection. 9 | ``` 10 | 11 | 数据集：闭眼、睁眼、闭嘴、哈欠、打电话、吸烟数据集 12 | 口罩数据集 13 | fer2013表情数据集 14 | 15 | Data set: closed eyes, open eyes, yawn, telephone, smoking data set 16 | Mask data set 17 | Fer2013 Emoji Data Set 18 | ``` 19 | 眨眼数据集http://parnec.nuaa.edu.cn/xtan/data/ClosedEyeDatabases.html 20 | 数据集 21 | 链接：https://pan.baidu.com/s/1FLfWZix-vsY5XSaV1oi-pA 22 | 提取码：pmh0 23 | ``` 24 | Done: 25 | 眨眼判断 26 | 哈欠判断 27 | 打电话识别 28 | 吸烟识别 29 | 预测准确率：93.46% 30 | 31 | Done: 32 | Blink of an eye to judge 33 | Yawn judgment 34 | Telephone identification 35 | Smoking is to identify 36 | Prediction accuracy: 93.46% 37 | ``` 38 | TODO： 39 | PERCLOS疲劳指标判断 40 | 将项目移植到树莓派 41 | TODO: 42 | PERCLOS fatigue index judgment 43 | Port the project to raspberry pie 44 | Thanks for https://github.com/ipazc/mtcnn/tree/master/mtcnn MTCNN module 45 | 感谢https://github.com/ipazc/mtcnn/tree/master/mtcnn的MTCNN模块 46 | ``` 47 | 如果有问题可以发邮箱aristochi@qq.com。 48 | -------------------------------------------------------------------------------- /SimpleVGGNet.py: -------------------------------------------------------------------------------- 1 | # 导入所需模块 2 | from keras.models import Sequential 3 | from keras.layers.normalization import BatchNormalization 4 | from keras.layers.convolutional import Conv2D 5 | from keras.layers.convolutional import MaxPooling2D 6 | from keras.initializers import TruncatedNormal 7 | from keras.layers.core import Activation 8 | from keras.layers.core import Flatten 9 | from keras.layers.core import Dropout 10 | from keras.layers.core import Dense 11 | from keras import backend as K 12 | 13 | class SimpleVGG: 14 | @staticmethod 15 | def build(width, height, depth, classes): 16 | model = Sequential() 17 | inputShape = (height, width, depth) 18 | chanDim = -1 19 | 20 | if K.image_data_format() == "channels_first": 21 | inputShape = (depth, height, width) 22 | chanDim = 1 23 | 24 | # CONV => RELU => POOL 25 | model.add(Conv2D(32, (3, 3), padding="same", 26 | input_shape=inputShape,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 27 | model.add(Activation("relu")) 28 | model.add(BatchNormalization(axis=chanDim)) 29 | model.add(MaxPooling2D(pool_size=(2, 2))) 30 | model.add(Dropout(0.25)) 31 | 32 | # (CONV => RELU) * 2 => POOL 33 | model.add(Conv2D(64, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 34 | model.add(Activation("relu")) 35 | model.add(BatchNormalization(axis=chanDim)) 36 | model.add(Conv2D(64, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 37 | model.add(Activation("relu")) 38 | model.add(BatchNormalization(axis=chanDim)) 39 | model.add(MaxPooling2D(pool_size=(2, 2))) 40 | model.add(Dropout(0.25)) 41 | 42 | # (CONV => RELU) * 3 => POOL 43 | model.add(Conv2D(128, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 44 | model.add(Activation("relu")) 45 | model.add(BatchNormalization(axis=chanDim)) 46 | model.add(Conv2D(128, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 47 | model.add(Activation("relu")) 48 | model.add(BatchNormalization(axis=chanDim)) 49 | model.add(Conv2D(128, (3, 3), padding="same",kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 50 | model.add(Activation("relu")) 51 | model.add(BatchNormalization(axis=chanDim)) 52 | model.add(MaxPooling2D(pool_size=(2, 2))) 53 | model.add(Dropout(0.25)) 54 | 55 | # FC层 56 | model.add(Flatten()) 57 | model.add(Dense(512,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 58 | model.add(Activation("relu")) 59 | model.add(BatchNormalization()) 60 | model.add(Dropout(0.6)) 61 | 62 | # softmax 分类 63 | model.add(Dense(classes,kernel_initializer=TruncatedNormal(mean=0.0, stddev=0.01))) 64 | model.add(Activation("softmax")) 65 | 66 | return model -------------------------------------------------------------------------------- /Train.py: -------------------------------------------------------------------------------- 1 | # set the matplotlib backend so figures can be saved in the background 2 | import keras 3 | import matplotlib 4 | from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint 5 | from keras.engine.saving import load_model 6 | from keras.utils import to_categorical 7 | from sklearn.metrics import classification_report 8 | 9 | from SimpleVGGNet import SimpleVGG 10 | from EAMNet import EAMNET 11 | 12 | matplotlib.use("Agg") 13 | 14 | # import the necessary packages 15 | from keras.preprocessing.image import ImageDataGenerator 16 | from keras.optimizers import Adam 17 | from keras.preprocessing.image import img_to_array 18 | from sklearn.preprocessing import LabelBinarizer 19 | from sklearn.model_selection import train_test_split 20 | import matplotlib.pyplot as plt 21 | from imutils import paths 22 | import numpy as np 23 | import argparse 24 | import random 25 | import pickle 26 | import cv2 27 | import os 28 | 29 | dataset="./dataset/" 30 | EPOCHS =100 31 | INIT_LR = 0.01 32 | BS = 64 33 | IMAGE_DIMS = (64, 64, 1) 34 | classnum=2 35 | # initialize the data and labels 36 | data = [] 37 | labels = [] 38 | 39 | # grab the image paths and randomly shuffle them 40 | print("[INFO] loading images...") 41 | imagePaths = sorted(list(paths.list_images(dataset))) 42 | # print(imagePaths) 43 | random.seed(10010) 44 | random.shuffle(imagePaths) 45 | 46 | # loop over the input images 47 | for imagePath in imagePaths: 48 | # load the image, pre-process it, and store it in the data list 49 | image = cv2.imread(imagePath,cv2.IMREAD_GRAYSCALE) 50 | print(imagePath) 51 | image = cv2.resize(image, (IMAGE_DIMS[1], IMAGE_DIMS[0])) 52 | image = img_to_array(image) 53 | data.append(image) 54 | 55 | # extract the class label from the image path and update the 56 | # labels list 57 | label = imagePath.split(os.path.sep)[-2] 58 | labels.append(label) 59 | # scale the raw pixel intensities to the range [0, 1] 60 | print(labels) 61 | data = np.array(data, dtype="float") / 255.0 62 | labels = np.array(labels) 63 | print("[INFO] data matrix: {:.2f}MB".format( 64 | data.nbytes / (1024 * 1000.0))) 65 | 66 | # 数据集切分 67 | (trainX, testX, trainY, testY) = train_test_split(data,labels, test_size=0.25, random_state=42) 68 | 69 | # 转换标签为one-hot encoding格式 70 | lb = LabelBinarizer() 71 | print(lb) 72 | trainY = lb.fit_transform(trainY) 73 | testY = lb.fit_transform(testY) 74 | 75 | #trainY = to_categorical(trainY) 76 | #testY = to_categorical(testY) 77 | # construct the image generator for data augmentation 78 | aug = ImageDataGenerator(rotation_range=25, width_shift_range=0.1, 79 | height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, 80 | horizontal_flip=True, fill_mode="nearest") 81 | # initialize the model 82 | print("[INFO] compiling model...") 83 | model = SimpleVGG.build(width=IMAGE_DIMS[1], height=IMAGE_DIMS[0], 84 | depth=IMAGE_DIMS[2], classes=classnum) 85 | # model=load_model('./model/best0428ep150.h5') 86 | opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS) 87 | model.summary() 88 | model.compile(loss="categorical_crossentropy", optimizer=opt, 89 | metrics=["accuracy"]) 90 | reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=1) 91 | early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1) 92 | # train the network 93 | filepath="./model/best0428ep150.h5" 94 | checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True,mode='max') 95 | 96 | H = model.fit_generator( 97 | aug.flow(trainX, trainY, batch_size=BS), 98 | validation_data=(testX, testY), 99 | steps_per_epoch=len(trainX) // BS, 100 | callbacks=[reduce_lr,checkpoint], 101 | epochs=EPOCHS, verbose=1) 102 | # save the model to disk 103 | model.save('./model/best0428ep150.h5') 104 | # plot the training loss and accuracy 105 | # 测试 106 | print("------测试网络------") 107 | predictions = model.predict(testX, batch_size=32) 108 | print(classification_report(testY.argmax(axis=1), 109 | predictions.argmax(axis=1), target_names=lb.classes_)) 110 | 111 | plt.style.use("ggplot") 112 | plt.figure() 113 | N = EPOCHS 114 | plt.plot(np.arange(0, N), H.history["loss"], label="train_loss") 115 | plt.plot(np.arange(0, N), H.history["val_loss"], label="val_loss") 116 | plt.plot(np.arange(0, N), H.history["acc"], label="train_acc") 117 | plt.plot(np.arange(0, N), H.history["val_acc"], label="val_acc") 118 | plt.title("Training Loss and Accuracy") 119 | plt.xlabel("Epoch #") 120 | plt.ylabel("Loss/Accuracy") 121 | plt.legend(loc="upper left") 122 | plt.savefig("./model/best0428ep150.png") 123 | f = open("./model/best0428ep150.pickle", "wb") 124 | f.write(pickle.dumps(lb)) 125 | f.close() 126 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # MIT License 5 | # 6 | # Copyright (c) 2019 Iván de Paz Centeno 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in all 16 | # copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | # SOFTWARE. 25 | 26 | from mtcnn.mtcnn import MTCNN 27 | 28 | 29 | __author__ = "Iván de Paz Centeno" 30 | __version__= "0.1.0" 31 | -------------------------------------------------------------------------------- /best0428ep150.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aristochi/MTCNN_CNN_DangerDrivingDetection/750659b7fa368e29c8da367f5ba14f50d36b40eb/best0428ep150.h5 -------------------------------------------------------------------------------- /layer_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | #MIT License 5 | # 6 | #Copyright (c) 2018 Iván de Paz Centeno 7 | # 8 | #Permission is hereby granted, free of charge, to any person obtaining a copy 9 | #of this software and associated documentation files (the "Software"), to deal 10 | #in the Software without restriction, including without limitation the rights 11 | #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | #copies of the Software, and to permit persons to whom the Software is 13 | #furnished to do so, subject to the following conditions: 14 | # 15 | #The above copyright notice and this permission notice shall be included in all 16 | #copies or substantial portions of the Software. 17 | # 18 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | #SOFTWARE. 25 | 26 | import tensorflow as tf 27 | from distutils.version import LooseVersion 28 | 29 | __author__ = "Iván de Paz Centeno" 30 | 31 | 32 | class LayerFactory(object): 33 | """ 34 | Allows to create stack layers for a given network. 35 | """ 36 | 37 | AVAILABLE_PADDINGS = ('SAME', 'VALID') 38 | 39 | def __init__(self, network): 40 | self.__network = network 41 | 42 | @staticmethod 43 | def __validate_padding(padding): 44 | if padding not in LayerFactory.AVAILABLE_PADDINGS: 45 | raise Exception("Padding {} not valid".format(padding)) 46 | 47 | @staticmethod 48 | def __validate_grouping(channels_input: int, channels_output: int, group: int): 49 | if channels_input % group != 0: 50 | raise Exception("The number of channels in the input does not match the group") 51 | 52 | if channels_output % group != 0: 53 | raise Exception("The number of channels in the output does not match the group") 54 | 55 | @staticmethod 56 | def vectorize_input(input_layer): 57 | input_shape = input_layer.get_shape() 58 | 59 | if input_shape.ndims == 4: 60 | # Spatial input, must be vectorized. 61 | dim = 1 62 | for x in input_shape[1:].as_list(): 63 | dim *= int(x) 64 | 65 | #dim = operator.mul(*(input_shape[1:].as_list())) 66 | vectorized_input = tf.reshape(input_layer, [-1, dim]) 67 | else: 68 | vectorized_input, dim = (input_layer, input_shape[-1]) 69 | 70 | return vectorized_input, dim 71 | 72 | def __make_var(self, name: str, shape: list): 73 | """ 74 | Creates a tensorflow variable with the given name and shape. 75 | :param name: name to set for the variable. 76 | :param shape: list defining the shape of the variable. 77 | :return: created TF variable. 78 | """ 79 | return tf.compat.v1.get_variable(name, shape, trainable=self.__network.is_trainable(), 80 | use_resource=False) 81 | 82 | def new_feed(self, name: str, layer_shape: tuple): 83 | """ 84 | Creates a feed layer. This is usually the first layer in the network. 85 | :param name: name of the layer 86 | :return: 87 | """ 88 | 89 | feed_data = tf.compat.v1.placeholder(tf.float32, layer_shape, 'input') 90 | self.__network.add_layer(name, layer_output=feed_data) 91 | 92 | def new_conv(self, name: str, kernel_size: tuple, channels_output: int, 93 | stride_size: tuple, padding: str='SAME', 94 | group: int=1, biased: bool=True, relu: bool=True, input_layer_name: str=None): 95 | """ 96 | Creates a convolution layer for the network. 97 | :param name: name for the layer 98 | :param kernel_size: tuple containing the size of the kernel (Width, Height) 99 | :param channels_output: ¿? Perhaps number of channels in the output? it is used as the bias size. 100 | :param stride_size: tuple containing the size of the stride (Width, Height) 101 | :param padding: Type of padding. Available values are: ('SAME', 'VALID') 102 | :param group: groups for the kernel operation. More info required. 103 | :param biased: boolean flag to set if biased or not. 104 | :param relu: boolean flag to set if ReLu should be applied at the end of the layer or not. 105 | :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of 106 | the network. 107 | """ 108 | 109 | # Verify that the padding is acceptable 110 | self.__validate_padding(padding) 111 | 112 | input_layer = self.__network.get_layer(input_layer_name) 113 | 114 | # Get the number of channels in the input 115 | channels_input = int(input_layer.get_shape()[-1]) 116 | 117 | # Verify that the grouping parameter is valid 118 | self.__validate_grouping(channels_input, channels_output, group) 119 | 120 | # Convolution for a given input and kernel 121 | convolve = lambda input_val, kernel: tf.nn.conv2d(input=input_val, 122 | filters=kernel, 123 | strides=[1, stride_size[1], stride_size[0], 1], 124 | padding=padding) 125 | 126 | with tf.compat.v1.variable_scope(name) as scope: 127 | kernel = self.__make_var('weights', shape=[kernel_size[1], kernel_size[0], channels_input // group, channels_output]) 128 | 129 | output = convolve(input_layer, kernel) 130 | 131 | # Add the biases, if required 132 | if biased: 133 | biases = self.__make_var('biases', [channels_output]) 134 | output = tf.nn.bias_add(output, biases) 135 | 136 | # Apply ReLU non-linearity, if required 137 | if relu: 138 | output = tf.nn.relu(output, name=scope.name) 139 | 140 | 141 | self.__network.add_layer(name, layer_output=output) 142 | 143 | def new_prelu(self, name: str, input_layer_name: str=None): 144 | """ 145 | Creates a new prelu layer with the given name and input. 146 | :param name: name for this layer. 147 | :param input_layer_name: name of the layer that serves as input for this one. 148 | """ 149 | input_layer = self.__network.get_layer(input_layer_name) 150 | 151 | with tf.compat.v1.variable_scope(name): 152 | channels_input = int(input_layer.get_shape()[-1]) 153 | alpha = self.__make_var('alpha', shape=[channels_input]) 154 | output = tf.nn.relu(input_layer) + tf.multiply(alpha, -tf.nn.relu(-input_layer)) 155 | 156 | self.__network.add_layer(name, layer_output=output) 157 | 158 | def new_max_pool(self, name:str, kernel_size: tuple, stride_size: tuple, padding='SAME', 159 | input_layer_name: str=None): 160 | """ 161 | Creates a new max pooling layer. 162 | :param name: name for the layer. 163 | :param kernel_size: tuple containing the size of the kernel (Width, Height) 164 | :param stride_size: tuple containing the size of the stride (Width, Height) 165 | :param padding: Type of padding. Available values are: ('SAME', 'VALID') 166 | :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of 167 | the network. 168 | """ 169 | 170 | self.__validate_padding(padding) 171 | 172 | input_layer = self.__network.get_layer(input_layer_name) 173 | 174 | output = tf.nn.max_pool2d(input=input_layer, 175 | ksize=[1, kernel_size[1], kernel_size[0], 1], 176 | strides=[1, stride_size[1], stride_size[0], 1], 177 | padding=padding, 178 | name=name) 179 | 180 | self.__network.add_layer(name, layer_output=output) 181 | 182 | def new_fully_connected(self, name: str, output_count: int, relu=True, input_layer_name: str=None): 183 | """ 184 | Creates a new fully connected layer. 185 | 186 | :param name: name for the layer. 187 | :param output_count: number of outputs of the fully connected layer. 188 | :param relu: boolean flag to set if ReLu should be applied at the end of this layer. 189 | :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of 190 | the network. 191 | """ 192 | 193 | with tf.compat.v1.variable_scope(name): 194 | input_layer = self.__network.get_layer(input_layer_name) 195 | vectorized_input, dimension = self.vectorize_input(input_layer) 196 | 197 | weights = self.__make_var('weights', shape=[dimension, output_count]) 198 | biases = self.__make_var('biases', shape=[output_count]) 199 | operation = tf.compat.v1.nn.relu_layer if relu else tf.compat.v1.nn.xw_plus_b 200 | 201 | fc = operation(vectorized_input, weights, biases, name=name) 202 | 203 | self.__network.add_layer(name, layer_output=fc) 204 | 205 | def new_softmax(self, name, axis, input_layer_name: str=None): 206 | """ 207 | Creates a new softmax layer 208 | :param name: name to set for the layer 209 | :param axis: 210 | :param input_layer_name: name of the input layer for this layer. If None, it will take the last added layer of 211 | the network. 212 | """ 213 | input_layer = self.__network.get_layer(input_layer_name) 214 | 215 | if LooseVersion(tf.__version__) < LooseVersion("1.5.0"): 216 | max_axis = tf.reduce_max(input_tensor=input_layer, axis=axis, keepdims=True) 217 | target_exp = tf.exp(input_layer - max_axis) 218 | normalize = tf.reduce_sum(input_tensor=target_exp, axis=axis, keepdims=True) 219 | else: 220 | max_axis = tf.reduce_max(input_tensor=input_layer, axis=axis, keepdims=True) 221 | target_exp = tf.exp(input_layer - max_axis) 222 | normalize = tf.reduce_sum(input_tensor=target_exp, axis=axis, keepdims=True) 223 | 224 | softmax = tf.math.divide(target_exp, normalize, name) 225 | 226 | self.__network.add_layer(name, layer_output=softmax) 227 | 228 | -------------------------------------------------------------------------------- /mtcnn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # MIT License 5 | # 6 | # Copyright (c) 2019 Iván de Paz Centeno 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in all 16 | # copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | # SOFTWARE. 25 | 26 | # 27 | # This code is derived from the MTCNN implementation of David Sandberg for Facenet 28 | # (https://github.com/davidsandberg/facenet/) 29 | # It has been rebuilt from scratch, taking the David Sandberg's implementation as a reference. 30 | # 31 | 32 | import cv2 33 | import numpy as np 34 | import pkg_resources 35 | 36 | from mtcnn.exceptions import InvalidImage 37 | from mtcnn.network.factory import NetworkFactory 38 | 39 | __author__ = "Iván de Paz Centeno" 40 | 41 | 42 | class StageStatus(object): 43 | """ 44 | Keeps status between MTCNN stages 45 | """ 46 | 47 | def __init__(self, pad_result: tuple = None, width=0, height=0): 48 | self.width = width 49 | self.height = height 50 | self.dy = self.edy = self.dx = self.edx = self.y = self.ey = self.x = self.ex = self.tmpw = self.tmph = [] 51 | 52 | if pad_result is not None: 53 | self.update(pad_result) 54 | 55 | def update(self, pad_result: tuple): 56 | s = self 57 | s.dy, s.edy, s.dx, s.edx, s.y, s.ey, s.x, s.ex, s.tmpw, s.tmph = pad_result 58 | 59 | 60 | class MTCNN(object): 61 | """ 62 | Allows to perform MTCNN Detection -> 63 | a) Detection of faces (with the confidence probability) 64 | b) Detection of keypoints (left eye, right eye, nose, mouth_left, mouth_right) 65 | """ 66 | 67 | def __init__(self, weights_file: str = None, min_face_size: int = 20, steps_threshold: list = None, 68 | scale_factor: float = 0.709): 69 | """ 70 | Initializes the MTCNN. 71 | :param weights_file: file uri with the weights of the P, R and O networks from MTCNN. By default it will load 72 | the ones bundled with the package. 73 | :param min_face_size: minimum size of the face to detect 74 | :param steps_threshold: step's thresholds values 75 | :param scale_factor: scale factor 76 | """ 77 | if steps_threshold is None: 78 | steps_threshold = [0.6, 0.7, 0.7] 79 | 80 | if weights_file is None: 81 | weights_file = pkg_resources.resource_stream('mtcnn', 'data/mtcnn_weights.npy') 82 | 83 | self._min_face_size = min_face_size 84 | self._steps_threshold = steps_threshold 85 | self._scale_factor = scale_factor 86 | 87 | self._pnet, self._rnet, self._onet = NetworkFactory().build_P_R_O_nets_from_file(weights_file) 88 | 89 | @property 90 | def min_face_size(self): 91 | return self._min_face_size 92 | 93 | @min_face_size.setter 94 | def min_face_size(self, mfc=20): 95 | try: 96 | self._min_face_size = int(mfc) 97 | except ValueError: 98 | self._min_face_size = 20 99 | 100 | def __compute_scale_pyramid(self, m, min_layer): 101 | scales = [] 102 | factor_count = 0 103 | 104 | while min_layer >= 12: 105 | scales += [m * np.power(self._scale_factor, factor_count)] 106 | min_layer = min_layer * self._scale_factor 107 | factor_count += 1 108 | 109 | return scales 110 | 111 | @staticmethod 112 | def __scale_image(image, scale: float): 113 | """ 114 | Scales the image to a given scale. 115 | :param image: 116 | :param scale: 117 | :return: 118 | """ 119 | height, width, _ = image.shape 120 | 121 | width_scaled = int(np.ceil(width * scale)) 122 | height_scaled = int(np.ceil(height * scale)) 123 | 124 | im_data = cv2.resize(image, (width_scaled, height_scaled), interpolation=cv2.INTER_AREA) 125 | 126 | # Normalize the image's pixels 127 | im_data_normalized = (im_data - 127.5) * 0.0078125 128 | 129 | return im_data_normalized 130 | 131 | @staticmethod 132 | def __generate_bounding_box(imap, reg, scale, t): 133 | 134 | # use heatmap to generate bounding boxes 135 | stride = 2 136 | cellsize = 12 137 | 138 | imap = np.transpose(imap) 139 | dx1 = np.transpose(reg[:, :, 0]) 140 | dy1 = np.transpose(reg[:, :, 1]) 141 | dx2 = np.transpose(reg[:, :, 2]) 142 | dy2 = np.transpose(reg[:, :, 3]) 143 | 144 | y, x = np.where(imap >= t) 145 | 146 | if y.shape[0] == 1: 147 | dx1 = np.flipud(dx1) 148 | dy1 = np.flipud(dy1) 149 | dx2 = np.flipud(dx2) 150 | dy2 = np.flipud(dy2) 151 | 152 | score = imap[(y, x)] 153 | reg = np.transpose(np.vstack([dx1[(y, x)], dy1[(y, x)], dx2[(y, x)], dy2[(y, x)]])) 154 | 155 | if reg.size == 0: 156 | reg = np.empty(shape=(0, 3)) 157 | 158 | bb = np.transpose(np.vstack([y, x])) 159 | 160 | q1 = np.fix((stride * bb + 1) / scale) 161 | q2 = np.fix((stride * bb + cellsize) / scale) 162 | boundingbox = np.hstack([q1, q2, np.expand_dims(score, 1), reg]) 163 | 164 | return boundingbox, reg 165 | 166 | @staticmethod 167 | def __nms(boxes, threshold, method): 168 | """ 169 | Non Maximum Suppression. 170 | 171 | :param boxes: np array with bounding boxes. 172 | :param threshold: 173 | :param method: NMS method to apply. Available values ('Min', 'Union') 174 | :return: 175 | """ 176 | if boxes.size == 0: 177 | return np.empty((0, 3)) 178 | 179 | x1 = boxes[:, 0] 180 | y1 = boxes[:, 1] 181 | x2 = boxes[:, 2] 182 | y2 = boxes[:, 3] 183 | s = boxes[:, 4] 184 | 185 | area = (x2 - x1 + 1) * (y2 - y1 + 1) 186 | sorted_s = np.argsort(s) 187 | 188 | pick = np.zeros_like(s, dtype=np.int16) 189 | counter = 0 190 | while sorted_s.size > 0: 191 | i = sorted_s[-1] 192 | pick[counter] = i 193 | counter += 1 194 | idx = sorted_s[0:-1] 195 | 196 | xx1 = np.maximum(x1[i], x1[idx]) 197 | yy1 = np.maximum(y1[i], y1[idx]) 198 | xx2 = np.minimum(x2[i], x2[idx]) 199 | yy2 = np.minimum(y2[i], y2[idx]) 200 | 201 | w = np.maximum(0.0, xx2 - xx1 + 1) 202 | h = np.maximum(0.0, yy2 - yy1 + 1) 203 | 204 | inter = w * h 205 | 206 | if method is 'Min': 207 | o = inter / np.minimum(area[i], area[idx]) 208 | else: 209 | o = inter / (area[i] + area[idx] - inter) 210 | 211 | sorted_s = sorted_s[np.where(o <= threshold)] 212 | 213 | pick = pick[0:counter] 214 | 215 | return pick 216 | 217 | @staticmethod 218 | def __pad(total_boxes, w, h): 219 | # compute the padding coordinates (pad the bounding boxes to square) 220 | tmpw = (total_boxes[:, 2] - total_boxes[:, 0] + 1).astype(np.int32) 221 | tmph = (total_boxes[:, 3] - total_boxes[:, 1] + 1).astype(np.int32) 222 | numbox = total_boxes.shape[0] 223 | 224 | dx = np.ones(numbox, dtype=np.int32) 225 | dy = np.ones(numbox, dtype=np.int32) 226 | edx = tmpw.copy().astype(np.int32) 227 | edy = tmph.copy().astype(np.int32) 228 | 229 | x = total_boxes[:, 0].copy().astype(np.int32) 230 | y = total_boxes[:, 1].copy().astype(np.int32) 231 | ex = total_boxes[:, 2].copy().astype(np.int32) 232 | ey = total_boxes[:, 3].copy().astype(np.int32) 233 | 234 | tmp = np.where(ex > w) 235 | edx.flat[tmp] = np.expand_dims(-ex[tmp] + w + tmpw[tmp], 1) 236 | ex[tmp] = w 237 | 238 | tmp = np.where(ey > h) 239 | edy.flat[tmp] = np.expand_dims(-ey[tmp] + h + tmph[tmp], 1) 240 | ey[tmp] = h 241 | 242 | tmp = np.where(x < 1) 243 | dx.flat[tmp] = np.expand_dims(2 - x[tmp], 1) 244 | x[tmp] = 1 245 | 246 | tmp = np.where(y < 1) 247 | dy.flat[tmp] = np.expand_dims(2 - y[tmp], 1) 248 | y[tmp] = 1 249 | 250 | return dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph 251 | 252 | @staticmethod 253 | def __rerec(bbox): 254 | # convert bbox to square 255 | height = bbox[:, 3] - bbox[:, 1] 256 | width = bbox[:, 2] - bbox[:, 0] 257 | max_side_length = np.maximum(width, height) 258 | bbox[:, 0] = bbox[:, 0] + width * 0.5 - max_side_length * 0.5 259 | bbox[:, 1] = bbox[:, 1] + height * 0.5 - max_side_length * 0.5 260 | bbox[:, 2:4] = bbox[:, 0:2] + np.transpose(np.tile(max_side_length, (2, 1))) 261 | return bbox 262 | 263 | @staticmethod 264 | def __bbreg(boundingbox, reg): 265 | # calibrate bounding boxes 266 | if reg.shape[1] == 1: 267 | reg = np.reshape(reg, (reg.shape[2], reg.shape[3])) 268 | 269 | w = boundingbox[:, 2] - boundingbox[:, 0] + 1 270 | h = boundingbox[:, 3] - boundingbox[:, 1] + 1 271 | b1 = boundingbox[:, 0] + reg[:, 0] * w 272 | b2 = boundingbox[:, 1] + reg[:, 1] * h 273 | b3 = boundingbox[:, 2] + reg[:, 2] * w 274 | b4 = boundingbox[:, 3] + reg[:, 3] * h 275 | boundingbox[:, 0:4] = np.transpose(np.vstack([b1, b2, b3, b4])) 276 | return boundingbox 277 | 278 | def detect_faces(self, img) -> list: 279 | """ 280 | Detects bounding boxes from the specified image. 281 | :param img: image to process 282 | :return: list containing all the bounding boxes detected with their keypoints. 283 | """ 284 | if img is None or not hasattr(img, "shape"): 285 | raise InvalidImage("Image not valid.") 286 | 287 | height, width, _ = img.shape 288 | stage_status = StageStatus(width=width, height=height) 289 | 290 | m = 12 / self._min_face_size 291 | min_layer = np.amin([height, width]) * m 292 | 293 | scales = self.__compute_scale_pyramid(m, min_layer) 294 | 295 | stages = [self.__stage1, self.__stage2, self.__stage3] 296 | result = [scales, stage_status] 297 | 298 | # We pipe here each of the stages 299 | for stage in stages: 300 | result = stage(img, result[0], result[1]) 301 | 302 | [total_boxes, points] = result 303 | 304 | bounding_boxes = [] 305 | 306 | for bounding_box, keypoints in zip(total_boxes, points.T): 307 | bounding_boxes.append({ 308 | 'box': [max(0, int(bounding_box[0])), max(0, int(bounding_box[1])), 309 | int(bounding_box[2] - bounding_box[0]), int(bounding_box[3] - bounding_box[1])], 310 | 'confidence': bounding_box[-1], 311 | 'keypoints': { 312 | 'left_eye': (int(keypoints[0]), int(keypoints[5])), 313 | 'right_eye': (int(keypoints[1]), int(keypoints[6])), 314 | 'nose': (int(keypoints[2]), int(keypoints[7])), 315 | 'mouth_left': (int(keypoints[3]), int(keypoints[8])), 316 | 'mouth_right': (int(keypoints[4]), int(keypoints[9])), 317 | } 318 | } 319 | ) 320 | 321 | return bounding_boxes 322 | 323 | def __stage1(self, image, scales: list, stage_status: StageStatus): 324 | """ 325 | First stage of the MTCNN. 326 | :param image: 327 | :param scales: 328 | :param stage_status: 329 | :return: 330 | """ 331 | total_boxes = np.empty((0, 9)) 332 | status = stage_status 333 | 334 | for scale in scales: 335 | scaled_image = self.__scale_image(image, scale) 336 | 337 | img_x = np.expand_dims(scaled_image, 0) 338 | img_y = np.transpose(img_x, (0, 2, 1, 3)) 339 | 340 | out = self._pnet.predict(img_y) 341 | 342 | out0 = np.transpose(out[0], (0, 2, 1, 3)) 343 | out1 = np.transpose(out[1], (0, 2, 1, 3)) 344 | 345 | boxes, _ = self.__generate_bounding_box(out1[0, :, :, 1].copy(), 346 | out0[0, :, :, :].copy(), scale, self._steps_threshold[0]) 347 | 348 | # inter-scale nms 349 | pick = self.__nms(boxes.copy(), 0.5, 'Union') 350 | if boxes.size > 0 and pick.size > 0: 351 | boxes = boxes[pick, :] 352 | total_boxes = np.append(total_boxes, boxes, axis=0) 353 | 354 | numboxes = total_boxes.shape[0] 355 | 356 | if numboxes > 0: 357 | pick = self.__nms(total_boxes.copy(), 0.7, 'Union') 358 | total_boxes = total_boxes[pick, :] 359 | 360 | regw = total_boxes[:, 2] - total_boxes[:, 0] 361 | regh = total_boxes[:, 3] - total_boxes[:, 1] 362 | 363 | qq1 = total_boxes[:, 0] + total_boxes[:, 5] * regw 364 | qq2 = total_boxes[:, 1] + total_boxes[:, 6] * regh 365 | qq3 = total_boxes[:, 2] + total_boxes[:, 7] * regw 366 | qq4 = total_boxes[:, 3] + total_boxes[:, 8] * regh 367 | 368 | total_boxes = np.transpose(np.vstack([qq1, qq2, qq3, qq4, total_boxes[:, 4]])) 369 | total_boxes = self.__rerec(total_boxes.copy()) 370 | 371 | total_boxes[:, 0:4] = np.fix(total_boxes[:, 0:4]).astype(np.int32) 372 | status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height), 373 | width=stage_status.width, height=stage_status.height) 374 | 375 | return total_boxes, status 376 | 377 | def __stage2(self, img, total_boxes, stage_status: StageStatus): 378 | """ 379 | Second stage of the MTCNN. 380 | :param img: 381 | :param total_boxes: 382 | :param stage_status: 383 | :return: 384 | """ 385 | 386 | num_boxes = total_boxes.shape[0] 387 | if num_boxes == 0: 388 | return total_boxes, stage_status 389 | 390 | # second stage 391 | tempimg = np.zeros(shape=(24, 24, 3, num_boxes)) 392 | 393 | for k in range(0, num_boxes): 394 | tmp = np.zeros((int(stage_status.tmph[k]), int(stage_status.tmpw[k]), 3)) 395 | 396 | tmp[stage_status.dy[k] - 1:stage_status.edy[k], stage_status.dx[k] - 1:stage_status.edx[k], :] = \ 397 | img[stage_status.y[k] - 1:stage_status.ey[k], stage_status.x[k] - 1:stage_status.ex[k], :] 398 | 399 | if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0: 400 | tempimg[:, :, :, k] = cv2.resize(tmp, (24, 24), interpolation=cv2.INTER_AREA) 401 | 402 | else: 403 | return np.empty(shape=(0,)), stage_status 404 | 405 | tempimg = (tempimg - 127.5) * 0.0078125 406 | tempimg1 = np.transpose(tempimg, (3, 1, 0, 2)) 407 | 408 | out = self._rnet.predict(tempimg1) 409 | 410 | out0 = np.transpose(out[0]) 411 | out1 = np.transpose(out[1]) 412 | 413 | score = out1[1, :] 414 | 415 | ipass = np.where(score > self._steps_threshold[1]) 416 | 417 | total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)]) 418 | 419 | mv = out0[:, ipass[0]] 420 | 421 | if total_boxes.shape[0] > 0: 422 | pick = self.__nms(total_boxes, 0.7, 'Union') 423 | total_boxes = total_boxes[pick, :] 424 | total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv[:, pick])) 425 | total_boxes = self.__rerec(total_boxes.copy()) 426 | 427 | return total_boxes, stage_status 428 | 429 | def __stage3(self, img, total_boxes, stage_status: StageStatus): 430 | """ 431 | Third stage of the MTCNN. 432 | 433 | :param img: 434 | :param total_boxes: 435 | :param stage_status: 436 | :return: 437 | """ 438 | num_boxes = total_boxes.shape[0] 439 | if num_boxes == 0: 440 | return total_boxes, np.empty(shape=(0,)) 441 | 442 | total_boxes = np.fix(total_boxes).astype(np.int32) 443 | 444 | status = StageStatus(self.__pad(total_boxes.copy(), stage_status.width, stage_status.height), 445 | width=stage_status.width, height=stage_status.height) 446 | 447 | tempimg = np.zeros((48, 48, 3, num_boxes)) 448 | 449 | for k in range(0, num_boxes): 450 | 451 | tmp = np.zeros((int(status.tmph[k]), int(status.tmpw[k]), 3)) 452 | 453 | tmp[status.dy[k] - 1:status.edy[k], status.dx[k] - 1:status.edx[k], :] = \ 454 | img[status.y[k] - 1:status.ey[k], status.x[k] - 1:status.ex[k], :] 455 | 456 | if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0: 457 | tempimg[:, :, :, k] = cv2.resize(tmp, (48, 48), interpolation=cv2.INTER_AREA) 458 | else: 459 | return np.empty(shape=(0,)), np.empty(shape=(0,)) 460 | 461 | tempimg = (tempimg - 127.5) * 0.0078125 462 | tempimg1 = np.transpose(tempimg, (3, 1, 0, 2)) 463 | 464 | out = self._onet.predict(tempimg1) 465 | out0 = np.transpose(out[0]) 466 | out1 = np.transpose(out[1]) 467 | out2 = np.transpose(out[2]) 468 | 469 | score = out2[1, :] 470 | 471 | points = out1 472 | 473 | ipass = np.where(score > self._steps_threshold[2]) 474 | 475 | points = points[:, ipass[0]] 476 | 477 | total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(), np.expand_dims(score[ipass].copy(), 1)]) 478 | 479 | mv = out0[:, ipass[0]] 480 | 481 | w = total_boxes[:, 2] - total_boxes[:, 0] + 1 482 | h = total_boxes[:, 3] - total_boxes[:, 1] + 1 483 | 484 | points[0:5, :] = np.tile(w, (5, 1)) * points[0:5, :] + np.tile(total_boxes[:, 0], (5, 1)) - 1 485 | points[5:10, :] = np.tile(h, (5, 1)) * points[5:10, :] + np.tile(total_boxes[:, 1], (5, 1)) - 1 486 | 487 | if total_boxes.shape[0] > 0: 488 | total_boxes = self.__bbreg(total_boxes.copy(), np.transpose(mv)) 489 | pick = self.__nms(total_boxes.copy(), 0.7, 'Min') 490 | total_boxes = total_boxes[pick, :] 491 | points = points[:, pick] 492 | 493 | return total_boxes, points 494 | -------------------------------------------------------------------------------- /network.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | 4 | #MIT License 5 | # 6 | #Copyright (c) 2018 Iván de Paz Centeno 7 | # 8 | #Permission is hereby granted, free of charge, to any person obtaining a copy 9 | #of this software and associated documentation files (the "Software"), to deal 10 | #in the Software without restriction, including without limitation the rights 11 | #to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | #copies of the Software, and to permit persons to whom the Software is 13 | #furnished to do so, subject to the following conditions: 14 | # 15 | #The above copyright notice and this permission notice shall be included in all 16 | #copies or substantial portions of the Software. 17 | # 18 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | #FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | #AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | #SOFTWARE. 25 | 26 | import tensorflow as tf 27 | 28 | __author__ = "Iván de Paz Centeno" 29 | 30 | 31 | class Network(object): 32 | 33 | def __init__(self, session, trainable: bool=True): 34 | """ 35 | Initializes the network. 36 | :param trainable: flag to determine if this network should be trainable or not. 37 | """ 38 | self._session = session 39 | self.__trainable = trainable 40 | self.__layers = {} 41 | self.__last_layer_name = None 42 | 43 | with tf.compat.v1.variable_scope(self.__class__.__name__.lower()): 44 | self._config() 45 | 46 | def _config(self): 47 | """ 48 | Configures the network layers. 49 | It is usually done using the LayerFactory() class. 50 | """ 51 | raise NotImplementedError("This method must be implemented by the network.") 52 | 53 | def add_layer(self, name: str, layer_output): 54 | """ 55 | Adds a layer to the network. 56 | :param name: name of the layer to add 57 | :param layer_output: output layer. 58 | """ 59 | self.__layers[name] = layer_output 60 | self.__last_layer_name = name 61 | 62 | def get_layer(self, name: str=None): 63 | """ 64 | Retrieves the layer by its name. 65 | :param name: name of the layer to retrieve. If name is None, it will retrieve the last added layer to the 66 | network. 67 | :return: layer output 68 | """ 69 | if name is None: 70 | name = self.__last_layer_name 71 | 72 | return self.__layers[name] 73 | 74 | def is_trainable(self): 75 | """ 76 | Getter for the trainable flag. 77 | """ 78 | return self.__trainable 79 | 80 | def set_weights(self, weights_values: dict, ignore_missing=False): 81 | """ 82 | Sets the weights values of the network. 83 | :param weights_values: dictionary with weights for each layer 84 | """ 85 | network_name = self.__class__.__name__.lower() 86 | 87 | with tf.compat.v1.variable_scope(network_name): 88 | for layer_name in weights_values: 89 | with tf.compat.v1.variable_scope(layer_name, reuse=True): 90 | for param_name, data in weights_values[layer_name].items(): 91 | try: 92 | var = tf.compat.v1.get_variable(param_name, use_resource=False) 93 | self._session.run(var.assign(data)) 94 | 95 | except ValueError: 96 | if not ignore_missing: 97 | raise 98 | 99 | def feed(self, image): 100 | """ 101 | Feeds the network with an image 102 | :param image: image (perhaps loaded with CV2) 103 | :return: network result 104 | """ 105 | network_name = self.__class__.__name__.lower() 106 | 107 | with tf.compat.v1.variable_scope(network_name): 108 | return self._feed(image) 109 | 110 | def _feed(self, image): 111 | raise NotImplementedError("Method not implemented.") -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import random 4 | import time 5 | import numpy as np 6 | import cv2 7 | from keras.applications.imagenet_utils import preprocess_input 8 | from keras.engine.saving import load_model 9 | from keras.preprocessing.image import img_to_array 10 | from mtcnn import MTCNN 11 | from math import * 12 | detector = MTCNN() 13 | 14 | 15 | 16 | 17 | 18 | # cap = cv2.VideoCapture(0) 19 | def get_MER(x1,x2,y1,y2,img): 20 | cropped = img[y1:y2, x1:x2] # 裁剪坐标为[y0:y1, x0:x1] 21 | return cropped 22 | 23 | 24 | 25 | modelpath='./model/best0428ep150.h5' 26 | model_cnn=load_model(modelpath, compile=False) 27 | classname=["closed_eye","closed_mouth","open_eye","open_mouth","smoke"] 28 | def get_label(img): 29 | img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 30 | img = cv2.resize(img, (32, 32)) 31 | img = img.astype("float") / 255.0 32 | img = img_to_array(img) 33 | img = np.expand_dims(img, axis=0) 34 | 35 | preds = model_cnn.predict(img) 36 | i = preds.argmax(axis=1)[0] 37 | label = classname[i] 38 | 39 | return label 40 | 41 | # img_count=0 42 | left_eye_count=0 43 | right_eye_count=0 44 | mouth_count=0 45 | frag=False 46 | blink=0 47 | path='./20200407_173126.mp4' 48 | 49 | cap = cv2.VideoCapture(path) 50 | 51 | start = time.time() 52 | while True: 53 | start = time.time() 54 | red,image=cap.read() 55 | Img=image.copy() 56 | 57 | # image = cv2.resize(image, (480, 360)) 58 | result = detector.detect_faces(image) 59 | if(len(result))>0: 60 | 61 | 62 | 63 | # Result is an array with all the bounding boxes detected. We know that for 'ivan.jpg' there is only one. 64 | # print(len(result[0]['box'])) 65 | bounding_box = result[0]['box'] 66 | keypoints = result[0]['keypoints'] 67 | # 68 | # print(face) 69 | left_eye=keypoints['left_eye'] 70 | right_eye=keypoints['right_eye'] 71 | nose=keypoints['nose'] 72 | mouth_left=keypoints['mouth_left'] 73 | mouth_right=keypoints['mouth_right'] 74 | arc = atan(abs(right_eye[1] - left_eye[1]) / abs(right_eye[0] - left_eye[0])) 75 | W = abs(right_eye[0] - left_eye[0]) / (2 * cos(arc)) 76 | H = W / 2 77 | ###########可去掉，只是避免裁剪出问题而已 78 | x1 = int(left_eye[0] - W / 2) 79 | if(x1<=0): 80 | x1=1 81 | x2 = int(left_eye[0] + W / 2) 82 | if(x2>=639): 83 | x2=638 84 | y1 = int(left_eye[1] - H / 2)-5 85 | if (y1<=0): 86 | y1=1 87 | y2 = int(left_eye[1] + H / 2) 88 | if(y2>=479): 89 | y2=478 90 | 91 | left=get_MER(x1, x2, y1, y2, image) 92 | label_left_eye = get_label(left) 93 | if label_left_eye=='closed_eye': 94 | left_state='closed' 95 | else: 96 | left_state='open' 97 | print(label_left_eye) 98 | cv2.putText(image, "{}".format(left_state), (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 255, 0), 99 | 1, 8) 100 | cv2.putText(image, "left_eye_state:{}".format(left_state), (5, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 1, 8) 101 | #cv2.imshow("left_eye", left) 102 | 103 | ##########裁剪左眼图像，可以扩充数据集 104 | # if len(left)>0: 105 | # cv2.imwrite(savePath1 + str(Img), left) 106 | # cv2.imwrite(savePath1 + str(Img), cv2.cvtColor(get_MER(x1, x2, y1, y2, image), cv2.COLOR_BGR2GRAY)) 107 | 108 | # 右眼 109 | rx1 = int(right_eye[0] - W / 2) 110 | if (rx1 <= 0): 111 | rx1 = 1 112 | rx2 = int(right_eye[0] + W / 2) 113 | if(rx2>=639): 114 | rx2=638 115 | ry1 = int(right_eye[1] - H / 2)-5 116 | if (ry1 <= 0): 117 | ry1 = 1 118 | ry2 = int(right_eye[1] + H / 2) 119 | if (ry2 >= 479): 120 | ry1 = 478 121 | right=get_MER(rx1, rx2, ry1, ry2, image) 122 | 123 | label_right_eye=get_label(right) 124 | if label_right_eye == 'closed_eye': 125 | right_state='close' 126 | blink = blink+1 127 | else: 128 | right_state='open' 129 | right_eye_count = right_eye_count + 1 130 | print(label_right_eye) 131 | cv2.putText(image, "{}".format(right_state), (rx1, ry1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (255, 255, 0), 1, 132 | 8) 133 | cv2.putText(image, "right_eye:{}".format(right_state), (5, 35), cv2.FONT_HERSHEY_SIMPLEX,0.7, (255, 255, 0), 1, 8) 134 | # cv2.imshow("right_eye",right) 135 | # if len(right)>0: 136 | cv2.rectangle(image, (rx1, ry1), (rx2, ry2), (255, 0, 0), 2) 137 | # cv2.imwrite(savePath2 + str(Img), cv2.cvtColor(get_MER(rx1, rx2, ry1, ry2, image), cv2.COLOR_BGR2GRAY)) 138 | 139 | 140 | 141 | 142 | 143 | 144 | D=(mouth_left[1]-nose[1])/cos(arc)-(mouth_left[1]-mouth_right[1])/(2*cos(arc)) 145 | m1=nose[1]+D/2+10 146 | m2=nose[1]+3*D/2+20 147 | xm1=int(mouth_left[0]) 148 | xm2=int(mouth_right[0])+10 149 | if(m2>=479): 150 | m2=478 151 | if(xm1<=1): 152 | xm1=2 153 | if(xm2>=639): 154 | xm2=638 155 | mouth=get_MER(int(mouth_left[0]), int(mouth_right[0]), int(m1), int(m2), image) 156 | mouth_label = get_label(mouth) 157 | if mouth_label=='open_mouth': 158 | mouth_count=mouth_count+1 159 | 160 | mouth_state='open_mouth' 161 | elif mouth_label=='closed_mouth': 162 | mouth_state='closed' 163 | mouth_count=0 164 | frag = False 165 | else: 166 | mouth_state='Eat or Smoke' 167 | 168 | if mouth_count>=10: 169 | mouth_count=0 170 | mouth_state = 'Yawd' 171 | frag=True 172 | 173 | 174 | # cv2.imshow("mouth", mouth) 175 | # if frag: 176 | # cv2.putText(image, "State:Yawd", (5, 105), 177 | # cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 160, 100), 2, 8) 178 | # 耳部，(上面那个模型没有训练打电话这个类别，单独训练了一个耳朵和打电话的二分类模型) 179 | # left_ear 180 | #lex1 = x1 - 3 * W 181 | #if int(lex1)<=10: 182 | # lex1=10 183 | #lex2 = bounding_box[0] 184 | #ley1 = y1 185 | #ley2 = bounding_box[1] + bounding_box[3] 186 | #left_ear=get_MER(int(lex1),int(lex2),int(ley1),int(ley2),image) 187 | #left_ear_label=get_call_label(left_ear) 188 | #print(left_ear_label) 189 | # cv2.imwrite(savePath2 + 'lear'+str(img_count)+'.jpg', left_ear) 190 | 191 | # print(left_ear_label) 192 | # if left_ear_label=='calling': 193 | # ear_state='Calling' 194 | # mouth_state='Talking' 195 | # cv2.putText(image, "other_state:{}".format(ear_state), (5, 75), 196 | # cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 100, 255), 1, 8) 197 | 198 | 199 | # cv2.imshow("left_ear", left_ear) 200 | # right_ear 201 | #rex2 =rx2 + 3 * W 202 | #if int(rex2)>=479: 203 | # rex2=478 204 | #if int(rex2) <= rex1: 205 | # rex2 = rex1+30 206 | #rey1 = y1 207 | #right_ear = get_MER(int(rex1), int(rex2), int(rey1), int(rey2), image) 208 | # cv2.imwrite(savePath2 + 'rear' + str(i)+'.jpg', right_ear) 209 | # i=i+1 210 | #right_ear_label = get_call_label(right_ear) 211 | # print(right_ear_label) 212 | # if right_ear_label=='calling': 213 | # ear_state='Calling' 214 | # mouth_state='Talking' 215 | # cv2.putText(image, "other_state:{}".format(ear_state), (5, 75), 216 | # cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 100, 255), 1, 8) 217 | # cv2.imshow("right_ear", right_ear) 218 | # 嘴部状态显示 219 | cv2.putText(image, "mouth_state:{}".format(mouth_state), (int(mouth_left[0]) - 20, int(m1) - 10), 220 | cv2.FONT_HERSHEY_SIMPLEX, 0.3, (100, 255, 255), 1, 8) 221 | cv2.putText(image, "mouth_state:{}".format(mouth_state), (5, 55), 222 | cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 1, 8) 223 | 224 | 225 | cv2.rectangle(image, 226 | (bounding_box[0], bounding_box[1]), 227 | (bounding_box[0] + bounding_box[2], bounding_box[1] + bounding_box[3]), 228 | (0, 155, 255), 229 | 2) 230 | cv2.rectangle(image, (int(mouth_left[0]), int(m1)), (int(mouth_right[0]), int(m2)), (0, 0, 255), 1) 231 | cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 1) 232 | cv2.rectangle(image, (rx1, ry1), (rx2, ry2), (0, 255, 0), 1) 233 | cv2.rectangle(image, (int(lex1), int(ley1)), (int(lex2), int(ley2)), (0, 255, 0), 1) 234 | cv2.rectangle(image, (int(rex1), int(rey1)), (int(rex2), int(rey2)), (0, 255, 0), 1) 235 | # print(mouth_label) 236 | # gray = cv2.cvtColor(mouth, cv2.COLOR_BGR2GRAY) 237 | image=cv2.resize(image,(640,480)) 238 | 239 | # cv2.imwrite(savepath + str(i)+'.jpg', gray) 240 | # i+=1 241 | T = time.time() - start 242 | fps = 1 / T # 实时在视频上显示fps 243 | # 244 | fps_txt = 'fps:%.2f' % (fps) 245 | cv2.putText(image, fps_txt, (0,180), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255), 1, 8) 246 | cv2.imshow("image",image) 247 | if cv2.waitKey(1)==27: 248 | cv2.destroyAllWindows() 249 | 250 | # print(time.time() - start) 251 | # print("FPS") 252 | # print(len(imagelist) / (time.time()-start)) 253 | # print("测试图片数量：{}".format(img_count)) 254 | # print("闭嘴图片预测数量{}".format(mouth_count)) 255 | # mv=mouth_count/img_count 256 | # print("闭嘴图片测试准确率{}".format(mv)) 257 | # print("左眼闭合预测数量{}".format(left_eye_count)) 258 | # print("右眼闭合的数量{}".format(right_eye_count)) 259 | # lv=left_eye_count/img_count 260 | # print("左眼测试准确率{}".format(lv)) 261 | # rv=right_eye_count/img_count 262 | # print("右眼测试准确率{}".format(rv)) 263 | 264 | --------------------------------------------------------------------------------