├── .coveragerc ├── .dockerignore ├── .github └── workflows │ ├── codeql-analysis.yml │ └── python-app.yml ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── data ├── Ubuntu-R.ttf ├── emojis │ ├── angry.png │ ├── disgusted.png │ ├── fearful.png │ ├── happy.png │ ├── neutral.png │ ├── sad.png │ └── surprised.png ├── media │ ├── 1.JPG │ ├── 2.JPG │ ├── 3.JPG │ ├── 4.JPG │ ├── model_plot.png │ ├── out3.jpg │ ├── out4.jpg │ ├── out5.jpg │ ├── out6.jpg │ └── output.gif └── sample │ ├── 1.jpg │ ├── 2.jpg │ ├── 3.jpg │ ├── 4.jpg │ ├── 5.jpg │ └── 6.jpg ├── emotion_analyzer ├── emotion_detector.py ├── emotion_detector_base.py ├── exceptions.py ├── face_detection_dlib.py ├── face_detection_mtcnn.py ├── face_detection_opencv.py ├── face_detector.py ├── logger.py ├── media_utils.py ├── model_utils.py └── validators.py ├── models ├── mmod_human_face_detector.dat ├── opencv_face_detector.pbtxt ├── opencv_face_detector_uint8.pb ├── shape_predictor_5_face_landmarks.dat └── weights.h5 ├── requirements.txt ├── tests ├── conftest.py ├── test_face_detection_dlib.py ├── test_face_detection_mtcnn.py ├── test_face_detection_opencv.py └── test_media_utils.py ├── training ├── data_prep.py ├── facial Emotions.ipynb └── preprocess.py └── video_main.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = video_main.py,tests//*,emotion_analyzer//model_utils.py, 3 | emotion_analyzer//emotion_detector_base.py,emotion_analyzer//media_utils.py, 4 | emotion_analyzer//exceptions.py,emotion_analyzer//face_detector.py,emotion_analyzer/validators.py -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **__pycache__ 2 | .benchmark/ 3 | .pytest_cache/ 4 | **.log 5 | .coverage 6 | .idea 7 | **.mp4 8 | .vscode 9 | data/media 10 | data/sample -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '34 23 * * 4' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'python' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Tests 5 | on: 6 | push: 7 | branches: [ master ] 8 | pull_request: 9 | branches: [ master ] 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python 3.9 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: 3.9 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install flake8 pytest 26 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 27 | - name: Lint with flake8 28 | run: | 29 | # stop the build if there are Python syntax errors or undefined names 30 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 31 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 32 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 33 | - name: Test with pytest 34 | run: | 35 | python -m pytest --cov=./ 36 | 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **__pycache__ 2 | .benchmark/ 3 | .pytest_cache/ 4 | **.log 5 | .coverage 6 | .idea 7 | **.mp4 8 | .vscode 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.9" 4 | 5 | # code climate 6 | before_script: 7 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 8 | - chmod +x ./cc-test-reporter 9 | 10 | install: 11 | - pip install -r requirements.txt 12 | 13 | # for codecoverage on codeclimate.com 14 | env: 15 | global: 16 | - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) 17 | - CODECLIMATE_REPO_TOKEN=[token] 18 | - CC_TEST_REPORTER_ID=[id] 19 | 20 | script: 21 | - python -m pytest --cov=./ 22 | after_success: 23 | - bash <(curl -s https://codecov.io/bash) 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | 3 | RUN apt-get -y update 4 | # Install FFmpeg and Cmake 5 | RUN apt-get install -y ffmpeg \ 6 | git \ 7 | build-essential \ 8 | cmake \ 9 | && apt-get clean && rm -rf /tmp/* /var/tmp/* 10 | 11 | WORKDIR /realtime-facial-emotion-analyzer 12 | 13 | COPY ./ ./ 14 | 15 | RUN pip3 install -r requirements.txt 16 | 17 | CMD ["bash"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Susanta Biswas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/susantabiswas/realtime-facial-emotion-analyzer.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/susantabiswas/realtime-facial-emotion-analyzer/context:python) 4 | [![Maintainability](https://api.codeclimate.com/v1/badges/8507a04fe1535a9c224a/maintainability)](https://codeclimate.com/github/susantabiswas/realtime-facial-emotion-analyzer/maintainability) 5 | ![Tests](https://github.com/susantabiswas/FaceRecog/workflows/Tests/badge.svg) 6 | [![Build Status](https://app.travis-ci.com/susantabiswas/realtime-facial-emotion-analyzer.svg?branch=master)](https://app.travis-ci.com/susantabiswas/realtime-facial-emotion-analyzer) 7 | [![codecov](https://codecov.io/gh/susantabiswas/realtime-facial-emotion-analyzer/branch/master/graph/badge.svg?token=O7CRXABZEA)](https://codecov.io/gh/susantabiswas/realtime-facial-emotion-analyzer) 8 | 9 | 10 | 11 | # Realtime Emotion Analysis from facial Expressions 12 | Real-time Human Emotion Analysis From facial expressions. It uses a deep Convolutional Neural Network. 13 | The model used achieved an accuracy of 63% on the test data. The realtime analyzer assigns a suitable emoji for the current emotion. 14 | 15 | There are 4 different face detectors for usage. Wrappers for video and webcam processing are provided for convenience.

16 | 17 | This emotion recognition library is built with ease and customization in mind. There are numerous control parameters to control how you want to use the features, be it face detection on videos, or with a webcam. 18 |
19 | 20 | ## Table of Contents 21 | - [Sample Output](#sample-output) 22 | - [Architecture](#architecture) 23 | - [Setup](#setup) 24 | - [Project Structure](#project-structure) 25 | - [Usage](#usage) 26 | - [References](#references) 27 | 28 | # Sample Output 29 | 30 | ## Processed Video 31 |
32 | 33 | ## Processed Images 34 | 35 | 36 | 37 | 38 | 39 | For emotion recognition, flow is: 40 | 41 | media -> frame -> face detection -> Facial ROI -> Convolutional Neural Network -> Emotion 42 | 43 | These are the major components: 44 | 1. **Face Detection**: There are 4 different face detectors with different cropping options. 45 | 2. **Emotion Recognition**: Responsible for handling emotion recognition related functionalities from an image. 46 | 3. **Utilities**: Methods for handling image, video operations, validations, etc. 47 | 48 |
49 | 50 | # Setup 51 | There are multiple ways to set this up. 52 | ### Clone the repo and install dependencies.
53 | ```python 54 | git clone https://github.com/susantabiswas/realtime-facial-emotion-analyzer.git 55 | pip install -r requirements.txt 56 | ``` 57 | 58 | ### Docker Image 59 | You can pull the docker image for this project and run the code there.
60 | ```docker pull susantabiswas/emotion-analyzer:latest``` 61 | 62 | ### Dockerfile 63 | You can build the docker image from the docker file present in the repo. 64 | 65 | ```docker build -t .``` 66 | 67 | 68 | # Project Structure 69 | ``` 70 | 71 | realtime-facial-emotion-analyzer/ 72 | ├── Dockerfile 73 | ├── LICENSE 74 | ├── README.md 75 | ├── data 76 | │   ├── Ubuntu-R.ttf 77 | │   ├── emojis 78 | │   │   ├── angry.png 79 | │   │   ├── disgusted.png 80 | │   │   ├── fearful.png 81 | │   │   ├── happy.png 82 | │   │   ├── neutral.png 83 | │   │   ├── sad.png 84 | │   │   └── surprised.png 85 | │   ├── media 86 | │   │   ├── 1.JPG 87 | │   │   ├── 2.JPG 88 | │   │   ├── 3.JPG 89 | │   │   ├── 4.JPG 90 | │   │   └── model_plot.png 91 | │   └── sample 92 | │   ├── 1.jpg 93 | │   └── 2.jpg 94 | ├── emotion_analyzer 95 | │   ├── emotion_detector.py 96 | │   ├── emotion_detector_base.py 97 | │   ├── exceptions.py 98 | │   ├── face_detection_dlib.py 99 | │   ├── face_detection_mtcnn.py 100 | │   ├── face_detection_opencv.py 101 | │   ├── face_detector.py 102 | │   ├── logger.py 103 | │   ├── media_utils.py 104 | │   ├── model_utils.py 105 | │   └── validators.py 106 | ├── models 107 | │   ├── mmod_human_face_detector.dat 108 | │   ├── opencv_face_detector.pbtxt 109 | │   ├── opencv_face_detector_uint8.pb 110 | │   └── shape_predictor_5_face_landmarks.dat 111 | ├── requirements.txt 112 | ├── tests 113 | │   ├── conftest.py 114 | │   ├── test_face_detection_dlib.py 115 | │   ├── test_face_detection_mtcnn.py 116 | │   ├── test_face_detection_opencv.py 117 | │   └── test_media_utils.py 118 | ├── training 119 | │   ├── data_prep.py 120 | │   ├── facial Emotions.ipynb 121 | │   └── preprocess.py 122 | └── video_main.py 123 | ``` 124 | 125 | # Usage 126 | 127 | ### Emotion Recognition 128 | Depending on the use case, whether to aim for accuracy and stability or speed etc., you can pick the face detector. Also, there are customization options inside face detectors to decide the facial ROI. 129 | 130 | 131 | ### To analyze facial emotion using a webcam 132 | ```python 133 | # Inside project root 134 | import video_main 135 | 136 | # You can pick a face detector depending on Acc/speed requirements 137 | emotion_recognizer = EmotionAnalysisVideo( 138 | face_detector="dlib", 139 | model_loc="models", 140 | face_detection_threshold=0.0, 141 | ) 142 | emotion_recognizer.emotion_analysis_video( 143 | video_path=None, 144 | detection_interval=1, 145 | save_output=False, 146 | preview=True, 147 | output_path="data/output.mp4", 148 | resize_scale=0.5, 149 | ) 150 | ``` 151 | 152 | ### To analyze facial emotion using a video file 153 | ```python 154 | # Inside project root 155 | import video_main 156 | 157 | # You can pick a face detector depending on Acc/speed requirements 158 | emotion_recognizer = EmotionAnalysisVideo( 159 | face_detector="dlib", 160 | model_loc="models", 161 | face_detection_threshold=0.0, 162 | ) 163 | emotion_recognizer.emotion_analysis_video( 164 | video_path='data/sample/test.mp4', 165 | detection_interval=1, 166 | save_output=False, 167 | preview=True, 168 | output_path="data/output.mp4", 169 | resize_scale=0.5, 170 | ) 171 | ``` 172 | 173 | ### Emotion recognition using an image 174 | ```python 175 | # Inside project root 176 | from emotion_analyzer.media_utils import load_image_path 177 | from emotion_analyzer.emotion_detector import EmotionDetector 178 | 179 | emotion_detector = EmotionDetector( 180 | model_loc="models", 181 | face_detection_threshold=0.8, 182 | face_detector="dlib", 183 | ) 184 | # Load the test image 185 | img = load_image_path("data/sample/1.jpg") 186 | emotion, emotion_conf = emotion_detector.detect_facial_emotion(img) 187 | ``` 188 | 189 | 190 | There are 4 face detectors namely dlib (HOG, MMOD), MTCNN, OpenCV (CNN). 191 | All the face detectors are based on a common abstract class and have a common detection interface **detect_faces(image)**. 192 | 193 | ```python 194 | # import the face detector you want, it follows absolute imports 195 | from emotion_analyzer.media_utils import load_image_path 196 | from emotion_analyzer.face_detection_dlib import FaceDetectorDlib 197 | 198 | face_detector = FaceDetectorDlib(model_type="hog") 199 | # Load the image in RGB format 200 | image = load_image_path("data/sample/1.jpg") 201 | # Returns a list of bounding box coordinates 202 | bboxes = face_detector.detect_faces(image) 203 | ``` 204 | 205 | 206 | # Architecture 207 | ![architecture](data/media/model_plot.png)
208 |
209 | 210 | # References 211 | The awesome work Davis E. King has done: 212 | http://dlib.net/cnn_face_detector.py.html, 213 | https://github.com/davisking/dlib-models
214 | You can find more about MTCNN from here: https://github.com/ipazc/mtcnn 215 |
216 | Dataset used was from Kaggle fer2013 Challenge [Challenges in Representation Learning: Facial Expression Recognition Challenge](https://www.kaggle.com/c/challenges-in-representation-learning-facial-expression-recognition-challenge/data) 217 |
218 | Emojis used were from https://emojiisland.com/ 219 |
220 | Ubuntu font license: https://ubuntu.com/legal/font-licence 221 | -------------------------------------------------------------------------------- /data/Ubuntu-R.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/Ubuntu-R.ttf -------------------------------------------------------------------------------- /data/emojis/angry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/angry.png -------------------------------------------------------------------------------- /data/emojis/disgusted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/disgusted.png -------------------------------------------------------------------------------- /data/emojis/fearful.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/fearful.png -------------------------------------------------------------------------------- /data/emojis/happy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/happy.png -------------------------------------------------------------------------------- /data/emojis/neutral.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/neutral.png -------------------------------------------------------------------------------- /data/emojis/sad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/sad.png -------------------------------------------------------------------------------- /data/emojis/surprised.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/emojis/surprised.png -------------------------------------------------------------------------------- /data/media/1.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/1.JPG -------------------------------------------------------------------------------- /data/media/2.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/2.JPG -------------------------------------------------------------------------------- /data/media/3.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/3.JPG -------------------------------------------------------------------------------- /data/media/4.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/4.JPG -------------------------------------------------------------------------------- /data/media/model_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/model_plot.png -------------------------------------------------------------------------------- /data/media/out3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/out3.jpg -------------------------------------------------------------------------------- /data/media/out4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/out4.jpg -------------------------------------------------------------------------------- /data/media/out5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/out5.jpg -------------------------------------------------------------------------------- /data/media/out6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/out6.jpg -------------------------------------------------------------------------------- /data/media/output.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/media/output.gif -------------------------------------------------------------------------------- /data/sample/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/1.jpg -------------------------------------------------------------------------------- /data/sample/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/2.jpg -------------------------------------------------------------------------------- /data/sample/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/3.jpg -------------------------------------------------------------------------------- /data/sample/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/4.jpg -------------------------------------------------------------------------------- /data/sample/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/5.jpg -------------------------------------------------------------------------------- /data/sample/6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/data/sample/6.jpg -------------------------------------------------------------------------------- /emotion_analyzer/emotion_detector.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Class for emotion analysis 6 | 7 | Usage: python -m emotion_analyzer.emotion_detector 8 | 9 | """ 10 | # =================================================== 11 | import numpy as np 12 | from emotion_analyzer.emotion_detector_base import EmotionDetectorBase 13 | from emotion_analyzer.exceptions import InvalidImage, ModelFileMissing, NoFaceDetected 14 | from emotion_analyzer.logger import LoggerFactory 15 | from emotion_analyzer.face_detection_dlib import FaceDetectorDlib 16 | from emotion_analyzer.face_detection_mtcnn import FaceDetectorMTCNN 17 | from emotion_analyzer.face_detection_opencv import FaceDetectorOpenCV 18 | import sys 19 | import os 20 | import cv2 21 | import dlib 22 | from decimal import Decimal 23 | from emotion_analyzer.media_utils import get_facial_ROI 24 | from emotion_analyzer.model_utils import define_model, load_model_weights 25 | from emotion_analyzer.validators import is_valid_img, path_exists 26 | 27 | # Load the custom logger 28 | logger = None 29 | try: 30 | logger_ob = LoggerFactory(logger_name=__name__) 31 | logger = logger_ob.get_logger() 32 | logger.info("{} loaded...".format(__name__)) 33 | # set exception hook for uncaught exceptions 34 | sys.excepthook = logger_ob.uncaught_exception_hook 35 | 36 | except Exception as exc: 37 | raise exc 38 | 39 | 40 | class EmotionDetector(EmotionDetectorBase): 41 | model_weights_filename = "weights.h5" 42 | keypoints_model_path = "shape_predictor_5_face_landmarks.dat" 43 | 44 | def __init__(self, 45 | model_loc: str='models', 46 | face_detection_threshold: int = 0.99, 47 | face_detector: str = "dlib",) -> None: 48 | 49 | # construct the model weights path 50 | model_weights_path = os.path.join(model_loc, EmotionDetector.model_weights_filename) 51 | # model path for facial keypoint detector, needed for dlib face detection 52 | keypoints_model_path = os.path.join( 53 | model_loc, EmotionDetector.keypoints_model_path 54 | ) 55 | 56 | if not ( 57 | path_exists(keypoints_model_path) or path_exists(model_weights_path) 58 | ): 59 | raise ModelFileMissing 60 | 61 | # load emotion model 62 | self.model = define_model() 63 | self.model = load_model_weights(self.model, model_weights_path) 64 | 65 | # select and load face detection model 66 | if face_detector == "opencv": 67 | self.face_detector = FaceDetectorOpenCV( 68 | model_loc=model_loc, crop_forehead=True, shrink_ratio=0.2 69 | ) 70 | elif face_detector == "mtcnn": 71 | self.face_detector = FaceDetectorMTCNN(crop_forehead=True, shrink_ratio=0.2) 72 | else: 73 | self.face_detector = FaceDetectorDlib() 74 | self.face_detection_threshold = face_detection_threshold 75 | 76 | self.keypoints_detector = dlib.shape_predictor(keypoints_model_path) 77 | 78 | 79 | def detect_emotion(self, img): 80 | """Detects emotion from faces in an image 81 | 82 | Img -> detect faces -> for each face: detect emotion 83 | Args: 84 | img (numpy matrix): input image 85 | 86 | Raises: 87 | InvalidImage: 88 | NoFaceDetected: 89 | 90 | Returns: 91 | str: emotion label 92 | """ 93 | # Check if image is valid 94 | if not is_valid_img(img): 95 | raise InvalidImage 96 | 97 | image = img.copy() 98 | 99 | emotions = [] 100 | try: 101 | bboxes = self.face_detector.detect_faces(image=image) 102 | if bboxes is None or len(bboxes) == 0: 103 | return emotions 104 | 105 | for bbox in bboxes: 106 | # extract the current face from image and run emotion detection 107 | face = get_facial_ROI(image, bbox) 108 | emotion, emotion_conf = self.detect_facial_emotion(face) 109 | facial_data = { "bbox": bbox, "emotion": emotion, "confidence_scores": emotion_conf} 110 | emotions.append(facial_data) 111 | 112 | except Exception as excep: 113 | raise excep 114 | 115 | return emotions 116 | 117 | 118 | def detect_facial_emotion(self, face) -> str: 119 | """Emotion detection on a assumed image of a facial region 120 | 121 | Args: 122 | face (numpy matrix): input image 123 | 124 | Raises: 125 | InvalidImage: 126 | 127 | Returns: 128 | emotion (str): detected emotion 129 | emotion_confidence (numpy array): prediction confidence of all the labels 130 | """ 131 | if not is_valid_img(face): 132 | raise InvalidImage 133 | 134 | # list of given emotions 135 | EMOTIONS = ['Angry', 'Disgusted', 'Fearful', 136 | 'Happy', 'Sad', 'Surprised', 'Neutral'] 137 | 138 | # detect emotion 139 | # resize image for the model 140 | face = cv2.resize(cv2.cvtColor(face, cv2.COLOR_BGR2GRAY), (48, 48)) 141 | face = np.reshape(face, (1, 48, 48, 1)) 142 | 143 | model_output = self.model.predict(face) 144 | detected_emotion = EMOTIONS[np.argmax(model_output[0])] 145 | 146 | # confidence for each emotion predication 147 | emotion_confidence = {} 148 | # Sum of all emotion confidence values 149 | total_sum = np.sum(model_output[0]) 150 | 151 | for index, emotion in enumerate(EMOTIONS): 152 | confidence = round(Decimal(model_output[0][index] / total_sum * 100), 2) 153 | emotion_confidence[emotion] = confidence 154 | 155 | return detected_emotion, emotion_confidence 156 | 157 | 158 | if __name__ == "__main__": 159 | # SAMPLE USAGE 160 | # from emotion_analyzer.media_utils import load_image_path 161 | 162 | # ob = EmotionDetector( 163 | # model_loc="models", 164 | # face_detector="dlib", 165 | # ) 166 | # img1 = load_image_path("data/sample/1.jpg") 167 | # emotion, emotion_conf = ob.detect_facial_emotion(img1) 168 | # print(emotion_conf) 169 | 170 | # emotions = ob.detect_emotion(img1) 171 | # print(emotions) 172 | pass -------------------------------------------------------------------------------- /emotion_analyzer/emotion_detector_base.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Base class for emotion detectcion 6 | 7 | Usage: python -m emotion_analyzer.emotion_detector_base 8 | 9 | """ 10 | # =================================================== 11 | from abc import ABC, abstractmethod 12 | 13 | 14 | class EmotionDetectorBase(ABC): 15 | @abstractmethod 16 | def detect_emotion(self): 17 | pass 18 | -------------------------------------------------------------------------------- /emotion_analyzer/exceptions.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Custom Exceptions""" 6 | # =================================================== 7 | 8 | 9 | class ModelFileMissing(Exception): 10 | """Exception raised when model related file is missing. 11 | 12 | Attributes: 13 | message: (str) Exception message 14 | """ 15 | 16 | def __init__(self): 17 | self.message = "Model file missing!!" 18 | 19 | 20 | class NoFaceDetected(Exception): 21 | """Raised when no face is detected in an image 22 | 23 | Attributes: 24 | message: (str) Exception message 25 | """ 26 | 27 | def __init__(self) -> None: 28 | self.message = "No face found in image!!" 29 | 30 | 31 | class MultipleFacesDetected(Exception): 32 | """Raised when multiple faces are detected in an image 33 | 34 | Attributes: 35 | message: (str) Exception message 36 | """ 37 | 38 | def __init__(self) -> None: 39 | self.message = "Multiple faces found in image!!" 40 | 41 | 42 | class InvalidImage(Exception): 43 | """Raised when an invalid image is encountered based on array dimension 44 | Attributes: 45 | message: (str) Exception message 46 | """ 47 | 48 | def __init__(self) -> None: 49 | self.message = "Invalid Image!!" 50 | 51 | 52 | class PathNotFound(Exception): 53 | """Raised when the path doesn't exist 54 | Attributes: 55 | message: (str) Exception message 56 | """ 57 | 58 | def __init__(self) -> None: 59 | self.message = "Path couldn't be found. Please check!!" 60 | 61 | 62 | class FaceMissing(Exception): 63 | """Raised when face is not found in an image 64 | Attributes: 65 | message: (str) Exception message 66 | """ 67 | 68 | def __init__(self) -> None: 69 | self.message = "Face not found!!" 70 | -------------------------------------------------------------------------------- /emotion_analyzer/face_detection_dlib.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Class for face detection. Uses face detectors 6 | from dlib. 7 | 8 | Usage: python -m emotion_analyzer.face_detection_dlib 9 | 10 | Ref: http://dlib.net/cnn_face_detector.py.html 11 | """ 12 | # =================================================== 13 | import os 14 | import sys 15 | from typing import List 16 | 17 | import dlib 18 | 19 | from emotion_analyzer.exceptions import InvalidImage, ModelFileMissing 20 | from emotion_analyzer.face_detector import FaceDetector 21 | from emotion_analyzer.logger import LoggerFactory 22 | from emotion_analyzer.validators import is_valid_img 23 | 24 | # Load the custom logger 25 | logger = None 26 | try: 27 | logger_ob = LoggerFactory(logger_name=__name__) 28 | logger = logger_ob.get_logger() 29 | logger.info("{} loaded...".format(__name__)) 30 | # set exception hook for uncaught exceptions 31 | sys.excepthook = logger_ob.uncaught_exception_hook 32 | except Exception as exc: 33 | raise exc 34 | 35 | 36 | class FaceDetectorDlib(FaceDetector): 37 | """Class for face detection. Uses face detectors from dlib. 38 | Raises: 39 | ModelFileMissing: [description] 40 | InvalidImage: [description] 41 | """ 42 | 43 | cnn_model_filename = "mmod_human_face_detector.dat" 44 | 45 | def __init__(self, model_loc: str = "models", model_type: str = "hog"): 46 | """Constructor 47 | 48 | Args: 49 | model_loc (str, optional): Path where the models are saved. 50 | Defaults to 'models'. 51 | model_type (str, optional): Supports HOG and MMOD based detectors. 52 | Defaults to 'hog'. 53 | 54 | Raises: 55 | ModelFileMissing: Raised when model file is not found 56 | """ 57 | try: 58 | # load the model 59 | if model_type == "hog": 60 | self.face_detector = dlib.get_frontal_face_detector() 61 | else: 62 | # MMOD model 63 | cnn_model_path = os.path.join( 64 | model_loc, FaceDetectorDlib.cnn_model_filename 65 | ) 66 | if not os.path.exists(cnn_model_path): 67 | raise ModelFileMissing 68 | self.face_detector = dlib.cnn_face_detection_model_v1(cnn_model_path) 69 | self.model_type = model_type 70 | logger.info("dlib: {} face detector loaded...".format(self.model_type)) 71 | except Exception as e: 72 | raise e 73 | 74 | 75 | def detect_faces(self, image, num_upscaling: int = 1) -> List[List[int]]: 76 | """Performs facial detection on an image. Works best with 77 | RGB image. Uses a dlib based detector either HOG or MMOD. 78 | 79 | Args: 80 | image (numpy array): 81 | num_upscaling (int, optional): Number of times to upscale 82 | while detecting faces. Defaults to 1. 83 | 84 | Raises: 85 | InvalidImage: When the image is either None or 86 | with wrong number of channels. 87 | 88 | Returns: 89 | List[List[int]]: List of bounding box coordinates 90 | """ 91 | if not is_valid_img(image): 92 | raise InvalidImage 93 | return [ 94 | self.dlib_rectangle_to_list(bbox) 95 | for bbox in self.face_detector(image, num_upscaling) 96 | ] 97 | 98 | def dlib_rectangle_to_list(self, dlib_bbox) -> List[int]: 99 | """Converts a dlib rectangle / mmod rectangle to 100 | List(top left x, top left y, bottom right x, bottom right y) 101 | 102 | Args: 103 | dlib_bbox (dlib.rectangle): 104 | 105 | Returns: 106 | List[int]: Bounding box coordinates 107 | """ 108 | # if it is MMOD type rectangle 109 | if type(dlib_bbox) == dlib.mmod_rectangle: 110 | dlib_bbox = dlib_bbox.rect 111 | # Top left corner 112 | x1, y1 = dlib_bbox.tl_corner().x, dlib_bbox.tl_corner().y 113 | width, height = dlib_bbox.width(), dlib_bbox.height() 114 | # Bottom right point 115 | x2, y2 = x1 + width, y1 + height 116 | 117 | return [x1, y1, x2, y2] 118 | 119 | def __repr__(self): 120 | return "FaceDetectorDlib" 121 | 122 | 123 | if __name__ == "__main__": 124 | 125 | # Sample Usage 126 | # ob = FaceDetectorDlib(model_type="hog") 127 | # img = cv2.imread("data/sample/2.jpg") 128 | # print(img.shape) 129 | # bbox = ob.detect_faces(convert_to_rgb(img)) 130 | # print(bbox) 131 | 132 | # draw_bounding_box(img, bbox) 133 | pass 134 | -------------------------------------------------------------------------------- /emotion_analyzer/face_detection_mtcnn.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Class for face detection. Uses a MTCNN 6 | based neural network to get the bounding box coordinates 7 | for a human face. 8 | 9 | Usage: python -m emotion_analyzer.face_detection_mtcnn 10 | 11 | You can install mtcnn using PIP by typing "pip install mtcnn" 12 | Ref: https://github.com/ipazc/mtcnn 13 | """ 14 | # =================================================== 15 | import sys 16 | from typing import List 17 | 18 | from mtcnn import MTCNN 19 | 20 | from emotion_analyzer.exceptions import InvalidImage 21 | from emotion_analyzer.face_detector import FaceDetector 22 | from emotion_analyzer.logger import LoggerFactory 23 | from emotion_analyzer.validators import is_valid_img 24 | 25 | # Load the custom logger 26 | logger = None 27 | try: 28 | logger_ob = LoggerFactory(logger_name=__name__) 29 | logger = logger_ob.get_logger() 30 | logger.info("{} loaded...".format(__name__)) 31 | # set exception hook for uncaught exceptions 32 | sys.excepthook = logger_ob.uncaught_exception_hook 33 | except Exception as exc: 34 | raise exc 35 | 36 | 37 | class FaceDetectorMTCNN(FaceDetector): 38 | """Class for face detection. Uses a MTCNN 39 | based neural network to get the bounding box coordinates 40 | for a human face. 41 | """ 42 | 43 | def __init__(self, crop_forehead: bool = True, shrink_ratio: int = 0.1): 44 | """Constructor 45 | 46 | Args: 47 | crop_forehead (bool, optional): Whether to trim the 48 | forehead in the detected facial ROI. Certain datasets 49 | like Dlib models are trained on cropped images without forehead. 50 | It can useful in those scenarios. 51 | Defaults to True. 52 | shrink_ratio (float, optional): Amount of height to shrink 53 | Defaults to 0.1 54 | """ 55 | try: 56 | # load the model 57 | self.face_detector = MTCNN() 58 | self.crop_forehead = crop_forehead 59 | self.shrink_ratio = shrink_ratio 60 | logger.info("MTCNN face detector loaded...") 61 | except Exception as e: 62 | raise e 63 | 64 | def detect_faces(self, image, conf_threshold: float = 0.7) -> List[List[int]]: 65 | """Performs facial detection on an image. Uses MTCNN. 66 | Args: 67 | image (numpy array): 68 | conf_threshold (float, optional): Threshold confidence to consider 69 | Raises: 70 | InvalidImage: When the image is either None or 71 | with wrong number of channels. 72 | 73 | Returns: 74 | List[List[int]]: List of bounding box coordinates 75 | """ 76 | if not is_valid_img(image): 77 | raise InvalidImage 78 | 79 | # Do a forward propagation with the blob created from input img 80 | detections = self.face_detector.detect_faces(image) 81 | # Bounding box coordinates of faces in image 82 | bboxes = [] 83 | for _, detection in enumerate(detections): 84 | conf = detection["confidence"] 85 | if conf >= conf_threshold: 86 | x, y, w, h = detection["box"] 87 | x1, y1, x2, y2 = x, y, x + w, y + h 88 | # Trim forehead area to match dlib style facial ROI 89 | if self.crop_forehead: 90 | y1 = y1 + int(h * self.shrink_ratio) 91 | bboxes.append([x1, y1, x2, y2]) 92 | 93 | return bboxes 94 | 95 | def dlib_face_crop(self, bbox: List[int], shrink_ratio: int = 0.2) -> List[int]: 96 | """ 97 | Crops an image in dlib styled facial ROI. 98 | Args: 99 | crop_forehead (bool, optional): Whether to trim the 100 | forehead in the detected facial ROI. Certain datasets 101 | like Dlib models are trained on cropped images without forehead. 102 | It can useful in those scenarios. 103 | Defaults to True. 104 | shrink_ratio (float, optional): Amount of height to shrink 105 | Defaults to 0.1. 106 | 107 | Returns: 108 | List[List[int]]: List of bounding box coordinates 109 | """ 110 | x1, y1, x2, y2 = bbox 111 | h, w = y2 - y1, x2 - x1 112 | # Shrink the height of box 113 | shift_y = int(shrink_ratio * h) 114 | return [x1, y1 + shift_y, x2, y2] 115 | 116 | def __repr__(self): 117 | return "FaceDetectorMTCNN" 118 | 119 | 120 | if __name__ == "__main__": 121 | 122 | # # Sample Usage 123 | # ob = FaceDetectorMTCNN(crop_forehead=False) 124 | # img = cv2.imread("data/sample/1.jpg") 125 | 126 | # # import numpy as np 127 | # # img = np.zeros((100,100,5), dtype='float32') 128 | # bbox = ob.detect_faces(convert_to_rgb(img)) 129 | # cv2.imwrite('data/out1.jpg', img) 130 | pass 131 | -------------------------------------------------------------------------------- /emotion_analyzer/face_detection_opencv.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Class for face detection. Uses a OpenCV's CNN 6 | model to get the bounding box coordinates for a human face. 7 | 8 | Usage: python -m emotion_analyzer.face_detection_opencv 9 | """ 10 | # =================================================== 11 | 12 | import os 13 | import sys 14 | from typing import List 15 | 16 | import cv2 17 | 18 | from emotion_analyzer.exceptions import InvalidImage, ModelFileMissing 19 | from emotion_analyzer.face_detector import FaceDetector 20 | from emotion_analyzer.logger import LoggerFactory 21 | from emotion_analyzer.validators import is_valid_img 22 | 23 | # Load the custom logger 24 | logger = None 25 | try: 26 | logger_ob = LoggerFactory(logger_name=__name__) 27 | logger = logger_ob.get_logger() 28 | logger.info("{} loaded...".format(__name__)) 29 | # set exception hook for uncaught exceptions 30 | sys.excepthook = logger_ob.uncaught_exception_hook 31 | except Exception as exc: 32 | raise exc 33 | 34 | 35 | class FaceDetectorOpenCV(FaceDetector): 36 | """Class for face detection. Uses a OpenCV's CNN 37 | model to get the bounding box coordinates for a human face. 38 | 39 | """ 40 | 41 | def __init__( 42 | self, model_loc="./models", crop_forehead: bool = True, shrink_ratio: int = 0.1 43 | ): 44 | """Constructor 45 | 46 | Args: 47 | model_loc (str, optional): Path where the models are saved. 48 | Defaults to 'models'. 49 | crop_forehead (bool, optional): Whether to trim the 50 | forehead in the detected facial ROI. Certain datasets 51 | like Dlib models are trained on cropped images without forehead. 52 | It can useful in those scenarios. 53 | Defaults to True. 54 | shrink_ratio (float, optional): Amount of height to shrink 55 | Defaults to 0.1 56 | Raises: 57 | ModelFileMissing: Raised when model file is not found 58 | """ 59 | # Model file and associated config path 60 | model_path = os.path.join(model_loc, "opencv_face_detector_uint8.pb") 61 | config_path = os.path.join(model_loc, "opencv_face_detector.pbtxt") 62 | 63 | self.crop_forehead = crop_forehead 64 | self.shrink_ratio = shrink_ratio 65 | if not os.path.exists(model_path) or not os.path.exists(config_path): 66 | raise ModelFileMissing 67 | try: 68 | # load the model 69 | self.face_detector = cv2.dnn.readNetFromTensorflow(model_path, config_path) 70 | except Exception as e: 71 | raise e 72 | 73 | 74 | def model_inference(self, image) -> List: 75 | # Run the face detection model on the image to get 76 | # bounding box coordinates 77 | # The model expects input as a blob, create input image blob 78 | img_blob = cv2.dnn.blobFromImage( 79 | image, 1.0, (300, 300), [104, 117, 123], False, False 80 | ) 81 | # Feed the input blob to NN and get the output layer predictions 82 | self.face_detector.setInput(img_blob) 83 | detections = self.face_detector.forward() 84 | 85 | return detections 86 | 87 | 88 | def detect_faces(self, image, conf_threshold: float = 0.7) -> List[List[int]]: 89 | """Performs facial detection on an image. Uses OpenCV DNN based face detector. 90 | Args: 91 | image (numpy array): 92 | conf_threshold (float, optional): Threshold confidence to consider 93 | Raises: 94 | InvalidImage: When the image is either None or 95 | with wrong number of channels. 96 | 97 | Returns: 98 | List[List[int]]: List of bounding box coordinates 99 | """ 100 | if not is_valid_img(image): 101 | raise InvalidImage 102 | # To prevent modification of orig img 103 | image = image.copy() 104 | height, width = image.shape[:2] 105 | 106 | # Do a forward propagation with the blob created from input img 107 | detections = self.model_inference(image) 108 | # Bounding box coordinates of faces in image 109 | bboxes = [] 110 | for idx in range(detections.shape[2]): 111 | conf = detections[0, 0, idx, 2] 112 | if conf >= conf_threshold: 113 | # Scale the bbox coordinates to suit image 114 | x1 = int(detections[0, 0, idx, 3] * width) 115 | y1 = int(detections[0, 0, idx, 4] * height) 116 | x2 = int(detections[0, 0, idx, 5] * width) 117 | y2 = int(detections[0, 0, idx, 6] * height) 118 | 119 | if self.crop_forehead: 120 | y1 = y1 + int(height * self.shrink_ratio) 121 | # openCv detector can give a lot of false bboxes 122 | # when the image is a zoomed in face / cropped face 123 | # This will get rid of atleast few, still there can be other 124 | # wrong detections present! 125 | if self.is_valid_bbox([x1, y1, x2, y2], height, width): 126 | bboxes.append([x1, y1, x2, y2]) 127 | 128 | return bboxes 129 | 130 | def is_valid_bbox(self, bbox: List[int], height: int, width: int) -> bool: 131 | """Checks if the bounding box exists in the image. 132 | 133 | Args: 134 | bbox (List[int]): Bounding box coordinates 135 | height (int): 136 | width (int): 137 | 138 | Returns: 139 | bool: Whether the bounding box is valid 140 | """ 141 | for idx in range(0, len(bbox), 2): 142 | if bbox[idx] < 0 or bbox[idx] >= width: 143 | return False 144 | for idx in range(1, len(bbox), 2): 145 | if bbox[idx] < 0 or bbox[idx] >= height: 146 | return False 147 | return True 148 | 149 | def __repr__(self): 150 | return "FaceDetectorOPENCV " 151 | 152 | 153 | if __name__ == "__main__": 154 | ############# Sample Usage ############# 155 | # ob = FaceDetectorOpenCV(model_loc="models", crop_forehead=False) 156 | # img = cv2.imread("data/media/8.jpg") 157 | 158 | # # import numpy as np 159 | # # img = np.zeros((100,100,5), dtype='float32') 160 | # bboxes = ob.detect_faces(convert_to_rgb(img), conf_threshold=0.99) 161 | 162 | # for bbox in bboxes: 163 | # cv2.imshow("Test", draw_bounding_box(img, bbox)) 164 | # cv2.waitKey(0) 165 | 166 | pass -------------------------------------------------------------------------------- /emotion_analyzer/face_detector.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Abstract class for face detectors""" 6 | # =================================================== 7 | from abc import ABC, abstractmethod 8 | 9 | 10 | class FaceDetector(ABC): 11 | @abstractmethod 12 | def detect_faces(self): 13 | pass 14 | -------------------------------------------------------------------------------- /emotion_analyzer/logger.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Custom logger factory with a deafult logger 6 | mathod to create a logger with file and console stream.""" 7 | # =================================================== 8 | import logging 9 | import sys 10 | import traceback 11 | from typing import List 12 | 13 | 14 | class LoggerFactory: 15 | """Custom logger factory with a deafult logger 16 | method to create a logger with file and console stream. 17 | """ 18 | 19 | def __init__(self, logger_name: str): 20 | """Creates a logger with default settings. 21 | 22 | Args: 23 | logger_name (str): logger name 24 | """ 25 | self.logger = self.create_logger(logger_name=logger_name) 26 | 27 | def create_formatter(self, format_pattern: str): 28 | """Creates a logger formatter with user defined/deafult format. 29 | 30 | Args: 31 | format_pattern (str, optional): Logger message format. Defaults to None. 32 | Returns: 33 | logger formatter 34 | """ 35 | format_pattern = ( 36 | format_pattern 37 | or "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s" 38 | ) 39 | return logging.Formatter(format_pattern) 40 | 41 | def get_console_handler(self, formatter, level=logging.INFO, stream=sys.stdout): 42 | """Returns a stream handler for logger 43 | 44 | Args: 45 | formatter : logger formatter object 46 | level (optional): Logger level. Defaults to logging.INFO. 47 | stream (stream, optional): Stream type. E.g: STDERR, STDOUT 48 | Defaults to sys.stdout. 49 | 50 | Returns: 51 | logger stream handler 52 | """ 53 | # create a stream handler, it can be for stdout or stderr 54 | console_handler = logging.StreamHandler(stream) 55 | console_handler.setFormatter(formatter) 56 | console_handler.setLevel(level) 57 | return console_handler 58 | 59 | def get_file_handler( 60 | self, formatter, level=logging.INFO, file_path: str = "data/app.log" 61 | ): 62 | """Returns a file handler for logger 63 | 64 | Args: 65 | formatter : logger formatter object 66 | level (optional): Logger level. Defaults to logging.INFO. 67 | file_path (str, optional): Path where the log file should be saved. 68 | Defaults to 'data/app.log'. 69 | 70 | Returns: 71 | logger file handler 72 | """ 73 | file_handler = logging.FileHandler(filename=file_path) 74 | file_handler.setFormatter(formatter) 75 | file_handler.setLevel(level) 76 | return file_handler 77 | 78 | def create_logger( 79 | self, 80 | logger_name, 81 | level=logging.DEBUG, 82 | format_pattern: str = None, 83 | file_path: str = "data/app.log", 84 | ): 85 | """Creates a logger with pre-defined settings 86 | 87 | Args: 88 | logger_name (str): Name of logger 89 | level (optional): Logger level. Defaults to logging.DEBUG. 90 | format_pattern (str, optional): Logger message format. Defaults to None. 91 | file_path (str, optional): Path where the log file should be saved. 92 | Defaults to 'data/app.log'. 93 | 94 | Returns: 95 | logger 96 | """ 97 | # Creates the default logger 98 | logger = logging.getLogger(logger_name) 99 | formatter = self.create_formatter(format_pattern=format_pattern) 100 | # Get the stream handlers, by default they are set at INFO level 101 | console_handler = self.get_console_handler( 102 | formatter=formatter, level=logging.INFO 103 | ) 104 | file_handler = self.get_file_handler( 105 | formatter=formatter, level=logging.INFO, file_path=file_path 106 | ) 107 | # Add all the stream handlers 108 | logger.addHandler(console_handler) 109 | logger.addHandler(file_handler) 110 | # Set the logger level, this is different from stream handler level 111 | # Stream handler levels are further filters when data reaches them 112 | logger.setLevel(level) 113 | logger.propagate = False 114 | return logger 115 | 116 | def get_logger(self): 117 | """Returns the created logger 118 | Returns: 119 | logger 120 | """ 121 | return self.logger 122 | 123 | def create_custom_logger( 124 | self, logger_name: str, handlers: List, propagate_error: bool = False 125 | ): 126 | """Creates a custom logger. 127 | 128 | Args: 129 | logger_name (str): Name of logger 130 | handlers (List) : Logger handlers. E.g: Stream handlers like file handler, console handler etc. 131 | propagate_error (bool, optional): Whether the errors should be propagated to the parent. 132 | Defaults to False. 133 | 134 | Returns: 135 | logger 136 | """ 137 | logger = logging.getLogger(logger_name) 138 | # Add all the stream handlers 139 | for handler in handlers: 140 | logger.addHandler(handler) 141 | logger.propagate = propagate_error 142 | return logger 143 | 144 | def uncaught_exception_hook(self, type, value, tb): 145 | """Handles uncaught exceptions and saves the details 146 | using the logger. So if the logger has console and file 147 | stream handlers, then the uncaught exception will be sent there. 148 | 149 | Args: 150 | logger (logger): Python native logger 151 | type (Exception type): 152 | value (Exception value): Exception message 153 | tb (traceback): 154 | 155 | """ 156 | # Returns a list of string sentences 157 | tb_message = traceback.extract_tb(tb).format() 158 | tb_message = "\n".join(tb_message) 159 | err_message = "Uncaught Exception raised! \n{}: {}\nMessage: {}".format( 160 | type, value, tb_message 161 | ) 162 | self.logger.critical(err_message) 163 | 164 | -------------------------------------------------------------------------------- /emotion_analyzer/media_utils.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Helper methods for media operations.""" 6 | # =================================================== 7 | from typing import List, Tuple 8 | 9 | import cv2 10 | import dlib 11 | import numpy as np 12 | from PIL import Image, ImageDraw, ImageFont 13 | 14 | from emotion_analyzer.exceptions import InvalidImage 15 | from emotion_analyzer.validators import is_valid_img 16 | 17 | 18 | # truetype font 19 | warning_font = ImageFont.truetype("data/Ubuntu-R.ttf", 20) 20 | annotation_font = ImageFont.truetype("data/Ubuntu-R.ttf", 16) 21 | bbox_font = ImageFont.truetype("data/Ubuntu-R.ttf", 16) 22 | 23 | def convert_to_rgb(image): 24 | """Converts an image to RGB format. 25 | 26 | Args: 27 | image (numpy array): [description] 28 | 29 | Raises: 30 | InvalidImage: [description] 31 | 32 | Returns: 33 | [type]: [description] 34 | """ 35 | if not is_valid_img(image): 36 | raise InvalidImage 37 | return cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 38 | 39 | 40 | def convert_to_dlib_rectangle(bbox): 41 | """Converts a bounding box coordinate list 42 | to dlib rectangle. 43 | 44 | Args: 45 | bbox (List[int]): Bounding box coordinates 46 | 47 | Returns: 48 | dlib.rectangle: Dlib rectangle 49 | """ 50 | return dlib.rectangle(bbox[0], bbox[1], bbox[2], bbox[3]) 51 | 52 | 53 | def load_image_path(img_path, mode: str = "rgb"): 54 | """Loads image from disk. Optional mode 55 | to load in RGB mode 56 | 57 | Args: 58 | img_path (numpy array): [description] 59 | mode (str, optional): Whether to load in RGB format. 60 | Defaults to 'rgb'. 61 | 62 | Raises: 63 | exc: [description] 64 | 65 | Returns: 66 | [type]: [description] 67 | """ 68 | try: 69 | img = cv2.imread(img_path) 70 | if mode == "rgb": 71 | return convert_to_rgb(img) 72 | return img 73 | except Exception as exc: 74 | raise exc 75 | 76 | 77 | def draw_bounding_box(image, bbox: List[int], color: Tuple = (0, 255, 0)): 78 | """Used for drawing bounding box on an image 79 | 80 | Args: 81 | image (numpy array): [description] 82 | bbox (List[int]): Bounding box coordinates 83 | color (Tuple, optional): [description]. Defaults to (0,255,0). 84 | 85 | Returns: 86 | [type]: [description] 87 | """ 88 | x1, y1, x2, y2 = bbox 89 | cv2.rectangle(image, (x1, y1), (x2, y2), color, 2) 90 | return image 91 | 92 | 93 | def draw_bounding_box_annotation(image, label: str, bbox: List[int], color: Tuple = (84, 247, 131)): 94 | """Used for drawing bounding box and label on an image 95 | 96 | Args: 97 | image (numpy array): [description] 98 | name (str): Label to annotate 99 | bbox (List[int]): Bounding box coordinates 100 | color (Tuple, optional): [description]. Defaults to (0,255,0). 101 | 102 | Returns: 103 | [type]: [description] 104 | """ 105 | x1, y1, x2, y2 = bbox 106 | # Make the bbox a bit taller than usual to cover the face properly 107 | draw_bounding_box(image, [x1, y1, x2, y2 + 20], color=color) 108 | 109 | # Draw the label with name below the face 110 | cv2.rectangle(image, (x1, y2), (x2, y2 + 20), color, cv2.FILLED) 111 | pil_img = Image.fromarray(image) 112 | 113 | # PIL drawing context 114 | draw = ImageDraw.Draw(pil_img) 115 | draw.text((x1 + 20, y2 + 2), label, (0, 0, 0), font=bbox_font) 116 | 117 | # Convert PIL img to numpy array type 118 | return np.array(pil_img) 119 | 120 | 121 | def annotate_warning(warning_text: str, img): 122 | """Draws warning text at the bottom of screen 123 | 124 | Args: 125 | warning_text (str): warning label 126 | img (numpy array): input image 127 | """ 128 | h, _, _ = img.shape 129 | x, y = 150, h - 100 130 | 131 | pil_img = Image.fromarray(img.copy()) 132 | 133 | # PIL drawing context 134 | draw = ImageDraw.Draw(pil_img) 135 | draw.text((x+1, y+1), warning_text, (0, 0, 0), font=warning_font) 136 | draw.text((x, y), warning_text, (12, 52, 242), font=warning_font) 137 | 138 | warning_text = "Emotion chart will be shown for one person only!" 139 | draw.text((x, y+31), warning_text, (255, 255, 255), font=warning_font) 140 | 141 | # Convert PIL img to numpy array type 142 | return np.array(pil_img) 143 | 144 | 145 | def annotate_emotion_stats(emotion_data, img): 146 | """Draws a bar chart of emotion labels on top of image 147 | 148 | Args: 149 | emotion_data (Dict): Emotions and their respective prediction confidence 150 | img (numpy array): input image 151 | """ 152 | 153 | for index, emotion in enumerate(emotion_data.keys()): 154 | # for drawing progress bar 155 | cv2.rectangle(img, (100, index * 20 + 20), (100 + int(emotion_data[emotion]), (index + 1) * 20 + 18), 156 | (255, 0, 0), -1) 157 | 158 | # convert to PIL format image 159 | pil_img = Image.fromarray(img.copy()) 160 | # PIL drawing context 161 | draw = ImageDraw.Draw(pil_img) 162 | 163 | for index, emotion in enumerate(emotion_data.keys()): 164 | # for putting emotion labels 165 | draw.text((10, index * 20 + 20), emotion, (7, 109, 16), font=annotation_font) 166 | 167 | emotion_confidence = str(emotion_data[emotion]) + "%" 168 | # for putting percentage confidence 169 | draw.text((105 + int(emotion_data[emotion]), index * 20 + 20), emotion_confidence, (255, 0, 0), font=annotation_font) 170 | 171 | # Convert PIL img to numpy array type 172 | return np.array(pil_img) 173 | 174 | 175 | def draw_emoji(emoji, img): 176 | """Puts an emoji img on top of another image. 177 | 178 | Args: 179 | emoji (numpy array): emoji picture 180 | img (numpy array): input image 181 | """ 182 | # overlay emoji on the frame for all the channels 183 | for c in range(0, 3): 184 | # for doing overlay we need to assign weights to both foreground and background 185 | foreground = emoji[:, :, c] * (emoji[:, :, 3] / 255.0) 186 | background = img[350:470, 10:130, c] * (1.0 - emoji[:, :, 3] / 255.0) 187 | img[350:470, 10:130, c] = foreground + background 188 | 189 | return img 190 | 191 | 192 | def get_facial_ROI(image, bbox: List[int]): 193 | """Extracts the facial region in an image 194 | using the bounding box coordinates. 195 | 196 | Args: 197 | image ([type]): [description] 198 | bbox (List[int]): [description] 199 | 200 | Raises: 201 | InvalidImage: [description] 202 | 203 | Returns: 204 | [type]: [description] 205 | """ 206 | if image is None or bbox is None: 207 | raise InvalidImage if image is None else ValueError 208 | return image[bbox[1] : bbox[3], bbox[0] : bbox[2], :] 209 | 210 | 211 | def get_video_writer(video_stream, output_filename: str = "data/output.mp4"): 212 | """Returns an OpenCV video writer with mp4 codec stream 213 | 214 | Args: 215 | video_stream (OpenCV video stream obj): Input video stream 216 | output_filename (str): 217 | 218 | Returns: 219 | OpenCV VideoWriter: 220 | """ 221 | try: 222 | fourcc = cv2.VideoWriter_fourcc(*"mp4v") 223 | FPS = video_stream.get(cv2.CAP_PROP_FPS) 224 | 225 | # (Width, Height) 226 | dims = (int(video_stream.get(3)), int(video_stream.get(4))) 227 | video_writer = cv2.VideoWriter(output_filename, fourcc, FPS, dims) 228 | return video_writer 229 | except Exception as exc: 230 | raise exc 231 | -------------------------------------------------------------------------------- /emotion_analyzer/model_utils.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Helper methods for Emotion model related work 6 | 7 | Usage: python -m emotion_analyzer.model_utils 8 | 9 | """ 10 | # =================================================== 11 | from tensorflow.keras.models import Sequential 12 | from tensorflow.keras.layers import Dense, Activation, Dropout, Flatten, BatchNormalization 13 | from tensorflow.keras.layers import Conv2D 14 | from tensorflow.keras.layers import MaxPool2D 15 | import os.path 16 | 17 | from emotion_analyzer.exceptions import ModelFileMissing 18 | 19 | 20 | def define_model(): 21 | """Creates a model from a predefined architecture 22 | 23 | Returns: 24 | model: Sequential keras model 25 | """ 26 | model = Sequential() 27 | 28 | # 1st stage 29 | model.add(Conv2D(32, 3, input_shape=(48, 48, 1), padding='same', 30 | activation='relu')) 31 | model.add(BatchNormalization()) 32 | model.add(Conv2D(32, 3, padding='same', 33 | activation='relu')) 34 | model.add(BatchNormalization()) 35 | model.add(MaxPool2D(pool_size=(2, 2), strides=2)) 36 | 37 | # 2nd stage 38 | model.add(Conv2D(64, 3, padding='same', 39 | activation='relu')) 40 | model.add(BatchNormalization()) 41 | model.add(Conv2D(64, 3, padding='same', 42 | activation='relu')) 43 | model.add(BatchNormalization()) 44 | model.add(MaxPool2D(pool_size=(2, 2), strides=2)) 45 | model.add(Dropout(0.25)) 46 | 47 | # 3rd stage 48 | model.add(Conv2D(128, 3, padding='same', 49 | activation='relu')) 50 | model.add(BatchNormalization()) 51 | model.add(Conv2D(128, 3, padding='same', 52 | activation='relu')) 53 | model.add(BatchNormalization()) 54 | model.add(MaxPool2D(pool_size=(2, 2), strides=2)) 55 | model.add(Dropout(0.25)) 56 | 57 | # FC layers 58 | model.add(Flatten()) 59 | model.add(Dense(256)) 60 | model.add(Activation("relu")) 61 | model.add(BatchNormalization()) 62 | model.add(Dropout(0.5)) 63 | 64 | model.add(Dense(256)) 65 | model.add(Activation("relu")) 66 | model.add(BatchNormalization()) 67 | model.add(Dropout(0.5)) 68 | 69 | model.add(Dense(256)) 70 | model.add(Activation("relu")) 71 | model.add(BatchNormalization()) 72 | model.add(Dropout(0.5)) 73 | 74 | model.add(Dense(7)) 75 | model.add(Activation('softmax')) 76 | 77 | # compile the model 78 | model.compile(loss='categorical_crossentropy', 79 | optimizer='adam', metrics=['accuracy']) 80 | 81 | return model 82 | 83 | 84 | def load_model_weights(model, model_path='./models/weights.h5'): 85 | """Loads trained model weights from model file. 86 | 87 | Args: 88 | model (keras model): [Untrained model with init weights] 89 | 90 | Returns: 91 | [keras model]: [Model loaded with trained weights] 92 | """ 93 | if os.path.exists(model_path): 94 | model.load_weights(model_path) 95 | else: 96 | raise ModelFileMissing 97 | return model -------------------------------------------------------------------------------- /emotion_analyzer/validators.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Helper methods for validation.""" 6 | # =================================================== 7 | import os 8 | 9 | 10 | def is_valid_img(image) -> bool: 11 | """Checks if an image is valid or not. 12 | 13 | Args: 14 | image (numpy array): [description] 15 | 16 | Returns: 17 | bool: [description] 18 | """ 19 | return image is None or not (len(image.shape) != 3 or image.shape[-1] != 3) 20 | 21 | 22 | def path_exists(path: str = None) -> bool: 23 | """Checks if a path exists. 24 | 25 | Args: 26 | path (str, optional): [description]. Defaults to None. 27 | 28 | Returns: 29 | bool: [description] 30 | """ 31 | if path and os.path.exists(path): 32 | return True 33 | return False 34 | -------------------------------------------------------------------------------- /models/mmod_human_face_detector.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/models/mmod_human_face_detector.dat -------------------------------------------------------------------------------- /models/opencv_face_detector.pbtxt: -------------------------------------------------------------------------------- 1 | node { 2 | name: "data" 3 | op: "Placeholder" 4 | attr { 5 | key: "dtype" 6 | value { 7 | type: DT_FLOAT 8 | } 9 | } 10 | } 11 | node { 12 | name: "data_bn/FusedBatchNorm" 13 | op: "FusedBatchNorm" 14 | input: "data:0" 15 | input: "data_bn/gamma" 16 | input: "data_bn/beta" 17 | input: "data_bn/mean" 18 | input: "data_bn/std" 19 | attr { 20 | key: "epsilon" 21 | value { 22 | f: 1.00099996416e-05 23 | } 24 | } 25 | } 26 | node { 27 | name: "data_scale/Mul" 28 | op: "Mul" 29 | input: "data_bn/FusedBatchNorm" 30 | input: "data_scale/mul" 31 | } 32 | node { 33 | name: "data_scale/BiasAdd" 34 | op: "BiasAdd" 35 | input: "data_scale/Mul" 36 | input: "data_scale/add" 37 | } 38 | node { 39 | name: "SpaceToBatchND/block_shape" 40 | op: "Const" 41 | attr { 42 | key: "value" 43 | value { 44 | tensor { 45 | dtype: DT_INT32 46 | tensor_shape { 47 | dim { 48 | size: 2 49 | } 50 | } 51 | int_val: 1 52 | int_val: 1 53 | } 54 | } 55 | } 56 | } 57 | node { 58 | name: "SpaceToBatchND/paddings" 59 | op: "Const" 60 | attr { 61 | key: "value" 62 | value { 63 | tensor { 64 | dtype: DT_INT32 65 | tensor_shape { 66 | dim { 67 | size: 2 68 | } 69 | dim { 70 | size: 2 71 | } 72 | } 73 | int_val: 3 74 | int_val: 3 75 | int_val: 3 76 | int_val: 3 77 | } 78 | } 79 | } 80 | } 81 | node { 82 | name: "Pad" 83 | op: "SpaceToBatchND" 84 | input: "data_scale/BiasAdd" 85 | input: "SpaceToBatchND/block_shape" 86 | input: "SpaceToBatchND/paddings" 87 | } 88 | node { 89 | name: "conv1_h/Conv2D" 90 | op: "Conv2D" 91 | input: "Pad" 92 | input: "conv1_h/weights" 93 | attr { 94 | key: "dilations" 95 | value { 96 | list { 97 | i: 1 98 | i: 1 99 | i: 1 100 | i: 1 101 | } 102 | } 103 | } 104 | attr { 105 | key: "padding" 106 | value { 107 | s: "VALID" 108 | } 109 | } 110 | attr { 111 | key: "strides" 112 | value { 113 | list { 114 | i: 1 115 | i: 2 116 | i: 2 117 | i: 1 118 | } 119 | } 120 | } 121 | } 122 | node { 123 | name: "conv1_h/BiasAdd" 124 | op: "BiasAdd" 125 | input: "conv1_h/Conv2D" 126 | input: "conv1_h/bias" 127 | } 128 | node { 129 | name: "BatchToSpaceND" 130 | op: "BatchToSpaceND" 131 | input: "conv1_h/BiasAdd" 132 | } 133 | node { 134 | name: "conv1_bn_h/FusedBatchNorm" 135 | op: "FusedBatchNorm" 136 | input: "BatchToSpaceND" 137 | input: "conv1_bn_h/gamma" 138 | input: "conv1_bn_h/beta" 139 | input: "conv1_bn_h/mean" 140 | input: "conv1_bn_h/std" 141 | attr { 142 | key: "epsilon" 143 | value { 144 | f: 1.00099996416e-05 145 | } 146 | } 147 | } 148 | node { 149 | name: "conv1_scale_h/Mul" 150 | op: "Mul" 151 | input: "conv1_bn_h/FusedBatchNorm" 152 | input: "conv1_scale_h/mul" 153 | } 154 | node { 155 | name: "conv1_scale_h/BiasAdd" 156 | op: "BiasAdd" 157 | input: "conv1_scale_h/Mul" 158 | input: "conv1_scale_h/add" 159 | } 160 | node { 161 | name: "Relu" 162 | op: "Relu" 163 | input: "conv1_scale_h/BiasAdd" 164 | } 165 | node { 166 | name: "conv1_pool/MaxPool" 167 | op: "MaxPool" 168 | input: "Relu" 169 | attr { 170 | key: "ksize" 171 | value { 172 | list { 173 | i: 1 174 | i: 3 175 | i: 3 176 | i: 1 177 | } 178 | } 179 | } 180 | attr { 181 | key: "padding" 182 | value { 183 | s: "SAME" 184 | } 185 | } 186 | attr { 187 | key: "strides" 188 | value { 189 | list { 190 | i: 1 191 | i: 2 192 | i: 2 193 | i: 1 194 | } 195 | } 196 | } 197 | } 198 | node { 199 | name: "layer_64_1_conv1_h/Conv2D" 200 | op: "Conv2D" 201 | input: "conv1_pool/MaxPool" 202 | input: "layer_64_1_conv1_h/weights" 203 | attr { 204 | key: "dilations" 205 | value { 206 | list { 207 | i: 1 208 | i: 1 209 | i: 1 210 | i: 1 211 | } 212 | } 213 | } 214 | attr { 215 | key: "padding" 216 | value { 217 | s: "SAME" 218 | } 219 | } 220 | attr { 221 | key: "strides" 222 | value { 223 | list { 224 | i: 1 225 | i: 1 226 | i: 1 227 | i: 1 228 | } 229 | } 230 | } 231 | } 232 | node { 233 | name: "layer_64_1_bn2_h/FusedBatchNorm" 234 | op: "BiasAdd" 235 | input: "layer_64_1_conv1_h/Conv2D" 236 | input: "layer_64_1_conv1_h/Conv2D_bn_offset" 237 | } 238 | node { 239 | name: "layer_64_1_scale2_h/Mul" 240 | op: "Mul" 241 | input: "layer_64_1_bn2_h/FusedBatchNorm" 242 | input: "layer_64_1_scale2_h/mul" 243 | } 244 | node { 245 | name: "layer_64_1_scale2_h/BiasAdd" 246 | op: "BiasAdd" 247 | input: "layer_64_1_scale2_h/Mul" 248 | input: "layer_64_1_scale2_h/add" 249 | } 250 | node { 251 | name: "Relu_1" 252 | op: "Relu" 253 | input: "layer_64_1_scale2_h/BiasAdd" 254 | } 255 | node { 256 | name: "layer_64_1_conv2_h/Conv2D" 257 | op: "Conv2D" 258 | input: "Relu_1" 259 | input: "layer_64_1_conv2_h/weights" 260 | attr { 261 | key: "dilations" 262 | value { 263 | list { 264 | i: 1 265 | i: 1 266 | i: 1 267 | i: 1 268 | } 269 | } 270 | } 271 | attr { 272 | key: "padding" 273 | value { 274 | s: "SAME" 275 | } 276 | } 277 | attr { 278 | key: "strides" 279 | value { 280 | list { 281 | i: 1 282 | i: 1 283 | i: 1 284 | i: 1 285 | } 286 | } 287 | } 288 | } 289 | node { 290 | name: "add" 291 | op: "Add" 292 | input: "layer_64_1_conv2_h/Conv2D" 293 | input: "conv1_pool/MaxPool" 294 | } 295 | node { 296 | name: "layer_128_1_bn1_h/FusedBatchNorm" 297 | op: "FusedBatchNorm" 298 | input: "add" 299 | input: "layer_128_1_bn1_h/gamma" 300 | input: "layer_128_1_bn1_h/beta" 301 | input: "layer_128_1_bn1_h/mean" 302 | input: "layer_128_1_bn1_h/std" 303 | attr { 304 | key: "epsilon" 305 | value { 306 | f: 1.00099996416e-05 307 | } 308 | } 309 | } 310 | node { 311 | name: "layer_128_1_scale1_h/Mul" 312 | op: "Mul" 313 | input: "layer_128_1_bn1_h/FusedBatchNorm" 314 | input: "layer_128_1_scale1_h/mul" 315 | } 316 | node { 317 | name: "layer_128_1_scale1_h/BiasAdd" 318 | op: "BiasAdd" 319 | input: "layer_128_1_scale1_h/Mul" 320 | input: "layer_128_1_scale1_h/add" 321 | } 322 | node { 323 | name: "Relu_2" 324 | op: "Relu" 325 | input: "layer_128_1_scale1_h/BiasAdd" 326 | } 327 | node { 328 | name: "layer_128_1_conv_expand_h/Conv2D" 329 | op: "Conv2D" 330 | input: "Relu_2" 331 | input: "layer_128_1_conv_expand_h/weights" 332 | attr { 333 | key: "dilations" 334 | value { 335 | list { 336 | i: 1 337 | i: 1 338 | i: 1 339 | i: 1 340 | } 341 | } 342 | } 343 | attr { 344 | key: "padding" 345 | value { 346 | s: "SAME" 347 | } 348 | } 349 | attr { 350 | key: "strides" 351 | value { 352 | list { 353 | i: 1 354 | i: 2 355 | i: 2 356 | i: 1 357 | } 358 | } 359 | } 360 | } 361 | node { 362 | name: "layer_128_1_conv1_h/Conv2D" 363 | op: "Conv2D" 364 | input: "Relu_2" 365 | input: "layer_128_1_conv1_h/weights" 366 | attr { 367 | key: "dilations" 368 | value { 369 | list { 370 | i: 1 371 | i: 1 372 | i: 1 373 | i: 1 374 | } 375 | } 376 | } 377 | attr { 378 | key: "padding" 379 | value { 380 | s: "SAME" 381 | } 382 | } 383 | attr { 384 | key: "strides" 385 | value { 386 | list { 387 | i: 1 388 | i: 2 389 | i: 2 390 | i: 1 391 | } 392 | } 393 | } 394 | } 395 | node { 396 | name: "layer_128_1_bn2/FusedBatchNorm" 397 | op: "BiasAdd" 398 | input: "layer_128_1_conv1_h/Conv2D" 399 | input: "layer_128_1_conv1_h/Conv2D_bn_offset" 400 | } 401 | node { 402 | name: "layer_128_1_scale2/Mul" 403 | op: "Mul" 404 | input: "layer_128_1_bn2/FusedBatchNorm" 405 | input: "layer_128_1_scale2/mul" 406 | } 407 | node { 408 | name: "layer_128_1_scale2/BiasAdd" 409 | op: "BiasAdd" 410 | input: "layer_128_1_scale2/Mul" 411 | input: "layer_128_1_scale2/add" 412 | } 413 | node { 414 | name: "Relu_3" 415 | op: "Relu" 416 | input: "layer_128_1_scale2/BiasAdd" 417 | } 418 | node { 419 | name: "layer_128_1_conv2/Conv2D" 420 | op: "Conv2D" 421 | input: "Relu_3" 422 | input: "layer_128_1_conv2/weights" 423 | attr { 424 | key: "dilations" 425 | value { 426 | list { 427 | i: 1 428 | i: 1 429 | i: 1 430 | i: 1 431 | } 432 | } 433 | } 434 | attr { 435 | key: "padding" 436 | value { 437 | s: "SAME" 438 | } 439 | } 440 | attr { 441 | key: "strides" 442 | value { 443 | list { 444 | i: 1 445 | i: 1 446 | i: 1 447 | i: 1 448 | } 449 | } 450 | } 451 | } 452 | node { 453 | name: "add_1" 454 | op: "Add" 455 | input: "layer_128_1_conv2/Conv2D" 456 | input: "layer_128_1_conv_expand_h/Conv2D" 457 | } 458 | node { 459 | name: "layer_256_1_bn1/FusedBatchNorm" 460 | op: "FusedBatchNorm" 461 | input: "add_1" 462 | input: "layer_256_1_bn1/gamma" 463 | input: "layer_256_1_bn1/beta" 464 | input: "layer_256_1_bn1/mean" 465 | input: "layer_256_1_bn1/std" 466 | attr { 467 | key: "epsilon" 468 | value { 469 | f: 1.00099996416e-05 470 | } 471 | } 472 | } 473 | node { 474 | name: "layer_256_1_scale1/Mul" 475 | op: "Mul" 476 | input: "layer_256_1_bn1/FusedBatchNorm" 477 | input: "layer_256_1_scale1/mul" 478 | } 479 | node { 480 | name: "layer_256_1_scale1/BiasAdd" 481 | op: "BiasAdd" 482 | input: "layer_256_1_scale1/Mul" 483 | input: "layer_256_1_scale1/add" 484 | } 485 | node { 486 | name: "Relu_4" 487 | op: "Relu" 488 | input: "layer_256_1_scale1/BiasAdd" 489 | } 490 | node { 491 | name: "SpaceToBatchND_1/paddings" 492 | op: "Const" 493 | attr { 494 | key: "value" 495 | value { 496 | tensor { 497 | dtype: DT_INT32 498 | tensor_shape { 499 | dim { 500 | size: 2 501 | } 502 | dim { 503 | size: 2 504 | } 505 | } 506 | int_val: 1 507 | int_val: 1 508 | int_val: 1 509 | int_val: 1 510 | } 511 | } 512 | } 513 | } 514 | node { 515 | name: "layer_256_1_conv_expand/Conv2D" 516 | op: "Conv2D" 517 | input: "Relu_4" 518 | input: "layer_256_1_conv_expand/weights" 519 | attr { 520 | key: "dilations" 521 | value { 522 | list { 523 | i: 1 524 | i: 1 525 | i: 1 526 | i: 1 527 | } 528 | } 529 | } 530 | attr { 531 | key: "padding" 532 | value { 533 | s: "SAME" 534 | } 535 | } 536 | attr { 537 | key: "strides" 538 | value { 539 | list { 540 | i: 1 541 | i: 2 542 | i: 2 543 | i: 1 544 | } 545 | } 546 | } 547 | } 548 | node { 549 | name: "conv4_3_norm/l2_normalize" 550 | op: "L2Normalize" 551 | input: "Relu_4:0" 552 | input: "conv4_3_norm/l2_normalize/Sum/reduction_indices" 553 | } 554 | node { 555 | name: "conv4_3_norm/mul_1" 556 | op: "Mul" 557 | input: "conv4_3_norm/l2_normalize" 558 | input: "conv4_3_norm/mul" 559 | } 560 | node { 561 | name: "conv4_3_norm_mbox_loc/Conv2D" 562 | op: "Conv2D" 563 | input: "conv4_3_norm/mul_1" 564 | input: "conv4_3_norm_mbox_loc/weights" 565 | attr { 566 | key: "dilations" 567 | value { 568 | list { 569 | i: 1 570 | i: 1 571 | i: 1 572 | i: 1 573 | } 574 | } 575 | } 576 | attr { 577 | key: "padding" 578 | value { 579 | s: "SAME" 580 | } 581 | } 582 | attr { 583 | key: "strides" 584 | value { 585 | list { 586 | i: 1 587 | i: 1 588 | i: 1 589 | i: 1 590 | } 591 | } 592 | } 593 | } 594 | node { 595 | name: "conv4_3_norm_mbox_loc/BiasAdd" 596 | op: "BiasAdd" 597 | input: "conv4_3_norm_mbox_loc/Conv2D" 598 | input: "conv4_3_norm_mbox_loc/bias" 599 | } 600 | node { 601 | name: "flatten/Reshape" 602 | op: "Flatten" 603 | input: "conv4_3_norm_mbox_loc/BiasAdd" 604 | } 605 | node { 606 | name: "conv4_3_norm_mbox_conf/Conv2D" 607 | op: "Conv2D" 608 | input: "conv4_3_norm/mul_1" 609 | input: "conv4_3_norm_mbox_conf/weights" 610 | attr { 611 | key: "dilations" 612 | value { 613 | list { 614 | i: 1 615 | i: 1 616 | i: 1 617 | i: 1 618 | } 619 | } 620 | } 621 | attr { 622 | key: "padding" 623 | value { 624 | s: "SAME" 625 | } 626 | } 627 | attr { 628 | key: "strides" 629 | value { 630 | list { 631 | i: 1 632 | i: 1 633 | i: 1 634 | i: 1 635 | } 636 | } 637 | } 638 | } 639 | node { 640 | name: "conv4_3_norm_mbox_conf/BiasAdd" 641 | op: "BiasAdd" 642 | input: "conv4_3_norm_mbox_conf/Conv2D" 643 | input: "conv4_3_norm_mbox_conf/bias" 644 | } 645 | node { 646 | name: "flatten_6/Reshape" 647 | op: "Flatten" 648 | input: "conv4_3_norm_mbox_conf/BiasAdd" 649 | } 650 | node { 651 | name: "Pad_1" 652 | op: "SpaceToBatchND" 653 | input: "Relu_4" 654 | input: "SpaceToBatchND/block_shape" 655 | input: "SpaceToBatchND_1/paddings" 656 | } 657 | node { 658 | name: "layer_256_1_conv1/Conv2D" 659 | op: "Conv2D" 660 | input: "Pad_1" 661 | input: "layer_256_1_conv1/weights" 662 | attr { 663 | key: "dilations" 664 | value { 665 | list { 666 | i: 1 667 | i: 1 668 | i: 1 669 | i: 1 670 | } 671 | } 672 | } 673 | attr { 674 | key: "padding" 675 | value { 676 | s: "VALID" 677 | } 678 | } 679 | attr { 680 | key: "strides" 681 | value { 682 | list { 683 | i: 1 684 | i: 2 685 | i: 2 686 | i: 1 687 | } 688 | } 689 | } 690 | } 691 | node { 692 | name: "layer_256_1_bn2/FusedBatchNorm" 693 | op: "BiasAdd" 694 | input: "layer_256_1_conv1/Conv2D" 695 | input: "layer_256_1_conv1/Conv2D_bn_offset" 696 | } 697 | node { 698 | name: "BatchToSpaceND_1" 699 | op: "BatchToSpaceND" 700 | input: "layer_256_1_bn2/FusedBatchNorm" 701 | } 702 | node { 703 | name: "layer_256_1_scale2/Mul" 704 | op: "Mul" 705 | input: "BatchToSpaceND_1" 706 | input: "layer_256_1_scale2/mul" 707 | } 708 | node { 709 | name: "layer_256_1_scale2/BiasAdd" 710 | op: "BiasAdd" 711 | input: "layer_256_1_scale2/Mul" 712 | input: "layer_256_1_scale2/add" 713 | } 714 | node { 715 | name: "Relu_5" 716 | op: "Relu" 717 | input: "layer_256_1_scale2/BiasAdd" 718 | } 719 | node { 720 | name: "layer_256_1_conv2/Conv2D" 721 | op: "Conv2D" 722 | input: "Relu_5" 723 | input: "layer_256_1_conv2/weights" 724 | attr { 725 | key: "dilations" 726 | value { 727 | list { 728 | i: 1 729 | i: 1 730 | i: 1 731 | i: 1 732 | } 733 | } 734 | } 735 | attr { 736 | key: "padding" 737 | value { 738 | s: "SAME" 739 | } 740 | } 741 | attr { 742 | key: "strides" 743 | value { 744 | list { 745 | i: 1 746 | i: 1 747 | i: 1 748 | i: 1 749 | } 750 | } 751 | } 752 | } 753 | node { 754 | name: "add_2" 755 | op: "Add" 756 | input: "layer_256_1_conv2/Conv2D" 757 | input: "layer_256_1_conv_expand/Conv2D" 758 | } 759 | node { 760 | name: "layer_512_1_bn1/FusedBatchNorm" 761 | op: "FusedBatchNorm" 762 | input: "add_2" 763 | input: "layer_512_1_bn1/gamma" 764 | input: "layer_512_1_bn1/beta" 765 | input: "layer_512_1_bn1/mean" 766 | input: "layer_512_1_bn1/std" 767 | attr { 768 | key: "epsilon" 769 | value { 770 | f: 1.00099996416e-05 771 | } 772 | } 773 | } 774 | node { 775 | name: "layer_512_1_scale1/Mul" 776 | op: "Mul" 777 | input: "layer_512_1_bn1/FusedBatchNorm" 778 | input: "layer_512_1_scale1/mul" 779 | } 780 | node { 781 | name: "layer_512_1_scale1/BiasAdd" 782 | op: "BiasAdd" 783 | input: "layer_512_1_scale1/Mul" 784 | input: "layer_512_1_scale1/add" 785 | } 786 | node { 787 | name: "Relu_6" 788 | op: "Relu" 789 | input: "layer_512_1_scale1/BiasAdd" 790 | } 791 | node { 792 | name: "layer_512_1_conv_expand_h/Conv2D" 793 | op: "Conv2D" 794 | input: "Relu_6" 795 | input: "layer_512_1_conv_expand_h/weights" 796 | attr { 797 | key: "dilations" 798 | value { 799 | list { 800 | i: 1 801 | i: 1 802 | i: 1 803 | i: 1 804 | } 805 | } 806 | } 807 | attr { 808 | key: "padding" 809 | value { 810 | s: "SAME" 811 | } 812 | } 813 | attr { 814 | key: "strides" 815 | value { 816 | list { 817 | i: 1 818 | i: 1 819 | i: 1 820 | i: 1 821 | } 822 | } 823 | } 824 | } 825 | node { 826 | name: "layer_512_1_conv1_h/Conv2D" 827 | op: "Conv2D" 828 | input: "Relu_6" 829 | input: "layer_512_1_conv1_h/weights" 830 | attr { 831 | key: "dilations" 832 | value { 833 | list { 834 | i: 1 835 | i: 1 836 | i: 1 837 | i: 1 838 | } 839 | } 840 | } 841 | attr { 842 | key: "padding" 843 | value { 844 | s: "SAME" 845 | } 846 | } 847 | attr { 848 | key: "strides" 849 | value { 850 | list { 851 | i: 1 852 | i: 1 853 | i: 1 854 | i: 1 855 | } 856 | } 857 | } 858 | } 859 | node { 860 | name: "layer_512_1_bn2_h/FusedBatchNorm" 861 | op: "BiasAdd" 862 | input: "layer_512_1_conv1_h/Conv2D" 863 | input: "layer_512_1_conv1_h/Conv2D_bn_offset" 864 | } 865 | node { 866 | name: "layer_512_1_scale2_h/Mul" 867 | op: "Mul" 868 | input: "layer_512_1_bn2_h/FusedBatchNorm" 869 | input: "layer_512_1_scale2_h/mul" 870 | } 871 | node { 872 | name: "layer_512_1_scale2_h/BiasAdd" 873 | op: "BiasAdd" 874 | input: "layer_512_1_scale2_h/Mul" 875 | input: "layer_512_1_scale2_h/add" 876 | } 877 | node { 878 | name: "Relu_7" 879 | op: "Relu" 880 | input: "layer_512_1_scale2_h/BiasAdd" 881 | } 882 | node { 883 | name: "layer_512_1_conv2_h/convolution/SpaceToBatchND" 884 | op: "SpaceToBatchND" 885 | input: "Relu_7" 886 | input: "layer_512_1_conv2_h/convolution/SpaceToBatchND/block_shape" 887 | input: "layer_512_1_conv2_h/convolution/SpaceToBatchND/paddings" 888 | } 889 | node { 890 | name: "layer_512_1_conv2_h/convolution" 891 | op: "Conv2D" 892 | input: "layer_512_1_conv2_h/convolution/SpaceToBatchND" 893 | input: "layer_512_1_conv2_h/weights" 894 | attr { 895 | key: "dilations" 896 | value { 897 | list { 898 | i: 1 899 | i: 1 900 | i: 1 901 | i: 1 902 | } 903 | } 904 | } 905 | attr { 906 | key: "padding" 907 | value { 908 | s: "VALID" 909 | } 910 | } 911 | attr { 912 | key: "strides" 913 | value { 914 | list { 915 | i: 1 916 | i: 1 917 | i: 1 918 | i: 1 919 | } 920 | } 921 | } 922 | } 923 | node { 924 | name: "layer_512_1_conv2_h/convolution/BatchToSpaceND" 925 | op: "BatchToSpaceND" 926 | input: "layer_512_1_conv2_h/convolution" 927 | input: "layer_512_1_conv2_h/convolution/BatchToSpaceND/block_shape" 928 | input: "layer_512_1_conv2_h/convolution/BatchToSpaceND/crops" 929 | } 930 | node { 931 | name: "add_3" 932 | op: "Add" 933 | input: "layer_512_1_conv2_h/convolution/BatchToSpaceND" 934 | input: "layer_512_1_conv_expand_h/Conv2D" 935 | } 936 | node { 937 | name: "last_bn_h/FusedBatchNorm" 938 | op: "FusedBatchNorm" 939 | input: "add_3" 940 | input: "last_bn_h/gamma" 941 | input: "last_bn_h/beta" 942 | input: "last_bn_h/mean" 943 | input: "last_bn_h/std" 944 | attr { 945 | key: "epsilon" 946 | value { 947 | f: 1.00099996416e-05 948 | } 949 | } 950 | } 951 | node { 952 | name: "last_scale_h/Mul" 953 | op: "Mul" 954 | input: "last_bn_h/FusedBatchNorm" 955 | input: "last_scale_h/mul" 956 | } 957 | node { 958 | name: "last_scale_h/BiasAdd" 959 | op: "BiasAdd" 960 | input: "last_scale_h/Mul" 961 | input: "last_scale_h/add" 962 | } 963 | node { 964 | name: "last_relu" 965 | op: "Relu" 966 | input: "last_scale_h/BiasAdd" 967 | } 968 | node { 969 | name: "conv6_1_h/Conv2D" 970 | op: "Conv2D" 971 | input: "last_relu" 972 | input: "conv6_1_h/weights" 973 | attr { 974 | key: "dilations" 975 | value { 976 | list { 977 | i: 1 978 | i: 1 979 | i: 1 980 | i: 1 981 | } 982 | } 983 | } 984 | attr { 985 | key: "padding" 986 | value { 987 | s: "SAME" 988 | } 989 | } 990 | attr { 991 | key: "strides" 992 | value { 993 | list { 994 | i: 1 995 | i: 1 996 | i: 1 997 | i: 1 998 | } 999 | } 1000 | } 1001 | } 1002 | node { 1003 | name: "conv6_1_h/BiasAdd" 1004 | op: "BiasAdd" 1005 | input: "conv6_1_h/Conv2D" 1006 | input: "conv6_1_h/bias" 1007 | } 1008 | node { 1009 | name: "conv6_1_h/Relu" 1010 | op: "Relu" 1011 | input: "conv6_1_h/BiasAdd" 1012 | } 1013 | node { 1014 | name: "conv6_2_h/Conv2D" 1015 | op: "Conv2D" 1016 | input: "conv6_1_h/Relu" 1017 | input: "conv6_2_h/weights" 1018 | attr { 1019 | key: "dilations" 1020 | value { 1021 | list { 1022 | i: 1 1023 | i: 1 1024 | i: 1 1025 | i: 1 1026 | } 1027 | } 1028 | } 1029 | attr { 1030 | key: "padding" 1031 | value { 1032 | s: "SAME" 1033 | } 1034 | } 1035 | attr { 1036 | key: "strides" 1037 | value { 1038 | list { 1039 | i: 1 1040 | i: 2 1041 | i: 2 1042 | i: 1 1043 | } 1044 | } 1045 | } 1046 | } 1047 | node { 1048 | name: "conv6_2_h/BiasAdd" 1049 | op: "BiasAdd" 1050 | input: "conv6_2_h/Conv2D" 1051 | input: "conv6_2_h/bias" 1052 | } 1053 | node { 1054 | name: "conv6_2_h/Relu" 1055 | op: "Relu" 1056 | input: "conv6_2_h/BiasAdd" 1057 | } 1058 | node { 1059 | name: "conv7_1_h/Conv2D" 1060 | op: "Conv2D" 1061 | input: "conv6_2_h/Relu" 1062 | input: "conv7_1_h/weights" 1063 | attr { 1064 | key: "dilations" 1065 | value { 1066 | list { 1067 | i: 1 1068 | i: 1 1069 | i: 1 1070 | i: 1 1071 | } 1072 | } 1073 | } 1074 | attr { 1075 | key: "padding" 1076 | value { 1077 | s: "SAME" 1078 | } 1079 | } 1080 | attr { 1081 | key: "strides" 1082 | value { 1083 | list { 1084 | i: 1 1085 | i: 1 1086 | i: 1 1087 | i: 1 1088 | } 1089 | } 1090 | } 1091 | } 1092 | node { 1093 | name: "conv7_1_h/BiasAdd" 1094 | op: "BiasAdd" 1095 | input: "conv7_1_h/Conv2D" 1096 | input: "conv7_1_h/bias" 1097 | } 1098 | node { 1099 | name: "conv7_1_h/Relu" 1100 | op: "Relu" 1101 | input: "conv7_1_h/BiasAdd" 1102 | } 1103 | node { 1104 | name: "Pad_2" 1105 | op: "SpaceToBatchND" 1106 | input: "conv7_1_h/Relu" 1107 | input: "SpaceToBatchND/block_shape" 1108 | input: "SpaceToBatchND_1/paddings" 1109 | } 1110 | node { 1111 | name: "conv7_2_h/Conv2D" 1112 | op: "Conv2D" 1113 | input: "Pad_2" 1114 | input: "conv7_2_h/weights" 1115 | attr { 1116 | key: "dilations" 1117 | value { 1118 | list { 1119 | i: 1 1120 | i: 1 1121 | i: 1 1122 | i: 1 1123 | } 1124 | } 1125 | } 1126 | attr { 1127 | key: "padding" 1128 | value { 1129 | s: "VALID" 1130 | } 1131 | } 1132 | attr { 1133 | key: "strides" 1134 | value { 1135 | list { 1136 | i: 1 1137 | i: 2 1138 | i: 2 1139 | i: 1 1140 | } 1141 | } 1142 | } 1143 | } 1144 | node { 1145 | name: "conv7_2_h/BiasAdd" 1146 | op: "BiasAdd" 1147 | input: "conv7_2_h/Conv2D" 1148 | input: "conv7_2_h/bias" 1149 | } 1150 | node { 1151 | name: "BatchToSpaceND_2" 1152 | op: "BatchToSpaceND" 1153 | input: "conv7_2_h/BiasAdd" 1154 | } 1155 | node { 1156 | name: "conv7_2_h/Relu" 1157 | op: "Relu" 1158 | input: "BatchToSpaceND_2" 1159 | } 1160 | node { 1161 | name: "conv8_1_h/Conv2D" 1162 | op: "Conv2D" 1163 | input: "conv7_2_h/Relu" 1164 | input: "conv8_1_h/weights" 1165 | attr { 1166 | key: "dilations" 1167 | value { 1168 | list { 1169 | i: 1 1170 | i: 1 1171 | i: 1 1172 | i: 1 1173 | } 1174 | } 1175 | } 1176 | attr { 1177 | key: "padding" 1178 | value { 1179 | s: "SAME" 1180 | } 1181 | } 1182 | attr { 1183 | key: "strides" 1184 | value { 1185 | list { 1186 | i: 1 1187 | i: 1 1188 | i: 1 1189 | i: 1 1190 | } 1191 | } 1192 | } 1193 | } 1194 | node { 1195 | name: "conv8_1_h/BiasAdd" 1196 | op: "BiasAdd" 1197 | input: "conv8_1_h/Conv2D" 1198 | input: "conv8_1_h/bias" 1199 | } 1200 | node { 1201 | name: "conv8_1_h/Relu" 1202 | op: "Relu" 1203 | input: "conv8_1_h/BiasAdd" 1204 | } 1205 | node { 1206 | name: "conv8_2_h/Conv2D" 1207 | op: "Conv2D" 1208 | input: "conv8_1_h/Relu" 1209 | input: "conv8_2_h/weights" 1210 | attr { 1211 | key: "dilations" 1212 | value { 1213 | list { 1214 | i: 1 1215 | i: 1 1216 | i: 1 1217 | i: 1 1218 | } 1219 | } 1220 | } 1221 | attr { 1222 | key: "padding" 1223 | value { 1224 | s: "SAME" 1225 | } 1226 | } 1227 | attr { 1228 | key: "strides" 1229 | value { 1230 | list { 1231 | i: 1 1232 | i: 1 1233 | i: 1 1234 | i: 1 1235 | } 1236 | } 1237 | } 1238 | } 1239 | node { 1240 | name: "conv8_2_h/BiasAdd" 1241 | op: "BiasAdd" 1242 | input: "conv8_2_h/Conv2D" 1243 | input: "conv8_2_h/bias" 1244 | } 1245 | node { 1246 | name: "conv8_2_h/Relu" 1247 | op: "Relu" 1248 | input: "conv8_2_h/BiasAdd" 1249 | } 1250 | node { 1251 | name: "conv9_1_h/Conv2D" 1252 | op: "Conv2D" 1253 | input: "conv8_2_h/Relu" 1254 | input: "conv9_1_h/weights" 1255 | attr { 1256 | key: "dilations" 1257 | value { 1258 | list { 1259 | i: 1 1260 | i: 1 1261 | i: 1 1262 | i: 1 1263 | } 1264 | } 1265 | } 1266 | attr { 1267 | key: "padding" 1268 | value { 1269 | s: "SAME" 1270 | } 1271 | } 1272 | attr { 1273 | key: "strides" 1274 | value { 1275 | list { 1276 | i: 1 1277 | i: 1 1278 | i: 1 1279 | i: 1 1280 | } 1281 | } 1282 | } 1283 | } 1284 | node { 1285 | name: "conv9_1_h/BiasAdd" 1286 | op: "BiasAdd" 1287 | input: "conv9_1_h/Conv2D" 1288 | input: "conv9_1_h/bias" 1289 | } 1290 | node { 1291 | name: "conv9_1_h/Relu" 1292 | op: "Relu" 1293 | input: "conv9_1_h/BiasAdd" 1294 | } 1295 | node { 1296 | name: "conv9_2_h/Conv2D" 1297 | op: "Conv2D" 1298 | input: "conv9_1_h/Relu" 1299 | input: "conv9_2_h/weights" 1300 | attr { 1301 | key: "dilations" 1302 | value { 1303 | list { 1304 | i: 1 1305 | i: 1 1306 | i: 1 1307 | i: 1 1308 | } 1309 | } 1310 | } 1311 | attr { 1312 | key: "padding" 1313 | value { 1314 | s: "SAME" 1315 | } 1316 | } 1317 | attr { 1318 | key: "strides" 1319 | value { 1320 | list { 1321 | i: 1 1322 | i: 1 1323 | i: 1 1324 | i: 1 1325 | } 1326 | } 1327 | } 1328 | } 1329 | node { 1330 | name: "conv9_2_h/BiasAdd" 1331 | op: "BiasAdd" 1332 | input: "conv9_2_h/Conv2D" 1333 | input: "conv9_2_h/bias" 1334 | } 1335 | node { 1336 | name: "conv9_2_h/Relu" 1337 | op: "Relu" 1338 | input: "conv9_2_h/BiasAdd" 1339 | } 1340 | node { 1341 | name: "conv9_2_mbox_loc/Conv2D" 1342 | op: "Conv2D" 1343 | input: "conv9_2_h/Relu" 1344 | input: "conv9_2_mbox_loc/weights" 1345 | attr { 1346 | key: "dilations" 1347 | value { 1348 | list { 1349 | i: 1 1350 | i: 1 1351 | i: 1 1352 | i: 1 1353 | } 1354 | } 1355 | } 1356 | attr { 1357 | key: "padding" 1358 | value { 1359 | s: "SAME" 1360 | } 1361 | } 1362 | attr { 1363 | key: "strides" 1364 | value { 1365 | list { 1366 | i: 1 1367 | i: 1 1368 | i: 1 1369 | i: 1 1370 | } 1371 | } 1372 | } 1373 | } 1374 | node { 1375 | name: "conv9_2_mbox_loc/BiasAdd" 1376 | op: "BiasAdd" 1377 | input: "conv9_2_mbox_loc/Conv2D" 1378 | input: "conv9_2_mbox_loc/bias" 1379 | } 1380 | node { 1381 | name: "flatten_5/Reshape" 1382 | op: "Flatten" 1383 | input: "conv9_2_mbox_loc/BiasAdd" 1384 | } 1385 | node { 1386 | name: "conv9_2_mbox_conf/Conv2D" 1387 | op: "Conv2D" 1388 | input: "conv9_2_h/Relu" 1389 | input: "conv9_2_mbox_conf/weights" 1390 | attr { 1391 | key: "dilations" 1392 | value { 1393 | list { 1394 | i: 1 1395 | i: 1 1396 | i: 1 1397 | i: 1 1398 | } 1399 | } 1400 | } 1401 | attr { 1402 | key: "padding" 1403 | value { 1404 | s: "SAME" 1405 | } 1406 | } 1407 | attr { 1408 | key: "strides" 1409 | value { 1410 | list { 1411 | i: 1 1412 | i: 1 1413 | i: 1 1414 | i: 1 1415 | } 1416 | } 1417 | } 1418 | } 1419 | node { 1420 | name: "conv9_2_mbox_conf/BiasAdd" 1421 | op: "BiasAdd" 1422 | input: "conv9_2_mbox_conf/Conv2D" 1423 | input: "conv9_2_mbox_conf/bias" 1424 | } 1425 | node { 1426 | name: "flatten_11/Reshape" 1427 | op: "Flatten" 1428 | input: "conv9_2_mbox_conf/BiasAdd" 1429 | } 1430 | node { 1431 | name: "conv8_2_mbox_loc/Conv2D" 1432 | op: "Conv2D" 1433 | input: "conv8_2_h/Relu" 1434 | input: "conv8_2_mbox_loc/weights" 1435 | attr { 1436 | key: "dilations" 1437 | value { 1438 | list { 1439 | i: 1 1440 | i: 1 1441 | i: 1 1442 | i: 1 1443 | } 1444 | } 1445 | } 1446 | attr { 1447 | key: "padding" 1448 | value { 1449 | s: "SAME" 1450 | } 1451 | } 1452 | attr { 1453 | key: "strides" 1454 | value { 1455 | list { 1456 | i: 1 1457 | i: 1 1458 | i: 1 1459 | i: 1 1460 | } 1461 | } 1462 | } 1463 | } 1464 | node { 1465 | name: "conv8_2_mbox_loc/BiasAdd" 1466 | op: "BiasAdd" 1467 | input: "conv8_2_mbox_loc/Conv2D" 1468 | input: "conv8_2_mbox_loc/bias" 1469 | } 1470 | node { 1471 | name: "flatten_4/Reshape" 1472 | op: "Flatten" 1473 | input: "conv8_2_mbox_loc/BiasAdd" 1474 | } 1475 | node { 1476 | name: "conv8_2_mbox_conf/Conv2D" 1477 | op: "Conv2D" 1478 | input: "conv8_2_h/Relu" 1479 | input: "conv8_2_mbox_conf/weights" 1480 | attr { 1481 | key: "dilations" 1482 | value { 1483 | list { 1484 | i: 1 1485 | i: 1 1486 | i: 1 1487 | i: 1 1488 | } 1489 | } 1490 | } 1491 | attr { 1492 | key: "padding" 1493 | value { 1494 | s: "SAME" 1495 | } 1496 | } 1497 | attr { 1498 | key: "strides" 1499 | value { 1500 | list { 1501 | i: 1 1502 | i: 1 1503 | i: 1 1504 | i: 1 1505 | } 1506 | } 1507 | } 1508 | } 1509 | node { 1510 | name: "conv8_2_mbox_conf/BiasAdd" 1511 | op: "BiasAdd" 1512 | input: "conv8_2_mbox_conf/Conv2D" 1513 | input: "conv8_2_mbox_conf/bias" 1514 | } 1515 | node { 1516 | name: "flatten_10/Reshape" 1517 | op: "Flatten" 1518 | input: "conv8_2_mbox_conf/BiasAdd" 1519 | } 1520 | node { 1521 | name: "conv7_2_mbox_loc/Conv2D" 1522 | op: "Conv2D" 1523 | input: "conv7_2_h/Relu" 1524 | input: "conv7_2_mbox_loc/weights" 1525 | attr { 1526 | key: "dilations" 1527 | value { 1528 | list { 1529 | i: 1 1530 | i: 1 1531 | i: 1 1532 | i: 1 1533 | } 1534 | } 1535 | } 1536 | attr { 1537 | key: "padding" 1538 | value { 1539 | s: "SAME" 1540 | } 1541 | } 1542 | attr { 1543 | key: "strides" 1544 | value { 1545 | list { 1546 | i: 1 1547 | i: 1 1548 | i: 1 1549 | i: 1 1550 | } 1551 | } 1552 | } 1553 | } 1554 | node { 1555 | name: "conv7_2_mbox_loc/BiasAdd" 1556 | op: "BiasAdd" 1557 | input: "conv7_2_mbox_loc/Conv2D" 1558 | input: "conv7_2_mbox_loc/bias" 1559 | } 1560 | node { 1561 | name: "flatten_3/Reshape" 1562 | op: "Flatten" 1563 | input: "conv7_2_mbox_loc/BiasAdd" 1564 | } 1565 | node { 1566 | name: "conv7_2_mbox_conf/Conv2D" 1567 | op: "Conv2D" 1568 | input: "conv7_2_h/Relu" 1569 | input: "conv7_2_mbox_conf/weights" 1570 | attr { 1571 | key: "dilations" 1572 | value { 1573 | list { 1574 | i: 1 1575 | i: 1 1576 | i: 1 1577 | i: 1 1578 | } 1579 | } 1580 | } 1581 | attr { 1582 | key: "padding" 1583 | value { 1584 | s: "SAME" 1585 | } 1586 | } 1587 | attr { 1588 | key: "strides" 1589 | value { 1590 | list { 1591 | i: 1 1592 | i: 1 1593 | i: 1 1594 | i: 1 1595 | } 1596 | } 1597 | } 1598 | } 1599 | node { 1600 | name: "conv7_2_mbox_conf/BiasAdd" 1601 | op: "BiasAdd" 1602 | input: "conv7_2_mbox_conf/Conv2D" 1603 | input: "conv7_2_mbox_conf/bias" 1604 | } 1605 | node { 1606 | name: "flatten_9/Reshape" 1607 | op: "Flatten" 1608 | input: "conv7_2_mbox_conf/BiasAdd" 1609 | } 1610 | node { 1611 | name: "conv6_2_mbox_loc/Conv2D" 1612 | op: "Conv2D" 1613 | input: "conv6_2_h/Relu" 1614 | input: "conv6_2_mbox_loc/weights" 1615 | attr { 1616 | key: "dilations" 1617 | value { 1618 | list { 1619 | i: 1 1620 | i: 1 1621 | i: 1 1622 | i: 1 1623 | } 1624 | } 1625 | } 1626 | attr { 1627 | key: "padding" 1628 | value { 1629 | s: "SAME" 1630 | } 1631 | } 1632 | attr { 1633 | key: "strides" 1634 | value { 1635 | list { 1636 | i: 1 1637 | i: 1 1638 | i: 1 1639 | i: 1 1640 | } 1641 | } 1642 | } 1643 | } 1644 | node { 1645 | name: "conv6_2_mbox_loc/BiasAdd" 1646 | op: "BiasAdd" 1647 | input: "conv6_2_mbox_loc/Conv2D" 1648 | input: "conv6_2_mbox_loc/bias" 1649 | } 1650 | node { 1651 | name: "flatten_2/Reshape" 1652 | op: "Flatten" 1653 | input: "conv6_2_mbox_loc/BiasAdd" 1654 | } 1655 | node { 1656 | name: "conv6_2_mbox_conf/Conv2D" 1657 | op: "Conv2D" 1658 | input: "conv6_2_h/Relu" 1659 | input: "conv6_2_mbox_conf/weights" 1660 | attr { 1661 | key: "dilations" 1662 | value { 1663 | list { 1664 | i: 1 1665 | i: 1 1666 | i: 1 1667 | i: 1 1668 | } 1669 | } 1670 | } 1671 | attr { 1672 | key: "padding" 1673 | value { 1674 | s: "SAME" 1675 | } 1676 | } 1677 | attr { 1678 | key: "strides" 1679 | value { 1680 | list { 1681 | i: 1 1682 | i: 1 1683 | i: 1 1684 | i: 1 1685 | } 1686 | } 1687 | } 1688 | } 1689 | node { 1690 | name: "conv6_2_mbox_conf/BiasAdd" 1691 | op: "BiasAdd" 1692 | input: "conv6_2_mbox_conf/Conv2D" 1693 | input: "conv6_2_mbox_conf/bias" 1694 | } 1695 | node { 1696 | name: "flatten_8/Reshape" 1697 | op: "Flatten" 1698 | input: "conv6_2_mbox_conf/BiasAdd" 1699 | } 1700 | node { 1701 | name: "fc7_mbox_loc/Conv2D" 1702 | op: "Conv2D" 1703 | input: "last_relu" 1704 | input: "fc7_mbox_loc/weights" 1705 | attr { 1706 | key: "dilations" 1707 | value { 1708 | list { 1709 | i: 1 1710 | i: 1 1711 | i: 1 1712 | i: 1 1713 | } 1714 | } 1715 | } 1716 | attr { 1717 | key: "padding" 1718 | value { 1719 | s: "SAME" 1720 | } 1721 | } 1722 | attr { 1723 | key: "strides" 1724 | value { 1725 | list { 1726 | i: 1 1727 | i: 1 1728 | i: 1 1729 | i: 1 1730 | } 1731 | } 1732 | } 1733 | } 1734 | node { 1735 | name: "fc7_mbox_loc/BiasAdd" 1736 | op: "BiasAdd" 1737 | input: "fc7_mbox_loc/Conv2D" 1738 | input: "fc7_mbox_loc/bias" 1739 | } 1740 | node { 1741 | name: "flatten_1/Reshape" 1742 | op: "Flatten" 1743 | input: "fc7_mbox_loc/BiasAdd" 1744 | } 1745 | node { 1746 | name: "mbox_loc" 1747 | op: "ConcatV2" 1748 | input: "flatten/Reshape" 1749 | input: "flatten_1/Reshape" 1750 | input: "flatten_2/Reshape" 1751 | input: "flatten_3/Reshape" 1752 | input: "flatten_4/Reshape" 1753 | input: "flatten_5/Reshape" 1754 | input: "mbox_loc/axis" 1755 | } 1756 | node { 1757 | name: "fc7_mbox_conf/Conv2D" 1758 | op: "Conv2D" 1759 | input: "last_relu" 1760 | input: "fc7_mbox_conf/weights" 1761 | attr { 1762 | key: "dilations" 1763 | value { 1764 | list { 1765 | i: 1 1766 | i: 1 1767 | i: 1 1768 | i: 1 1769 | } 1770 | } 1771 | } 1772 | attr { 1773 | key: "padding" 1774 | value { 1775 | s: "SAME" 1776 | } 1777 | } 1778 | attr { 1779 | key: "strides" 1780 | value { 1781 | list { 1782 | i: 1 1783 | i: 1 1784 | i: 1 1785 | i: 1 1786 | } 1787 | } 1788 | } 1789 | } 1790 | node { 1791 | name: "fc7_mbox_conf/BiasAdd" 1792 | op: "BiasAdd" 1793 | input: "fc7_mbox_conf/Conv2D" 1794 | input: "fc7_mbox_conf/bias" 1795 | } 1796 | node { 1797 | name: "flatten_7/Reshape" 1798 | op: "Flatten" 1799 | input: "fc7_mbox_conf/BiasAdd" 1800 | } 1801 | node { 1802 | name: "mbox_conf" 1803 | op: "ConcatV2" 1804 | input: "flatten_6/Reshape" 1805 | input: "flatten_7/Reshape" 1806 | input: "flatten_8/Reshape" 1807 | input: "flatten_9/Reshape" 1808 | input: "flatten_10/Reshape" 1809 | input: "flatten_11/Reshape" 1810 | input: "mbox_conf/axis" 1811 | } 1812 | node { 1813 | name: "mbox_conf_reshape" 1814 | op: "Reshape" 1815 | input: "mbox_conf" 1816 | input: "reshape_before_softmax" 1817 | } 1818 | node { 1819 | name: "mbox_conf_softmax" 1820 | op: "Softmax" 1821 | input: "mbox_conf_reshape" 1822 | attr { 1823 | key: "axis" 1824 | value { 1825 | i: 2 1826 | } 1827 | } 1828 | } 1829 | node { 1830 | name: "mbox_conf_flatten" 1831 | op: "Flatten" 1832 | input: "mbox_conf_softmax" 1833 | } 1834 | node { 1835 | name: "PriorBox_0" 1836 | op: "PriorBox" 1837 | input: "conv4_3_norm/mul_1" 1838 | input: "data" 1839 | attr { 1840 | key: "aspect_ratio" 1841 | value { 1842 | tensor { 1843 | dtype: DT_FLOAT 1844 | tensor_shape { 1845 | dim { 1846 | size: 1 1847 | } 1848 | } 1849 | float_val: 2.0 1850 | } 1851 | } 1852 | } 1853 | attr { 1854 | key: "clip" 1855 | value { 1856 | b: false 1857 | } 1858 | } 1859 | attr { 1860 | key: "flip" 1861 | value { 1862 | b: true 1863 | } 1864 | } 1865 | attr { 1866 | key: "max_size" 1867 | value { 1868 | i: 60 1869 | } 1870 | } 1871 | attr { 1872 | key: "min_size" 1873 | value { 1874 | i: 30 1875 | } 1876 | } 1877 | attr { 1878 | key: "offset" 1879 | value { 1880 | f: 0.5 1881 | } 1882 | } 1883 | attr { 1884 | key: "step" 1885 | value { 1886 | f: 8.0 1887 | } 1888 | } 1889 | attr { 1890 | key: "variance" 1891 | value { 1892 | tensor { 1893 | dtype: DT_FLOAT 1894 | tensor_shape { 1895 | dim { 1896 | size: 4 1897 | } 1898 | } 1899 | float_val: 0.10000000149 1900 | float_val: 0.10000000149 1901 | float_val: 0.20000000298 1902 | float_val: 0.20000000298 1903 | } 1904 | } 1905 | } 1906 | } 1907 | node { 1908 | name: "PriorBox_1" 1909 | op: "PriorBox" 1910 | input: "last_relu" 1911 | input: "data" 1912 | attr { 1913 | key: "aspect_ratio" 1914 | value { 1915 | tensor { 1916 | dtype: DT_FLOAT 1917 | tensor_shape { 1918 | dim { 1919 | size: 2 1920 | } 1921 | } 1922 | float_val: 2.0 1923 | float_val: 3.0 1924 | } 1925 | } 1926 | } 1927 | attr { 1928 | key: "clip" 1929 | value { 1930 | b: false 1931 | } 1932 | } 1933 | attr { 1934 | key: "flip" 1935 | value { 1936 | b: true 1937 | } 1938 | } 1939 | attr { 1940 | key: "max_size" 1941 | value { 1942 | i: 111 1943 | } 1944 | } 1945 | attr { 1946 | key: "min_size" 1947 | value { 1948 | i: 60 1949 | } 1950 | } 1951 | attr { 1952 | key: "offset" 1953 | value { 1954 | f: 0.5 1955 | } 1956 | } 1957 | attr { 1958 | key: "step" 1959 | value { 1960 | f: 16.0 1961 | } 1962 | } 1963 | attr { 1964 | key: "variance" 1965 | value { 1966 | tensor { 1967 | dtype: DT_FLOAT 1968 | tensor_shape { 1969 | dim { 1970 | size: 4 1971 | } 1972 | } 1973 | float_val: 0.10000000149 1974 | float_val: 0.10000000149 1975 | float_val: 0.20000000298 1976 | float_val: 0.20000000298 1977 | } 1978 | } 1979 | } 1980 | } 1981 | node { 1982 | name: "PriorBox_2" 1983 | op: "PriorBox" 1984 | input: "conv6_2_h/Relu" 1985 | input: "data" 1986 | attr { 1987 | key: "aspect_ratio" 1988 | value { 1989 | tensor { 1990 | dtype: DT_FLOAT 1991 | tensor_shape { 1992 | dim { 1993 | size: 2 1994 | } 1995 | } 1996 | float_val: 2.0 1997 | float_val: 3.0 1998 | } 1999 | } 2000 | } 2001 | attr { 2002 | key: "clip" 2003 | value { 2004 | b: false 2005 | } 2006 | } 2007 | attr { 2008 | key: "flip" 2009 | value { 2010 | b: true 2011 | } 2012 | } 2013 | attr { 2014 | key: "max_size" 2015 | value { 2016 | i: 162 2017 | } 2018 | } 2019 | attr { 2020 | key: "min_size" 2021 | value { 2022 | i: 111 2023 | } 2024 | } 2025 | attr { 2026 | key: "offset" 2027 | value { 2028 | f: 0.5 2029 | } 2030 | } 2031 | attr { 2032 | key: "step" 2033 | value { 2034 | f: 32.0 2035 | } 2036 | } 2037 | attr { 2038 | key: "variance" 2039 | value { 2040 | tensor { 2041 | dtype: DT_FLOAT 2042 | tensor_shape { 2043 | dim { 2044 | size: 4 2045 | } 2046 | } 2047 | float_val: 0.10000000149 2048 | float_val: 0.10000000149 2049 | float_val: 0.20000000298 2050 | float_val: 0.20000000298 2051 | } 2052 | } 2053 | } 2054 | } 2055 | node { 2056 | name: "PriorBox_3" 2057 | op: "PriorBox" 2058 | input: "conv7_2_h/Relu" 2059 | input: "data" 2060 | attr { 2061 | key: "aspect_ratio" 2062 | value { 2063 | tensor { 2064 | dtype: DT_FLOAT 2065 | tensor_shape { 2066 | dim { 2067 | size: 2 2068 | } 2069 | } 2070 | float_val: 2.0 2071 | float_val: 3.0 2072 | } 2073 | } 2074 | } 2075 | attr { 2076 | key: "clip" 2077 | value { 2078 | b: false 2079 | } 2080 | } 2081 | attr { 2082 | key: "flip" 2083 | value { 2084 | b: true 2085 | } 2086 | } 2087 | attr { 2088 | key: "max_size" 2089 | value { 2090 | i: 213 2091 | } 2092 | } 2093 | attr { 2094 | key: "min_size" 2095 | value { 2096 | i: 162 2097 | } 2098 | } 2099 | attr { 2100 | key: "offset" 2101 | value { 2102 | f: 0.5 2103 | } 2104 | } 2105 | attr { 2106 | key: "step" 2107 | value { 2108 | f: 64.0 2109 | } 2110 | } 2111 | attr { 2112 | key: "variance" 2113 | value { 2114 | tensor { 2115 | dtype: DT_FLOAT 2116 | tensor_shape { 2117 | dim { 2118 | size: 4 2119 | } 2120 | } 2121 | float_val: 0.10000000149 2122 | float_val: 0.10000000149 2123 | float_val: 0.20000000298 2124 | float_val: 0.20000000298 2125 | } 2126 | } 2127 | } 2128 | } 2129 | node { 2130 | name: "PriorBox_4" 2131 | op: "PriorBox" 2132 | input: "conv8_2_h/Relu" 2133 | input: "data" 2134 | attr { 2135 | key: "aspect_ratio" 2136 | value { 2137 | tensor { 2138 | dtype: DT_FLOAT 2139 | tensor_shape { 2140 | dim { 2141 | size: 1 2142 | } 2143 | } 2144 | float_val: 2.0 2145 | } 2146 | } 2147 | } 2148 | attr { 2149 | key: "clip" 2150 | value { 2151 | b: false 2152 | } 2153 | } 2154 | attr { 2155 | key: "flip" 2156 | value { 2157 | b: true 2158 | } 2159 | } 2160 | attr { 2161 | key: "max_size" 2162 | value { 2163 | i: 264 2164 | } 2165 | } 2166 | attr { 2167 | key: "min_size" 2168 | value { 2169 | i: 213 2170 | } 2171 | } 2172 | attr { 2173 | key: "offset" 2174 | value { 2175 | f: 0.5 2176 | } 2177 | } 2178 | attr { 2179 | key: "step" 2180 | value { 2181 | f: 100.0 2182 | } 2183 | } 2184 | attr { 2185 | key: "variance" 2186 | value { 2187 | tensor { 2188 | dtype: DT_FLOAT 2189 | tensor_shape { 2190 | dim { 2191 | size: 4 2192 | } 2193 | } 2194 | float_val: 0.10000000149 2195 | float_val: 0.10000000149 2196 | float_val: 0.20000000298 2197 | float_val: 0.20000000298 2198 | } 2199 | } 2200 | } 2201 | } 2202 | node { 2203 | name: "PriorBox_5" 2204 | op: "PriorBox" 2205 | input: "conv9_2_h/Relu" 2206 | input: "data" 2207 | attr { 2208 | key: "aspect_ratio" 2209 | value { 2210 | tensor { 2211 | dtype: DT_FLOAT 2212 | tensor_shape { 2213 | dim { 2214 | size: 1 2215 | } 2216 | } 2217 | float_val: 2.0 2218 | } 2219 | } 2220 | } 2221 | attr { 2222 | key: "clip" 2223 | value { 2224 | b: false 2225 | } 2226 | } 2227 | attr { 2228 | key: "flip" 2229 | value { 2230 | b: true 2231 | } 2232 | } 2233 | attr { 2234 | key: "max_size" 2235 | value { 2236 | i: 315 2237 | } 2238 | } 2239 | attr { 2240 | key: "min_size" 2241 | value { 2242 | i: 264 2243 | } 2244 | } 2245 | attr { 2246 | key: "offset" 2247 | value { 2248 | f: 0.5 2249 | } 2250 | } 2251 | attr { 2252 | key: "step" 2253 | value { 2254 | f: 300.0 2255 | } 2256 | } 2257 | attr { 2258 | key: "variance" 2259 | value { 2260 | tensor { 2261 | dtype: DT_FLOAT 2262 | tensor_shape { 2263 | dim { 2264 | size: 4 2265 | } 2266 | } 2267 | float_val: 0.10000000149 2268 | float_val: 0.10000000149 2269 | float_val: 0.20000000298 2270 | float_val: 0.20000000298 2271 | } 2272 | } 2273 | } 2274 | } 2275 | node { 2276 | name: "mbox_priorbox" 2277 | op: "ConcatV2" 2278 | input: "PriorBox_0" 2279 | input: "PriorBox_1" 2280 | input: "PriorBox_2" 2281 | input: "PriorBox_3" 2282 | input: "PriorBox_4" 2283 | input: "PriorBox_5" 2284 | input: "mbox_loc/axis" 2285 | } 2286 | node { 2287 | name: "detection_out" 2288 | op: "DetectionOutput" 2289 | input: "mbox_loc" 2290 | input: "mbox_conf_flatten" 2291 | input: "mbox_priorbox" 2292 | attr { 2293 | key: "background_label_id" 2294 | value { 2295 | i: 0 2296 | } 2297 | } 2298 | attr { 2299 | key: "code_type" 2300 | value { 2301 | s: "CENTER_SIZE" 2302 | } 2303 | } 2304 | attr { 2305 | key: "confidence_threshold" 2306 | value { 2307 | f: 0.00999999977648 2308 | } 2309 | } 2310 | attr { 2311 | key: "keep_top_k" 2312 | value { 2313 | i: 200 2314 | } 2315 | } 2316 | attr { 2317 | key: "nms_threshold" 2318 | value { 2319 | f: 0.449999988079 2320 | } 2321 | } 2322 | attr { 2323 | key: "num_classes" 2324 | value { 2325 | i: 2 2326 | } 2327 | } 2328 | attr { 2329 | key: "share_location" 2330 | value { 2331 | b: true 2332 | } 2333 | } 2334 | attr { 2335 | key: "top_k" 2336 | value { 2337 | i: 400 2338 | } 2339 | } 2340 | } 2341 | node { 2342 | name: "reshape_before_softmax" 2343 | op: "Const" 2344 | attr { 2345 | key: "value" 2346 | value { 2347 | tensor { 2348 | dtype: DT_INT32 2349 | tensor_shape { 2350 | dim { 2351 | size: 3 2352 | } 2353 | } 2354 | int_val: 0 2355 | int_val: -1 2356 | int_val: 2 2357 | } 2358 | } 2359 | } 2360 | } 2361 | library { 2362 | } 2363 | -------------------------------------------------------------------------------- /models/opencv_face_detector_uint8.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/models/opencv_face_detector_uint8.pb -------------------------------------------------------------------------------- /models/shape_predictor_5_face_landmarks.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/models/shape_predictor_5_face_landmarks.dat -------------------------------------------------------------------------------- /models/weights.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/susantabiswas/realtime-facial-emotion-analyzer/e9d5ee19aebe7602de91ba57ade15a6655944ed6/models/weights.h5 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest-cov==3.0.0 2 | mtcnn==0.1.1 3 | dlib==19.19.0 4 | numpy==1.19.3 5 | opencv_python==4.5.1.48 6 | tensorflow==2.7.0 7 | Pillow==8.4.0 -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from emotion_analyzer.media_utils import load_image_path 2 | import pytest 3 | 4 | 5 | @pytest.fixture 6 | def img1_data(): 7 | img_loc = "data/sample/1.jpg" 8 | return load_image_path(img_path=img_loc, mode="rgb") 9 | 10 | 11 | @pytest.fixture 12 | def img2_data(): 13 | img_loc = "data/sample/2.jpg" 14 | return load_image_path(img_path=img_loc, mode="rgb") 15 | 16 | 17 | @pytest.fixture 18 | def img2_keypoints(): 19 | keypoints = [[139, 92], [125, 93], [91, 89], [105, 91], [114, 122]] 20 | 21 | return keypoints 22 | 23 | 24 | @pytest.fixture 25 | def img2_facebox_dlib_hog(): 26 | return [[66, 66, 156, 157]] 27 | 28 | 29 | @pytest.fixture 30 | def img2_facebox_dlib_mmod(): 31 | return [[70, 61, 152, 144]] 32 | 33 | 34 | @pytest.fixture 35 | def img2_facebox_opencv(): 36 | # With dlib style fore head cropping 37 | return [[74, 72, 153, 160]] 38 | 39 | 40 | @pytest.fixture 41 | def img2_facebox_mtcnn(): 42 | # With dlib style fore head cropping 43 | return [[76, 60, 151, 155]] 44 | 45 | -------------------------------------------------------------------------------- /tests/test_face_detection_dlib.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Tests for dlib face detector.""" 6 | # =================================================== 7 | from emotion_analyzer.exceptions import ModelFileMissing, InvalidImage 8 | from emotion_analyzer.face_detection_dlib import FaceDetectorDlib 9 | import pytest 10 | import numpy as np 11 | 12 | def test_invalid_image(): 13 | model_loc = "./models" 14 | ob = FaceDetectorDlib(model_loc=model_loc, model_type="hog") 15 | img = np.zeros((100,100,5), dtype='float32') 16 | 17 | with pytest.raises(InvalidImage): 18 | ob.detect_faces(img) 19 | 20 | def test_correct_model_path(): 21 | """ 22 | Test object init with the correct model path 23 | """ 24 | ob = None 25 | model_loc = "./models" 26 | try: 27 | ob = FaceDetectorDlib(model_loc=model_loc, model_type="mmod") 28 | except Exception: 29 | pass 30 | finally: 31 | assert isinstance(ob, FaceDetectorDlib) 32 | 33 | 34 | def test_incorrect_model_path(): 35 | """ 36 | Test object init with the incorrect model path 37 | """ 38 | incorrect_model_loc = "./wrong_models" 39 | with pytest.raises(ModelFileMissing): 40 | ob = FaceDetectorDlib(model_loc=incorrect_model_loc, model_type="mmod") 41 | 42 | 43 | def test_detect_face_hog(img2_data, img2_facebox_dlib_hog): 44 | model_loc = "./models" 45 | ob = FaceDetectorDlib(model_loc=model_loc, model_type="hog") 46 | assert img2_facebox_dlib_hog == ob.detect_faces(img2_data) 47 | 48 | 49 | def test_detect_face_mmod(img2_data, img2_facebox_dlib_mmod): 50 | model_loc = "./models" 51 | ob = FaceDetectorDlib(model_loc=model_loc, model_type="mmod") 52 | assert img2_facebox_dlib_mmod == ob.detect_faces(img2_data) 53 | -------------------------------------------------------------------------------- /tests/test_face_detection_mtcnn.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Tests for mtcnn face detector.""" 6 | # =================================================== 7 | 8 | from emotion_analyzer.exceptions import InvalidImage 9 | from emotion_analyzer.face_detection_mtcnn import FaceDetectorMTCNN 10 | import numpy as np 11 | import pytest 12 | 13 | def test_detect_face(img2_data, img2_facebox_mtcnn): 14 | ob = FaceDetectorMTCNN() 15 | assert img2_facebox_mtcnn == ob.detect_faces(img2_data) 16 | 17 | def test_invalid_image(): 18 | ob = FaceDetectorMTCNN() 19 | img = np.zeros((100,100,5), dtype='float32') 20 | with pytest.raises(InvalidImage): 21 | ob.detect_faces(img) -------------------------------------------------------------------------------- /tests/test_face_detection_opencv.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Tests for opencv face detector.""" 6 | # =================================================== 7 | 8 | from emotion_analyzer.exceptions import ModelFileMissing, InvalidImage 9 | from emotion_analyzer.face_detection_opencv import FaceDetectorOpenCV 10 | import pytest 11 | import numpy as np 12 | 13 | def test_invalid_image(): 14 | model_loc = "./models" 15 | ob = FaceDetectorOpenCV(model_loc=model_loc) 16 | img = np.zeros((100,100,5), dtype='float32') 17 | 18 | with pytest.raises(InvalidImage): 19 | ob.detect_faces(img) 20 | 21 | 22 | def test_bbox_outside_img(): 23 | model_loc = "./models" 24 | ob = FaceDetectorOpenCV(model_loc=model_loc) 25 | 26 | assert ob.is_valid_bbox([0, 0, 100, 100], 10, 10) == False 27 | 28 | 29 | def test_correct_model_path(): 30 | """ 31 | Test object init with the correct model path 32 | """ 33 | ob = None 34 | model_loc = "./models" 35 | try: 36 | ob = FaceDetectorOpenCV(model_loc=model_loc) 37 | except Exception: 38 | pass 39 | finally: 40 | assert isinstance(ob, FaceDetectorOpenCV) 41 | 42 | 43 | def test_incorrect_model_path(): 44 | """ 45 | Test object init with the incorrect model path 46 | """ 47 | inccorrect_model_loc = "./wrong_models" 48 | with pytest.raises(ModelFileMissing): 49 | _ = FaceDetectorOpenCV(model_loc=inccorrect_model_loc) 50 | 51 | 52 | def test_detect_face(img2_data, img2_facebox_opencv): 53 | model_loc = "./models" 54 | ob = FaceDetectorOpenCV(model_loc=model_loc) 55 | assert img2_facebox_opencv == ob.detect_faces(img2_data) 56 | -------------------------------------------------------------------------------- /tests/test_media_utils.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Tests for media utils.""" 6 | # =================================================== 7 | 8 | from emotion_analyzer.exceptions import InvalidImage 9 | import pytest 10 | from emotion_analyzer.media_utils import ( 11 | convert_to_dlib_rectangle, 12 | convert_to_rgb, 13 | load_image_path, 14 | ) 15 | import numpy as np 16 | import cv2 17 | import dlib 18 | 19 | 20 | def test_convert_to_dlib_rectangle(): 21 | """ Check if dlib rectangle is created properly""" 22 | bbox = [1, 2, 3, 4] 23 | dlib_box = dlib.rectangle(bbox[0], bbox[1], bbox[2], bbox[3]) 24 | assert convert_to_dlib_rectangle(bbox) == dlib_box 25 | 26 | 27 | def test_load_image_path(): 28 | """ Check if exception is thrown when an invalid array is given""" 29 | path = "data/sample/1.jpg" 30 | img = cv2.imread(path) 31 | img = convert_to_rgb(img) 32 | loaded_img = load_image_path(path) 33 | assert np.all(loaded_img == img) == True 34 | 35 | 36 | def test_convert_to_rgb_exception(): 37 | """ Check if exception is thrown when an invalid array is given""" 38 | # create a dummy image 39 | img = np.zeros((100, 100, 5)) 40 | with pytest.raises(InvalidImage): 41 | convert_to_rgb(img) 42 | 43 | 44 | def test_convert_to_rgb(img1_data): 45 | """ Check if RGB conversion happens correctly""" 46 | rgb = cv2.cvtColor(img1_data, cv2.COLOR_BGR2RGB) 47 | converted_img = convert_to_rgb(img1_data) 48 | assert np.all(rgb == converted_img) == True 49 | -------------------------------------------------------------------------------- /training/data_prep.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | def load_images(start_idx, end_idx, base_path): 5 | # training images 6 | images = [] 7 | for name in range(start_idx, end_idx): 8 | img = cv2.imread(base_path + str(name) + '.jpg', 0) 9 | if img is not None: 10 | images.append(img) 11 | 12 | return images 13 | 14 | # read images from folder 15 | def load_images_folder(): 16 | # training images 17 | images_train = load_images(0, 28710, 'output/Training/') 18 | # validation images 19 | images_cv = load_images(28710, 32299, 'output/PublicTest/') 20 | # test images 21 | images_test = load_images(32299, 35888, 'output/PrivateTest/') 22 | 23 | return images_train, images_cv, images_test 24 | 25 | 26 | # load the images 27 | images_train, images_cv, images_test = load_images_folder() 28 | 29 | # change to numpy matrix 30 | images_train = np.array(images_train) 31 | images_cv = np.array(images_cv) 32 | images_test = np.array(images_test) 33 | 34 | # save the numpy matrix 35 | np.save('dataset/train_raw.npy', images_train) 36 | np.save('dataset/cv_raw.npy', images_cv) 37 | np.save('dataset/test_raw.npy', images_test) 38 | -------------------------------------------------------------------------------- /training/preprocess.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | ''' 4 | This script creates 3-channel gray images from FER 2013 dataset. 5 | It has been done so that the CNNs designed for RGB images can 6 | be used without modifying the input shape. 7 | 8 | This script requires two command line parameters: 9 | 1. The path to the CSV file 10 | 2. The output directory 11 | 12 | It generates the images and saves them in three directories inside 13 | the output directory - Training, PublicTest, and PrivateTest. 14 | These are the three original splits in the dataset. 15 | ''' 16 | 17 | 18 | import os 19 | import csv 20 | import argparse 21 | import numpy as np 22 | import scipy.misc 23 | import pandas as pd 24 | import numpy as np 25 | 26 | parser = argparse.ArgumentParser() 27 | parser.add_argument('-f', '--file', required=True, help="path of the csv file") 28 | parser.add_argument('-o', '--output', required=True, help="path of the output directory") 29 | 30 | args = parser.parse_args() 31 | 32 | w, h = 48, 48 33 | image = np.zeros((h, w), dtype=np.uint8) 34 | id = 1 35 | emo_list = [] 36 | with open(args.file, 'r') as csvfile: 37 | datareader = csv.reader(csvfile, delimiter =',') 38 | headers = next(datareader) 39 | print(headers) 40 | for row in datareader: 41 | emotion = row[0] 42 | pixels = map(int, row[1].split()) 43 | usage = row[2] 44 | 45 | pixels_array = np.asarray(list(pixels)) 46 | 47 | image = np.reshape(pixels_array, (w, h)) 48 | 49 | stacked_image = np.dstack((image,) * 3) 50 | 51 | # add emotion to the list 52 | emo_list += emotion 53 | 54 | image_folder = os.path.join(args.output, usage) 55 | if not os.path.exists(image_folder): 56 | os.makedirs(image_folder) 57 | image_file = os.path.join(image_folder , str(id) + '.jpg') 58 | scipy.misc.imsave(image_file, stacked_image) 59 | id += 1 60 | if id % 100 == 0: 61 | print('Processed {} images'.format(id)) 62 | 63 | np.savetxt("emotions.csv", emo_list, delimiter=",", fmt='%s') 64 | print("Finished processing {} images".format(id)) -------------------------------------------------------------------------------- /video_main.py: -------------------------------------------------------------------------------- 1 | # ---- coding: utf-8 ---- 2 | # =================================================== 3 | # Author: Susanta Biswas 4 | # =================================================== 5 | """Description: Class with methods to do emotion analysis 6 | on video or webcam feed. 7 | 8 | Usage: python video_main""" 9 | # =================================================== 10 | import sys 11 | import time 12 | from typing import Dict, List 13 | 14 | import cv2 15 | import numpy as np 16 | 17 | from emotion_analyzer.emotion_detector import EmotionDetector 18 | from emotion_analyzer.logger import LoggerFactory 19 | from emotion_analyzer.media_utils import ( 20 | annotate_emotion_stats, 21 | annotate_warning, 22 | convert_to_rgb, 23 | draw_bounding_box_annotation, 24 | draw_emoji, 25 | get_video_writer, 26 | ) 27 | from emotion_analyzer.validators import path_exists 28 | 29 | # Load the custom logger 30 | logger = None 31 | try: 32 | logger_ob = LoggerFactory(logger_name=__name__) 33 | logger = logger_ob.get_logger() 34 | logger.info("{} loaded...".format(__name__)) 35 | # set exception hook for uncaught exceptions 36 | sys.excepthook = logger_ob.uncaught_exception_hook 37 | except Exception as exc: 38 | raise exc 39 | 40 | 41 | class EmotionAnalysisVideo: 42 | """Class with methods to do emotion analysis on video or webcam feed.""" 43 | 44 | emoji_foldername = "emojis" 45 | 46 | def __init__( 47 | self, 48 | face_detector: str = "dlib", 49 | model_loc: str = "models", 50 | face_detection_threshold: float = 0.8, 51 | emoji_loc: str = "data", 52 | ) -> None: 53 | 54 | # construct the path to emoji folder 55 | self.emoji_path = os.path.join(emoji_loc, EmotionAnalysisVideo.emoji_foldername) 56 | # Load the emojis 57 | self.emojis = self.load_emojis(emoji_path=self.emoji_path) 58 | 59 | self.emotion_detector = EmotionDetector( 60 | model_loc=model_loc, 61 | face_detection_threshold=face_detection_threshold, 62 | face_detector=face_detector, 63 | ) 64 | 65 | def emotion_analysis_video( 66 | self, 67 | video_path: str = None, 68 | detection_interval: int = 15, 69 | save_output: bool = False, 70 | preview: bool = False, 71 | output_path: str = "data/output.mp4", 72 | resize_scale: float = 0.5, 73 | ) -> None: 74 | 75 | # if video_path is None: 76 | # If no video source is given, try 77 | # switching to webcam 78 | video_path = 0 if video_path is None else video_path 79 | 80 | if not path_exists(video_path): 81 | raise FileNotFoundError 82 | 83 | cap, video_writer = None, None 84 | 85 | try: 86 | cap = cv2.VideoCapture(video_path) 87 | # To save the video file, get the opencv video writer 88 | video_writer = get_video_writer(cap, output_path) 89 | frame_num = 1 90 | 91 | t1 = time.time() 92 | logger.info("Enter q to exit...") 93 | 94 | emotions = None 95 | 96 | while True: 97 | status, frame = cap.read() 98 | if not status: 99 | break 100 | 101 | try: 102 | # Flip webcam feed so that it looks mirrored 103 | if video_path == 0: 104 | frame = cv2.flip(frame, 2) 105 | 106 | if frame_num % detection_interval == 0: 107 | # Scale down the image to increase model 108 | # inference time. 109 | smaller_frame = convert_to_rgb( 110 | cv2.resize(frame, (0, 0), fx=resize_scale, fy=resize_scale) 111 | ) 112 | # Detect emotion 113 | emotions = self.emotion_detector.detect_emotion(smaller_frame) 114 | 115 | # Annotate the current frame with emotion detection data 116 | frame = self.annotate_emotion_data(emotions, frame, resize_scale) 117 | 118 | if save_output: 119 | video_writer.write(frame) 120 | if preview: 121 | cv2.imshow("Preview", cv2.resize(frame, (680, 480))) 122 | 123 | key = cv2.waitKey(1) & 0xFF 124 | if key == ord("q"): 125 | break 126 | 127 | except Exception as exc: 128 | raise exc 129 | frame_num += 1 130 | 131 | t2 = time.time() 132 | logger.info("Time:{}".format((t2 - t1) / 60)) 133 | logger.info("Total frames: {}".format(frame_num)) 134 | logger.info("Time per frame: {}".format((t2 - t1) / frame_num)) 135 | 136 | except Exception as exc: 137 | raise exc 138 | finally: 139 | cv2.destroyAllWindows() 140 | cap.release() 141 | video_writer.release() 142 | 143 | 144 | def load_emojis(self, emoji_path: str = "data//emoji") -> List: 145 | emojis = {} 146 | 147 | # list of given emotions 148 | EMOTIONS = [ 149 | "Angry", 150 | "Disgusted", 151 | "Fearful", 152 | "Happy", 153 | "Sad", 154 | "Surprised", 155 | "Neutral", 156 | ] 157 | 158 | # store the emoji coreesponding to different emotions 159 | for _, emotion in enumerate(EMOTIONS): 160 | emoji_path = os.path.join(self.emoji_path, emotion.lower() + ".png") 161 | emojis[emotion] = cv2.imread(emoji_path, -1) 162 | 163 | logger.info("Finished loading emojis...") 164 | 165 | return emojis 166 | 167 | 168 | def annotate_emotion_data( 169 | self, emotion_data: List[Dict], image, resize_scale: float 170 | ) -> None: 171 | 172 | # draw bounding boxes for each detected person 173 | for data in emotion_data: 174 | image = draw_bounding_box_annotation( 175 | image, data["emotion"], int(1 / resize_scale) * np.array(data["bbox"]) 176 | ) 177 | 178 | # If there are more than one person in frame, the emoji can be shown for 179 | # only one, so show a warning. In case of multiple people the stats are shown 180 | # for just one person 181 | WARNING_TEXT = "Warning ! More than one person detected !" 182 | 183 | if len(emotion_data) > 1: 184 | image = annotate_warning(WARNING_TEXT, image) 185 | 186 | if len(emotion_data) > 0: 187 | # draw emotion confidence stats 188 | image = annotate_emotion_stats(emotion_data[0]["confidence_scores"], image) 189 | # draw the emoji corresponding to the emotion 190 | image = draw_emoji(self.emojis[emotion_data[0]["emotion"]], image) 191 | 192 | return image 193 | 194 | 195 | if __name__ == "__main__": 196 | import os 197 | 198 | # SAMPLE USAGE 199 | from emotion_analyzer.media_utils import load_image_path 200 | 201 | ob = EmotionAnalysisVideo( 202 | face_detector="dlib", 203 | model_loc="models", 204 | face_detection_threshold=0.0, 205 | ) 206 | 207 | img1 = load_image_path("data/sample/1.jpg") 208 | ob.emotion_analysis_video( 209 | video_path="data/sample/test3.mp4", 210 | detection_interval=1, 211 | save_output=True, 212 | preview=False, 213 | output_path="data/output.mp4", 214 | resize_scale=0.5, 215 | ) 216 | --------------------------------------------------------------------------------