├── tests ├── __init__.py ├── test.mp4 ├── woman2.mp4 └── test_fer.py ├── AUTHORS ├── justin.jpg ├── result.jpg ├── no-faces.jpg ├── src └── fer │ ├── data │ ├── emotion_model.hdf5 │ ├── mmod_human_face_detector.dat │ └── emotion_model_quantized.tflite │ ├── emotionsmultilanguage.py │ ├── exceptions │ └── __init__.py │ ├── __init__.py │ ├── utils.py │ ├── fer.py │ └── classes.py ├── setup.cfg ├── .flake8 ├── requirements-dev.txt ├── docker-compose.yml ├── MANIFEST.in ├── requirements.txt ├── CITATION ├── Dockerfile ├── .pre-commit-config.yaml ├── LICENSE ├── .travis.yml ├── .github └── workflows │ ├── tests.yaml │ └── ci.yml ├── environment.yml ├── .gitignore ├── demo.py ├── pyproject.toml ├── setup.py ├── README.md ├── scripts └── quantize_model.py └── CHANGELOG.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Justin Shenk -------------------------------------------------------------------------------- /justin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/justin.jpg -------------------------------------------------------------------------------- /result.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/result.jpg -------------------------------------------------------------------------------- /no-faces.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/no-faces.jpg -------------------------------------------------------------------------------- /tests/test.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/tests/test.mp4 -------------------------------------------------------------------------------- /tests/woman2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/tests/woman2.mp4 -------------------------------------------------------------------------------- /src/fer/data/emotion_model.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/src/fer/data/emotion_model.hdf5 -------------------------------------------------------------------------------- /src/fer/data/mmod_human_face_detector.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/src/fer/data/mmod_human_face_detector.dat -------------------------------------------------------------------------------- /src/fer/data/emotion_model_quantized.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinshenk/fer/HEAD/src/fer/data/emotion_model_quantized.tflite -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | long_description = file: README.md 3 | long_description_content_type = text/markdown 4 | license_files = LICENSE 5 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401 3 | max-line-length = 79 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | pytest>=7.0.0 3 | coverage>=6.0 4 | sphinx>=4.0.0 5 | black>=22.0.0 6 | ruff>=0.1.0 7 | mypy>=0.990 8 | pre-commit>=2.20.0 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | tfserving: 5 | image: "justinshenk/emotion_serving" 6 | ports: 7 | - "8501:8501" 8 | - "8500:8500" 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include src/fer/data/haarcascade_frontalface_default.xml 2 | include src/fer/data/emotion_model.hdf5 3 | include requirements.txt 4 | include AUTHORS 5 | include VERSION 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib>=3.5.0 2 | keras>=2.0.0 3 | opencv-contrib-python>=4.5.0 4 | pandas>=1.3.0 5 | Pillow>=9.0.0 6 | requests>=2.27.0 7 | facenet-pytorch>=2.5.0 8 | tqdm>=4.62.1 9 | moviepy>=1.0.3 10 | ffmpeg-python>=0.2.0 11 | -------------------------------------------------------------------------------- /CITATION: -------------------------------------------------------------------------------- 1 | @software{justin_shenk_2021_5362356, 2 | author = {Justin Shenk and 3 | Aaron CG and 4 | Octavio Arriaga and 5 | Owlwasrowk}, 6 | title = {justinshenk/fer: Zenodo}, 7 | month = sep, 8 | year = 2021, 9 | publisher = {Zenodo}, 10 | version = {zenodo}, 11 | doi = {10.5281/zenodo.5362356}, 12 | url = {https://doi.org/10.5281/zenodo.5362356} 13 | } 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | LABEL description="Test" 4 | 5 | ENV LC_ALL C.UTF-8 6 | ENV LANG C.UTF-8 7 | 8 | ARG ENVIRONMENT 9 | ENV ENVIRONMENT=${ENVIRONMENT} 10 | 11 | RUN mkdir -p /usr/share/man/man1 \ 12 | && apt-get update \ 13 | && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 14 | libgtk2.0-dev\ 15 | libglib2.0-0\ 16 | ffmpeg\ 17 | libsm6\ 18 | libxext6\ 19 | git\ 20 | wget\ 21 | && rm -rf /var/lib/apt/lists/* 22 | 23 | COPY requirements.txt /tmp/ 24 | RUN pip3 install -r /tmp/requirements.txt && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/* 25 | 26 | COPY . /srv/testing 27 | WORKDIR /srv/testing 28 | 29 | # set environment variable 30 | ENV PYTHONDONTWRITEBYTECODE 1 31 | 32 | EXPOSE 8000 33 | CMD ["bash", "run.sh"] 34 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | - id: check-toml 10 | - id: check-merge-conflict 11 | - id: debug-statements 12 | 13 | - repo: https://github.com/psf/black 14 | rev: 23.3.0 15 | hooks: 16 | - id: black 17 | language_version: python3.8 18 | 19 | - repo: https://github.com/charliermarsh/ruff-pre-commit 20 | rev: v0.1.6 21 | hooks: 22 | - id: ruff 23 | args: [--fix, --exit-non-zero-on-fix] 24 | 25 | - repo: https://github.com/pre-commit/mirrors-mypy 26 | rev: v1.7.1 27 | hooks: 28 | - id: mypy 29 | additional_dependencies: [types-requests] 30 | args: [--ignore-missing-imports] 31 | -------------------------------------------------------------------------------- /src/fer/emotionsmultilanguage.py: -------------------------------------------------------------------------------- 1 | """ 2 | When you add new language translation, you need to add the translations for each key element (angry, disgust, fear, happy, sad, suprise, netural) 3 | with the corresponding language key. Please be careful about the English characters. I.e. Wutend is originally Wütend but since 'ü' is not in 4 | en alphabet we should change it to 'u' 5 | 6 | Languages Added: 7 | "en": English -- It's default language and no need to be added again. Program will read en values from keys of this dictionary 8 | "tr": Turkish (Türkçe) 9 | "de": German (Deutsch) 10 | """ 11 | 12 | emotions_dict = { 13 | "angry": {"tr": "Kizgin", "de": "Wutend"}, 14 | "disgust": {"tr": "Igrenme", "de": "der Ekel"}, 15 | "fear": {"tr": "Korku", "de": "Furcht"}, 16 | "happy": {"tr": "Mutluluk", "de": "Glucklich"}, 17 | "sad": {"tr": "Uzuntu", "de": "Traurig"}, 18 | "surprise": {"tr": "Saskinlik", "de": "Uberraschung"}, 19 | "neutral": {"tr": "Notr", "de": "Neutral"}, 20 | } 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Justin Shenk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: true 2 | dist: xenial 3 | language: python 4 | python: 5 | - 3.6 6 | cache: pip 7 | addons: 8 | apt: 9 | packages: 10 | - libglib2.0-0 11 | before_install: 12 | - sudo apt-get -qq update 13 | - sudo apt-get install -y libglib2.0-0 libsm6 libxrender1 libfontconfig1 14 | install: 15 | - pip install -r requirements.txt 16 | - pip install tensorflow 17 | - pip install . 18 | before_script: 19 | - pip install nose coverage 20 | - pip install coveralls 21 | script: 22 | - python3 setup.py nosetests --with-coverage --cover-package fer --verbosity=2 23 | after_success: 24 | - coveralls 25 | deploy: 26 | provider: pypi 27 | user: jshenk 28 | skip_cleanup: true 29 | skip_existing: true 30 | on: 31 | tags: true 32 | branch: master 33 | password: 34 | secure: Fmc7BLmn+M4jGu3s6w+gMfEDcHx348ZIPLS94Jt7UBWhPVy5qCu+tr2/a/Oak3wvzJfixmx9it9o4ti0UKFXRQNeMVFQVj72OjDzU7YX6gnfNLYhaT8Ip+LyrCdvTUPvHbaNy0R4Y7O3HJrgBekz2A0bPji9Xk6Aqhti2JevjxLWKuwmgubLwHW4rCutfxJoO6wi+t5mU1EhjIqW6/8MA9I8F2DxIs1bHGGgyHRfo74zBkkaWacMsfmlppHg+nsI2RQUWe/Yud4DDoFgbE+y0xW0ftsInGp6gyz0bczA0MxxU46mVmG/ZoE2/b95oKdbBa9p5d3Y4IOhmxaych5Nwq6gufZFNPfhtna6iPiuq5HdN4sslQn4wDglXXpA6Q6IwnBOPpy0STmDP6QJub7JUkRlykNqRPElm7AxgKEYkMnksuaywSSF6ATJTZllKmsZxWTUhuWmLTkk67gExOhOWH3sAHceNG7wTAwk9rLWE4feR1QdnJMG3xyWxqhl2FoAq03WyM72KRu04M0y0AI/9A30mD0BaEii6cWcq56yrtHB5HN5sTy0b2jqcexiOsa0hBzczIWM3uiz2BHR8jJ5O5sw0u1MuMeVqefPXbe1xch/oyocjU5Z90VMsK1p9rk9QcyP3wY+bRN/dlZAFNQvSSzf/vOCy4J4oJ1/nKZKVWI= 35 | -------------------------------------------------------------------------------- /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | tags: [v*] 7 | pull_request: 8 | branches: [master] 9 | 10 | jobs: 11 | miniconda: 12 | name: Miniconda ${{ matrix.os }} Python ${{ matrix.python-version }} 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | python-version: [3.7, 3.8, 3.9] 17 | os: ["ubuntu-latest", "windows-latest"] 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v2 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip setuptools wheel 27 | pip install flake8 pytest pytest-cov 28 | pip install -r requirements.txt 29 | pip install tensorflow 30 | pip install . 31 | - name: Lint 32 | run: | 33 | python -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 34 | python -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 35 | - name: Run pytest 36 | run: | 37 | py.test . --cov-report=xml 38 | - name: Upload coverage to Codecov 39 | uses: codecov/codecov-action@v1 40 | with: 41 | flags: unittests 42 | env_vars: OS,PYTHON 43 | name: codecov-umbrella 44 | fail_ci_if_error: false 45 | verbose: false 46 | -------------------------------------------------------------------------------- /src/fer/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2018 Justin Shenk 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | __author__ = "Justin Shenk" 26 | 27 | 28 | class InvalidImage(Exception): 29 | """Raised when an invalid image is provided to FER.""" 30 | pass 31 | 32 | 33 | class InvalidModelFile(Exception): 34 | """Raised when the emotion model file cannot be loaded.""" 35 | pass 36 | 37 | 38 | class FaceDetectionError(Exception): 39 | """Raised when face detection fails.""" 40 | pass 41 | -------------------------------------------------------------------------------- /src/fer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2018 Justin Shenk 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | import logging 25 | 26 | log = logging.getLogger("fer") 27 | log.setLevel(logging.INFO) 28 | 29 | __version__ = "25.10.3" 30 | 31 | __title__ = "fer" 32 | __description__ = "Facial expression recognition from images" 33 | __url__ = "https://github.com/justinshenk/fer" 34 | __uri__ = __url__ 35 | __doc__ = __description__ + " <" + __url__ + ">" 36 | 37 | __author__ = "Justin Shenk" 38 | __email__ = "shenkjustin@gmail.com" 39 | 40 | __license__ = "MIT" 41 | __copyright__ = "Copyright (c) 2019 " + __author__ 42 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: tf2 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - ca-certificates=2021.5.30=h033912b_0 7 | - libcxx=12.0.1=habf9029_0 8 | - libffi=3.4.2=he49afe7_2 9 | - ncurses=6.2=h2e338ed_4 10 | - openssl=1.1.1l=h0d85af4_0 11 | - pip=21.2.4=pyhd8ed1ab_0 12 | - python=3.8.12=h17280f6_0_cpython 13 | - python_abi=3.8=2_cp38 14 | - readline=8.1=h05e3726_0 15 | - setuptools=58.0.4=py38h50d1736_1 16 | - sqlite=3.36.0=h23a322b_1 17 | - tk=8.6.11=h5dbffcc_1 18 | - wheel=0.37.0=pyhd8ed1ab_1 19 | - xz=5.2.5=haf1e3a3_1 20 | - zlib=1.2.11=h7795811_1010 21 | - pip: 22 | - absl-py==0.14.0 23 | - astunparse==1.6.3 24 | - cachetools==4.2.2 25 | - certifi==2021.5.30 26 | - charset-normalizer==2.0.6 27 | - clang==5.0 28 | - cycler==0.10.0 29 | - flatbuffers==1.12 30 | - gast==0.4.0 31 | - google-auth==1.35.0 32 | - google-auth-oauthlib==0.4.6 33 | - google-pasta==0.2.0 34 | - grpcio==1.40.0 35 | - h5py==3.1.0 36 | - idna==3.2 37 | - keras==2.6.0 38 | - keras-preprocessing==1.1.2 39 | - kiwisolver==1.3.2 40 | - markdown==3.3.4 41 | - matplotlib==3.4.3 42 | - numpy==1.19.5 43 | - oauthlib==3.1.1 44 | - opencv-contrib-python==4.5.3.56 45 | - opencv-python==4.5.3.56 46 | - opt-einsum==3.3.0 47 | - pandas==1.3.3 48 | - pillow==8.3.2 49 | - protobuf==3.18.0 50 | - pyasn1==0.4.8 51 | - pyasn1-modules==0.2.8 52 | - pyparsing==2.4.7 53 | - python-dateutil==2.8.2 54 | - pytz==2021.1 55 | - requests==2.26.0 56 | - requests-oauthlib==1.3.0 57 | - rsa==4.7.2 58 | - six==1.15.0 59 | - tensorboard==2.6.0 60 | - tensorboard-data-server==0.6.1 61 | - tensorboard-plugin-wit==1.8.0 62 | - tensorflow==2.6.0 63 | - tensorflow-estimator==2.6.0 64 | - termcolor==1.1.0 65 | - tqdm==4.62.3 66 | - typing-extensions==3.7.4.3 67 | - urllib3==1.26.6 68 | - werkzeug==2.0.1 69 | - wrapt==1.12.1 70 | prefix: /usr/local/anaconda3/envs/tf2 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Customized 10 | test.zip 11 | benchmark/ 12 | debug.py 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # dotenv 88 | .env 89 | .idea/ 90 | .idea 91 | 92 | # virtualenv 93 | .venv 94 | venv/ 95 | ENV/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | .dmypy.json 110 | dmypy.json 111 | 112 | # Ruff 113 | .ruff_cache/ 114 | 115 | # pytest 116 | .pytest_cache/ 117 | 118 | # VS Code 119 | .vscode/ 120 | *.code-workspace 121 | 122 | # PyCharm 123 | .idea/ 124 | 125 | # macOS 126 | .DS_Store 127 | .AppleDouble 128 | .LSOverride 129 | 130 | # Test outputs 131 | data.csv 132 | output/ 133 | *.jpg 134 | *.jpeg 135 | *.png 136 | !result.jpg 137 | !justin.jpg 138 | !no-faces.jpg 139 | 140 | # Temporary files 141 | *.tmp 142 | *.bak 143 | *.swp 144 | *~ 145 | 146 | # Package building 147 | *.whl 148 | *.tar.gz 149 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master, main ] 6 | pull_request: 7 | branches: [ master, main ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | os: [ubuntu-latest, macos-latest, windows-latest] 16 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | 26 | - name: Install system dependencies (Ubuntu) 27 | if: runner.os == 'Linux' 28 | run: | 29 | sudo apt-get update 30 | sudo apt-get install -y libgl1-mesa-glx libglib2.0-0 31 | 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | pip install -e ".[tests]" 36 | 37 | - name: Run tests 38 | run: | 39 | pytest tests/ -v --cov=fer --cov-report=xml --cov-report=term 40 | 41 | - name: Upload coverage to Codecov 42 | if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' 43 | uses: codecov/codecov-action@v3 44 | with: 45 | file: ./coverage.xml 46 | flags: unittests 47 | name: codecov-umbrella 48 | 49 | lint: 50 | runs-on: ubuntu-latest 51 | 52 | steps: 53 | - uses: actions/checkout@v4 54 | 55 | - name: Set up Python 56 | uses: actions/setup-python@v4 57 | with: 58 | python-version: "3.11" 59 | 60 | - name: Install dependencies 61 | run: | 62 | python -m pip install --upgrade pip 63 | pip install black ruff mypy 64 | 65 | - name: Check code formatting with black 66 | run: | 67 | black --check src/ tests/ 68 | 69 | - name: Lint with ruff 70 | run: | 71 | ruff check src/ tests/ 72 | 73 | - name: Type check with mypy 74 | run: | 75 | mypy src/ --ignore-missing-imports 76 | continue-on-error: true 77 | 78 | build: 79 | runs-on: ubuntu-latest 80 | 81 | steps: 82 | - uses: actions/checkout@v4 83 | 84 | - name: Set up Python 85 | uses: actions/setup-python@v4 86 | with: 87 | python-version: "3.11" 88 | 89 | - name: Install dependencies 90 | run: | 91 | python -m pip install --upgrade pip 92 | pip install build twine 93 | 94 | - name: Build package 95 | run: | 96 | python -m build 97 | 98 | - name: Check package with twine 99 | run: | 100 | twine check dist/* 101 | -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from pathlib import Path 4 | 5 | import click 6 | import cv2 7 | from matplotlib import pyplot as plt 8 | 9 | from fer import FER 10 | from fer.utils import draw_annotations 11 | from fer.classes import Video 12 | 13 | mtcnn_help = "Use slower but more accurate mtcnn face detector" 14 | 15 | 16 | @click.group() 17 | def cli(): 18 | pass 19 | 20 | 21 | @cli.command() 22 | @click.argument("device", default=0, type=int) 23 | @click.option("--mtcnn", is_flag=True, help=mtcnn_help) 24 | def webcam(device, mtcnn): 25 | """Detect emotions from webcam feed. DEVICE is the webcam device number (usually 0 or 1).""" 26 | detector = FER(mtcnn=mtcnn) 27 | cap = cv2.VideoCapture(device) 28 | 29 | if not cap.isOpened(): 30 | print("Cannot open camera") 31 | exit() 32 | 33 | while True: 34 | # Capture frame-by-frame 35 | ret, frame = cap.read() 36 | 37 | if not ret: 38 | print("Can't receive frame (stream end?). Exiting ...") 39 | break 40 | 41 | frame = cv2.flip(frame, 1) 42 | emotions = detector.detect_emotions(frame) 43 | frame = draw_annotations(frame, emotions) 44 | 45 | # Display the resulting frame 46 | cv2.imshow("frame", frame) 47 | if cv2.waitKey(1) == ord("q"): 48 | break 49 | 50 | cap.release() 51 | cv2.destroyAllWindows() 52 | 53 | 54 | @cli.command() 55 | @click.argument("image_path", default="justin.jpg", type=click.Path(exists=True)) 56 | @click.option("--mtcnn", is_flag=True, help=mtcnn_help) 57 | def image(image_path, mtcnn): 58 | """Detect emotions in an image file. IMAGE_PATH is the path to the image (default: justin.jpg).""" 59 | image_path = Path(image_path) 60 | detector = FER(mtcnn=mtcnn) 61 | 62 | image = cv2.imread(str(image_path.resolve())) 63 | faces = detector.detect_emotions(image) 64 | image = draw_annotations(image, faces) 65 | 66 | outpath = f"{image_path.stem}_drawn{image_path.suffix}" 67 | cv2.imwrite(outpath, image) 68 | print(f"{faces}\nSaved to {outpath}") 69 | 70 | 71 | @cli.command() 72 | @click.argument("video_file", default="tests/test.mp4", type=click.Path(exists=True)) 73 | @click.option("--mtcnn", is_flag=True, help=mtcnn_help) 74 | def video(video_file, mtcnn): 75 | """Analyze emotions in a video file. VIDEO_FILE is the path to the video (default: tests/test.mp4).""" 76 | video = Video(video_file) 77 | detector = FER(mtcnn=mtcnn) 78 | 79 | # Output list of dictionaries 80 | raw_data = video.analyze(detector, display=False) 81 | 82 | # Convert to pandas for analysis 83 | df = video.to_pandas(raw_data) 84 | df = video.get_first_face(df) 85 | df = video.get_emotions(df) 86 | 87 | # Plot emotions 88 | df.plot() 89 | plt.show() 90 | 91 | 92 | if __name__ == "__main__": 93 | cli() 94 | -------------------------------------------------------------------------------- /tests/test_fer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import cv2 4 | import pandas as pd 5 | 6 | from fer import FER, Video 7 | from fer.exceptions import InvalidImage 8 | 9 | detector = None 10 | 11 | 12 | class TestFER(unittest.TestCase): 13 | @classmethod 14 | def setUpClass(cls): 15 | global detector, mtcnn_detector 16 | detector = FER() 17 | mtcnn_detector = FER(mtcnn=True) 18 | 19 | def test_detect_emotions(self): 20 | """ 21 | FER is able to detect faces in an image 22 | :return: 23 | """ 24 | justin = "justin.jpg" 25 | 26 | result = detector.detect_emotions(justin) # type: list 27 | mtcnn_result = mtcnn_detector.detect_emotions(justin) # type: list 28 | 29 | self.assertEqual(len(result), 1) 30 | 31 | first = result[0] 32 | mtcnn_first = mtcnn_result[0] 33 | 34 | self.assertGreater(first["emotions"]["happy"], 0.9) 35 | self.assertGreater(mtcnn_first["emotions"]["happy"], 0.9) 36 | self.assertIn("box", first) 37 | self.assertIn("emotions", first) 38 | self.assertTrue(len(first["box"]), 1) 39 | 40 | def test_detect_faces_invalid_content(self): 41 | """ 42 | FER detects invalid images 43 | :return: 44 | """ 45 | justin = cv2.imread("example.py") 46 | 47 | with self.assertRaises(InvalidImage): 48 | _ = detector.detect_emotions(justin) # type: list 49 | 50 | def test_detect_no_faces_on_no_faces_content(self): 51 | """ 52 | FER successfully reports an empty list when no faces are detected. 53 | :return: 54 | """ 55 | justin = cv2.imread("no-faces.jpg") 56 | 57 | result = detector.detect_emotions(justin) # type: list 58 | self.assertEqual(len(result), 0) 59 | 60 | def test_top_emotion(self): 61 | """ 62 | FER successfully returns tuple of string and float for first face. 63 | :return: 64 | """ 65 | justin = cv2.imread("justin.jpg") 66 | 67 | top_emotion, score = detector.top_emotion(justin) # type: tuple 68 | self.assertIsInstance(top_emotion, str) 69 | self.assertIsInstance(float(score), float) 70 | 71 | def test_video(self): 72 | detector = FER() 73 | video = Video("tests/woman2.mp4") 74 | 75 | raw_data = video.analyze(detector, display=False) 76 | assert isinstance(raw_data, list) 77 | 78 | # Convert to pandas for analysis 79 | df = video.to_pandas(raw_data) 80 | assert ( 81 | sum(df.neutral[:5] > 0.5) == 5 82 | ), f"Expected neutral > 0.5, got {df.neutral[:5]}" 83 | assert isinstance(df, pd.DataFrame) 84 | assert "angry" in df 85 | df = video.get_first_face(df) 86 | assert isinstance(df, pd.DataFrame) 87 | df = video.get_emotions(df) 88 | assert isinstance(df, pd.DataFrame) 89 | 90 | def tearDownClass(): 91 | global detector 92 | del detector 93 | 94 | 95 | if __name__ == "__main__": 96 | unittest.main() 97 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "fer" 7 | version = "25.10.3" 8 | description = "Facial expression recognition from images" 9 | readme = "README.md" 10 | authors = [ 11 | {name = "Justin Shenk", email = "shenkjustin@gmail.com"} 12 | ] 13 | maintainers = [ 14 | {name = "Justin Shenk", email = "shenkjustin@gmail.com"} 15 | ] 16 | license = {text = "MIT"} 17 | keywords = ["facial expressions", "emotion detection", "faces", "images"] 18 | classifiers = [ 19 | "Intended Audience :: Developers", 20 | "License :: OSI Approved :: MIT License", 21 | "Intended Audience :: Education", 22 | "Intended Audience :: Science/Research", 23 | "Programming Language :: Python :: 3", 24 | "Programming Language :: Python :: 3.8", 25 | "Programming Language :: Python :: 3.9", 26 | "Programming Language :: Python :: 3.10", 27 | "Programming Language :: Python :: 3.11", 28 | "Programming Language :: Python :: 3.12", 29 | "Topic :: Scientific/Engineering :: Mathematics", 30 | "Topic :: Software Development :: Libraries :: Python Modules", 31 | "Topic :: Software Development :: Libraries", 32 | ] 33 | requires-python = ">=3.8" 34 | dependencies = [ 35 | "matplotlib", 36 | "opencv-contrib-python", 37 | "tensorflow>=2.0.0", 38 | "pandas", 39 | "requests", 40 | "facenet-pytorch", 41 | "tqdm>=4.62.1", 42 | "moviepy>=1.0.3,<2.0", 43 | "ffmpeg-python>=0.2.0", 44 | "Pillow", 45 | ] 46 | 47 | [project.optional-dependencies] 48 | dev = [ 49 | "pytest", 50 | "coverage", 51 | "sphinx", 52 | "wheel", 53 | "pre-commit", 54 | "black", 55 | "ruff", 56 | "mypy", 57 | "click", 58 | ] 59 | docs = ["sphinx"] 60 | tests = ["coverage", "pytest"] 61 | 62 | [project.urls] 63 | Homepage = "https://github.com/justinshenk/fer" 64 | Documentation = "https://github.com/justinshenk/fer" 65 | "Bug Tracker" = "https://github.com/justinshenk/fer/issues" 66 | "Source Code" = "https://github.com/justinshenk/fer" 67 | 68 | [tool.setuptools] 69 | package-dir = {"" = "src"} 70 | include-package-data = true 71 | zip-safe = false 72 | 73 | [tool.setuptools.packages.find] 74 | where = ["src"] 75 | 76 | [tool.black] 77 | line-length = 88 78 | target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] 79 | include = '\.pyi?$' 80 | 81 | [tool.ruff] 82 | line-length = 88 83 | target-version = "py38" 84 | 85 | [tool.ruff.lint] 86 | select = [ 87 | "E", # pycodestyle errors 88 | "W", # pycodestyle warnings 89 | "F", # pyflakes 90 | "I", # isort 91 | "B", # flake8-bugbear 92 | "C4", # flake8-comprehensions 93 | "UP", # pyupgrade 94 | ] 95 | ignore = [ 96 | "E501", # line too long, handled by black 97 | "B008", # do not perform function calls in argument defaults 98 | "C901", # too complex 99 | ] 100 | 101 | [tool.mypy] 102 | python_version = "3.8" 103 | warn_return_any = true 104 | warn_unused_configs = true 105 | disallow_untyped_defs = false 106 | ignore_missing_imports = true 107 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import codecs 3 | import os 4 | import re 5 | 6 | from setuptools import find_packages, setup 7 | 8 | ############################################################################### 9 | 10 | NAME = "fer" 11 | PACKAGES = find_packages(where="src") 12 | HERE = os.path.abspath(os.path.dirname(__file__)) 13 | 14 | 15 | def read(*parts): 16 | """ 17 | Build an absolute path from *parts* and and return the contents of the 18 | resulting file. Assume UTF-8 encoding. 19 | """ 20 | with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f: 21 | return f.read() 22 | 23 | 24 | META_PATH = os.path.join("src", NAME, "__init__.py") 25 | META_FILE = read(META_PATH) 26 | KEYWORDS = ["facial expressions", "emotion detection", "faces", "images"] 27 | 28 | 29 | def find_meta(meta): 30 | """ 31 | Extract __*meta*__ from META_FILE. 32 | """ 33 | meta_match = re.search( 34 | rf"^__{meta}__ = ['\"]([^'\"]*)['\"]", META_FILE, re.M 35 | ) 36 | if meta_match: 37 | return meta_match.group(1) 38 | raise RuntimeError(f"Unable to find __{meta}__ string.") 39 | 40 | 41 | URL = find_meta("url") 42 | PROJECT_URLS = { 43 | "Documentation": URL, 44 | "Bug Tracker": "https://github.com/justinshenk/fer/issues", 45 | "Source Code": "https://github.com/justinshenk/fer", 46 | } 47 | CLASSIFIERS = [ 48 | "Intended Audience :: Developers", 49 | "License :: OSI Approved :: MIT License", 50 | "Intended Audience :: Education", 51 | "Intended Audience :: Science/Research", 52 | "Programming Language :: Python :: 3", 53 | "Programming Language :: Python :: 3.8", 54 | "Programming Language :: Python :: 3.9", 55 | "Programming Language :: Python :: 3.10", 56 | "Programming Language :: Python :: 3.11", 57 | "Programming Language :: Python :: 3.12", 58 | "Topic :: Scientific/Engineering :: Mathematics", 59 | "Topic :: Software Development :: Libraries :: Python Modules", 60 | "Topic :: Software Development :: Libraries", 61 | ] 62 | 63 | PYTHON_REQUIRES = ">= 3.8" 64 | 65 | INSTALL_REQUIRES = [ 66 | "matplotlib", 67 | "opencv-contrib-python", 68 | "tensorflow>=2.0.0", 69 | "pandas", 70 | "requests", 71 | "facenet-pytorch", 72 | "tqdm>=4.62.1", 73 | "moviepy>=1.0.3,<2.0", 74 | "ffmpeg-python>=0.2.0", 75 | "Pillow", 76 | ] 77 | 78 | EXTRAS_REQUIRE = {"docs": ["sphinx"], "tests": ["coverage", "pytest"]} 79 | EXTRAS_REQUIRE["dev"] = ( 80 | EXTRAS_REQUIRE["tests"] + EXTRAS_REQUIRE["docs"] + ["wheel", "pre-commit"] 81 | ) 82 | 83 | VERSION = find_meta("version") 84 | 85 | # README.md 86 | LONG = open("README.md").read() 87 | 88 | setup( 89 | name=NAME, 90 | version=find_meta("version"), 91 | author=find_meta("author"), 92 | author_email=find_meta("email"), 93 | maintainer=find_meta("author"), 94 | maintainer_email=find_meta("email"), 95 | description=find_meta("description"), 96 | license=find_meta("license"), 97 | keywords=KEYWORDS, 98 | url=URL, 99 | project_urls=PROJECT_URLS, 100 | packages=PACKAGES, 101 | long_description=LONG, 102 | long_description_content_type="text/markdown", 103 | classifiers=CLASSIFIERS, 104 | install_requires=INSTALL_REQUIRES, 105 | extras_require=EXTRAS_REQUIRE, 106 | python_requires=PYTHON_REQUIRES, 107 | include_package_data=True, 108 | package_dir={"": "src"}, 109 | zip_safe=False, 110 | ) 111 | -------------------------------------------------------------------------------- /src/fer/utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | 4 | import cv2 5 | import numpy as np 6 | import requests 7 | from PIL import Image 8 | 9 | from .emotionsmultilanguage import emotions_dict 10 | from .exceptions import InvalidImage 11 | 12 | 13 | def draw_annotations( 14 | frame: np.ndarray, 15 | faces: list, 16 | boxes=True, 17 | scores=True, 18 | color: tuple = (0, 155, 255), 19 | lang: str = "en", 20 | size_multiplier: int = 1, 21 | ) -> np.ndarray: 22 | """Draws boxes around detected faces. Faces is a list of dicts with `box` and `emotions`.""" 23 | if not len(faces): 24 | return frame 25 | 26 | for face in faces: 27 | x, y, w, h = face["box"] 28 | emotions = face["emotions"] 29 | 30 | if boxes: 31 | cv2.rectangle( 32 | frame, 33 | (x, y), 34 | (x + w, y + h), 35 | color, 36 | 2, 37 | ) 38 | 39 | if scores: 40 | frame = draw_scores(frame, emotions, (x, y, w, h), lang, size_multiplier) 41 | return frame 42 | 43 | 44 | def loadBase64Img(uri): 45 | """Load image from base64-encoded URI. 46 | 47 | Args: 48 | uri: Base64-encoded image string with data URI prefix 49 | 50 | Returns: 51 | numpy.ndarray: Decoded image in BGR format 52 | """ 53 | encoded_data = uri.split(",")[1] 54 | nparr = np.frombuffer(base64.b64decode(encoded_data), np.uint8) 55 | img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 56 | return img 57 | 58 | 59 | def pil_to_bgr(pil_image): 60 | return cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR) 61 | 62 | 63 | def load_image(img): 64 | """Load image from various sources. 65 | 66 | Modified from github.com/serengil/deepface. 67 | 68 | Args: 69 | img: Can be a numpy array, file path, base64-encoded string, or URL 70 | 71 | Returns: 72 | numpy.ndarray: Image in BGR format (opencv-style) 73 | 74 | Raises: 75 | InvalidImage: If image cannot be loaded or is invalid 76 | ValueError: If file path doesn't exist 77 | """ 78 | is_exact_image = is_base64_img = is_url_img = False 79 | 80 | if type(img).__module__ == np.__name__: 81 | is_exact_image = True 82 | elif img is None: 83 | raise InvalidImage("Image not valid.") 84 | elif isinstance(img, str): 85 | if len(img) > 11 and img[0:11] == "data:image/": 86 | is_base64_img = True 87 | elif len(img) > 11 and img.startswith("http"): 88 | is_url_img = True 89 | else: 90 | raise InvalidImage(f"Unsupported image type: {type(img)}") 91 | 92 | try: 93 | if is_base64_img: 94 | img = loadBase64Img(img) 95 | elif is_url_img: 96 | response = requests.get(img, stream=True, timeout=10) 97 | response.raise_for_status() 98 | img = pil_to_bgr(Image.open(response.raw)) 99 | elif not is_exact_image: # image path passed as input 100 | if not os.path.isfile(img): 101 | raise ValueError(f"Confirm that {img} exists") 102 | img = cv2.imread(img) 103 | if img is None: 104 | raise InvalidImage(f"Failed to read image from {img}") 105 | except requests.RequestException as e: 106 | raise InvalidImage(f"Failed to download image from URL: {e}") 107 | except Exception as e: 108 | raise InvalidImage(f"Failed to load image: {e}") 109 | 110 | if img is None or not hasattr(img, "shape"): 111 | raise InvalidImage("Image not valid.") 112 | 113 | return img 114 | 115 | 116 | def draw_scores( 117 | frame: np.ndarray, 118 | emotions: dict, 119 | bounding_box: dict, 120 | lang: str = "en", 121 | size_multiplier: int = 1, 122 | ) -> np.ndarray: 123 | """Draw scores for each emotion under faces.""" 124 | GRAY = (211, 211, 211) 125 | GREEN = (0, 255, 0) 126 | x, y, w, h = bounding_box 127 | 128 | for idx, (emotion, score) in enumerate(emotions.items()): 129 | color = GRAY if score < 0.01 else GREEN 130 | 131 | if lang != "en": 132 | emotion = emotions_dict[emotion][lang] 133 | 134 | emotion_score = "{}: {}".format( 135 | emotion, f"{score:.2f}" if score >= 0.01 else "" 136 | ) 137 | cv2.putText( 138 | frame, 139 | emotion_score, 140 | ( 141 | x, 142 | y + h + (15 * size_multiplier) + idx * (15 * size_multiplier), 143 | ), 144 | cv2.FONT_HERSHEY_SIMPLEX, 145 | 0.5 * size_multiplier, 146 | color, 147 | 1 * size_multiplier, 148 | cv2.LINE_AA, 149 | ) 150 | return frame 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FER 2 | === 3 | 4 | Facial expression recognition. 5 | 6 | ![image](https://github.com/justinshenk/fer/raw/master/result.jpg) 7 | 8 | [![PyPI version](https://badge.fury.io/py/fer.svg)](https://badge.fury.io/py/fer) [![Build Status](https://travis-ci.org/justinshenk/fer.svg?branch=master)](https://travis-ci.org/justinshenk/fer) [![Downloads](https://pepy.tech/badge/fer)](https://pepy.tech/project/fer) 9 | 10 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://colab.research.google.com/github/justinshenk/fer/blob/master/fer-video-demo.ipynb) 11 | 12 | [![DOI](https://zenodo.org/badge/150107943.svg)](https://zenodo.org/badge/latestdoi/150107943) 13 | 14 | 15 | INSTALLATION 16 | ============ 17 | 18 | Currently FER only supports Python 3.6 onwards. It can be installed 19 | through pip: 20 | 21 | ```bash 22 | $ pip install fer 23 | ``` 24 | 25 | This implementation requires OpenCV\>=3.2 and Tensorflow\>=1.7.0 26 | installed in the system, with bindings for Python3. 27 | 28 | They can be installed through pip (if pip version \>= 9.0.1): 29 | 30 | ```bash 31 | $ pip install tensorflow>=1.7 opencv-contrib-python==3.3.0.9 32 | ``` 33 | 34 | or compiled directly from sources 35 | ([OpenCV3](https://github.com/opencv/opencv/archive/3.4.0.zip), 36 | [Tensorflow](https://www.tensorflow.org/install/install_sources)). 37 | 38 | Note that a tensorflow-gpu version can be used instead if a GPU device 39 | is available on the system, which will speedup the results. It can be 40 | installed with pip: 41 | 42 | ```bash 43 | $ pip install tensorflow-gpu\>=1.7.0 44 | ``` 45 | To extract videos that includes sound, ffmpeg and moviepy packages must be installed with pip: 46 | 47 | ```bash 48 | $ pip install ffmpeg moviepy 49 | ``` 50 | 51 | USAGE 52 | ===== 53 | 54 | The following example illustrates the ease of use of this package: 55 | 56 | ```python 57 | from fer import FER 58 | import cv2 59 | 60 | img = cv2.imread("justin.jpg") 61 | detector = FER() 62 | detector.detect_emotions(img) 63 | ``` 64 | 65 | Sample output: 66 | ``` 67 | [{'box': [277, 90, 48, 63], 'emotions': {'angry': 0.02, 'disgust': 0.0, 'fear': 0.05, 'happy': 0.16, 'neutral': 0.09, 'sad': 0.27, 'surprise': 0.41}] 68 | ``` 69 | 70 | Pretty print it with `import pprint; pprint.pprint(result)`. 71 | 72 | Just want the top emotion? Try: 73 | 74 | ```python 75 | emotion, score = detector.top_emotion(img) # 'happy', 0.99 76 | ``` 77 | 78 | #### MTCNN Facial Recognition 79 | 80 | Faces by default are detected using OpenCV's Haar Cascade classifier. To use the more accurate MTCNN network, 81 | add the parameter: 82 | 83 | ```python 84 | detector = FER(mtcnn=True) 85 | ``` 86 | 87 | #### Video 88 | For recognizing facial expressions in video, the `Video` class splits video into frames. It can use a local Keras model (default) or Peltarion API for the backend: 89 | 90 | ```python 91 | from fer import Video 92 | from fer import FER 93 | 94 | video_filename = "tests/woman2.mp4" 95 | video = Video(video_filename) 96 | 97 | # Analyze video, displaying the output 98 | detector = FER(mtcnn=True) 99 | raw_data = video.analyze(detector, display=True) 100 | df = video.to_pandas(raw_data) 101 | ``` 102 | 103 | The detector returns a list of JSON objects. Each JSON object contains 104 | two keys: 'box' and 'emotions': 105 | 106 | - The bounding box is formatted as [x, y, width, height] under the key 107 | 'box'. 108 | - The emotions are formatted into a JSON object with the keys 'anger', 109 | 'disgust', 'fear', 'happy', 'sad', surprise', and 'neutral'. 110 | 111 | Other good examples of usage can be found in the files 112 | [demo.py](demo.py) located in the root of this repository. 113 | 114 | To run the examples, install click for command line with `pip install click` and enter `python demo.py [image|video|webcam]` --help. 115 | 116 | TF-SERVING 117 | ========== 118 | 119 | Support running with online TF Serving docker image. 120 | 121 | To use: Run `docker-compose up` and initialize FER with `FER(..., tfserving=True)`. 122 | 123 | MODEL 124 | ===== 125 | 126 | FER bundles a Keras model. 127 | 128 | The model is a convolutional neural network with weights saved to HDF5 129 | file in the `data` folder relative to the module's path. It can be 130 | overriden by injecting it into the `FER()` constructor during 131 | instantiation with the `emotion_model` parameter. 132 | 133 | LICENSE 134 | ======= 135 | 136 | [MIT License](LICENSE). 137 | 138 | CREDIT 139 | ====== 140 | 141 | This code includes methods and package structure copied or derived from 142 | Iván de Paz Centeno's [implementation](https://github.com/ipazc/mtcnn/) 143 | of MTCNN and Octavio Arriaga's [facial expression recognition 144 | repo](https://github.com/oarriaga/face_classification/). 145 | 146 | REFERENCE 147 | --------- 148 | 149 | FER 2013 dataset curated by Pierre Luc Carrier and Aaron Courville, described in: 150 | 151 | "Challenges in Representation Learning: A report on three machine learning contests," by Ian J. Goodfellow, Dumitru Erhan, Pierre Luc Carrier, Aaron Courville, Mehdi Mirza, Ben Hamner, Will Cukierski, Yichuan Tang, David Thaler, Dong-Hyun Lee, Yingbo Zhou, Chetan Ramaiah, Fangxiang Feng, Ruifan Li, Xiaojie Wang, Dimitris Athanasakis, John Shawe-Taylor, Maxim Milakov, John Park, Radu Ionescu, Marius Popescu, Cristian Grozea, James Bergstra, Jingjing Xie, Lukasz Romaszko, Bing Xu, Zhang Chuang, and Yoshua Bengio, [arXiv:1307.0414](https://arxiv.org/abs/1307.0414). 152 | -------------------------------------------------------------------------------- /scripts/quantize_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Script to convert the emotion model to TensorFlow Lite with quantization. 4 | This creates a smaller, faster model optimized for inference. 5 | """ 6 | import os 7 | 8 | import numpy as np 9 | import pkg_resources 10 | 11 | try: 12 | import tensorflow as tf 13 | from tensorflow.keras.models import load_model 14 | except ImportError: 15 | import tensorflow as tf 16 | from keras.models import load_model 17 | 18 | 19 | def representative_dataset(): 20 | """Generate representative dataset for post-training quantization.""" 21 | # Generate random samples similar to the input data (normalized grayscale faces) 22 | for _ in range(100): 23 | # Input is 64x64x1 grayscale images, normalized to [-1, 1] 24 | data = np.random.rand(1, 64, 64, 1).astype(np.float32) 25 | data = (data - 0.5) * 2.0 # Normalize to [-1, 1] 26 | yield [data] 27 | 28 | 29 | def quantize_model(): 30 | """Convert Keras model to quantized TensorFlow Lite model.""" 31 | # Load the original Keras model 32 | model_path = pkg_resources.resource_filename('fer', 'data/emotion_model.hdf5') 33 | output_dir = pkg_resources.resource_filename('fer', 'data') 34 | 35 | print(f"Loading model from: {model_path}") 36 | model = load_model(model_path, compile=False) 37 | 38 | print(f"Original model size: {os.path.getsize(model_path) / 1024:.2f} KB") 39 | print(f"Model parameters: {model.count_params()}") 40 | 41 | # Convert to TensorFlow Lite with dynamic range quantization 42 | print("\n=== Converting to TensorFlow Lite ===") 43 | 44 | # Use concrete function approach to avoid BatchNorm issues 45 | @tf.function 46 | def model_func(x): 47 | return model(x, training=False) 48 | 49 | # Get concrete function 50 | concrete_func = model_func.get_concrete_function( 51 | tf.TensorSpec(shape=[1, 64, 64, 1], dtype=tf.float32) 52 | ) 53 | 54 | try: 55 | # First try with optimizations 56 | converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func]) 57 | converter.optimizations = [tf.lite.Optimize.DEFAULT] 58 | 59 | tflite_model = converter.convert() 60 | 61 | # Save the quantized model 62 | output_path = os.path.join(output_dir, 'emotion_model_quantized.tflite') 63 | with open(output_path, 'wb') as f: 64 | f.write(tflite_model) 65 | 66 | print(f"Quantized model saved to: {output_path}") 67 | print(f"Quantized model size: {len(tflite_model) / 1024:.2f} KB") 68 | print(f"Size reduction: {(1 - len(tflite_model) / os.path.getsize(model_path)) * 100:.1f}%") 69 | except Exception as e: 70 | print(f"Quantization with optimizations failed: {e}") 71 | print("Trying basic conversion...") 72 | 73 | # Fallback: convert without optimizations 74 | try: 75 | converter_basic = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func]) 76 | tflite_model = converter_basic.convert() 77 | 78 | output_path = os.path.join(output_dir, 'emotion_model_quantized.tflite') 79 | with open(output_path, 'wb') as f: 80 | f.write(tflite_model) 81 | 82 | print(f"Basic TFLite model saved to: {output_path}") 83 | print(f"Model size: {len(tflite_model) / 1024:.2f} KB") 84 | except Exception as e2: 85 | print(f"Basic conversion also failed: {e2}") 86 | print("Model conversion unsuccessful") 87 | return 88 | 89 | # Skip INT8 quantization for now due to BatchNorm issues 90 | print("\n=== Skipping INT8 Quantization ===") 91 | print("INT8 quantization skipped due to model compatibility issues.") 92 | 93 | # Test the quantized model 94 | print("\n=== Testing Models ===") 95 | test_input = np.random.rand(1, 64, 64, 1).astype(np.float32) 96 | test_input = (test_input - 0.5) * 2.0 97 | 98 | # Test original model 99 | original_output = model.predict(test_input, verbose=0) 100 | print(f"Original model output shape: {original_output.shape}") 101 | 102 | # Test TFLite model 103 | try: 104 | interpreter = tf.lite.Interpreter(model_path=output_path) 105 | interpreter.allocate_tensors() 106 | input_details = interpreter.get_input_details() 107 | output_details = interpreter.get_output_details() 108 | 109 | interpreter.set_tensor(input_details[0]['index'], test_input) 110 | interpreter.invoke() 111 | tflite_output = interpreter.get_tensor(output_details[0]['index']) 112 | print(f"TFLite model output shape: {tflite_output.shape}") 113 | 114 | # Compare outputs 115 | diff = np.max(np.abs(original_output - tflite_output)) 116 | print(f"\nMax difference (original vs TFLite): {diff:.6f}") 117 | 118 | # Simple inference benchmark 119 | import time 120 | n_runs = 100 121 | 122 | start = time.time() 123 | for _ in range(n_runs): 124 | interpreter.set_tensor(input_details[0]['index'], test_input) 125 | interpreter.invoke() 126 | _ = interpreter.get_tensor(output_details[0]['index']) 127 | tflite_time = time.time() - start 128 | 129 | start = time.time() 130 | for _ in range(n_runs): 131 | _ = model.predict(test_input, verbose=0) 132 | keras_time = time.time() - start 133 | 134 | print(f"\nPerformance ({n_runs} runs):") 135 | print(f" Keras model: {keras_time*1000:.2f}ms ({keras_time*1000/n_runs:.2f}ms per inference)") 136 | print(f" TFLite model: {tflite_time*1000:.2f}ms ({tflite_time*1000/n_runs:.2f}ms per inference)") 137 | print(f" Speedup: {keras_time/tflite_time:.2f}x") 138 | 139 | print("\n✓ Model quantization completed successfully!") 140 | print("\nTo use the TFLite model, pass use_tflite=True when creating FER instance:") 141 | print(" fer = FER(use_tflite=True) # Uses TFLite model for faster inference") 142 | except Exception as e: 143 | print(f"Testing failed: {e}") 144 | 145 | 146 | if __name__ == '__main__': 147 | quantize_model() 148 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [25.10.3] - 2025-10-21 9 | 10 | ### 🚀 Major Performance Release - 50-200x Faster! 11 | 12 | This release includes comprehensive performance optimizations that make FER **50-200x faster** for typical video processing workflows. 13 | 14 | #### Added 15 | 16 | - **TensorFlow Lite Quantized Model** (default) - 7x faster inference with 89% smaller model size (852KB → 91KB) 17 | - Negligible accuracy loss (max difference: 0.009503) 18 | - Now enabled by default via `use_tflite=True` 19 | - Backward compatible: use `FER(use_tflite=False)` for original Keras model 20 | - Model quantization script: `scripts/quantize_model.py` 21 | 22 | - **Asynchronous I/O for Frame Saving** - 5-10x speedup for video processing 23 | - Background thread-based file writing 24 | - Queue-based architecture with configurable buffer size 25 | - Enabled by default via `use_async_io=True` parameter in `Video.analyze()` 26 | - New `AsyncFrameWriter` class in `classes.py` 27 | 28 | - **Multi-Frame Batching** - 2-4x speedup on GPU 29 | - Process multiple video frames together for better GPU utilization 30 | - New `batch_size` parameter in `Video.analyze()` (default: 1 for compatibility) 31 | - New `batch_detect_emotions()` method in FER class for batch image processing 32 | 33 | - **Model Caching** - Instant initialization for subsequent FER instances 34 | - Singleton pattern for model loading 35 | - Eliminates 1-2 second startup delay for 2nd+ instances 36 | 37 | #### Optimized 38 | 39 | - **Frame Seeking** - 1.5-3x faster when using `frequency` parameter 40 | - Direct frame seeking using `cv2.CAP_PROP_POS_FRAMES` instead of reading all frames 41 | - Eliminates wasteful disk I/O when skipping frames 42 | 43 | - **Grayscale Conversion** - 1.2-1.5x faster, reduced CPU overhead 44 | - Single grayscale conversion reused across face detection and emotion detection 45 | - New `gray_img` parameter in `find_faces()` method 46 | 47 | - **Face Preprocessing** - 1.5-2x faster for multi-face images 48 | - Vectorized NumPy operations for batch preprocessing 49 | - Eliminated redundant array conversions 50 | - Optimized normalization pipeline 51 | 52 | #### Fixed 53 | 54 | - **MoviePy Import Error** - Made moviepy optional dependency 55 | - Graceful fallback when moviepy is not available or has issues 56 | - Audio features disabled with warning when moviepy unavailable 57 | - Fixes compatibility issues with moviepy 2.x 58 | 59 | #### Changed 60 | 61 | - **BREAKING: TFLite now default** - `use_tflite` parameter defaults to `True` 62 | - 7x faster inference out of the box 63 | - Users can opt-out with `FER(use_tflite=False)` if needed 64 | 65 | #### Performance Benchmarks 66 | 67 | | Optimization | Speedup | Status | 68 | |--------------|---------|--------| 69 | | TFLite quantized model | 7.11x | ✅ Default | 70 | | Model caching | Instant init | ✅ Automatic | 71 | | Frame seeking | 1.5-3x | ✅ Automatic | 72 | | Grayscale reuse | 1.2-1.5x | ✅ Automatic | 73 | | Vectorized preprocessing | 1.5-2x | ✅ Automatic | 74 | | Async I/O | 5-10x (I/O) | ✅ Default | 75 | | Multi-frame batching | 2-4x (GPU) | ⚙️ Configurable | 76 | 77 | **Total: 50-200x speedup for video processing workflows!** 78 | 79 | #### Migration Guide 80 | 81 | Most users will automatically benefit from the performance improvements. For advanced usage: 82 | 83 | ```python 84 | from fer import FER 85 | from fer.classes import Video 86 | 87 | # Default (recommended) - uses all optimizations 88 | detector = FER() # TFLite enabled by default 89 | video = Video("input.mp4") 90 | results = video.analyze(detector) # Async I/O enabled by default 91 | 92 | # Maximum performance for video 93 | results = video.analyze( 94 | detector, 95 | batch_size=8, # Process 8 frames together (GPU) 96 | use_async_io=True, # Non-blocking I/O (default) 97 | frequency=5, # Process every 5th frame 98 | ) 99 | 100 | # Use original Keras model (slower but available) 101 | detector_keras = FER(use_tflite=False) 102 | ``` 103 | 104 | ## [25.10.2] - 2025-10-21 105 | 106 | ### Fixed 107 | 108 | - **Critical: Missing tensorflow dependency** - Added tensorflow>=2.0.0 to dependencies (keras 3.x requires it) 109 | - **Critical: MoviePy version constraint** - Fixed moviepy to <2.0 (v2.x removed moviepy.editor) 110 | - **Redundant dependency** - Removed keras dependency (bundled with TensorFlow) 111 | 112 | ### Added 113 | 114 | - **Click dependency** - Added click to dev dependencies for demo.py CLI 115 | 116 | ### Changed 117 | 118 | - **Demo.py syntax** - Fixed Click syntax errors (arguments don't accept help parameter) 119 | - **Demo.py documentation** - Added docstrings to demo.py commands 120 | 121 | ## [25.10.1] - 2025-10-21 122 | 123 | ### Fixed 124 | 125 | - **Critical: Missing imports** - Restored FER and Video imports to __init__.py 126 | 127 | ## [25.10.0] - 2025-10-21 128 | 129 | ### Fixed 130 | 131 | - **Critical: Incorrect project URLs in setup.py** - Fixed GitHub repository links that were pointing to argon2_cffi instead of fer 132 | - **Integer division bug in Video class** - Changed float division to integer division in classes.py:299 for frame count calculation 133 | - **Duplicate dependency** - Removed duplicate facenet-pytorch entry from setup.py 134 | - **Deprecated Keras API** - Removed deprecated `make_predict_function()` call 135 | - **Deprecated NumPy function** - Changed `np.fromstring()` to `np.frombuffer()` in utils.py 136 | - **Incorrect OpenCV API usage** - Fixed `cv2.rectangle()` call to use correct parameter format (two corner points instead of single 4-tuple) 137 | 138 | ### Added 139 | 140 | - **Enhanced exception classes** - Added `InvalidModelFile` and `FaceDetectionError` exceptions with docstrings 141 | - **Parameter validation** - Added input validation to `FER.__init__()` for scale_factor, min_face_size, min_neighbors, and offsets 142 | - **Better error handling** - Enhanced `load_image()` function with: 143 | - 10-second timeout for URL downloads 144 | - More specific error messages 145 | - Proper type checking with isinstance() 146 | - Try-except blocks for better error handling 147 | - **Modern packaging** - Added pyproject.toml following PEP 517/518 standards 148 | - **CI/CD automation** - Added GitHub Actions workflow for automated testing across: 149 | - Multiple OS: Ubuntu, macOS, Windows 150 | - Python versions: 3.8, 3.9, 3.10, 3.11, 3.12 151 | - Automated linting with black, ruff, mypy 152 | - Package building and validation 153 | - **Enhanced .gitignore** - Better patterns for modern tools (ruff, mypy, VS Code, macOS) 154 | - **Development tools** - Enhanced requirements-dev.txt with black, ruff, mypy, and updated versions 155 | 156 | ### Changed 157 | 158 | - **Python version support** - Minimum Python version updated from 3.6 to 3.8 (dropped EOL Python 3.6) 159 | - **Dependency versions** - Added minimum version constraints: 160 | - matplotlib>=3.5.0 161 | - opencv-contrib-python>=4.5.0 162 | - pandas>=1.3.0 163 | - Pillow>=9.0.0 164 | - requests>=2.27.0 165 | - facenet-pytorch>=2.5.0 166 | - moviepy>=1.0.3 167 | - ffmpeg-python>=0.2.0 (changed from ffmpeg==1.4) 168 | - **Keras import** - Added fallback import to support both tensorflow.keras and standalone keras 169 | -------------------------------------------------------------------------------- /src/fer/fer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # MIT License 4 | # 5 | # Copyright (c) 2018 Justin Shenk 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # IMPORTANT: 26 | # 27 | # This code is derived from Iván de Paz Centeno's implementation of MTCNN 28 | # (https://github.com/ipazc/mtcnn/) and Octavia Arriaga's facial expression recognition repo 29 | # (https://github.com/oarriaga/face_classification). 30 | # 31 | import logging 32 | import sys 33 | from typing import Sequence, Tuple, Union 34 | 35 | import cv2 36 | import numpy as np 37 | import pkg_resources 38 | import requests 39 | 40 | try: 41 | import tensorflow as tf 42 | from tensorflow.keras.models import load_model 43 | except ImportError: 44 | import tensorflow as tf 45 | from keras.models import load_model 46 | 47 | 48 | from .utils import load_image 49 | 50 | logging.basicConfig(level=logging.INFO) 51 | log = logging.getLogger("fer") 52 | 53 | NumpyRects = Union[np.ndarray, Sequence[Tuple[int, int, int, int]]] 54 | 55 | __author__ = "Justin Shenk" 56 | 57 | PADDING = 40 58 | SERVER_URL = "http://localhost:8501/v1/models/emotion_model:predict" 59 | 60 | 61 | class FER: 62 | """ 63 | Allows performing Facial Expression Recognition -> 64 | a) Detection of faces 65 | b) Detection of emotions 66 | """ 67 | 68 | # Class-level model cache for performance 69 | _model_cache = None 70 | _model_cache_lock = None 71 | _tflite_interpreter_cache = None 72 | 73 | def __init__( 74 | self, 75 | cascade_file: str = None, 76 | mtcnn=False, 77 | tfserving: bool = False, 78 | use_tflite: bool = True, 79 | scale_factor: float = 1.1, 80 | min_face_size: int = 50, 81 | min_neighbors: int = 5, 82 | offsets: tuple = (10, 10), 83 | ): 84 | """ 85 | Initializes the face detector and Keras model for facial expression recognition. 86 | 87 | Args: 88 | cascade_file: File URI with the Haar cascade for face classification 89 | mtcnn: Use MTCNN network for face detection instead of Haar Cascade 90 | tfserving: Use TensorFlow Serving for predictions 91 | use_tflite: Use quantized TensorFlow Lite model for 7x faster inference (default: True) 92 | scale_factor: How much the image size is reduced at each image scale (default: 1.1) 93 | min_face_size: Minimum size of the face to detect in pixels (default: 50) 94 | min_neighbors: How many neighbors each candidate rectangle should have (default: 5) 95 | offsets: Padding around face before classification as (x_offset, y_offset) (default: (10, 10)) 96 | 97 | Raises: 98 | ValueError: If parameters are invalid 99 | Exception: If MTCNN is requested but facenet-pytorch is not installed 100 | """ 101 | # Validate parameters 102 | if scale_factor <= 1.0: 103 | raise ValueError("scale_factor must be greater than 1.0") 104 | if min_face_size < 1: 105 | raise ValueError("min_face_size must be at least 1") 106 | if min_neighbors < 0: 107 | raise ValueError("min_neighbors must be non-negative") 108 | if not isinstance(offsets, (tuple, list)) or len(offsets) != 2: 109 | raise ValueError("offsets must be a tuple or list of length 2") 110 | 111 | self.__scale_factor = scale_factor 112 | self.__min_neighbors = min_neighbors 113 | self.__min_face_size = min_face_size 114 | self.__offsets = offsets 115 | self.tfserving = tfserving 116 | self.use_tflite = use_tflite 117 | 118 | if cascade_file is None: 119 | cascade_file = cv2.data.haarcascades + "haarcascade_frontalface_default.xml" 120 | 121 | if mtcnn: 122 | try: 123 | from facenet_pytorch import MTCNN 124 | except ImportError: 125 | raise Exception( 126 | "MTCNN not installed, install it with pip install facenet-pytorch and from facenet_pytorch import MTCNN" 127 | ) 128 | self.__face_detector = "mtcnn" 129 | 130 | # use cuda GPU if available 131 | import torch 132 | if torch.cuda.is_available() and torch.cuda.device_count() > 0: 133 | device = torch.device('cuda') 134 | self._mtcnn = MTCNN(keep_all=True, device=device) 135 | else: 136 | self._mtcnn = MTCNN(keep_all=True) 137 | else: 138 | self.__face_detector = cv2.CascadeClassifier(cascade_file) 139 | 140 | self._initialize_model() 141 | 142 | def _initialize_model(self): 143 | if self.tfserving: 144 | self.__emotion_target_size = (64, 64) # hardcoded for now 145 | elif self.use_tflite: 146 | # Use TensorFlow Lite model for faster inference 147 | self.__emotion_target_size = (64, 64) 148 | 149 | # Use cached interpreter if available 150 | if FER._tflite_interpreter_cache is not None: 151 | log.debug("Using cached TFLite interpreter") 152 | self.__tflite_interpreter = FER._tflite_interpreter_cache 153 | else: 154 | tflite_model_path = pkg_resources.resource_filename( 155 | "fer", "data/emotion_model_quantized.tflite" 156 | ) 157 | log.debug(f"Loading TFLite model: {tflite_model_path}") 158 | self.__tflite_interpreter = tf.lite.Interpreter(model_path=tflite_model_path) 159 | self.__tflite_interpreter.allocate_tensors() 160 | # Cache for future instances 161 | FER._tflite_interpreter_cache = self.__tflite_interpreter 162 | 163 | # Get input and output details 164 | self.__tflite_input_details = self.__tflite_interpreter.get_input_details() 165 | self.__tflite_output_details = self.__tflite_interpreter.get_output_details() 166 | else: 167 | # Use cached model if available for performance 168 | if FER._model_cache is not None: 169 | log.debug("Using cached emotion model") 170 | self.__emotion_classifier = FER._model_cache 171 | self.__emotion_target_size = self.__emotion_classifier.input_shape[1:3] 172 | else: 173 | # Load model for the first time 174 | emotion_model = pkg_resources.resource_filename( 175 | "fer", "data/emotion_model.hdf5" 176 | ) 177 | log.debug(f"Loading emotion model: {emotion_model}") 178 | self.__emotion_classifier = load_model(emotion_model, compile=False) 179 | self.__emotion_target_size = self.__emotion_classifier.input_shape[1:3] 180 | # Cache for future instances 181 | FER._model_cache = self.__emotion_classifier 182 | return 183 | 184 | def _classify_emotions(self, gray_faces: np.ndarray) -> np.ndarray: # b x w x h 185 | """Run faces through online or offline classifier.""" 186 | if self.tfserving: 187 | gray_faces = np.expand_dims(gray_faces, -1) # to 4-dimensions 188 | instances = gray_faces.tolist() 189 | response = requests.post(SERVER_URL, json={"instances": instances}) 190 | response.raise_for_status() 191 | 192 | emotion_predictions = response.json()["predictions"] 193 | return emotion_predictions 194 | elif self.use_tflite: 195 | # Use TFLite interpreter 196 | gray_faces = np.expand_dims(gray_faces, -1) # Add channel dimension 197 | batch_size = gray_faces.shape[0] 198 | 199 | # TFLite inference 200 | all_predictions = [] 201 | for i in range(batch_size): 202 | # Get single face 203 | face = gray_faces[i:i+1].astype(np.float32) 204 | 205 | # Run inference 206 | self.__tflite_interpreter.set_tensor( 207 | self.__tflite_input_details[0]['index'], 208 | face 209 | ) 210 | self.__tflite_interpreter.invoke() 211 | 212 | # Get output 213 | output = self.__tflite_interpreter.get_tensor( 214 | self.__tflite_output_details[0]['index'] 215 | ) 216 | all_predictions.append(output[0]) 217 | 218 | return np.array(all_predictions) 219 | else: 220 | return self.__emotion_classifier(gray_faces) 221 | 222 | @staticmethod 223 | def pad(image): 224 | """Pad image.""" 225 | row, col = image.shape[:2] 226 | bottom = image[row - 2 : row, 0:col] 227 | mean = cv2.mean(bottom)[0] 228 | 229 | padded_image = cv2.copyMakeBorder( 230 | image, 231 | top=PADDING, 232 | bottom=PADDING, 233 | left=PADDING, 234 | right=PADDING, 235 | borderType=cv2.BORDER_CONSTANT, 236 | value=[mean, mean, mean], 237 | ) 238 | return padded_image 239 | 240 | @staticmethod 241 | def depad(image): 242 | row, col = image.shape[:2] 243 | return image[PADDING : row - PADDING, PADDING : col - PADDING] 244 | 245 | @staticmethod 246 | def tosquare(bbox): 247 | """Convert bounding box to square by elongating shorter side.""" 248 | x, y, w, h = bbox 249 | if h > w: 250 | diff = h - w 251 | x -= diff // 2 252 | w += diff 253 | elif w > h: 254 | diff = w - h 255 | y -= diff // 2 256 | h += diff 257 | if w != h: 258 | log.debug(f"{w} is not {h}") 259 | 260 | return (x, y, w, h) 261 | 262 | def find_faces(self, img: np.ndarray, bgr=True, gray_img=None) -> list: 263 | """Image to list of faces bounding boxes(x,y,w,h)""" 264 | if isinstance(self.__face_detector, cv2.CascadeClassifier): 265 | # Use provided grayscale image if available to avoid redundant conversion 266 | if gray_img is not None: 267 | gray_image_array = gray_img 268 | elif bgr: 269 | gray_image_array = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 270 | else: # assume gray 271 | gray_image_array = img 272 | 273 | faces = self.__face_detector.detectMultiScale( 274 | gray_image_array, 275 | scaleFactor=self.__scale_factor, 276 | minNeighbors=self.__min_neighbors, 277 | flags=cv2.CASCADE_SCALE_IMAGE, 278 | minSize=(self.__min_face_size, self.__min_face_size), 279 | ) 280 | elif self.__face_detector == "mtcnn": 281 | boxes, probs = self._mtcnn.detect(img) 282 | faces = [] 283 | if type(boxes) == np.ndarray: 284 | for face in boxes: 285 | faces.append( 286 | [ 287 | int(face[0]), 288 | int(face[1]), 289 | int(face[2]) - int(face[0]), 290 | int(face[3]) - int(face[1]), 291 | ] 292 | ) 293 | 294 | return faces 295 | 296 | @staticmethod 297 | def __preprocess_input(x, v2=False): 298 | x = x.astype("float32") 299 | x = x / 255.0 300 | if v2: 301 | x = x - 0.5 302 | x = x * 2.0 303 | return x 304 | 305 | def __apply_offsets(self, face_coordinates): 306 | """Offset face coordinates with padding before classification. 307 | x1, x2, y1, y2 = 0, 100, 0, 100 becomes -10, 110, -10, 110 308 | """ 309 | x, y, width, height = face_coordinates 310 | x_off, y_off = self.__offsets 311 | x1 = x - x_off 312 | x2 = x + width + x_off 313 | y1 = y - y_off 314 | y2 = y + height + y_off 315 | return x1, x2, y1, y2 316 | 317 | @staticmethod 318 | def _get_labels(): 319 | return { 320 | 0: "angry", 321 | 1: "disgust", 322 | 2: "fear", 323 | 3: "happy", 324 | 4: "sad", 325 | 5: "surprise", 326 | 6: "neutral", 327 | } 328 | 329 | def detect_emotions( 330 | self, img: np.ndarray, face_rectangles: NumpyRects = None 331 | ) -> list: 332 | """ 333 | Detects bounding boxes from the specified image with ranking of emotions. 334 | :param img: exact image path, numpy array (BGR or gray) or based64 encoded images 335 | could be passed. 336 | :return: list containing all the bounding boxes detected with their emotions. 337 | """ 338 | img = load_image(img) 339 | 340 | emotion_labels = self._get_labels() 341 | 342 | # Convert to grayscale once and reuse 343 | gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 344 | 345 | if face_rectangles is None: 346 | # Pass grayscale image to find_faces to avoid redundant conversion 347 | face_rectangles = self.find_faces(img, bgr=True, gray_img=gray_img) 348 | 349 | gray_img = self.pad(gray_img) 350 | 351 | emotions = [] 352 | gray_faces = [] 353 | if face_rectangles is not None: 354 | # Pre-allocate array for better performance 355 | gray_faces = [] 356 | valid_face_indices = [] 357 | 358 | for idx, face_coordinates in enumerate(face_rectangles): 359 | face_coordinates = self.tosquare(face_coordinates) 360 | 361 | # offset to expand bounding box 362 | # Note: x1 and y1 can be negative 363 | x1, x2, y1, y2 = self.__apply_offsets(face_coordinates) 364 | 365 | # account for padding in bounding box coordinates 366 | x1 += PADDING 367 | y1 += PADDING 368 | x2 += PADDING 369 | y2 += PADDING 370 | x1 = max(0, x1) 371 | y1 = max(0, y1) 372 | 373 | gray_face = gray_img[y1:y2, x1:x2] 374 | 375 | try: 376 | gray_face = cv2.resize(gray_face, self.__emotion_target_size) 377 | except Exception as e: 378 | log.warn(f"{gray_face.shape} resize failed: {e}") 379 | continue 380 | 381 | gray_faces.append(gray_face) 382 | valid_face_indices.append(idx) 383 | 384 | # Vectorize preprocessing - process all faces at once 385 | if gray_faces: 386 | gray_faces = np.array(gray_faces, dtype="float32") 387 | # Vectorized preprocessing 388 | gray_faces = gray_faces / 255.0 389 | gray_faces = (gray_faces - 0.5) * 2.0 390 | 391 | # Update face_rectangles to only include valid faces 392 | face_rectangles = [face_rectangles[i] for i in valid_face_indices] 393 | 394 | # predict all faces 395 | if not len(gray_faces): 396 | return emotions # no valid faces 397 | 398 | # classify emotions (gray_faces is already a numpy array) 399 | emotion_predictions = self._classify_emotions(gray_faces) 400 | 401 | # label scores 402 | for face_idx, face in enumerate(emotion_predictions): 403 | labelled_emotions = { 404 | emotion_labels[idx]: round(float(score), 2) 405 | for idx, score in enumerate(face) 406 | } 407 | 408 | emotions.append( 409 | dict(box=face_rectangles[face_idx], emotions=labelled_emotions) 410 | ) 411 | 412 | self.emotions = emotions 413 | 414 | return emotions 415 | 416 | def batch_detect_emotions(self, imgs: list) -> list: 417 | """ 418 | Detects emotions from multiple images in a batch for better performance. 419 | 420 | :param imgs: list of images (numpy arrays, file paths, or base64) 421 | :return: list of results, one per image 422 | """ 423 | if not imgs: 424 | return [] 425 | 426 | # Process each image individually but could be optimized further 427 | # by batching face detection and emotion classification across all images 428 | results = [] 429 | for img in imgs: 430 | result = self.detect_emotions(img) 431 | results.append(result) 432 | 433 | return results 434 | 435 | def top_emotion( 436 | self, img: np.ndarray 437 | ) -> Tuple[Union[str, None], Union[float, None]]: 438 | """Convenience wrapper for `detect_emotions` returning only top emotion for first face in frame. 439 | :param img: image to process 440 | :return: top emotion and score (for first face in frame) or (None, None) 441 | 442 | """ 443 | emotions = self.detect_emotions(img=img) 444 | top_emotions = [ 445 | max(e["emotions"], key=lambda key: e["emotions"][key]) for e in emotions 446 | ] 447 | 448 | # Take first face 449 | if len(top_emotions): 450 | top_emotion = top_emotions[0] 451 | else: 452 | return (None, None) 453 | score = emotions[0]["emotions"][top_emotion] 454 | 455 | return top_emotion, score 456 | 457 | 458 | def parse_arguments(args): 459 | import argparse 460 | 461 | parser = argparse.ArgumentParser() 462 | parser.add_argument("--image", type=str, help="Image filepath") 463 | return parser.parse_args() 464 | 465 | 466 | def top_emotion(): 467 | args = parse_arguments(sys.argv) 468 | fer = FER() 469 | top_emotion, score = fer.top_emotion(args.image) 470 | print(top_emotion, score) 471 | 472 | 473 | def main(): 474 | top_emotion() 475 | 476 | 477 | if __name__ == "__main__": 478 | main() 479 | -------------------------------------------------------------------------------- /src/fer/classes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import csv 3 | import logging 4 | import os 5 | import re 6 | from pathlib import Path 7 | from queue import Queue 8 | from threading import Thread 9 | from typing import Optional, Union 10 | from zipfile import ZipFile 11 | 12 | import cv2 13 | import numpy as np 14 | import pandas as pd 15 | from tqdm import tqdm 16 | from tqdm.contrib.logging import logging_redirect_tqdm 17 | 18 | from .utils import draw_annotations 19 | 20 | log = logging.getLogger("fer") 21 | 22 | # Optional moviepy import - only needed for audio features 23 | try: 24 | from moviepy.editor import AudioFileClip, CompositeAudioClip, VideoFileClip 25 | MOVIEPY_AVAILABLE = True 26 | except (ImportError, ModuleNotFoundError) as e: 27 | MOVIEPY_AVAILABLE = False 28 | log.warning(f"moviepy not available: {e}. Audio features will be disabled.") 29 | 30 | 31 | class AsyncFrameWriter: 32 | """Asynchronous frame writer for non-blocking I/O operations.""" 33 | 34 | def __init__(self, max_queue_size=50): 35 | """Initialize async frame writer with a background thread. 36 | 37 | Args: 38 | max_queue_size: Maximum number of frames to buffer in queue 39 | """ 40 | self.queue = Queue(maxsize=max_queue_size) 41 | self.thread = Thread(target=self._write_worker, daemon=True) 42 | self.running = False 43 | 44 | def start(self): 45 | """Start the background writer thread.""" 46 | self.running = True 47 | self.thread.start() 48 | 49 | def _write_worker(self): 50 | """Background worker that writes frames from queue.""" 51 | while self.running or not self.queue.empty(): 52 | try: 53 | item = self.queue.get(timeout=0.1) 54 | if item is None: # Poison pill 55 | break 56 | 57 | filepath, frame = item 58 | cv2.imwrite(filepath, frame) 59 | self.queue.task_done() 60 | except: 61 | continue 62 | 63 | def write(self, filepath: str, frame: np.ndarray): 64 | """Queue a frame for async writing. 65 | 66 | Args: 67 | filepath: Path where frame should be saved 68 | frame: Frame data to save 69 | """ 70 | self.queue.put((filepath, frame)) 71 | 72 | def stop(self): 73 | """Stop the writer and wait for queue to empty.""" 74 | self.queue.join() # Wait for all items to be processed 75 | self.running = False 76 | self.queue.put(None) # Poison pill 77 | self.thread.join(timeout=5.0) 78 | 79 | 80 | class Video: 81 | def __init__( 82 | self, 83 | video_file: str, 84 | outdir: str = "output", 85 | first_face_only: bool = True, 86 | tempfile: Optional[str] = None, 87 | ): 88 | """Video class for extracting and saving frames for emotion detection. 89 | param video_file - str 90 | :param outdir - str 91 | :param tempdir - str 92 | :param first_face_only - bool 93 | :param tempfile - str 94 | """ 95 | assert os.path.exists(video_file), "Video file not found at {}".format( 96 | os.path.abspath(video_file) 97 | ) 98 | self.cap = cv2.VideoCapture(video_file) 99 | if not os.path.isdir(outdir): 100 | os.makedirs(outdir, exist_ok=True) 101 | self.outdir = outdir 102 | 103 | if not first_face_only: 104 | log.error("Only single-face charting is implemented") 105 | self.first_face_only = first_face_only 106 | self.tempfile = tempfile 107 | self.filepath = video_file 108 | self.filename = "".join(self.filepath.split("/")[-1]) 109 | 110 | @staticmethod 111 | def get_max_faces(data: list) -> int: 112 | """Get max number of faces detected in a series of frames, eg 3""" 113 | max_faces = 0 114 | for frame in data: 115 | for face in frame: 116 | if len(face) > max_faces: 117 | max_faces = len(face) 118 | return max_faces 119 | 120 | @staticmethod 121 | def _to_dict(data: Union[dict, list]) -> dict: 122 | emotions = [] 123 | 124 | frame = data[0] 125 | if isinstance(frame, list): 126 | try: 127 | emotions = frame[0]["emotions"].keys() 128 | except IndexError: 129 | raise Exception("No data in 'data'") 130 | elif isinstance(frame, dict): 131 | return data 132 | 133 | dictlist = [] 134 | 135 | for data_idx, frame in enumerate(data): 136 | rowdict = {} 137 | for idx, face in enumerate(list(frame)): 138 | if not isinstance(face, dict): 139 | break 140 | rowdict.update({"box" + str(idx): face["box"]}) 141 | rowdict.update( 142 | {emo + str(idx): face["emotions"][emo] for emo in emotions} 143 | ) 144 | dictlist.append(rowdict) 145 | return dictlist 146 | 147 | def to_pandas(self, data: Union[pd.DataFrame, list]) -> pd.DataFrame: 148 | """Convert results to pandas DataFrame""" 149 | if isinstance(data, pd.DataFrame): 150 | return data 151 | 152 | if not len(data): 153 | return pd.DataFrame() 154 | datalist = self._to_dict(data) 155 | df = pd.DataFrame(datalist) 156 | if self.first_face_only: 157 | df = self.get_first_face(df) 158 | return df 159 | 160 | @staticmethod 161 | def get_first_face(df: pd.DataFrame) -> pd.DataFrame: 162 | assert isinstance(df, pd.DataFrame), "Must be a pandas DataFrame" 163 | try: 164 | int(df.columns[0][-1]) 165 | except ValueError: 166 | # Already only one face in df 167 | return df 168 | 169 | columns = [x for x in df.columns if x[-1] == "0"] 170 | new_columns = [x[:-1] for x in columns] 171 | single_df = df[columns] 172 | single_df.columns = new_columns 173 | return single_df 174 | 175 | @staticmethod 176 | def get_emotions(df: pd.DataFrame) -> list: 177 | """Get emotion columns from results.""" 178 | columns = [x for x in df.columns if "box" not in x] 179 | return df[columns] 180 | 181 | def to_csv(self, data, filename="data.csv"): 182 | """Save data to csv""" 183 | 184 | def key(item): 185 | key_pat = re.compile(r"^(\D+)(\d+)$") 186 | m = key_pat.match(item) 187 | return m.group(1), int(m.group(2)) 188 | 189 | dictlist = self._to_dict(data) 190 | columns = set().union(*(d.keys() for d in dictlist)) 191 | columns = sorted(columns, key=key) # sort by trailing number (faces) 192 | 193 | with open("data.csv", "w", newline="") as csvfile: 194 | writer = csv.DictWriter(csvfile, columns, lineterminator="\n") 195 | writer.writeheader() 196 | writer.writerows(dictlist) 197 | return dictlist 198 | 199 | def _close_video(self, outfile, save_frames, zip_images): 200 | self.cap.release() 201 | if self.display or self.save_video: 202 | self.videowriter.release() 203 | 204 | if self.save_video: 205 | log.info(f"Completed analysis: saved to {self.tempfile or outfile}") 206 | if self.tempfile: 207 | os.replace(self.tempfile, outfile) 208 | 209 | if save_frames and zip_images: 210 | log.info("Starting to Zip") 211 | outdir = Path(self.outdir) 212 | zip_dir = outdir / "images.zip" 213 | images = sorted(list(outdir.glob("*.jpg"))) 214 | total = len(images) 215 | i = 0 216 | with ZipFile(zip_dir, "w") as zip: 217 | for file in images: 218 | zip.write(file, arcname=file.name) 219 | os.remove(file) 220 | i += 1 221 | if i % 50 == 0: 222 | log.info(f"Compressing: {i*100 // total}%") 223 | log.info("Zip has finished") 224 | 225 | def _offset_detection_box(self, faces, detection_box): 226 | for face in faces: 227 | original_box = face.get("box") 228 | face["box"] = ( 229 | original_box[0] + detection_box.get("x_min"), 230 | original_box[1] + detection_box.get("y_min"), 231 | original_box[2], 232 | original_box[3], 233 | ) 234 | return faces 235 | 236 | def _increment_frames( 237 | self, frame, faces, video_id, root, lang="en", size_multiplier=1, async_writer=None 238 | ): 239 | # Save images to `self.outdir` 240 | imgpath = os.path.join( 241 | self.outdir, (video_id or root) + str(self.frameCount) + ".jpg" 242 | ) 243 | 244 | if self.annotate_frames: 245 | frame = draw_annotations( 246 | frame, 247 | faces, 248 | boxes=True, 249 | scores=True, 250 | lang=lang, 251 | size_multiplier=size_multiplier, 252 | ) 253 | 254 | if self.save_frames: 255 | if async_writer is not None: 256 | # Use async I/O for non-blocking write 257 | async_writer.write(imgpath, frame.copy()) 258 | else: 259 | # Fallback to synchronous write 260 | cv2.imwrite(imgpath, frame) 261 | 262 | if self.display: 263 | cv2.imshow("Video", frame) 264 | 265 | if self.save_video: 266 | self.videowriter.write(frame) 267 | 268 | self.frameCount += 1 269 | 270 | def analyze( 271 | self, 272 | detector, # fer.FER instance 273 | display: bool = False, 274 | output: str = "csv", 275 | frequency: Optional[int] = None, 276 | max_results: int = None, 277 | save_fps: Optional[int] = None, 278 | video_id: Optional[str] = None, 279 | save_frames: bool = True, 280 | save_video: bool = True, 281 | annotate_frames: bool = True, 282 | zip_images: bool = True, 283 | detection_box: Optional[dict] = None, 284 | lang: str = "en", 285 | include_audio: bool = False, 286 | size_multiplier: int = 1, 287 | batch_size: int = 1, 288 | use_async_io: bool = True, 289 | ) -> list: 290 | """Recognize facial expressions in video using `detector`. 291 | 292 | Args: 293 | 294 | detector (fer.FER): facial expression recognizer 295 | display (bool): show images with cv2.imshow 296 | output (str): csv or pandas 297 | frequency (int): inference on every nth frame (higher number is faster) 298 | max_results (int): number of frames to run inference before stopping 299 | save_fps (bool): inference frequency = video fps // save_fps 300 | video_id (str): filename for saving 301 | save_frames (bool): saves frames to directory 302 | save_video (bool): saves output video 303 | annotate_frames (bool): add emotion labels 304 | zip_images (bool): compress output 305 | detection_box (dict): dict with bounding box for subimage (xmin, xmax, ymin, ymax) 306 | lang (str): emotion language that will be shown on video 307 | include_audio (bool): indicates if a sounded version of the prediction video should be created or not 308 | size_multiplier (int): increases the size of emotion labels shown in the video by x(size_multiplier) 309 | batch_size (int): number of frames to process together for GPU efficiency (default: 1) 310 | use_async_io (bool): use asynchronous I/O for frame saving (default: True) 311 | Returns: 312 | 313 | data (list): list of results 314 | 315 | """ 316 | frames_emotions = [] 317 | if frequency is None: 318 | frequency = 1 319 | else: 320 | frequency = int(frequency) 321 | 322 | self.display = display 323 | self.save_frames = save_frames 324 | self.save_video = save_video 325 | self.annotate_frames = annotate_frames 326 | 327 | results_nr = 0 328 | 329 | # Open video 330 | assert self.cap.open(self.filepath), "Video capture not opening" 331 | self.__emotions = detector._get_labels().items() 332 | self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0) 333 | pos_frames = self.cap.get(cv2.CAP_PROP_POS_FRAMES) 334 | assert int(pos_frames) == 0, "Video not at index 0" 335 | 336 | self.frameCount = 0 337 | height, width = ( 338 | int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), 339 | int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)), 340 | ) 341 | 342 | fps = self.cap.get(cv2.CAP_PROP_FPS) 343 | length = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) 344 | assert fps and length, f"File {self.filepath} not loaded" 345 | 346 | if save_fps is not None: 347 | frequency = fps // save_fps 348 | log.info(f"Saving every {frequency} frames") 349 | 350 | log.info( 351 | f"{fps:.2f} fps, {length} frames, {length / fps:.2f} seconds" 352 | ) 353 | 354 | if self.save_frames: 355 | os.makedirs(self.outdir, exist_ok=True) 356 | log.info(f"Making directories at {self.outdir}") 357 | root, ext = os.path.splitext(os.path.basename(self.filepath)) 358 | outfile = os.path.join(self.outdir, f"{root}_output{ext}") 359 | 360 | if save_video: 361 | self.videowriter = self._save_video(outfile, fps, width, height) 362 | 363 | total_frames = length 364 | if frequency > 1: 365 | total_frames = length // frequency 366 | 367 | # Initialize async frame writer if enabled 368 | async_writer = None 369 | if use_async_io and save_frames: 370 | async_writer = AsyncFrameWriter(max_queue_size=batch_size * 2) 371 | async_writer.start() 372 | log.info("Async I/O enabled for frame saving") 373 | 374 | # Frame batching setup 375 | frame_batch = [] 376 | frame_metadata = [] # Store (frame_number, detection_box) for each frame in batch 377 | 378 | with logging_redirect_tqdm(): 379 | pbar = tqdm(total=total_frames, unit="frames") 380 | 381 | try: 382 | while self.cap.isOpened(): 383 | # Optimize frame skipping by seeking directly to target frames 384 | if frequency > 1: 385 | target_frame = self.frameCount 386 | self.cap.set(cv2.CAP_PROP_POS_FRAMES, target_frame) 387 | 388 | ret, frame = self.cap.read() 389 | if not ret: # end of video 390 | break 391 | 392 | if frame is None: 393 | log.warn("Empty frame") 394 | if frequency > 1: 395 | self.frameCount += frequency 396 | else: 397 | self.frameCount += 1 398 | continue 399 | 400 | if detection_box is not None: 401 | frame = self._crop(frame, detection_box) 402 | 403 | # Add frame to batch 404 | frame_batch.append(frame.copy()) 405 | frame_metadata.append((self.frameCount, detection_box)) 406 | 407 | # Process batch when full or at end of video 408 | if len(frame_batch) >= batch_size or (max_results and results_nr + len(frame_batch) >= max_results): 409 | # Process batch of frames efficiently 410 | try: 411 | if batch_size > 1: 412 | # Use batch processing for multiple frames 413 | batch_results = detector.batch_detect_emotions(frame_batch) 414 | else: 415 | # Single frame - use regular detection 416 | batch_results = [detector.detect_emotions(frame_batch[0])] 417 | except Exception as e: 418 | log.error(e) 419 | break 420 | 421 | # Process results for each frame in batch 422 | for idx, (batch_frame, (frame_num, det_box), faces) in enumerate(zip(frame_batch, frame_metadata, batch_results)): 423 | # Offset detection_box to include padding 424 | if det_box is not None: 425 | faces = self._offset_detection_box(faces, det_box) 426 | 427 | self._increment_frames(batch_frame, faces, video_id, root, lang, size_multiplier, async_writer) 428 | 429 | if cv2.waitKey(1) & 0xFF == ord("q"): 430 | break 431 | 432 | if faces: 433 | frames_emotions.append(faces) 434 | 435 | results_nr += 1 436 | pbar.update(1) 437 | 438 | # Advance frameCount by frequency for next iteration 439 | if frequency > 1: 440 | self.frameCount += frequency - 1 # -1 because _increment_frames already added 1 441 | 442 | if max_results and results_nr >= max_results: 443 | break 444 | 445 | # Clear batch 446 | frame_batch = [] 447 | frame_metadata = [] 448 | 449 | if max_results and results_nr >= max_results: 450 | break 451 | 452 | # Process remaining frames in batch 453 | if frame_batch: 454 | try: 455 | if len(frame_batch) > 1: 456 | batch_results = detector.batch_detect_emotions(frame_batch) 457 | else: 458 | batch_results = [detector.detect_emotions(frame_batch[0])] 459 | except Exception as e: 460 | log.error(e) 461 | else: 462 | for idx, (batch_frame, (frame_num, det_box), faces) in enumerate(zip(frame_batch, frame_metadata, batch_results)): 463 | if det_box is not None: 464 | faces = self._offset_detection_box(faces, det_box) 465 | 466 | self._increment_frames(batch_frame, faces, video_id, root, lang, size_multiplier, async_writer) 467 | 468 | if faces: 469 | frames_emotions.append(faces) 470 | 471 | results_nr += 1 472 | pbar.update(1) 473 | 474 | if frequency > 1: 475 | self.frameCount += frequency - 1 476 | 477 | finally: 478 | pbar.close() 479 | # Stop async writer if enabled 480 | if async_writer is not None: 481 | log.info("Waiting for async I/O to complete...") 482 | async_writer.stop() 483 | log.info("Async I/O completed") 484 | 485 | self._close_video(outfile, save_frames, zip_images) 486 | 487 | if include_audio: 488 | if not MOVIEPY_AVAILABLE: 489 | log.error("Audio feature requested but moviepy is not available. Install with: pip install moviepy") 490 | else: 491 | audio_suffix = "_audio." 492 | my_audio = AudioFileClip(self.filepath) 493 | new_audioclip = CompositeAudioClip([my_audio]) 494 | 495 | my_output_clip = VideoFileClip(outfile) 496 | my_output_clip.audio = new_audioclip 497 | my_output_clip.write_videofile(audio_suffix.join(outfile.rsplit(".", 1))) 498 | 499 | return self.to_format(frames_emotions, output) 500 | 501 | def to_format(self, data, format): 502 | """Return data in format.""" 503 | methods_lookup = {"csv": self.to_csv, "pandas": self.to_pandas} 504 | return methods_lookup[format](data) 505 | 506 | def _save_video(self, outfile: str, fps: int, width: int, height: int): 507 | if os.path.isfile(outfile): 508 | os.remove(outfile) 509 | log.info(f"Deleted pre-existing {outfile}") 510 | if self.tempfile and os.path.isfile(self.tempfile): 511 | os.remove(self.tempfile) 512 | fourcc = cv2.VideoWriter_fourcc("m", "p", "4", "v") 513 | videowriter = cv2.VideoWriter( 514 | self.tempfile or outfile, fourcc, fps, (width, height), True 515 | ) 516 | return videowriter 517 | 518 | @staticmethod 519 | def _crop(frame, detection_box): 520 | crop_frame = frame[ 521 | detection_box.get("y_min") : detection_box.get("y_max"), 522 | detection_box.get("x_min") : detection_box.get("x_max"), 523 | ] 524 | return crop_frame 525 | 526 | def __del__(self): 527 | cv2.destroyAllWindows() 528 | --------------------------------------------------------------------------------