├── migrations ├── README ├── script.py.mako ├── alembic.ini ├── versions │ └── 88f3cf82b5d7_.py └── env.py ├── boot.sh ├── .db.env ├── hiitpi ├── assets │ ├── img │ │ └── pixels_art.jpg │ ├── font │ │ └── Comfortaa │ │ │ └── Comfortaa-Regular.ttf │ ├── models │ │ └── posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite │ └── css │ │ ├── style.css │ │ └── base.css ├── model.py ├── redisclient.py ├── config.py ├── annotation.py ├── camera.py ├── pose.py ├── layout.py ├── __init__.py └── workout.py ├── .gitignore ├── .dockerignore ├── app.py ├── .env ├── requirements.txt ├── LICENSE ├── docker-compose.yml ├── Dockerfile └── README.md /migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | flask db upgrade 4 | exec python app.py 5 | -------------------------------------------------------------------------------- /.db.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=hiitpi 2 | POSTGRES_PASSWORD=hiitpi 3 | POSTGRES_DB=hiitpi -------------------------------------------------------------------------------- /hiitpi/assets/img/pixels_art.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jingw222/hiitpi/HEAD/hiitpi/assets/img/pixels_art.jpg -------------------------------------------------------------------------------- /hiitpi/assets/font/Comfortaa/Comfortaa-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jingw222/hiitpi/HEAD/hiitpi/assets/font/Comfortaa/Comfortaa-Regular.ttf -------------------------------------------------------------------------------- /hiitpi/assets/models/posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jingw222/hiitpi/HEAD/hiitpi/assets/models/posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # System 2 | .DS_Store 3 | 4 | # Python 5 | .ipynb_checkpoints/ 6 | __pycache__/ 7 | *.ipynb 8 | *.py[cod] 9 | 10 | # Env 11 | .vscode/ 12 | .venv/ 13 | .python-version 14 | 15 | # Misc 16 | *.sqlite 17 | rpi_camera_webstream.py 18 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # System 2 | .DS_Store 3 | 4 | # Python 5 | .ipynb_checkpoints/ 6 | __pycache__/ 7 | *.ipynb 8 | *.py[cod] 9 | 10 | # Env 11 | .vscode/ 12 | .venv/ 13 | .python-version 14 | 15 | # Misc 16 | .git 17 | *.sqlite 18 | rpi_camera_webstream.py 19 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import os 2 | from hiitpi import create_app 3 | 4 | 5 | app = create_app(os.getenv("FLASK_CONFIG") or "default") 6 | server = app.server 7 | 8 | if __name__ == "__main__": 9 | app.run_server(debug=False, host="0.0.0.0", port=8050, use_reloader=False) 10 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | FLASK_APP=app.py 2 | FLASK_ENV=production 3 | FLASK_CONFIG=production 4 | SQLALCHEMY_DATABASE_URI=postgresql://hiitpi:hiitpi@db:5432/hiitpi 5 | CACHE_REDIS_URL=redis://redis:6379/0 6 | SESSION_REDIS_URL=redis://redis:6379/1 7 | REDIS_HOST=redis 8 | REDIS_PORT=6379 9 | REDIS_DB=2 10 | MODEL_FILE=posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | picamera[array] 4 | Pillow 5 | opencv-python 6 | Flask 7 | Flask-Caching 8 | Flask-Session 9 | Flask-Migrate 10 | Flask-SQLAlchemy 11 | plotly 12 | dash 13 | redis 14 | psycopg2-binary 15 | https://dl.google.com/coral/edgetpu_api/edgetpu-2.14.1-py3-none-any.whl 16 | https://github.com/google-coral/pycoral/releases/download/release-frogfish/tflite_runtime-2.5.0-cp37-cp37m-linux_armv7l.whl -------------------------------------------------------------------------------- /hiitpi/model.py: -------------------------------------------------------------------------------- 1 | import time 2 | import datetime 3 | from . import db 4 | 5 | 6 | class WorkoutSession(db.Model): 7 | __tablename__ = "workout_session" 8 | 9 | id = db.Column(db.Integer, primary_key=True) 10 | created_date = db.Column(db.DateTime(), default=datetime.datetime.utcnow) 11 | user_name = db.Column(db.String(80), nullable=False) 12 | workout = db.Column(db.String(80), nullable=False) 13 | reps = db.Column(db.Integer(), nullable=False) 14 | pace = db.Column(db.Float(), nullable=False) 15 | 16 | def __repr__(self): 17 | return f"" 18 | -------------------------------------------------------------------------------- /migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [handler_console] 38 | class = StreamHandler 39 | args = (sys.stderr,) 40 | level = NOTSET 41 | formatter = generic 42 | 43 | [formatter_generic] 44 | format = %(levelname)-5.5s [%(name)s] %(message)s 45 | datefmt = %H:%M:%S 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 James Wong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /migrations/versions/88f3cf82b5d7_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 88f3cf82b5d7 4 | Revises: 5 | Create Date: 2020-11-22 20:11:52.775287 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '88f3cf82b5d7' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('workout_session', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('created_date', sa.DateTime(), nullable=True), 24 | sa.Column('user_name', sa.String(length=80), nullable=False), 25 | sa.Column('workout', sa.String(length=80), nullable=False), 26 | sa.Column('reps', sa.Integer(), nullable=False), 27 | sa.Column('pace', sa.Float(), nullable=False), 28 | sa.PrimaryKeyConstraint('id') 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade(): 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.drop_table('workout_session') 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /hiitpi/redisclient.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | 4 | class RedisClient(object): 5 | """Sets up a Redis database client for data storage and transmission""" 6 | 7 | def __init__(self, host, port, db): 8 | self.pool = redis.BlockingConnectionPool(host=host, port=port, db=db) 9 | 10 | @property 11 | def conn(self): 12 | if not hasattr(self, "_conn"): 13 | self.get_connection() 14 | return self._conn 15 | 16 | def get_connection(self): 17 | self._conn = redis.StrictRedis(connection_pool=self.pool) 18 | 19 | self._conn.set_response_callback( 20 | "get", lambda i: float(i) if i is not None else None 21 | ) 22 | self._conn.set_response_callback( 23 | "lpop", lambda i: float(i) if i is not None else None 24 | ) 25 | self._conn.set_response_callback("lrange", lambda l: [float(i) for i in l]) 26 | 27 | def set(self, key, value): 28 | self.conn.set(key, value) 29 | 30 | def get(self, key): 31 | return self.conn.get(key) 32 | 33 | def lpush(self, key, value, max_size=None): 34 | self.conn.lpush(key, value) 35 | if max_size is not None and self.conn.llen(key) > max_size: 36 | self.conn.ltrim(key, 0, max_size - 1) 37 | 38 | def lpop(self, key): 39 | return self.conn.lpop(key) 40 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | web: 5 | build: 6 | context: ./ 7 | dockerfile: Dockerfile 8 | image: hiitpi:latest 9 | container_name: "web-hiitpi" 10 | restart: unless-stopped 11 | privileged: true 12 | ports: 13 | - "8050:8050" 14 | env_file: 15 | - ./.env 16 | networks: 17 | - network-hiitpi 18 | volumes: 19 | - ./:/hiitpi/ 20 | - /dev/bus/usb:/dev/bus/usb 21 | devices: 22 | - /dev/bus/usb:/dev/bus/usb 23 | depends_on: 24 | - db 25 | - redis 26 | 27 | db: 28 | image: postgres:latest 29 | container_name: "db-hiitpi" 30 | volumes: 31 | - postgres_data:/var/lib/postgresql/data 32 | restart: unless-stopped 33 | expose: 34 | - "5432" 35 | env_file: 36 | - ./.db.env 37 | networks: 38 | - network-hiitpi 39 | 40 | redis: 41 | image: redis:latest 42 | container_name: "redis-hiitpi" 43 | restart: unless-stopped 44 | expose: 45 | - "6379" 46 | networks: 47 | - network-hiitpi 48 | 49 | networks: 50 | network-hiitpi: 51 | driver: bridge 52 | 53 | volumes: 54 | postgres_data: -------------------------------------------------------------------------------- /hiitpi/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import redis 3 | import secrets 4 | 5 | 6 | BASE_PATH = os.path.dirname(__file__) 7 | 8 | 9 | class Config: 10 | DEBUG = False 11 | DEVELOPMENT = False 12 | TESTING = False 13 | SQLALCHEMY_TRACK_MODIFICATIONS = False 14 | SECRET_KEY = os.environ.get("SECRET_KEY", "this is a secret") 15 | 16 | MODEL_DIR = os.path.join(BASE_PATH, "assets", "models") 17 | MODEL_FILE = os.environ.get("MODEL_FILE") 18 | MODEL_PATH = os.path.join(MODEL_DIR, MODEL_FILE) 19 | 20 | SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI") 21 | 22 | CACHE_TYPE = "redis" 23 | CACHE_REDIS_URL = os.environ.get("CACHE_REDIS_URL") 24 | 25 | SESSION_TYPE = "redis" 26 | SESSION_REDIS_URL = os.environ.get("SESSION_REDIS_URL") 27 | SESSION_REDIS = redis.from_url(SESSION_REDIS_URL) 28 | 29 | REDIS_HOST = os.environ.get("REDIS_HOST") 30 | REDIS_PORT = os.environ.get("REDIS_PORT") 31 | REDIS_DB = os.environ.get("REDIS_DB") 32 | 33 | 34 | class DevelopmentConfig(Config): 35 | ENV = "development" 36 | DEBUG = True 37 | DEVELOPMENT = True 38 | TEMPLATES_AUTO_RELOAD = True 39 | 40 | 41 | class TestingConfig(Config): 42 | ENV = "testing" 43 | TESTING = True 44 | 45 | 46 | class ProductionConfig(Config): 47 | ENV = "production" 48 | SECRET_KEY = os.environ.get("SECRET_KEY", secrets.token_urlsafe(16)) 49 | 50 | 51 | config = { 52 | "development": DevelopmentConfig, 53 | "testing": TestingConfig, 54 | "production": ProductionConfig, 55 | "default": ProductionConfig, 56 | } 57 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberrypi3-debian-python:3.7.9-buster 2 | 3 | LABEL maintainer="James Wong " 4 | 5 | ARG DEBIAN_FRONTEND=noninteractive 6 | 7 | ENV LC_ALL=C.UTF-8 \ 8 | LANG=C.UTF-8 \ 9 | PYTHONDONTWRITEBYTECODE=1 10 | 11 | # Install dependencies 12 | RUN apt-get update && \ 13 | apt-get install -y build-essential cmake pkg-config apt-utils && \ 14 | apt-get install -y libjpeg-dev libtiff5-dev libjasper-dev libpng-dev libilmbase23 libopenexr-dev libgtk-3-dev && \ 15 | apt-get install -y libavcodec-dev libavformat-dev libswscale-dev libv4l-dev && \ 16 | apt-get install -y libxvidcore-dev libx264-dev && \ 17 | apt-get install -y gnupg2 apt-transport-https curl libatlas-base-dev gfortran python3-dev libpq-dev 18 | 19 | # Add Coral Edge TPU package repository 20 | RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list && \ 21 | curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add 22 | 23 | # Install the Edge TPU runtime 24 | RUN apt-get update && apt-get install -y libedgetpu1-legacy-std 25 | 26 | # Add pip package index urls 27 | RUN echo '[global]' >> /etc/pip.conf && \ 28 | echo 'index-url = https://www.piwheels.org/simple' >> /etc/pip.conf && \ 29 | echo 'extra-index-url = https://pypi.python.org/simple' >> /etc/pip.conf 30 | 31 | WORKDIR /hiitpi 32 | 33 | COPY hiitpi hiitpi 34 | COPY migrations migrations 35 | COPY app.py boot.sh requirements.txt ./ 36 | 37 | RUN pip install --upgrade pip && \ 38 | pip install --upgrade setuptools wheel && \ 39 | pip install -r requirements.txt 40 | 41 | RUN chmod +x boot.sh 42 | 43 | EXPOSE 8050 44 | 45 | ENTRYPOINT ["./boot.sh"] 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HIIT PI 2 | 3 | HIIT PI is a [Dash](https://dash.plot.ly/) app that uses machine learning (specifically pose estimation) on edge devices to help track your [HIIT](https://en.wikipedia.org/wiki/High-intensity_interval_training) workout progress in real time (~30fps). The backend runs everything locally on a [Raspberry Pi](https://www.raspberrypi.org/) while you interact with the app wherever there is a web browser connecting to the same local network as the Pi does. 4 | 5 | ## How it works 6 | * [Video demo](https://youtu.be/eErfWulipiA) 7 | * [Blog post](https://iamjameswong.com/2020-04-06-building-a-home-hiit-workout-trainer/) 8 | 9 | ## Hardwares 10 | * [Raspberry Pi](https://www.raspberrypi.org/products/raspberry-pi-4-model-b/) *(Pi 4 recommended)* 11 | * [Raspberry Pi Camera Module v2](https://www.raspberrypi.org/products/camera-module-v2/) 12 | * [Google's Coral USB Accelerator](https://coral.ai/products/accelerator/) (Edge TPU) 13 | 14 | ## Softwares 15 | * [Raspbian 10 Buster](https://www.raspberrypi.org/downloads/raspbian/) 16 | * [Docker](https://docs.docker.com/engine/install/debian/) & [Docker Compose](https://docs.docker.com/compose/install/) 17 | * [Python 3.7+](https://www.python.org/) 18 | 19 | ## Usage 20 | 1. SSH into your Raspberry Pi and clone the repository. 21 | 2. Install Docker & Docker Compose. 22 | 3. Build our Docker images and spawn up the containers with 23 | ``` 24 | $ docker-compose -d --build up 25 | ``` 26 | 4. *(Optional)* For maximum performance, swap the standard Edge TPU runtime library `libedgetpu1-legacy-std` with `libedgetpu1-legacy-max` 27 | 1. get into the shell of container `web-hiitpi` by 28 | ``` 29 | $ docker exec -it web-hiitpi bash 30 | ``` 31 | 2. run the following inside the container 32 | ``` 33 | $ DEBIAN_FRONTEND=dialog apt-get install -y libedgetpu1-legacy-max 34 | ``` 35 | *Note: select `yes` and hit `ENTER` in the interactive installation process.* 36 | 3. restart the `web` service after the above install finishes 37 | ``` 38 | $ docker-compose restart web 39 | ``` 40 | 41 | 5. Go to `:8050` on a device in the same LAN as the Pi's, and then enter a player name in the welcome page to get started. 42 | 6. The live-updating line graphs show the model inferencing time (~50fps) and pose score frame by frame, which indicates how likely the camera senses a person in front. 43 | 7. Selecting a workout from the dropdown menu starts a training session, where your training session stats (`reps` & `pace`) are updating in the widgets below as the workout progresses. Tap the `DONE!` button to complete the session, or `EXIT?` to switch a player. Click `LEADERBOARD` to view total reps accomplished by top players. 44 | 45 | ## Notes 46 | * This project currently has implemented a couple of workouts to play with, and we're planning to expand our workout repertoire as it evolves over time. -------------------------------------------------------------------------------- /migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import logging 4 | from logging.config import fileConfig 5 | 6 | from sqlalchemy import engine_from_config 7 | from sqlalchemy import pool 8 | 9 | from alembic import context 10 | 11 | # this is the Alembic Config object, which provides 12 | # access to the values within the .ini file in use. 13 | config = context.config 14 | 15 | # Interpret the config file for Python logging. 16 | # This line sets up loggers basically. 17 | fileConfig(config.config_file_name) 18 | logger = logging.getLogger('alembic.env') 19 | 20 | # add your model's MetaData object here 21 | # for 'autogenerate' support 22 | # from myapp import mymodel 23 | # target_metadata = mymodel.Base.metadata 24 | from flask import current_app 25 | config.set_main_option( 26 | 'sqlalchemy.url', 27 | str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) 28 | target_metadata = current_app.extensions['migrate'].db.metadata 29 | 30 | # other values from the config, defined by the needs of env.py, 31 | # can be acquired: 32 | # my_important_option = config.get_main_option("my_important_option") 33 | # ... etc. 34 | 35 | 36 | def run_migrations_offline(): 37 | """Run migrations in 'offline' mode. 38 | 39 | This configures the context with just a URL 40 | and not an Engine, though an Engine is acceptable 41 | here as well. By skipping the Engine creation 42 | we don't even need a DBAPI to be available. 43 | 44 | Calls to context.execute() here emit the given string to the 45 | script output. 46 | 47 | """ 48 | url = config.get_main_option("sqlalchemy.url") 49 | context.configure( 50 | url=url, target_metadata=target_metadata, literal_binds=True 51 | ) 52 | 53 | with context.begin_transaction(): 54 | context.run_migrations() 55 | 56 | 57 | def run_migrations_online(): 58 | """Run migrations in 'online' mode. 59 | 60 | In this scenario we need to create an Engine 61 | and associate a connection with the context. 62 | 63 | """ 64 | 65 | # this callback is used to prevent an auto-migration from being generated 66 | # when there are no changes to the schema 67 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 68 | def process_revision_directives(context, revision, directives): 69 | if getattr(config.cmd_opts, 'autogenerate', False): 70 | script = directives[0] 71 | if script.upgrade_ops.is_empty(): 72 | directives[:] = [] 73 | logger.info('No changes in schema detected.') 74 | 75 | connectable = engine_from_config( 76 | config.get_section(config.config_ini_section), 77 | prefix='sqlalchemy.', 78 | poolclass=pool.NullPool, 79 | ) 80 | 81 | with connectable.connect() as connection: 82 | context.configure( 83 | connection=connection, 84 | target_metadata=target_metadata, 85 | process_revision_directives=process_revision_directives, 86 | **current_app.extensions['migrate'].configure_args 87 | ) 88 | 89 | with context.begin_transaction(): 90 | context.run_migrations() 91 | 92 | 93 | if context.is_offline_mode(): 94 | run_migrations_offline() 95 | else: 96 | run_migrations_online() 97 | -------------------------------------------------------------------------------- /hiitpi/annotation.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | import logging 4 | import collections 5 | import numpy as np 6 | from PIL import Image, ImageDraw 7 | 8 | 9 | logging.basicConfig( 10 | stream=sys.stdout, 11 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 12 | datefmt=" %I:%M:%S ", 13 | level="INFO", 14 | ) 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | EDGES = ( 19 | ("nose", "left eye"), 20 | ("nose", "right eye"), 21 | ("nose", "left ear"), 22 | ("nose", "right ear"), 23 | ("left ear", "left eye"), 24 | ("right ear", "right eye"), 25 | ("left eye", "right eye"), 26 | ("left shoulder", "right shoulder"), 27 | ("left shoulder", "left elbow"), 28 | ("left shoulder", "left hip"), 29 | ("right shoulder", "right elbow"), 30 | ("right shoulder", "right hip"), 31 | ("left elbow", "left wrist"), 32 | ("right elbow", "right wrist"), 33 | ("left hip", "right hip"), 34 | ("left hip", "left knee"), 35 | ("right hip", "right knee"), 36 | ("left knee", "left ankle"), 37 | ("right knee", "right ankle"), 38 | ) 39 | 40 | 41 | class Annotator(object): 42 | """Annotates video streaming output with a drawing overlay.""" 43 | 44 | def __init__(self): 45 | self._init_time = time.perf_counter() 46 | self._rendering_time = collections.deque([self._init_time], maxlen=30) 47 | 48 | def annotate(self, output): 49 | self._rendering_time.append(time.perf_counter()) 50 | rendering_fps = len(self._rendering_time) / ( 51 | self._rendering_time[-1] - self._rendering_time[0] 52 | ) 53 | 54 | img = Image.fromarray(output["array"]) 55 | draw = ImageDraw.Draw(img, "RGBA") 56 | 57 | self.draw_pose(draw, output["pose"]) 58 | 59 | text_lines = [ 60 | f'Inference time: {output["inference_time"]:.1f}ms ({1000 / output["inference_time"]:.1f}fps)' 61 | f"Rendering time: {1000 / rendering_fps:.1f}ms ({rendering_fps:.1f}fps)", 62 | "", 63 | ] 64 | 65 | workout = output["workout"] 66 | if workout.stats is not None: 67 | text_lines.extend([f"{k}: {v:.1f}" for k, v in workout.stats.items()]) 68 | 69 | self.draw_text(draw, 10, 10, text="\n".join(text_lines)) 70 | 71 | return np.asarray(img) 72 | 73 | def draw_text(self, draw, x, y, text): 74 | draw.text(xy=(x + 1, y + 1), text=text, fill="black") 75 | draw.text(xy=(x, y), text=text, fill="lightgray") 76 | 77 | def draw_circle(self, draw, x, y, r, width, alpha): 78 | draw.ellipse( 79 | [(x - r, y - r), (x + r, y + r)], 80 | fill=(0, 255, 255, int(255 * alpha)), 81 | outline="yellow", 82 | width=width, 83 | ) 84 | 85 | def draw_line(self, draw, xy): 86 | draw.line(xy, fill="yellow", width=2) 87 | 88 | def draw_pose(self, draw, pose, threshold=0.2): 89 | if pose: 90 | xys = {} 91 | for label, keypoint in pose.keypoints.items(): 92 | if keypoint.score < threshold: 93 | continue 94 | 95 | y = int(keypoint.yx[0]) 96 | x = int(keypoint.yx[1]) 97 | 98 | xys[label] = (x, y) 99 | self.draw_circle(draw, x, y, r=3, width=1, alpha=keypoint.score) 100 | 101 | for a, b in EDGES: 102 | if a not in xys or b not in xys: 103 | continue 104 | ax, ay = xys[a] 105 | bx, by = xys[b] 106 | self.draw_line(draw, [(ax, ay), (bx, by)]) 107 | -------------------------------------------------------------------------------- /hiitpi/camera.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import picamera 4 | import picamera.array 5 | 6 | 7 | WIDTH, HEIGHT = 640, 480 8 | FRAMERATE = 24 9 | HFLIP = True 10 | ZOOM = (0.0, 0.0, 1.0, 1.0) 11 | EV = 0 12 | 13 | 14 | logging.basicConfig( 15 | stream=sys.stdout, 16 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 17 | datefmt=" %I:%M:%S ", 18 | level="INFO", 19 | ) 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class StreamOutput(picamera.array.PiRGBAnalysis): 24 | """Custom streaming output for the PiCamera""" 25 | 26 | def setup(self, model, redis): 27 | """ 28 | Args: 29 | model: PoseEngine for TensorFlow Lite models. 30 | redis: RedisClient. 31 | """ 32 | self.array = None 33 | self.pose = None 34 | self.inference_time = None 35 | self.model = model 36 | self.redis = redis 37 | self.redis.set("reps", 0) 38 | self.redis.set("pace", 0) 39 | 40 | def analyze(self, array): 41 | """While recording is in progress, analyzes incoming array data""" 42 | self.array = array 43 | self.pose, self.inference_time = self.model.DetectPosesInImage(self.array) 44 | if self.pose: 45 | self.pose = max(self.pose, key=lambda pose: pose.score) 46 | self.redis.lpush("pose_score", self.pose.score.item(), max_size=5) 47 | self.redis.lpush("inference_time", self.inference_time, max_size=5) 48 | 49 | 50 | class VideoStream(object): 51 | def __init__( 52 | self, 53 | resolution=(WIDTH, HEIGHT), 54 | framerate=FRAMERATE, 55 | hflip=HFLIP, 56 | zoom=ZOOM, 57 | ev=EV, 58 | ): 59 | """Creates a VideoStream from picamera for streaming and analyzing incoming data. 60 | Args: 61 | resolution: tuple, the resolution at which video recordings will be captured, in (width, height). 62 | framerate: int, the framerate video recordings will run (fps). 63 | hflip: flip view horizontally. 64 | zoom: the zoom applied to the camera’s input. 65 | ev: the exposure compensation level of the camera. 66 | """ 67 | # PiCamera configurations 68 | self.resolution = resolution 69 | self.framerate = framerate 70 | self.hflip = hflip 71 | self.zoom = zoom 72 | self.ev = ev 73 | logger.info( 74 | f"PiCamera configurations: " 75 | f"resolution={self.resolution}, framerate={self.framerate}, " 76 | f"hflip={self.hflip}, zoom={self.zoom}, ev={self.ev}" 77 | ) 78 | self.closed = None 79 | 80 | def setup(self, model, redis): 81 | """Initiates a PiCamera, attaches a StreamOutput and starts recording. 82 | Args: 83 | model: PoseEngine for TensorFlow Lite models. 84 | redis: RedisClient. 85 | """ 86 | 87 | # Builds and sets up a PiCamera 88 | self.camera = picamera.PiCamera() 89 | self.camera.resolution = self.resolution 90 | self.camera.framerate = self.framerate 91 | self.camera.hflip = self.hflip 92 | self.camera.zoom = self.zoom 93 | self.camera.exposure_compensation = self.ev 94 | 95 | # Creates and sets up a StreamOutput 96 | self.stream = StreamOutput(self.camera) 97 | self.stream.setup(model=model, redis=redis) 98 | 99 | self.closed = False 100 | 101 | def start(self): 102 | """Starts recording to the stream.""" 103 | self.camera.start_recording(self.stream, format="rgb") 104 | self.camera.wait_recording(2) 105 | logger.info("Recording started.") 106 | 107 | def close(self): 108 | """Closes the camera and the stream.""" 109 | self.camera.stop_recording() 110 | self.camera.close() 111 | self.stream.close() 112 | logger.info("Recording stopped.") 113 | 114 | self.closed = True 115 | 116 | def update(self): 117 | """Streams outputs from the camera.""" 118 | while not self.closed: 119 | yield { 120 | "array": self.stream.array, 121 | "pose": self.stream.pose, 122 | "inference_time": self.stream.inference_time, 123 | } 124 | -------------------------------------------------------------------------------- /hiitpi/pose.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from edgetpu.basic.basic_engine import BasicEngine 3 | 4 | 5 | KEYPOINTS = ( 6 | "nose", 7 | "left eye", 8 | "right eye", 9 | "left ear", 10 | "right ear", 11 | "left shoulder", 12 | "right shoulder", 13 | "left elbow", 14 | "right elbow", 15 | "left wrist", 16 | "right wrist", 17 | "left hip", 18 | "right hip", 19 | "left knee", 20 | "right knee", 21 | "left ankle", 22 | "right ankle", 23 | ) 24 | 25 | 26 | class Keypoint: 27 | __slots__ = ["k", "yx", "score"] 28 | 29 | def __init__(self, k, yx, score=None): 30 | self.k = k 31 | self.yx = yx 32 | self.score = score 33 | 34 | def __repr__(self): 35 | return f"Keypoint(<{self.k}>, {self.yx}, {self.score})" 36 | 37 | 38 | class Pose: 39 | __slots__ = ["keypoints", "score"] 40 | 41 | def __init__(self, keypoints, score=None): 42 | assert len(keypoints) == len(KEYPOINTS) 43 | self.keypoints = keypoints 44 | self.score = score 45 | 46 | def __repr__(self): 47 | return f"Pose({self.keypoints}, {self.score})" 48 | 49 | 50 | class PoseEngine(BasicEngine): 51 | def __init__(self, model_path, mirror=False): 52 | """Creates a PoseEngine with given model. 53 | Args: 54 | model_path: String, path to TF-Lite Flatbuffer file. 55 | mirror: Flip keypoints horizontally 56 | Raises: 57 | ValueError: An error occurred when model output is invalid. 58 | """ 59 | BasicEngine.__init__(self, model_path) 60 | self._mirror = mirror 61 | 62 | self._input_tensor_shape = self.get_input_tensor_shape() 63 | if ( 64 | self._input_tensor_shape.size != 4 65 | or self._input_tensor_shape[3] != 3 66 | or self._input_tensor_shape[0] != 1 67 | ): 68 | raise ValueError( 69 | ( 70 | "Image model should have input shape [1, height, width, 3]!" 71 | f" This model has {self._input_tensor_shape}." 72 | ) 73 | ) 74 | ( 75 | _, 76 | self.image_height, 77 | self.image_width, 78 | self.image_depth, 79 | ) = self.get_input_tensor_shape() 80 | 81 | # The API returns all the output tensors flattened and concatenated. We 82 | # have to figure out the boundaries from the tensor shapes & sizes. 83 | offset = 0 84 | self._output_offsets = [0] 85 | for size in self.get_all_output_tensors_sizes(): 86 | offset += size 87 | self._output_offsets.append(offset) 88 | 89 | def DetectPosesInImage(self, img): 90 | """Detects poses in a given image. 91 | For ideal results make sure the image fed to this function is close to the 92 | expected input size - it is the caller's responsibility to resize the 93 | image accordingly. 94 | Args: 95 | img: numpy array containing image 96 | """ 97 | 98 | # Extend or crop the input to match the input shape of the network. 99 | if img.shape[0] < self.image_height or img.shape[1] < self.image_width: 100 | img = np.pad( 101 | img, 102 | [ 103 | [0, max(0, self.image_height - img.shape[0])], 104 | [0, max(0, self.image_width - img.shape[1])], 105 | [0, 0], 106 | ], 107 | mode="constant", 108 | ) 109 | img = img[0 : self.image_height, 0 : self.image_width] 110 | assert img.shape == tuple(self._input_tensor_shape[1:]) 111 | 112 | # Run the inference (API expects the data to be flattened) 113 | return self.ParseOutput(self.run_inference(img.flatten())) 114 | 115 | def ParseOutput(self, output): 116 | inference_time, output = output 117 | outputs = [ 118 | output[i:j] for i, j in zip(self._output_offsets, self._output_offsets[1:]) 119 | ] 120 | 121 | keypoints = outputs[0].reshape(-1, len(KEYPOINTS), 2) 122 | keypoint_scores = outputs[1].reshape(-1, len(KEYPOINTS)) 123 | pose_scores = outputs[2] 124 | nposes = int(outputs[3][0]) 125 | assert nposes < outputs[0].shape[0] 126 | 127 | # Convert the poses to a friendlier format of keypoints with associated 128 | # scores. 129 | poses = [] 130 | for pose_i in range(nposes): 131 | keypoint_dict = {} 132 | for point_i, point in enumerate(keypoints[pose_i]): 133 | keypoint = Keypoint( 134 | KEYPOINTS[point_i], point, keypoint_scores[pose_i, point_i] 135 | ) 136 | if self._mirror: 137 | keypoint.yx[1] = self.image_width - keypoint.yx[1] 138 | keypoint_dict[KEYPOINTS[point_i]] = keypoint 139 | poses.append(Pose(keypoint_dict, pose_scores[pose_i])) 140 | 141 | return poses, inference_time 142 | -------------------------------------------------------------------------------- /hiitpi/assets/css/style.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: Comfortaa; 3 | src: url("../font/Comfortaa/Comfortaa-Regular.ttf"); 4 | } 5 | 6 | html,body { 7 | font-family: "Comfortaa"; 8 | background-color: #1E1E1E; 9 | color: #d8d8d8; 10 | width: 100%; 11 | height: 100%; 12 | margin: 0; 13 | padding: 0; 14 | } 15 | 16 | body { 17 | overflow: hidden; 18 | } 19 | 20 | .page-background-image { 21 | width: 100%; 22 | height: 100%; 23 | background: #a7cece url("../img/pixels_art.jpg"); 24 | background-repeat: repeat; 25 | } 26 | 27 | h1, h2, h3, h4, h5 { 28 | font-family: "Comfortaa"; 29 | font-weight: bold; 30 | } 31 | p { 32 | font-family: "Comfortaa"; 33 | padding-left: 12px; 34 | padding-right: 12px; 35 | margin-top: 1rem; 36 | } 37 | a { 38 | text-decoration: none; 39 | } 40 | 41 | .flex-display { 42 | display: flex; 43 | align-items: center; 44 | justify-content: center; 45 | } 46 | 47 | .center { 48 | text-align: center; 49 | } 50 | 51 | img{ 52 | width: 100%; 53 | height: auto; 54 | display: inline-block; 55 | object-fit: contain; 56 | } 57 | 58 | .app__config_panel { 59 | display: inline-block; 60 | padding: 10px; 61 | } 62 | 63 | .app__video_image { 64 | margin: 0; 65 | display: inline-block; 66 | justify-content: center; 67 | align-items: center; 68 | text-align: center; 69 | width: 100%; 70 | height: auto; 71 | } 72 | 73 | .app__header { 74 | display: block; 75 | text-align: center; 76 | } 77 | 78 | .header_container { 79 | margin: auto; 80 | padding: 10% 5% 20% 5%; 81 | } 82 | 83 | .app__dropdown { 84 | padding: 15px; 85 | } 86 | 87 | .app__container { 88 | margin: 0 0 0 0; 89 | width: 100%; 90 | } 91 | 92 | .pretty_container { 93 | border-radius: 5px; 94 | background-color: #1E1E1E; 95 | margin: 0.5rem; 96 | padding: 1rem; 97 | position: relative; 98 | border: 0.5px solid #4B4B4B 99 | } 100 | 101 | .indicator { 102 | flex: 1; 103 | padding: 0.75rem 1.75rem; 104 | text-align: center; 105 | } 106 | 107 | .indicator_value { 108 | color: #D3D3D3; 109 | font-size: 3rem; 110 | font-weight: bold; 111 | margin: 0; 112 | } 113 | 114 | .indicator_text { 115 | color: #A9A9A9; 116 | font-size: 1rem; 117 | margin: 0; 118 | } 119 | 120 | .indicators { 121 | padding: 0 15px 15px 15px; 122 | } 123 | 124 | #workout-dropdown { 125 | width: 100%; 126 | height: auto; 127 | border-radius: 8px 0 0 8px; 128 | } 129 | 130 | #workout-stop-btn { 131 | font-family: "Comfortaa"; 132 | font-weight: bold; 133 | color: #212121; 134 | background-color: #fc6408; 135 | border-radius: 0; 136 | border-color: #fc6408; 137 | margin: 0; 138 | width: 50%; 139 | } 140 | 141 | #user-logout-btn { 142 | font-family: "Comfortaa"; 143 | font-weight: bold; 144 | color: white; 145 | background-color: royalblue; 146 | border-radius: 0; 147 | border-color: royalblue; 148 | margin: 0; 149 | width: 50%; 150 | } 151 | 152 | #user-name-input { 153 | font-family: "Comfortaa"; 154 | font-weight: bold; 155 | color: white; 156 | background-color: transparent; 157 | border-radius: 8px 0 0 8px; 158 | border: none; 159 | } 160 | 161 | .form-inline { 162 | display: flex; 163 | flex-flow: row wrap; 164 | align-items: center; 165 | } 166 | 167 | .form-inline input { 168 | font-family: "Comfortaa"; 169 | font-weight: bold; 170 | font-size: 1rem; 171 | vertical-align: middle; 172 | padding: 10px; 173 | background-color: lightgray; 174 | border: none; 175 | border-radius: 8px 0 0 8px; 176 | height: 38px; 177 | outline: none; 178 | } 179 | 180 | .form-inline button { 181 | font-family: "Comfortaa"; 182 | font-weight: bold; 183 | font-size: 1rem; 184 | background-color: royalblue; 185 | border: none; 186 | color: white; 187 | height: 38px; 188 | box-shadow: 0px 8px 15px rgba(0, 0, 0, 0.1); 189 | transition: all 0.3s ease 0s; 190 | cursor: pointer; 191 | } 192 | 193 | .form-inline button:hover { 194 | background-color: dodgerblue; 195 | box-shadow: 0px 15px 20px rgba(46, 229, 157, 0.4); 196 | color: #fff; 197 | transform: translateY(-3px); 198 | } 199 | 200 | #user-login-btn { 201 | font-family: "Comfortaa"; 202 | font-weight: bold; 203 | color: white; 204 | background-color: royalblue; 205 | border-radius: 0 8px 8px 0; 206 | border: none; 207 | margin: 0; 208 | width: 100px; 209 | } 210 | 211 | #welcome-intro-md { 212 | color: lightgray; 213 | font-size: 1.5rem; 214 | } 215 | 216 | #welcome-intro-sub-md { 217 | color: darkgray; 218 | font-size: 1.0rem; 219 | } 220 | 221 | .welcome_login_form { 222 | width: 100%; 223 | height: 100%; 224 | background-color: transparent; 225 | position: absolute; 226 | top:0; 227 | bottom: 0; 228 | left: 0; 229 | right: 0; 230 | margin: auto; 231 | } 232 | 233 | #update-leaderboard-btn { 234 | font-family: "Comfortaa"; 235 | font-weight: bold; 236 | color: white; 237 | background-color: #282828; 238 | border-radius: 0; 239 | border-color: #282828; 240 | margin: 0; 241 | width: 100%; 242 | } 243 | 244 | #leaderboard-graph { 245 | height: 150px; 246 | } 247 | 248 | .Select-control, .Select-menu-outer, .Select-multi-value-wrapper, .Select-input, .select-up, .is-open .Select-control { 249 | background-color: #282828; 250 | color: white; 251 | border-color: #282828; 252 | border-radius: 0; 253 | margin: 0; 254 | } 255 | 256 | .Select-control .Select-arrow-zone { 257 | padding: 0; 258 | } 259 | 260 | .has-value.Select--single>.Select-control .Select-value .Select-value-label, .has-value.is-pseudo-focused.Select--single>.Select-control .Select-value .Select-value-label{ 261 | color: #d8d8d8; 262 | } 263 | 264 | .Select-input>input { 265 | color: #d8d8d8; 266 | } 267 | 268 | hr { background-color: #404040; height: 1px; border: 0; margin: 15px} 269 | 270 | 271 | /* Hide scrollbar */ 272 | .example::-webkit-scrollbar { 273 | display: none; 274 | } 275 | .example { 276 | -ms-overflow-style: none; 277 | } 278 | 279 | /* For Mobile Phones and small screens ––––––––––––––––––––––––––––––––––––––––––––––––––*/ 280 | @media only screen and (max-width: 768px) { 281 | .four, .eight { 282 | min-width: 100%; 283 | } 284 | h1, h2, h3, p { 285 | text-align: center; 286 | } 287 | body { 288 | display: block; 289 | margin: 0px; 290 | overflow-y: scroll; 291 | } 292 | .app__content { 293 | display: block; 294 | } 295 | .app__container { 296 | margin: 0 0 0 0; 297 | } 298 | .header_container { 299 | margin: auto; 300 | padding: 0 5% 20% 5%; 301 | } 302 | .app__video_image { 303 | margin: 0 0 0 0; 304 | padding: 0; 305 | justify-content: center; 306 | display: block; 307 | max-width: 100%; 308 | height: auto; 309 | } 310 | } 311 | 312 | /* width */ 313 | ::-webkit-scrollbar { 314 | width: 10px !important; 315 | display: block !important; 316 | } 317 | 318 | /* Track */ 319 | ::-webkit-scrollbar-track { 320 | background: #1e1e1e !important; 321 | border-radius: 10px !important; 322 | display: block !important; 323 | } 324 | 325 | /* Handle */ 326 | ::-webkit-scrollbar-thumb { 327 | background: transparent; 328 | } 329 | 330 | /* Handle on hover */ 331 | ::-webkit-scrollbar-thumb:hover { 332 | background: #d8d8d870 !important; 333 | } 334 | 335 | -------------------------------------------------------------------------------- /hiitpi/layout.py: -------------------------------------------------------------------------------- 1 | import plotly.express as px 2 | import plotly.graph_objects as go 3 | import dash_core_components as dcc 4 | import dash_html_components as html 5 | 6 | from .workout import WORKOUTS 7 | 8 | 9 | COLORS = {"graph_bg": "#1E1E1E", "text": "#696969"} 10 | 11 | 12 | def layout_config_panel(current_user): 13 | """The Dash app layout for the user config panel""" 14 | 15 | title = html.Div([html.H5(f"HIIT PI")], className="app__header") 16 | subtitle = html.Div([html.P(f"Welcome, {current_user}!")], className="app__header") 17 | dropdown_options = [{"label": "Random", "value": "random"}] + [ 18 | {"label": v.name, "value": k} for k, v in WORKOUTS.items() 19 | ] 20 | dropdown_menu = html.Div( 21 | [ 22 | html.Div( 23 | [ 24 | dcc.Dropdown( 25 | id="workout-dropdown", 26 | options=dropdown_options, 27 | placeholder="Select a workout", 28 | searchable=True, 29 | clearable=False, 30 | ) 31 | ], 32 | className="six columns flex-display", 33 | ), 34 | html.Div( 35 | [ 36 | html.Button( 37 | id="workout-stop-btn", 38 | n_clicks=0, 39 | children="Done!", 40 | className="button", 41 | ), 42 | html.A( 43 | id="user-logout-btn", 44 | n_clicks=0, 45 | children="Exit?", 46 | className="button", 47 | href="/user_logout", 48 | ), 49 | ], 50 | className="six columns flex-display", 51 | ), 52 | ], 53 | className="row app__dropdown flex-display", 54 | ) 55 | 56 | lines_graph = html.Div( 57 | [ 58 | dcc.Graph( 59 | id="live-update-graph", 60 | figure={ 61 | "data": [ 62 | { 63 | "name": "Inference Time", 64 | "type": "scatter", 65 | "y": [], 66 | "mode": "lines", 67 | "line": {"color": "#e6af19"}, 68 | }, 69 | { 70 | "name": "Pose Score", 71 | "type": "scatter", 72 | "y": [], 73 | "yaxis": "y2", 74 | "mode": "lines", 75 | "line": {"color": "#6145bf"}, 76 | }, 77 | ], 78 | "layout": { 79 | "margin": {"l": 60, "r": 60, "b": 10, "t": 20}, 80 | "height": 180, 81 | "autosize": True, 82 | "font": { 83 | "family": "Comfortaa", 84 | "color": COLORS["text"], 85 | "size": 10, 86 | }, 87 | "plot_bgcolor": COLORS["graph_bg"], 88 | "paper_bgcolor": COLORS["graph_bg"], 89 | "xaxis": { 90 | "ticks": "", 91 | "showticklabels": False, 92 | "showgrid": False, 93 | }, 94 | "yaxis": { 95 | # "autorange": True, 96 | "range": [10, 30], 97 | "title": "Inference Time (ms)", 98 | }, 99 | "yaxis2": { 100 | # "autorange": True, 101 | "range": [0, 1], 102 | "title": "Pose Score", 103 | "overlaying": "y", 104 | "side": "right", 105 | }, 106 | "legend": { 107 | "x": 0.5, 108 | "y": -0.2, 109 | "xanchor": "center", 110 | "yanchor": "middle", 111 | "orientation": "h", 112 | }, 113 | }, 114 | }, 115 | config={ 116 | "displayModeBar": False, 117 | "responsive": True, 118 | "scrollZoom": True, 119 | }, 120 | ) 121 | ] 122 | ) 123 | 124 | workout_name = html.Div([html.P(id="workout_name")], className="app__header") 125 | 126 | def indicator(id_value, text): 127 | return html.Div( 128 | [ 129 | html.P(id=id_value, className="indicator_value"), 130 | html.P(text, className="twelve columns indicator_text"), 131 | ], 132 | className="six columns indicator pretty_container", 133 | ) 134 | 135 | indicators = html.Div( 136 | [ 137 | indicator("indicator-reps", "REPS"), 138 | indicator("indicator-pace", "PACE (/30s)"), 139 | ], 140 | className="row indicators flex-display", 141 | ) 142 | 143 | live_update_graph = html.Div( 144 | [ 145 | lines_graph, 146 | dcc.Interval(id="live-update-interval", interval=50, n_intervals=0), 147 | ] 148 | ) 149 | 150 | bars_graph = html.Div( 151 | [ 152 | html.Button( 153 | id="update-leaderboard-btn", 154 | n_clicks=0, 155 | children="Leaderboard", 156 | className="button", 157 | ), 158 | dcc.Graph( 159 | id="leaderboard-graph", 160 | config={ 161 | "displayModeBar": False, 162 | "responsive": True, 163 | "scrollZoom": True, 164 | }, 165 | ), 166 | ] 167 | ) 168 | 169 | return html.Div( 170 | [ 171 | title, 172 | subtitle, 173 | live_update_graph, 174 | dropdown_menu, 175 | workout_name, 176 | indicators, 177 | bars_graph, 178 | ], 179 | className="four columns app__config_panel", 180 | ) 181 | 182 | 183 | def layout_videostream(): 184 | """The Dash app layout for the video stream""" 185 | videostream = html.Img(id="videostream") 186 | return html.Div([videostream], className="eight columns app__video_image") 187 | 188 | 189 | def layout_homepage(current_user): 190 | """The Dash app home page layout""" 191 | return html.Div( 192 | [layout_videostream(), layout_config_panel(current_user)], 193 | className="row app__container", 194 | ) 195 | 196 | 197 | def layout_login(): 198 | """The Dash app login oage layout""" 199 | header = html.Div( 200 | [ 201 | html.H2("HIIT PI"), 202 | dcc.Markdown( 203 | id="welcome-intro-md", 204 | children=""" 205 | A workout trainer [Dash](https://dash.plot.ly/) app 206 | that helps track your [HIIT](https://en.wikipedia.org/wiki/High-intensity_interval_training) workouts 207 | by analyzing real-time video streaming from your sweet [Pi](https://www.raspberrypi.org/). 208 | """, 209 | ), 210 | dcc.Markdown( 211 | id="welcome-intro-sub-md", 212 | children="Powered by [TensorFlow Lite](https://www.tensorflow.org/lite) and [Edge TPU](https://cloud.google.com/edge-tpu) with ❤️.", 213 | ), 214 | ], 215 | className="app__header", 216 | ) 217 | 218 | login_form = html.Div( 219 | [ 220 | html.Form( 221 | [ 222 | dcc.Input( 223 | id="user_name_input", 224 | name="user_name_form", 225 | type="text", 226 | placeholder="PLAYER NAME", 227 | maxLength=20, 228 | minLength=1, 229 | required=True, 230 | ), 231 | html.Button( 232 | id="user-login-btn", 233 | children="ENTER", 234 | type="submit", 235 | n_clicks=0, 236 | className="button", 237 | ), 238 | ], 239 | action="/user_login", 240 | method="POST", 241 | autoComplete="off", 242 | className="form-inline", 243 | title="Enter your player name.", 244 | ) 245 | ], 246 | className="flex-display", 247 | style={"margin-top": "4rem"}, 248 | ) 249 | 250 | welcome_jumbotron = html.Div([header, login_form], className="header_container") 251 | return html.Div( 252 | [welcome_jumbotron], 253 | className="welcome_login_form page-background-image flex-display", 254 | ) 255 | 256 | 257 | def layout(): 258 | return html.Div([dcc.Location(id="url", refresh=True), html.Div(id="page-content")]) 259 | 260 | -------------------------------------------------------------------------------- /hiitpi/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import datetime 4 | import random 5 | import cv2 6 | import pandas as pd 7 | from flask import Response, request, redirect, session 8 | from flask_migrate import Migrate 9 | from flask_sqlalchemy import SQLAlchemy 10 | from flask_caching import Cache 11 | from flask_session import Session 12 | import plotly.express as px 13 | import dash 14 | from dash.dependencies import Input, Output, State 15 | 16 | 17 | logging.basicConfig( 18 | stream=sys.stdout, 19 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 20 | datefmt=" %I:%M:%S ", 21 | level="INFO", 22 | ) 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | COLORS = {"graph_bg": "#1E1E1E", "text": "#696969"} 27 | 28 | sess = Session() 29 | cache = Cache() 30 | db = SQLAlchemy() 31 | migrate = Migrate() 32 | 33 | 34 | def create_app(config_name): 35 | """Create a Dash app.""" 36 | 37 | from .config import config 38 | from .model import WorkoutSession 39 | from .pose import PoseEngine 40 | from .camera import VideoStream 41 | from .redisclient import RedisClient 42 | from .workout import WORKOUTS 43 | from .annotation import Annotator 44 | from .layout import layout_homepage, layout_login, layout 45 | 46 | app = dash.Dash( 47 | __name__, 48 | meta_tags=[ 49 | {"name": "charset", "content": "UTF-8"}, 50 | { 51 | "name": "viewport", 52 | "content": "width=device-width, initial-scale=1, maximum-scale=1, shrink-to-fit=no", 53 | }, 54 | {"name": "author", "content": "James Wong"}, 55 | { 56 | "name": "description", 57 | "content": "A HIIT Workout Trainer Dash App on Your Raspberry Pi", 58 | }, 59 | ], 60 | ) 61 | app.title = "HIIT PI" 62 | app.config.suppress_callback_exceptions = True 63 | app.layout = layout() 64 | 65 | server = app.server 66 | server.config.from_object(config[config_name]) 67 | 68 | with server.app_context(): 69 | db.init_app(server) 70 | migrate.init_app(server, db) 71 | 72 | sess.init_app(server) 73 | 74 | cache.init_app(server) 75 | cache.clear() 76 | 77 | video = VideoStream() 78 | model = PoseEngine(model_path=server.config["MODEL_PATH"]) 79 | redis = RedisClient( 80 | host=server.config["REDIS_HOST"], 81 | port=server.config["REDIS_PORT"], 82 | db=server.config["REDIS_DB"], 83 | ) 84 | 85 | def gen(video, workout): 86 | """Streams and analyzes video contents while overlaying stats info 87 | Args: 88 | video: a VideoStream object. 89 | workout: str, a workout name or "None". 90 | Returns: 91 | bytes, the output image data 92 | """ 93 | if workout != "None": 94 | # Initiates the Workout object from the workout name 95 | workout = WORKOUTS[workout]() 96 | workout.setup(redis=redis) 97 | annotator = Annotator() 98 | 99 | for output in video.update(): 100 | # Computes pose stats 101 | workout.update(output["pose"]) 102 | output["workout"] = workout 103 | 104 | # Annotates the image and encodes the raw RGB data into JPEG format 105 | output["array"] = annotator.annotate(output) 106 | img = cv2.cvtColor(output["array"], cv2.COLOR_RGB2BGR) 107 | _, buf = cv2.imencode(".jpeg", img) 108 | yield ( 109 | b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" 110 | + buf.tobytes() 111 | + b"\r\n\r\n" 112 | ) 113 | else: 114 | # Renders a blurring effect while on standby with no workout 115 | for output in video.update(): 116 | img = cv2.blur(output["array"], (32, 32)) 117 | img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 118 | ret, buf = cv2.imencode(".jpeg", img) 119 | yield ( 120 | b"--frame\r\nContent-Type: image/jpeg\r\n\r\n" 121 | + buf.tobytes() 122 | + b"\r\n\r\n" 123 | ) 124 | 125 | @app.callback( 126 | [Output("videostream", "src"), Output("workout_name", "children")], 127 | [Input("workout-dropdown", "value")], 128 | ) 129 | def start_workout(workout): 130 | if workout is not None: 131 | if workout == "random": 132 | workout = random.choice(list(WORKOUTS)) 133 | workout_name = WORKOUTS[workout].name 134 | session["workout"] = workout_name 135 | else: 136 | workout_name = "Select a workout to get started." 137 | session["workout"] = None 138 | logger.info(f'Current workout: {session.get("workout")}') 139 | return f"/videostream/{workout}", workout_name 140 | 141 | @app.callback( 142 | Output("workout-dropdown", "value"), 143 | [Input("workout-stop-btn", "n_clicks")], 144 | [State("workout-dropdown", "value")], 145 | ) 146 | def stop_workout(n_clicks, workout): 147 | if workout is not None: 148 | ws = WorkoutSession( 149 | user_name=session.get("user_name"), 150 | workout=session.get("workout"), 151 | reps=redis.get("reps"), 152 | pace=redis.get("pace"), 153 | ) 154 | db.session.add(ws) 155 | db.session.commit() 156 | logger.info(f"{ws} inserted into db") 157 | return None 158 | 159 | @app.callback( 160 | Output("leaderboard-graph", "figure"), 161 | [Input("update-leaderboard-btn", "n_clicks")], 162 | [State("workout-dropdown", "value")], 163 | ) 164 | def update_leaderboard_graph(n_clicks, workout): 165 | if n_clicks > 0: 166 | current_time = datetime.datetime.utcnow() 167 | a_week_ago = current_time - datetime.timedelta(weeks=1) 168 | 169 | query = ( 170 | db.session.query( 171 | WorkoutSession.user_name, 172 | WorkoutSession.workout, 173 | db.func.sum(WorkoutSession.reps).label("reps"), 174 | ) 175 | .filter(WorkoutSession.created_date >= a_week_ago) 176 | .group_by(WorkoutSession.user_name, WorkoutSession.workout) 177 | .order_by(db.func.sum(WorkoutSession.reps).desc()) 178 | .all() 179 | ) 180 | 181 | df = pd.DataFrame(query, columns=["user_name", "workout", "reps"]) 182 | layout = { 183 | "barmode": "stack", 184 | "margin": {"l": 0, "r": 0, "b": 0, "t": 40}, 185 | "autosize": True, 186 | "font": {"family": "Comfortaa", "color": COLORS["text"], "size": 10}, 187 | "plot_bgcolor": COLORS["graph_bg"], 188 | "paper_bgcolor": COLORS["graph_bg"], 189 | "xaxis": { 190 | "ticks": "", 191 | "showgrid": False, 192 | "title": "", 193 | "automargin": True, 194 | "zeroline": False, 195 | }, 196 | "yaxis": { 197 | "showgrid": False, 198 | "title": "", 199 | "automargin": True, 200 | "categoryorder": "total ascending", 201 | "linewidth": 1, 202 | "linecolor": "#282828", 203 | "zeroline": False, 204 | }, 205 | "title": { 206 | "text": "Last 7 Days", 207 | "y": 0.9, 208 | "x": 0.5, 209 | "xanchor": "center", 210 | "yanchor": "top", 211 | }, 212 | "legend": { 213 | "x": 1.0, 214 | "y": -0.2, 215 | "xanchor": "right", 216 | "yanchor": "top", 217 | "title": "", 218 | "orientation": "h", 219 | "itemclick": "toggle", 220 | "itemdoubleclick": "toggleothers", 221 | }, 222 | "showlegend": True, 223 | } 224 | fig = px.bar( 225 | df, 226 | x="reps", 227 | y="user_name", 228 | color="workout", 229 | orientation="h", 230 | color_discrete_sequence=px.colors.qualitative.Plotly, 231 | ) 232 | fig.update_layout(layout) 233 | fig.update_traces(marker_line_width=0, width=0.5) 234 | return fig 235 | else: 236 | return { 237 | "data": [], 238 | "layout": { 239 | "plot_bgcolor": COLORS["graph_bg"], 240 | "paper_bgcolor": COLORS["graph_bg"], 241 | "xaxis": { 242 | "showgrid": False, 243 | "showline": False, 244 | "zeroline": False, 245 | "showticklabels": False, 246 | }, 247 | "yaxis": { 248 | "showgrid": False, 249 | "showline": False, 250 | "zeroline": False, 251 | "showticklabels": False, 252 | }, 253 | }, 254 | } 255 | 256 | @server.route("/videostream/", methods=["GET"]) 257 | def videiostream(workout): 258 | user_name = session.get("user_name") 259 | logger.info(f"Current player: {user_name}") 260 | return Response( 261 | gen(video, workout), mimetype="multipart/x-mixed-replace; boundary=frame" 262 | ) 263 | 264 | @app.callback( 265 | [ 266 | Output("live-update-graph", "extendData"), 267 | Output("indicator-reps", "children"), 268 | Output("indicator-pace", "children"), 269 | ], 270 | [Input("live-update-interval", "n_intervals")], 271 | ) 272 | def update_workout_graph(n_intervals): 273 | inference_time = redis.lpop("inference_time") 274 | pose_score = redis.lpop("pose_score") 275 | data = [{"y": [[inference_time], [pose_score]]}, [0, 1], 200] 276 | 277 | reps = redis.get("reps") 278 | pace = redis.get("pace") 279 | 280 | return data, f"{reps:.0f}", f"{pace*30:.1f}" if pace > 0 else "/" 281 | 282 | @server.route("/user_login", methods=["POST"]) 283 | def user_login(): 284 | user_name = request.form.get("user_name_form") 285 | session["user_name"] = user_name 286 | logger.info(f"Player {user_name} logged in") 287 | 288 | if video.closed is None or video.closed: 289 | video.setup(model=model, redis=redis) 290 | video.start() 291 | 292 | return redirect("/home") 293 | 294 | @server.route("/user_logout") 295 | def user_logout(): 296 | user_name = session.pop("user_name") 297 | if user_name is not None: 298 | session.clear() 299 | logger.info(f"Player {user_name} logged out") 300 | 301 | if not video.closed: 302 | video.close() 303 | 304 | return redirect("/") 305 | 306 | @app.callback(Output("page-content", "children"), [Input("url", "pathname")]) 307 | def display_page(pathname): 308 | if pathname == "/home": 309 | current_user = session.get("user_name") 310 | return layout_homepage(current_user) 311 | else: 312 | return layout_login() 313 | 314 | return app 315 | -------------------------------------------------------------------------------- /hiitpi/assets/css/base.css: -------------------------------------------------------------------------------- 1 | /* Table of contents 2 | –––––––––––––––––––––––––––––––––––––––––––––––––– 3 | - Plotly.js 4 | - Grid 5 | - Base Styles 6 | - Typography 7 | - Links 8 | - Buttons 9 | - Forms 10 | - Lists 11 | - Code 12 | - Tables 13 | - Spacing 14 | - Utilities 15 | - Clearing 16 | - Media Queries 17 | */ 18 | 19 | /* PLotly.js 20 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 21 | /* plotly.js's modebar's z-index is 1001 by default 22 | * https://github.com/plotly/plotly.js/blob/7e4d8ab164258f6bd48be56589dacd9bdd7fded2/src/css/_modebar.scss#L5 23 | * In case a dropdown is above the graph, the dropdown's options 24 | * will be rendered below the modebar 25 | * Increase the select option's z-index 26 | */ 27 | 28 | /* This was actually not quite right - 29 | dropdowns were overlapping each other (edited October 26) 30 | 31 | .Select { 32 | z-index: 1002; 33 | }*/ 34 | 35 | 36 | /* Grid 37 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 38 | .container { 39 | position: relative; 40 | width: 100%; 41 | max-width: 960px; 42 | margin: 0 auto; 43 | padding: 0 20px; 44 | box-sizing: border-box; } 45 | .column, 46 | .columns { 47 | width: 100%; 48 | float: left; 49 | box-sizing: border-box; } 50 | 51 | /* For devices larger than 400px */ 52 | @media (min-width: 400px) { 53 | .container { 54 | width: 85%; 55 | padding: 0; } 56 | } 57 | 58 | /* For devices larger than 550px */ 59 | @media (min-width: 550px) { 60 | .container { 61 | width: 80%; } 62 | .column, 63 | .columns { 64 | margin-left: 0; } 65 | /* .column:first-child, 66 | .columns:first-child { 67 | margin-left: 0; } */ 68 | 69 | .one.column, 70 | .one.columns { width: 4.66666666667%; } 71 | .two.columns { width: 13.3333333333%; } 72 | .three.columns { width: 22%; } 73 | .four.columns { width: 33.3333333333%; } 74 | .five.columns { width: 39.3333333333%; } 75 | .six.columns { width: 50%; } 76 | .seven.columns { width: 56.6666666667%; } 77 | .eight.columns { width: 66.66666666667%; } 78 | .nine.columns { width: 74.0%; } 79 | .ten.columns { width: 82.6666666667%; } 80 | .eleven.columns { width: 91.3333333333%; } 81 | .twelve.columns { width: 100%; margin-left: 0; } 82 | 83 | .one-third.column { width: 30.6666666667%; } 84 | .two-thirds.column { width: 65.3333333333%; } 85 | 86 | .one-half.column { width: 48%; } 87 | 88 | /* Offsets */ 89 | .offset-by-one.column, 90 | .offset-by-one.columns { margin-left: 8.66666666667%; } 91 | .offset-by-two.column, 92 | .offset-by-two.columns { margin-left: 17.3333333333%; } 93 | .offset-by-three.column, 94 | .offset-by-three.columns { margin-left: 26%; } 95 | .offset-by-four.column, 96 | .offset-by-four.columns { margin-left: 34.6666666667%; } 97 | .offset-by-five.column, 98 | .offset-by-five.columns { margin-left: 43.3333333333%; } 99 | .offset-by-six.column, 100 | .offset-by-six.columns { margin-left: 52%; } 101 | .offset-by-seven.column, 102 | .offset-by-seven.columns { margin-left: 60.6666666667%; } 103 | .offset-by-eight.column, 104 | .offset-by-eight.columns { margin-left: 69.3333333333%; } 105 | .offset-by-nine.column, 106 | .offset-by-nine.columns { margin-left: 78.0%; } 107 | .offset-by-ten.column, 108 | .offset-by-ten.columns { margin-left: 86.6666666667%; } 109 | .offset-by-eleven.column, 110 | .offset-by-eleven.columns { margin-left: 95.3333333333%; } 111 | 112 | .offset-by-one-third.column, 113 | .offset-by-one-third.columns { margin-left: 34.6666666667%; } 114 | .offset-by-two-thirds.column, 115 | .offset-by-two-thirds.columns { margin-left: 69.3333333333%; } 116 | 117 | .offset-by-one-half.column, 118 | .offset-by-one-half.columns { margin-left: 52%; } 119 | 120 | } 121 | 122 | 123 | /* Base Styles 124 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 125 | /* NOTE 126 | html is set to 62.5% so that all the REM measurements throughout Skeleton 127 | are based on 10px sizing. So basically 1.5rem = 15px :) */ 128 | html { 129 | font-size: 62.5%; } 130 | body { 131 | font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */ 132 | line-height: 1.6; 133 | font-weight: 400; 134 | font-family: "Open Sans", "HelveticaNeue", "Helvetica Neue", Helvetica, Arial, sans-serif; 135 | color: rgb(50, 50, 50); } 136 | 137 | 138 | /* Typography 139 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 140 | h1, h2, h3, h4, h5, h6 { 141 | margin-top: 0; 142 | margin-bottom: 0; 143 | font-weight: 300; } 144 | h1 { font-size: 4.5rem; line-height: 1.2; letter-spacing: -.1rem; margin-bottom: 2rem; } 145 | h2 { font-size: 3.6rem; line-height: 1.25; letter-spacing: -.1rem; margin-bottom: 1.8rem; margin-top: 1.8rem;} 146 | h3 { font-size: 3.0rem; line-height: 1.3; letter-spacing: -.1rem; margin-bottom: 1.5rem; margin-top: 1.5rem;} 147 | h4 { font-size: 2.6rem; line-height: 1.35; letter-spacing: -.08rem; margin-bottom: 1.2rem; margin-top: 1.2rem;} 148 | h5 { font-size: 2.2rem; line-height: 1.5; letter-spacing: -.05rem; margin-bottom: 0.6rem; margin-top: 0.6rem;} 149 | h6 { font-size: 2.0rem; line-height: 1.6; letter-spacing: 0; margin-bottom: 0.75rem; margin-top: 0.75rem;} 150 | 151 | p { 152 | margin-top: 0; } 153 | 154 | 155 | /* Blockquotes 156 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 157 | blockquote { 158 | border-left: 4px lightgrey solid; 159 | padding-left: 1rem; 160 | margin-top: 2rem; 161 | margin-bottom: 2rem; 162 | margin-left: 0rem; 163 | } 164 | 165 | 166 | /* Links 167 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 168 | a { 169 | color: #d8d8d8; 170 | text-decoration: underline; 171 | cursor: pointer;} 172 | a:hover { 173 | color: #d8d8d8; } 174 | 175 | 176 | /* Buttons 177 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 178 | .button, 179 | button, 180 | input[type="submit"], 181 | input[type="reset"], 182 | input[type="button"] { 183 | display: inline-block; 184 | height: 36px; 185 | padding: 0 15px; 186 | color: #555; 187 | text-align: center; 188 | font-size: 11px; 189 | font-weight: 600; 190 | line-height: 36px; 191 | letter-spacing: .1rem; 192 | text-transform: uppercase; 193 | text-decoration: none; 194 | white-space: nowrap; 195 | background-color: transparent; 196 | border-radius: 4px; 197 | border: 1px solid #bbb; 198 | cursor: pointer; 199 | box-sizing: border-box; } 200 | .button:hover, 201 | button:hover, 202 | input[type="submit"]:hover, 203 | input[type="reset"]:hover, 204 | input[type="button"]:hover, 205 | .button:focus, 206 | button:focus, 207 | input[type="submit"]:focus, 208 | input[type="reset"]:focus, 209 | input[type="button"]:focus { 210 | color: #333; 211 | border-color: #888; 212 | outline: 0; } 213 | .button.button-primary, 214 | button.button-primary, 215 | input[type="submit"].button-primary, 216 | input[type="reset"].button-primary, 217 | input[type="button"].button-primary { 218 | color: #FFF; 219 | background-color: #33C3F0; 220 | border-color: #33C3F0; } 221 | .button.button-primary:hover, 222 | button.button-primary:hover, 223 | input[type="submit"].button-primary:hover, 224 | input[type="reset"].button-primary:hover, 225 | input[type="button"].button-primary:hover, 226 | .button.button-primary:focus, 227 | button.button-primary:focus, 228 | input[type="submit"].button-primary:focus, 229 | input[type="reset"].button-primary:focus, 230 | input[type="button"].button-primary:focus { 231 | color: #FFF; 232 | background-color: #1EAEDB; 233 | border-color: #1EAEDB; } 234 | 235 | 236 | /* Forms 237 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 238 | input[type="email"], 239 | input[type="number"], 240 | input[type="search"], 241 | input[type="text"], 242 | input[type="tel"], 243 | input[type="url"], 244 | input[type="password"], 245 | textarea, 246 | select { 247 | height: 38px; 248 | padding: 6px 10px; /* The 6px vertically centers text on FF, ignored by Webkit */ 249 | background-color: #fff; 250 | border: 1px solid #D1D1D1; 251 | border-radius: 4px; 252 | box-shadow: none; 253 | box-sizing: border-box; 254 | font-family: inherit; 255 | font-size: inherit; /*https://stackoverflow.com/questions/6080413/why-doesnt-input-inherit-the-font-from-body*/} 256 | /* Removes awkward default styles on some inputs for iOS */ 257 | input[type="email"], 258 | input[type="number"], 259 | input[type="search"], 260 | input[type="text"], 261 | input[type="tel"], 262 | input[type="url"], 263 | input[type="password"], 264 | textarea { 265 | -webkit-appearance: none; 266 | -moz-appearance: none; 267 | appearance: none; } 268 | textarea { 269 | min-height: 65px; 270 | padding-top: 6px; 271 | padding-bottom: 6px; } 272 | input[type="email"]:focus, 273 | input[type="number"]:focus, 274 | input[type="search"]:focus, 275 | input[type="text"]:focus, 276 | input[type="tel"]:focus, 277 | input[type="url"]:focus, 278 | input[type="password"]:focus, 279 | /* textarea:focus, 280 | select:focus { 281 | border: 1px solid #33C3F0; 282 | outline: 0; } */ 283 | label, 284 | legend { 285 | display: block; 286 | margin-bottom: 0px; } 287 | fieldset { 288 | padding: 0; 289 | border-width: 0; } 290 | input[type="checkbox"], 291 | input[type="radio"] { 292 | display: inline; } 293 | label > .label-body { 294 | display: inline-block; 295 | margin-left: .5rem; 296 | font-weight: normal; } 297 | 298 | 299 | /* Lists 300 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 301 | ul { 302 | list-style: circle inside; } 303 | ol { 304 | list-style: decimal inside; } 305 | ol, ul { 306 | padding-left: 0; 307 | margin-top: 0; } 308 | ul ul, 309 | ul ol, 310 | ol ol, 311 | ol ul { 312 | margin: 1.5rem 0 1.5rem 3rem; 313 | font-size: 90%; } 314 | li { 315 | margin-bottom: 1rem; } 316 | 317 | 318 | /* Tables 319 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 320 | table { 321 | border-collapse: collapse; 322 | } 323 | th:not(.CalendarDay), 324 | td:not(.CalendarDay) { 325 | padding: 12px 15px; 326 | text-align: left; 327 | border-bottom: 1px solid #E1E1E1; } 328 | th:first-child:not(.CalendarDay), 329 | td:first-child:not(.CalendarDay) { 330 | padding-left: 0; } 331 | th:last-child:not(.CalendarDay), 332 | td:last-child:not(.CalendarDay) { 333 | padding-right: 0; } 334 | 335 | 336 | /* Spacing 337 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 338 | button, 339 | .button { 340 | margin-bottom: 0rem; } 341 | input, 342 | textarea, 343 | select, 344 | fieldset { 345 | margin-bottom: 0rem; } 346 | pre, 347 | dl, 348 | figure, 349 | table, 350 | form { 351 | margin-bottom: 0rem; } 352 | p, 353 | ul, 354 | ol { 355 | margin-bottom: 0.75rem; } 356 | 357 | /* Utilities 358 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 359 | .u-full-width { 360 | width: 100%; 361 | box-sizing: border-box; } 362 | .u-max-full-width { 363 | max-width: 100%; 364 | box-sizing: border-box; } 365 | .u-pull-right { 366 | float: right; } 367 | .u-pull-left { 368 | float: left; } 369 | 370 | 371 | /* Misc 372 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 373 | hr { 374 | margin-top: 3rem; 375 | margin-bottom: 3.5rem; 376 | border-width: 0; 377 | border-top: 1px solid #E1E1E1; } 378 | 379 | 380 | /* Clearing 381 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 382 | 383 | /* Self Clearing Goodness */ 384 | .container:after, 385 | .row:after, 386 | .u-cf { 387 | content: ""; 388 | display: table; 389 | clear: both; } 390 | 391 | 392 | /* Media Queries 393 | –––––––––––––––––––––––––––––––––––––––––––––––––– */ 394 | /* 395 | Note: The best way to structure the use of media queries is to create the queries 396 | near the relevant code. For example, if you wanted to change the styles for buttons 397 | on small devices, paste the mobile query code up in the buttons section and style it 398 | there. 399 | */ 400 | 401 | 402 | /* Larger than mobile */ 403 | @media (min-width: 400px) {} 404 | 405 | /* Larger than phablet (also point when grid becomes active) */ 406 | @media (min-width: 550px) {} 407 | 408 | /* Larger than tablet */ 409 | @media (min-width: 750px) {} 410 | 411 | /* Larger than desktop */ 412 | @media (min-width: 1000px) {} 413 | 414 | /* Larger than Desktop HD */ 415 | @media (min-width: 1200px) {} -------------------------------------------------------------------------------- /hiitpi/workout.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import time 4 | import itertools 5 | import collections 6 | import numpy as np 7 | 8 | 9 | logging.basicConfig( 10 | stream=sys.stdout, 11 | format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", 12 | datefmt=" %I:%M:%S ", 13 | level="DEBUG", 14 | ) 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class Edge: 20 | __slots__ = ["k_a", "k_b", "vec", "norm", "center", "score"] 21 | 22 | def __init__(self, k_a, k_b): 23 | self.k_a = k_a 24 | self.k_b = k_b 25 | self.vec = self.k_a.yx - self.k_b.yx 26 | self.norm = np.linalg.norm(self.vec) 27 | self.center = (self.k_a.yx + self.k_b.yx) / 2 28 | self.score = min(self.k_a.score, self.k_b.score) 29 | 30 | def __repr__(self): 31 | return f"Edge({self.k_a.k} -> {self.k_b.k}, {self.vec})" 32 | 33 | def __invert__(self): 34 | return Edge(self.k_b, self.k_a) 35 | 36 | def find_angle(self, edge): 37 | cosang = np.dot(self.vec, edge.vec) 38 | sinang = np.linalg.norm(np.cross(self.vec, edge.vec)) 39 | angle = np.arctan2(sinang, cosang) 40 | return np.degrees(angle) 41 | 42 | 43 | class Joint: 44 | __slots__ = ["e_a", "e_b", "angle"] 45 | 46 | def __init__(self, e_a, e_b): 47 | self.e_a = e_a 48 | self.e_b = e_b 49 | self.angle = self.e_a.find_angle(self.e_b) 50 | 51 | def __repr__(self): 52 | return f"Joint({self.e_a}, {self.e_b}, {self.angle})" 53 | 54 | 55 | class Workout: 56 | """Base class for tracking workout progress with movement analysis""" 57 | 58 | def __init__(self, n_keystates): 59 | self.THRESHOLD = 0.2 60 | self.N_KEYSTATES = n_keystates 61 | self.KEYSTATES = itertools.cycle(range(1, self.N_KEYSTATES + 1)) 62 | self._prev_state = None 63 | self._next_state = next(self.KEYSTATES) 64 | self.stats = None 65 | self.reps = 0 66 | self.pace = 0 67 | self._init_time = time.perf_counter() 68 | self._reps_time = collections.deque([], maxlen=32) 69 | 70 | def setup(self, redis): 71 | """ 72 | Args: 73 | redis: RedisClient. 74 | """ 75 | self.redis = redis 76 | 77 | self.redis.set("reps", self.reps) 78 | self.redis.set("pace", self.pace) 79 | 80 | def get_stats(self, pose): 81 | raise NotImplemented 82 | 83 | def get_state(self, stats): 84 | raise NotImplemented 85 | 86 | def update(self, pose): 87 | if pose: 88 | self.stats = self.get_stats(pose) 89 | state = self.get_state(self.stats) 90 | 91 | if state != 0 and state != self._prev_state and state == self._next_state: 92 | self._prev_state = state 93 | self._next_state = next(self.KEYSTATES) 94 | if state == self.N_KEYSTATES: 95 | self._reps_time.append(time.perf_counter()) 96 | if self.reps > 1: 97 | self.pace = (len(self._reps_time) - 1) / ( 98 | self._reps_time[-1] - self._reps_time[0] 99 | ) 100 | self.reps += 1 101 | self.redis.set("reps", self.reps) 102 | self.redis.set("pace", self.pace) 103 | 104 | 105 | class ToeTap(Workout): 106 | name = "Toe Tap" 107 | 108 | def __init__(self, *args, **kwargs): 109 | super().__init__(n_keystates=2, *args, **kwargs) 110 | self.KEYPOINTS = [ 111 | "left hip", 112 | "right hip", 113 | "left ankle", 114 | "right ankle", 115 | "left elbow", 116 | "left shoulder", 117 | "left wrist", 118 | "right elbow", 119 | "right shoulder", 120 | "right wrist", 121 | ] 122 | 123 | def get_stats(self, pose): 124 | kps = pose.keypoints 125 | 126 | if all(kps[k].score > self.THRESHOLD for k in self.KEYPOINTS): 127 | 128 | e_hips = Edge(kps["left hip"], kps["right hip"]) 129 | e_ankles = Edge(kps["left ankle"], kps["right ankle"]) 130 | e_lelbow_lshoulder = Edge(kps["left elbow"], kps["left shoulder"]) 131 | e_lelbow_lwrist = Edge(kps["left elbow"], kps["left wrist"]) 132 | e_relbow_rshoulder = Edge(kps["right elbow"], kps["right shoulder"]) 133 | e_relbow_rwrist = Edge(kps["right elbow"], kps["right wrist"]) 134 | 135 | j_lelbow = Joint(e_lelbow_lshoulder, e_lelbow_lwrist) 136 | j_relbow = Joint(e_relbow_rshoulder, e_relbow_rwrist) 137 | 138 | return { 139 | "e_hips_norm": e_hips.norm, 140 | "e_ankles_norm": e_ankles.norm, 141 | "j_lelbow_angle": j_lelbow.angle, 142 | "j_relbow_angle": j_relbow.angle, 143 | } 144 | else: 145 | return None 146 | 147 | def get_state(self, stats): 148 | if stats is not None: 149 | if stats["e_ankles_norm"] <= stats["e_hips_norm"]: 150 | if stats["j_lelbow_angle"] >= 90 and stats["j_relbow_angle"] <= 90: 151 | return 1 152 | elif stats["j_relbow_angle"] >= 90 and stats["j_lelbow_angle"] <= 90: 153 | return 2 154 | else: 155 | return 0 156 | else: 157 | return 0 158 | else: 159 | return 0 160 | 161 | 162 | class JumpingJacks(Workout): 163 | name = "Jumping Jacks" 164 | 165 | def __init__(self, *args, **kwargs): 166 | super().__init__(n_keystates=2, *args, **kwargs) 167 | self.KEYPOINTS = [ 168 | "left hip", 169 | "right hip", 170 | "left ankle", 171 | "right ankle", 172 | "left elbow", 173 | "left shoulder", 174 | "right elbow", 175 | "right shoulder", 176 | ] 177 | 178 | def get_stats(self, pose): 179 | kps = pose.keypoints 180 | 181 | if all(kps[k].score > self.THRESHOLD for k in self.KEYPOINTS): 182 | 183 | e_hips = Edge(kps["left hip"], kps["right hip"]) 184 | e_ankles = Edge(kps["left ankle"], kps["right ankle"]) 185 | e_lshoulder_lelbow = Edge(kps["left shoulder"], kps["left elbow"]) 186 | e_lshoulder_lhip = Edge(kps["left shoulder"], kps["left hip"]) 187 | e_rshoulder_relbow = Edge(kps["right shoulder"], kps["right elbow"]) 188 | e_rshoulder_rhip = Edge(kps["right shoulder"], kps["right hip"]) 189 | 190 | j_lshoulder = Joint(e_lshoulder_lelbow, e_lshoulder_lhip) 191 | j_rshoulder = Joint(e_rshoulder_relbow, e_rshoulder_rhip) 192 | 193 | return { 194 | "e_hips_norm": e_hips.norm, 195 | "e_ankles_norm": e_ankles.norm, 196 | "j_lshoulder_angle": j_lshoulder.angle, 197 | "j_rshoulder_angle": j_rshoulder.angle, 198 | } 199 | else: 200 | return None 201 | 202 | def get_state(self, stats): 203 | if stats is not None: 204 | if ( 205 | stats["e_ankles_norm"] <= stats["e_hips_norm"] 206 | and stats["j_lshoulder_angle"] <= 30 207 | and stats["j_rshoulder_angle"] <= 30 208 | ): 209 | return 1 210 | elif ( 211 | stats["e_ankles_norm"] >= stats["e_hips_norm"] * 1.5 212 | and stats["j_lshoulder_angle"] >= 110 213 | and stats["j_rshoulder_angle"] >= 110 214 | ): 215 | return 2 216 | else: 217 | return 0 218 | else: 219 | return 0 220 | 221 | 222 | class PushUp(Workout): 223 | name = "Push Up" 224 | 225 | def __init__(self, *args, **kwargs): 226 | super().__init__(n_keystates=2, *args, **kwargs) 227 | self.KEYPOINTS = [ 228 | "left elbow", 229 | "left shoulder", 230 | "left wrist", 231 | "right elbow", 232 | "right shoulder", 233 | "right wrist", 234 | ] 235 | 236 | def get_stats(self, pose): 237 | kps = pose.keypoints 238 | 239 | if all(kps[k].score > self.THRESHOLD for k in self.KEYPOINTS): 240 | 241 | e_lelbow_lshoulder = Edge(kps["left elbow"], kps["left shoulder"]) 242 | e_lelbow_lwrist = Edge(kps["left elbow"], kps["left wrist"]) 243 | e_relbow_rshoulder = Edge(kps["right elbow"], kps["right shoulder"]) 244 | e_relbow_rwrist = Edge(kps["right elbow"], kps["right wrist"]) 245 | e_lshoulder_rshoulder = Edge(kps["left shoulder"], kps["right shoulder"]) 246 | 247 | j_lelbow = Joint(e_lelbow_lshoulder, e_lelbow_lwrist) 248 | j_relbow = Joint(e_relbow_rshoulder, e_relbow_rwrist) 249 | j_lshoulder = Joint(e_lshoulder_rshoulder, ~e_lelbow_lshoulder) 250 | j_rshoulder = Joint(~e_lshoulder_rshoulder, ~e_relbow_rshoulder) 251 | 252 | return { 253 | "j_lelbow_angle": j_lelbow.angle, 254 | "j_relbow_angle": j_relbow.angle, 255 | "j_lshoulder_angle": j_lshoulder.angle, 256 | "j_rshoulder_angle": j_rshoulder.angle, 257 | } 258 | else: 259 | return None 260 | 261 | def get_state(self, stats): 262 | if stats is not None: 263 | if ( 264 | stats["j_lelbow_angle"] >= 150 265 | and stats["j_relbow_angle"] >= 150 266 | and stats["j_lshoulder_angle"] <= 120 267 | and stats["j_rshoulder_angle"] <= 120 268 | ): 269 | return 1 270 | elif ( 271 | stats["j_lelbow_angle"] <= 100 272 | and stats["j_relbow_angle"] <= 100 273 | and stats["j_lshoulder_angle"] >= 150 274 | and stats["j_rshoulder_angle"] >= 150 275 | ): 276 | return 2 277 | else: 278 | return 0 279 | else: 280 | return 0 281 | 282 | 283 | class SideSquatJump(Workout): 284 | name = "Side Squat Jump" 285 | 286 | def __init__(self, *args, **kwargs): 287 | super().__init__(n_keystates=3, *args, **kwargs) 288 | self.KEYPOINTS = [ 289 | "left elbow", 290 | "left shoulder", 291 | "left wrist", 292 | "right elbow", 293 | "right shoulder", 294 | "right wrist", 295 | "left hip", 296 | "right hip", 297 | "left knee", 298 | "right knee", 299 | "left ankle", 300 | "right ankle", 301 | ] 302 | 303 | def get_stats(self, pose): 304 | kps = pose.keypoints 305 | 306 | if all(kps[k].score > self.THRESHOLD for k in self.KEYPOINTS): 307 | 308 | e_lelbow_lshoulder = Edge(kps["left elbow"], kps["left shoulder"]) 309 | e_lelbow_lwrist = Edge(kps["left elbow"], kps["left wrist"]) 310 | e_relbow_rshoulder = Edge(kps["right elbow"], kps["right shoulder"]) 311 | e_relbow_rwrist = Edge(kps["right elbow"], kps["right wrist"]) 312 | 313 | e_lshoulder_rshoulder = Edge(kps["left shoulder"], kps["right shoulder"]) 314 | e_lwrist_rwrist = Edge(kps["left wrist"], kps["right wrist"]) 315 | 316 | e_hips = Edge(kps["left hip"], kps["right hip"]) 317 | e_ankles = Edge(kps["left ankle"], kps["right ankle"]) 318 | 319 | j_lelbow = Joint(e_lelbow_lshoulder, e_lelbow_lwrist) 320 | j_relbow = Joint(e_relbow_rshoulder, e_relbow_rwrist) 321 | 322 | return { 323 | "e_shoulders_norm": e_lshoulder_rshoulder.norm, 324 | "e_wrists_norm": e_lwrist_rwrist.norm, 325 | "e_hips_norm": e_hips.norm, 326 | "e_ankles_norm": e_ankles.norm, 327 | "j_lelbow_angle": j_lelbow.angle, 328 | "j_relbow_angle": j_relbow.angle, 329 | } 330 | else: 331 | return None 332 | 333 | def get_state(self, stats): 334 | if stats is not None: 335 | if ( 336 | stats["e_wrists_norm"] <= stats["e_shoulders_norm"] 337 | and stats["j_lelbow_angle"] <= 90 338 | and stats["j_relbow_angle"] <= 90 339 | ): 340 | if stats["e_ankles_norm"] <= stats["e_hips_norm"] * 1.5: 341 | return 1 342 | elif stats["e_ankles_norm"] >= stats["e_hips_norm"] * 2.0: 343 | return 2 344 | else: 345 | return 0 346 | elif ( 347 | stats["e_wrists_norm"] >= stats["e_shoulders_norm"] 348 | and stats["e_ankles_norm"] <= stats["e_hips_norm"] * 1.5 349 | and stats["j_lelbow_angle"] >= 150 350 | and stats["j_relbow_angle"] >= 150 351 | ): 352 | return 3 353 | else: 354 | return 0 355 | else: 356 | return 0 357 | 358 | 359 | WORKOUTS = { 360 | "toe_tap": ToeTap, 361 | "jumping_jacks": JumpingJacks, 362 | "push_up": PushUp, 363 | "side_squat_jump": SideSquatJump, 364 | } 365 | --------------------------------------------------------------------------------