├── .vscode └── settings.json ├── assets ├── logo.png ├── logo.min.png ├── moves │ ├── down.png │ ├── left.png │ ├── tag.png │ ├── up.png │ ├── block.png │ ├── right.png │ ├── throw.png │ ├── back_kick.png │ ├── back_punch.png │ ├── front_kick.png │ └── front_punch.png ├── pose_landmark.jpg └── pose_landmark.min.jpg ├── .env.template ├── requirements.txt ├── LICENSE ├── constants.py ├── run.py ├── .gitignore ├── translate.py ├── README.md └── utils.py /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "black" 3 | } -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/logo.png -------------------------------------------------------------------------------- /assets/logo.min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/logo.min.png -------------------------------------------------------------------------------- /assets/moves/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/down.png -------------------------------------------------------------------------------- /assets/moves/left.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/left.png -------------------------------------------------------------------------------- /assets/moves/tag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/tag.png -------------------------------------------------------------------------------- /assets/moves/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/up.png -------------------------------------------------------------------------------- /assets/moves/block.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/block.png -------------------------------------------------------------------------------- /assets/moves/right.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/right.png -------------------------------------------------------------------------------- /assets/moves/throw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/throw.png -------------------------------------------------------------------------------- /assets/moves/back_kick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/back_kick.png -------------------------------------------------------------------------------- /assets/pose_landmark.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/pose_landmark.jpg -------------------------------------------------------------------------------- /assets/moves/back_punch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/back_punch.png -------------------------------------------------------------------------------- /assets/moves/front_kick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/front_kick.png -------------------------------------------------------------------------------- /assets/moves/front_punch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/moves/front_punch.png -------------------------------------------------------------------------------- /assets/pose_landmark.min.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ra101/Pose2Input-MK9/HEAD/assets/pose_landmark.min.jpg -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | # Pose2Input Variables 2 | # ------------------ 3 | CAMERA_PORT= 4 | DELAY_TIME= 5 | LOG_FPS= 6 | MOTION_THRESHOLD_FACTOR= 7 | 8 | 9 | # PyAutoGUI Constants 10 | # ------------------ 11 | PYAUTO_PAUSE= 12 | 13 | 14 | # Input Config 15 | # ------------------ 16 | UP= 17 | DOWN= 18 | LEFT= 19 | RIGHT= 20 | FRONT_PUNCH= 21 | BACK_PUNCH= 22 | FRONT_KICK= 23 | BACK_KICK= 24 | THROW= 25 | TAG= 26 | BLOCK= 27 | FLIP_STANCE= 28 | PAUSE= 29 | BACK= 30 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # DotENV 2 | python-dotenv==0.18.0 3 | 4 | # OpenCV 5 | numpy==1.21.1 6 | opencv-python==4.5.3.56 7 | 8 | # MediaPipe 9 | absl-py==0.13.0 10 | attrs==21.2.0 11 | cycler==0.10.0 12 | kiwisolver==1.3.1 13 | matplotlib==3.4.2 14 | mediapipe==0.8.6 15 | opencv-contrib-python==4.5.3.56 16 | Pillow==8.3.1 17 | protobuf==3.17.3 18 | pyparsing==2.4.7 19 | python-dateutil==2.8.2 20 | six==1.16.0 21 | 22 | # PyAutoGUI 23 | MouseInfo==0.1.3 24 | PyAutoGUI==0.9.53 25 | PyGetWindow==0.0.9 26 | PyMsgBox==1.0.9 27 | pyperclip==1.8.2 28 | PyRect==0.1.4 29 | PyScreeze==0.1.27 30 | python3-xlib==0.15 31 | PyTweening==1.0.3 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Parth Agarwal 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | # ./constants.py 2 | 3 | import os 4 | import enum 5 | 6 | from dotenv import load_dotenv 7 | 8 | 9 | load_dotenv() 10 | 11 | 12 | @enum.unique 13 | class InputConfig(enum.Enum): 14 | ''' 15 | Config for the gameplay, Takes input from .env 16 | Value should be something tha can be used by pyAutoGUI 17 | If not available then, uses default input config (mine) 18 | and Yes I use arrow keys, deal with it! 19 | ''' 20 | DEFAULT = '' 21 | UP = os.getenv('UP', 'up').lower() 22 | DOWN = os.getenv('DOWN', 'down').lower() 23 | LEFT = os.getenv('LEFT', 'left').lower() 24 | RIGHT = os.getenv('RIGHT', 'right').lower() 25 | FRONT_PUNCH = os.getenv('FRONT_PUNCH', 'a').lower() 26 | BACK_PUNCH = os.getenv('BACK_PUNCH', 's').lower() 27 | FRONT_KICK = os.getenv('FRONT_KICK', 'z').lower() 28 | BACK_KICK = os.getenv('BACK_KICK', 'x').lower() 29 | THROW = os.getenv('THROW', 'd').lower() 30 | TAG = os.getenv('TAG', 'c').lower() 31 | BLOCK = os.getenv('BLOCK', 'space').lower() 32 | FLIP_STANCE = os.getenv('FLIP_STANCE', 'ctrlright').lower() 33 | PAUSE = os.getenv('PAUSE', 'tab').lower() 34 | BACK = os.getenv('BACK', 'backspace').lower() 35 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | # ./run.py 2 | 3 | import os 4 | import time 5 | import argparse 6 | 7 | import pyautogui 8 | from dotenv import load_dotenv 9 | 10 | from translate import translate 11 | 12 | 13 | if __name__ == "__main__": 14 | 15 | load_dotenv() 16 | 17 | # Time (sec) to pause after each PyAuto Function Call 18 | pyautogui.PAUSE = float(os.getenv('PYAUTO_PAUSE', 0.1)) 19 | 20 | parser = argparse.ArgumentParser( 21 | prog="Pose2Input-MK9", 22 | description="Translate the Video Input to Keystrokes" 23 | ) 24 | parser.add_argument( 25 | '-d', '--debug_level', type=int, metavar='', default=0, 26 | help='<0, 1, 2, 3> set different levels of information for logs or live feed' 27 | ) 28 | parser.add_argument( 29 | '-l', '--log_flag', action='store_true', 30 | help='stores the video_log in "logs" folder' 31 | ) 32 | parser.add_argument( 33 | '-L', '--live_flag', action='store_true', 34 | help='displays the captured video' 35 | ) 36 | args = parser.parse_args() 37 | 38 | # A Delay before Starting the Program 39 | time.sleep(int(os.getenv('DELAY_TIME', 0))) 40 | 41 | live_flag = args.live_flag 42 | log_flag = args.log_flag 43 | debug_level = args.debug_level 44 | 45 | # If Debug > 0 and no flag is selected 46 | # Then log_flag is automatically set to True 47 | if (debug_level > 0) and not (log_flag or live_flag): 48 | log_flag = True 49 | 50 | if not os.path.exists('logs'): 51 | os.mkdir('logs') 52 | 53 | translate( 54 | log_flag=log_flag, 55 | live_flag=live_flag, 56 | debug_level=debug_level, 57 | log_fps=int(os.getenv('LOG_FPS', 20)), 58 | camera_port=os.getenv('CAMERA_PORT', '0'), 59 | motion_threshold_factor=int( 60 | os.getenv('MOTION_THRESHOLD_FACTOR', 64) 61 | ), 62 | ) 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # log Files and Folder 132 | logs/ 133 | log/ 134 | log_* 135 | -------------------------------------------------------------------------------- /translate.py: -------------------------------------------------------------------------------- 1 | # ./translate.py 2 | 3 | import time 4 | from datetime import datetime 5 | 6 | import cv2 7 | import mediapipe.python.solutions as mp 8 | 9 | from utils import TranslatePose, moves_to_keystroke, input_keys, UNUSED_LANDMARKS 10 | 11 | 12 | def translate( 13 | log_flag=True, live_flag=False, debug_level=0, log_fps=20, 14 | camera_port="0", motion_threshold_factor=48 15 | ): 16 | """ 17 | function translate input settings for camera, captures frames accordingly, 18 | process the captured frame and types the associated keystroke. 19 | 20 | log_flag : if True, it stores the video_log in "logs" folder 21 | live_flag : if True, it displays the captured video 22 | log_fps : FPS Setting for video_log 23 | camera_port : select the port, can be url address or port number in str 24 | motion_threshold_factor : More the value is, More the Motion is Captured 25 | debug_level -> <0 | 1 | 2 | 3>: 26 | - 0: Raw Video Footage 27 | - 1: 0 + FPS and Output Moves 28 | - 2: 1 + Virtual Exoskeleton of Body Parts Found 29 | - 3: 2 + Black Screen if no motion found 30 | """ 31 | 32 | camera_port = int(camera_port) if camera_port.isdigit() else camera_port 33 | 34 | translate_pose = TranslatePose() 35 | 36 | camera = cv2.VideoCapture(camera_port, cv2.CAP_DSHOW) 37 | success, prv_img = camera.read() 38 | 39 | if not success: 40 | raise cv2.error("Invalid Video Source") 41 | 42 | height, width, channel = prv_img.shape 43 | 44 | # More the value is, More the Motion is Captured 45 | motion_theshold = height * width * channel * 255 // motion_threshold_factor 46 | 47 | # MediaPipe Pose 48 | pose = mp.pose.Pose() 49 | 50 | if log_flag: 51 | # Intialise VideoWriter for logs in `./logs/` folder 52 | video_log = cv2.VideoWriter( 53 | f"logs/log_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.avi", 54 | cv2.VideoWriter_fourcc(*"XVID"), log_fps, (width, height) 55 | ) 56 | video_log.write(prv_img) 57 | 58 | if debug_level > 0: 59 | # FPS and Output Moves setting 60 | cur_time = 0 61 | cv_font_vars_outline = (cv2.FONT_HERSHEY_PLAIN, 1.7, (192, 44, 44), 3) 62 | cv_font_vars = (cv2.FONT_HERSHEY_PLAIN, 1.7, (245,66,66), 2) 63 | fps_coord = (width - 100, 50) 64 | movelist_coord = (50, height - 50) 65 | 66 | 67 | while cv2.waitKey(1) & 0xFF != 27: 68 | if debug_level > 0: 69 | # Update prv_time for FPS 70 | prv_time = cur_time 71 | 72 | # Set to false for Each Frame 73 | motion_detected = False 74 | 75 | # Grabs image form Capturing Device 76 | _, img = camera.read() 77 | 78 | # Frame differencing for motion detection 79 | diff_img = cv2.absdiff(img, prv_img) 80 | 81 | # Compare the sum of pixel after differencing with initalized theshold 82 | if diff_img.flatten().sum() > motion_theshold: 83 | motion_detected = True 84 | 85 | # same img for next comparsion 86 | prv_img = img.copy() 87 | 88 | if motion_detected: 89 | # MediaPipe finds marker for all body parts 90 | pose_landmarks = pose.process( 91 | cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 92 | ).pose_landmarks 93 | 94 | movelist = [] 95 | if motion_detected and pose_landmarks: 96 | 97 | # removing lankmarks that will not be used futher: 98 | for mark in UNUSED_LANDMARKS: 99 | pose_landmarks.landmark[mark].visibility = 0 100 | 101 | # Using Coordinates, Deduces the Move(s) 102 | movelist = translate_pose.process(pose_landmarks.landmark) 103 | 104 | # Using Move(s), returns the associated key 105 | inputs = moves_to_keystroke(movelist) 106 | 107 | # Uses pyAutoGUI to input the key(s) 108 | input_keys(inputs) 109 | 110 | if debug_level > 0: 111 | cur_time = time.time() 112 | 113 | if debug_level > 1 and motion_detected and pose_landmarks: 114 | mp.drawing_utils.draw_landmarks( 115 | img, pose_landmarks, mp.pose.POSE_CONNECTIONS, 116 | mp.drawing_utils.DrawingSpec(color=(66,66,245)), 117 | mp.drawing_utils.DrawingSpec(color=(66,245,66)) 118 | ) 119 | 120 | if debug_level > 2 and not motion_detected: 121 | img = diff_img 122 | 123 | # FPS: 1 frame / time taken to process a whole frame 124 | fps = 1/(cur_time - prv_time) if (cur_time - prv_time) !=0 else 0 125 | 126 | # FPS and Output Moves Output 127 | cv2.putText(img, f"{fps:.2f}", fps_coord, *cv_font_vars_outline) 128 | cv2.putText(img, f"{fps:.2f}", fps_coord, *cv_font_vars) 129 | 130 | cv2.putText(img, f"{list(movelist)}", movelist_coord, *cv_font_vars_outline) 131 | cv2.putText(img, f"{list(movelist)}", movelist_coord, *cv_font_vars) 132 | 133 | # live_flag is set to True, Display the captured image 134 | if live_flag: 135 | cv2.imshow("Pose2Input-MK9", img) 136 | 137 | # log_flag is set to True, Store the captured image 138 | if log_flag: 139 | video_log.write(img) 140 | 141 | # Closes Capturing Device 142 | camera.release() 143 | 144 | if log_flag: 145 | # Closes Video Writer 146 | video_log.release() 147 | 148 | # Closes all the frames 149 | cv2.destroyAllWindows() 150 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 |

5 | 6 | 7 | Made with Python Stars Forks Open Issues Open Source Love Built with Love 8 | 9 |
10 | 11 | Play **Mortal Kombat** with the camera as your input device and your body/pose as a controller. 12 | 13 | 14 | 15 | **Video Tutorial:** [LRBY]() | [YouTube]() 16 | 17 |
18 | 19 |
20 | 21 | ## 📈Workflow 22 | 23 | It converts given image of pose into keystroke (accorging to the given config). This is done broadly in 4 simple steps: 24 | 25 | - **OpenCV** capture image if any motion is detected (via frame differencing) 26 | - **MediaPipe** returns `(x, y)` coordinates of the each body part (if found) 27 | - We use *Mathematics* to determine what pose is and which key that pose is associated with 28 | - **PyAutoGUI** to input the keystroke 29 | 30 |
31 | 32 | ### 📷 Image Processing 33 | 34 | 35 | 36 |
37 | 38 | ### :punch:MoveList 39 | 40 | | Move | Pose Image | Mathematics | 41 | | :-------------: | :--: | :------ | 42 | | **UP** | | • *ankles should be above knees* | 43 | | **DOWN** | | • *dist(hip->nose) : dist(hip->knee) > **7 : 10*** | 44 | | **LEFT** | | • *(perpendicular of r_ankle->l_ankle) should be right of **right_hip***
• *slope(**right_hip**->ankle) for both ankles should be b/w -65° and -115°* | 45 | | **RIGHT** | | • *same as above but with **left_hip*** | 46 | | **FRONT_PUNCH** | | • *slope(left_shoulder->left_wirst) should be b/w -10.5° and 10.5°*
• *angle(left_shoulder<-left_elbow->left_wirst) should be b/w 169.5° and 190.5°*
**TLDR: arm should parallel to ground** | 47 | | **BACK_PUNCH** | | • *same as above but with **right arm*** | 48 | | **FRONT_KICK** | | • *slope(left_ankle->left_hip) should be b/w 45° and 135°* | 49 | | **BACK_KICK** | | • *same as above but with **right leg*** | 50 | | **THROW** | | • *wrist should be above elbow of same arm, they both should be above nose* | 51 | | **TAG** | | • *dist(wrists) : dist(elbows) < 3 : 10*
• *angle(left_elbow<-wirsts->right_elbow) should be b/w 169.5° and 190.5°*
**TLDR: forearms should parallel to ground** | 52 | | **BLOCK** | | • *wrist should be above nose, they both should be above elbow of same arm* | 53 | 54 |
55 | 56 | ## ⚙Development 57 | 58 | ### :floppy_disk:Setup 59 | 60 | Lets start the standard procedure for python project setup. 61 | 62 | - Clone the repository 63 | 64 | ```bash 65 | $ git clone https://github.com/ra101/Pose2Input-MK9.git 66 | ``` 67 | 68 | - Create the virtualenv and activate it 69 | 70 | ```bash 71 | $ cd Pose2Input-MK9 72 | $ virtualenv venv 73 | $ source ./venv/bin/activate # unix 74 | $ .\venv\Scripts\activate.bat # windows 75 | ``` 76 | 77 | - Install requirements 78 | 79 | ```bash 80 | $ pip install -r requirements.txt 81 | ``` 82 | 83 | - copy content of .env.template into .env *(one can use [dump-env](https://github.com/sobolevn/dump-env) as well)* 84 | 85 | ```bash 86 | $ cat .env.template > .env 87 | ``` 88 | 89 | 90 | 91 | **Env Help Guide:** 92 | 93 | | Env Variables | Defination | Default Value | 94 | | ------------------------------ | ----------------------------------- | ------------- | 95 | | **Pose2Input Variables**: | | | 96 | | CAMERA_PORT | Camera Port for OpenCV | 0 | 97 | | DELAY_TIME | A Delay before Starting the Program | 0 | 98 | | LOG_FPS | FPS Setting for logs | 20 | 99 | | MOTION_THRESHOLD_FACTOR | More the value is, More the Motion is Captured | 64 | 100 | | | | | 101 | | **PyAutoGUI Constants:** | | | 102 | | PYAUTO_PAUSE | Time (sec) to pause after each PyAuto Function Call | 0.1 | 103 | | | | | 104 | | **Input Config:** | | | 105 | | UP | KeyStroke for UP (used by PyAuto) | up | 106 | | DOWN | KeyStroke for Down (used by PyAuto) | down | 107 | | `` | KeyStroke for `` (used by PyAuto) | `` | 108 | 109 |
110 | 111 | ### 💻Run 112 | 113 | One can simply run the application by this 114 | 115 | ```bash 116 | $ python run.py 117 | ``` 118 | 119 | but for calibration, optional arguments are provided 120 | 121 | | Argument | Alias | Purpose | Deafult | 122 | | ---------------------------- | ----------------- | ------------------------------------------------------------ | ------- | 123 | | --help | --h | Shows the available options | - | 124 | | --debug_level <0, 1, 2, 3> | --d <0, 1, 2, 3> | Set Different Levels of Information for Logs or Live feed (explained below this table) | `0` | 125 | | --log_flag | --l | Stores the `video_log` in "logs" folder (.avi) | `False` | 126 | | --live_flag | -L | Displays the Captured Video | `False` | 127 | 128 | **Debug**: 129 | 130 | - Levels: 131 | - **0**: Raw Video Footage 132 | - **1**: `0` + FPS and Output Moves 133 | - **2**: `1` + Virtual Exoskeleton of Body Parts Found 134 | - **3**: `2` + Black Screen if no motion found 135 | - If `debug_level` > 0 and no flag is selected, then `log_flag` is automatically set to `True` 136 | 137 | 138 | 139 | **Example of all flags being used**: 140 | 141 | ```bash 142 | $ python run.py --debug_level 3 --live_flag --log_flag 143 | ``` 144 | 145 |

146 | 147 | ## 📃Breakdown of `requirements.txt` 148 | 149 | | Dependency | Usage | 150 | | ---------------- | ------------------------------------------------------------ | 151 | | Python-DotENV | Reads the key-value pair from `.env` file and adds them to environment variable. | 152 | | OpenCV-Python | Image Processing library which uses NumPy containers| 153 | | MediaPipe | Offers cross-platform, customizable ML solutions for live and streaming media. | 154 | | PyAutoGUI | It can control the mouse and keyboard to automate interactions with other applications. | 155 | 156 | 157 |

158 | 159 | ## 🎁Donations 160 | 161 | [![Sponsor](https://img.shields.io/badge/Buy_Me_A_Coffee-Sponsor-ff9933?style=for-the-badge&logo=buy-me-a-coffee&logoColor=orange)](https://www.buymeacoffee.com/ra101) 162 | 163 |
164 | 165 | ## 🌟Credit/Acknowledgment 166 | 167 | [![Contributors](https://img.shields.io/github/contributors/ra101/Pose2Input-MK9?style=for-the-badge)](https://github.com/ra101/Pose2Input-MK9/graphs/contributors) 168 | 169 |
170 | 171 | ## 📜License 172 | 173 | [![License](https://img.shields.io/github/license/ra101/Pose2Input-MK9?style=for-the-badge)](https://github.com/ra101/Pose2Input-MK9/blob/core/LICENSE) 174 | 175 |
176 | 177 | ## 🤙Contact Me 178 | 179 | [![Protonmail](https://img.shields.io/badge/Protonmail-Email-ab44fe?style=for-the-badge&logo=protonmail)](mailto://agarwal.parth.101@protonmail.com) [![Telegram](https://img.shields.io/badge/Telegram-Chat-informational?style=for-the-badge&logo=telegram)](https://telegram.me/ra_101) 180 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # ./utils.py 2 | 3 | import pyautogui 4 | import numpy as np 5 | 6 | from constants import InputConfig 7 | from mediapipe.python.solutions.pose import ( 8 | PoseLandmark as LandmarkIndexEnum 9 | ) 10 | 11 | 12 | UNUSED_LANDMARKS = ( 13 | LandmarkIndexEnum.LEFT_EYE, LandmarkIndexEnum.RIGHT_EYE, 14 | LandmarkIndexEnum.LEFT_EAR, LandmarkIndexEnum.RIGHT_EAR, 15 | LandmarkIndexEnum.LEFT_HEEL, LandmarkIndexEnum.RIGHT_HEEL, 16 | LandmarkIndexEnum.MOUTH_LEFT, LandmarkIndexEnum.MOUTH_RIGHT, 17 | LandmarkIndexEnum.LEFT_PINKY, LandmarkIndexEnum.RIGHT_PINKY, 18 | LandmarkIndexEnum.LEFT_INDEX, LandmarkIndexEnum.RIGHT_INDEX, 19 | LandmarkIndexEnum.LEFT_THUMB, LandmarkIndexEnum.RIGHT_THUMB, 20 | LandmarkIndexEnum.LEFT_EYE_INNER, LandmarkIndexEnum.RIGHT_EYE_INNER, 21 | LandmarkIndexEnum.LEFT_EYE_OUTER, LandmarkIndexEnum.RIGHT_EYE_OUTER, 22 | LandmarkIndexEnum.LEFT_FOOT_INDEX, LandmarkIndexEnum.RIGHT_FOOT_INDEX, 23 | ) 24 | 25 | 26 | def input_keys(inputs): 27 | """ 28 | press the given list of keys in ordered fashion 29 | pyautogui: uses keyDown and keyUp 30 | Not using pyautogui.hotkey as it not recognize by many applications 31 | 32 | input > 33 | """ 34 | 35 | for key in inputs: 36 | pyautogui.keyDown(key) 37 | for key in reversed(inputs): 38 | pyautogui.keyUp(key) 39 | 40 | def moves_to_keystroke(movelist): 41 | """ 42 | Uses InputConfig to return list of keys from given moves in movelist 43 | 44 | movelist > 45 | """ 46 | 47 | return [ 48 | getattr(InputConfig, move, 'DEFAULT').value for move in movelist 49 | ] 50 | 51 | class TranslatePose: 52 | """ 53 | Using Coordinates of body parts, this class can deduces the Move(s)/Pose(s) 54 | """ 55 | 56 | def __init__(self): 57 | """ 58 | Initalise a list of all the functions that starts with "move_" 59 | all these fuctions return bool for associated moves. 60 | functions and moves are associated in the format move_ 61 | help(TranslatePose.move_sample) to get how to make a move function 62 | """ 63 | 64 | self.all_transaltion_funcs = [ 65 | getattr(self, i) for i in dir(self) if i.startswith('move_') 66 | ] 67 | self.all_transaltion_funcs.remove(self.move_sample) 68 | self.visibility_threshold = 0.9 69 | 70 | def process(self, landmark_list): 71 | """ 72 | pass landmark_list to all the functions in 73 | self.all_transaltion_funcs, and create a set of possible moves 74 | """ 75 | 76 | return set([ 77 | func.__name__[5:].upper() 78 | for func in self.all_transaltion_funcs if func(landmark_list) 79 | ]) 80 | 81 | def move_sample(self, landmark_list): 82 | """ 83 | sample function, bases for all `move_` functions 84 | 85 | # Input 86 | landmark_list >: is a list of positional markers 87 | of each body part, Index for each body part can be found at 88 | mediapipe.python.solutions.pose.PoseLandmark 89 | which can be now used as nose = landmark_list[LandmarkIndexEnum.NOSE] 90 | this nose is also NamedTuple, which has `x`, `y`, and `visiblity` fields 91 | 92 | # Output 93 | : True or False, whether the `sample` move is portrayed by the 94 | given landmarks 95 | 96 | # Processing 97 | ## Initalizing 98 | initalize all the body parts required for the mathematical 99 | operations required to deduce the pose 100 | # Visibility Check 101 | check if the average of visiblity of all initialized body 102 | are greater than self.visibility_threshold 103 | eg: numpy.average([part_1.visibility, part_2.visibility]) < 0.9 104 | # Mathematics 105 | perform mathematical operations on x,y cordinates to determine pose 106 | always usew ratio or angles, as they are independent of camera distance 107 | # return True or False accordingly 108 | """ 109 | 110 | def move_up(self, landmark_list): 111 | """ 112 | ankles should be above knees 113 | """ 114 | 115 | r_ankle = landmark_list[LandmarkIndexEnum.RIGHT_ANKLE] 116 | l_ankle=landmark_list[LandmarkIndexEnum.LEFT_ANKLE] 117 | r_knee = landmark_list[LandmarkIndexEnum.RIGHT_KNEE] 118 | l_knee=landmark_list[LandmarkIndexEnum.LEFT_KNEE] 119 | 120 | if np.average([ 121 | l_ankle.visibility, r_ankle.visibility, 122 | l_knee.visibility, r_knee.visibility 123 | ]) < self.visibility_threshold: 124 | return False 125 | 126 | if (l_ankle.y + r_ankle.y) < (r_knee.y + l_knee.y): 127 | return True 128 | return False 129 | 130 | def move_down(self, landmark_list): 131 | """ 132 | dist(hip->nose) : dist(hip->knee) > 7 : 10 133 | """ 134 | 135 | r_hip = landmark_list[LandmarkIndexEnum.RIGHT_HIP] 136 | l_hip=landmark_list[LandmarkIndexEnum.LEFT_HIP] 137 | r_knee = landmark_list[LandmarkIndexEnum.RIGHT_KNEE] 138 | l_knee=landmark_list[LandmarkIndexEnum.LEFT_KNEE] 139 | nose=landmark_list[LandmarkIndexEnum.NOSE] 140 | 141 | if np.average([ 142 | r_hip.visibility, l_hip.visibility, r_knee.visibility, 143 | l_knee.visibility, nose.visibility 144 | ]) < self.visibility_threshold: 145 | return False 146 | 147 | hip = np.complex(0, (r_hip.y + l_hip.y)/2) 148 | knee = np.complex(0, (r_knee.y + l_knee.y)/2) 149 | nose = np.complex(0, nose.y) 150 | 151 | knee_to_nose = knee - nose 152 | hip_to_nose = hip - nose 153 | 154 | if abs(hip_to_nose)/abs(knee_to_nose) > 0.7: 155 | return True 156 | return False 157 | 158 | def move_left(self, landmark_list): 159 | """ 160 | (perpendicular of r_ankle->l_ankle) should be right of right hip 161 | slope(right_hip->ankle) for both ankles should be b/w -65° and -115° 162 | """ 163 | 164 | return self._move_x_direction( 165 | r_ankle=landmark_list[LandmarkIndexEnum.RIGHT_ANKLE], 166 | l_ankle=landmark_list[LandmarkIndexEnum.LEFT_ANKLE], 167 | hip=landmark_list[LandmarkIndexEnum.RIGHT_HIP], 168 | right=False 169 | ) 170 | 171 | def move_right(self, landmark_list): 172 | """ 173 | (perpendicular of r_ankle->l_ankle) should be left of left hip 174 | slope(left_hip->ankle) for both ankles should be b/w -65° and -115° 175 | """ 176 | 177 | return self._move_x_direction( 178 | r_ankle=landmark_list[LandmarkIndexEnum.RIGHT_ANKLE], 179 | l_ankle=landmark_list[LandmarkIndexEnum.LEFT_ANKLE], 180 | hip=landmark_list[LandmarkIndexEnum.LEFT_HIP], 181 | right=True 182 | ) 183 | 184 | def _move_x_direction( 185 | self, r_ankle, l_ankle, hip, right=True 186 | ): 187 | """ 188 | common implementation for move_left and move_right 189 | """ 190 | 191 | if np.average([ 192 | r_ankle.visibility, l_ankle.visibility, hip.visibility 193 | ]) < self.visibility_threshold: 194 | return False 195 | 196 | median_of_body_x = (r_ankle.x + l_ankle.x)/2 197 | 198 | l_ankle_to_hip = np.complex(hip.x - l_ankle.x, hip.y - l_ankle.y) 199 | r_ankle_to_hip = np.complex(hip.x - r_ankle.x, hip.y - r_ankle.y) 200 | 201 | if np.average(np.abs([ 202 | np.sin(np.angle(l_ankle_to_hip)) , np.sin(np.angle(r_ankle_to_hip)) 203 | ])) > self.visibility_threshold: 204 | if right and median_of_body_x > hip.x: 205 | return True 206 | if not right and median_of_body_x < hip.x: 207 | return True 208 | return False 209 | 210 | def move_front_punch(self, landmark_list): 211 | """ 212 | slope(left_shoulder->left_wirst) should be b/w -10.5° and 10.5° 213 | angle(left_shoulder<-left_elbow->left_wirst) should be b/w 169.5° and 190.5° 214 | TLDR: arm should parallel to ground 215 | """ 216 | 217 | return self._move_punch( 218 | wrist=landmark_list[LandmarkIndexEnum.LEFT_WRIST], 219 | elbow=landmark_list[LandmarkIndexEnum.LEFT_ELBOW], 220 | shoulder=landmark_list[LandmarkIndexEnum.LEFT_SHOULDER] 221 | ) 222 | 223 | def move_back_punch(self, landmark_list): 224 | """ 225 | slope(right_shoulder->right_wirst) should be b/w -10.5° and 10.5° 226 | angle(right_shoulder<-right_elbow->right_wirst) should be b/w 169.5° and 190.5° 227 | TLDR: arm should parallel to ground 228 | """ 229 | 230 | return self._move_punch( 231 | wrist=landmark_list[LandmarkIndexEnum.RIGHT_WRIST], 232 | elbow=landmark_list[LandmarkIndexEnum.RIGHT_ELBOW], 233 | shoulder=landmark_list[LandmarkIndexEnum.RIGHT_SHOULDER] 234 | ) 235 | 236 | def _move_punch(self, wrist, elbow, shoulder): 237 | """ 238 | common implementation for move_front_punch and move_back_punch 239 | """ 240 | 241 | if np.average([ 242 | elbow.visibility, wrist.visibility, shoulder.visibility 243 | ]) < self.visibility_threshold: 244 | return False 245 | 246 | elbow_to_wrist = np.complex(wrist.x - elbow.x, wrist.y - elbow.y) 247 | elbow_to_shoulder = np.complex(shoulder.x - elbow.x , shoulder.y - elbow.y) 248 | wrist_to_shoulder = np.complex(shoulder.x - wrist.x , shoulder.y - wrist.y) 249 | 250 | if abs(np.sin( 251 | np.angle(elbow_to_wrist) + np.angle(elbow_to_shoulder) 252 | )) < 0.18: 253 | if abs(np.sin(np.angle(wrist_to_shoulder))) < 0.18: 254 | return True 255 | return False 256 | 257 | def move_front_kick(self, landmark_list): 258 | """ 259 | slope(left_ankle->left_hip) should be b/w 45° and 135° 260 | """ 261 | 262 | return self._move_kick( 263 | hip=landmark_list[LandmarkIndexEnum.LEFT_HIP], 264 | ankle=landmark_list[LandmarkIndexEnum.LEFT_ANKLE], 265 | ) 266 | 267 | def move_back_kick(self, landmark_list): 268 | """ 269 | slope(right_ankle->right_hip) should be b/w 45° and 135° 270 | """ 271 | 272 | return self._move_kick( 273 | hip=landmark_list[LandmarkIndexEnum.RIGHT_HIP], 274 | ankle=landmark_list[LandmarkIndexEnum.RIGHT_ANKLE], 275 | ) 276 | 277 | def _move_kick(self, hip, ankle): 278 | """ 279 | common implementation for move_front_kick and move_back_kick 280 | """ 281 | 282 | if np.average([hip.visibility, ankle.visibility]) < self.visibility_threshold: 283 | return False 284 | 285 | ankle_to_hip = np.complex(hip.x - ankle.x, hip.y - ankle.y) 286 | 287 | if abs(np.sin(np.angle(ankle_to_hip))) < 0.7: 288 | return True 289 | return False 290 | 291 | def move_throw(self, landmark_list): 292 | """ 293 | wrist should be above elbow of same arm, they both should be above nose 294 | """ 295 | 296 | nose = landmark_list[LandmarkIndexEnum.NOSE] 297 | if nose.visibility < self.visibility_threshold: 298 | return False 299 | 300 | r_wrist = landmark_list[LandmarkIndexEnum.RIGHT_WRIST] 301 | r_elbow = landmark_list[LandmarkIndexEnum.RIGHT_ELBOW] 302 | l_wrist = landmark_list[LandmarkIndexEnum.LEFT_WRIST] 303 | l_elbow = landmark_list[LandmarkIndexEnum.LEFT_ELBOW] 304 | 305 | if np.average([r_wrist.visibility, r_elbow.visibility]) > self.visibility_threshold: 306 | if r_wrist.y < r_elbow.y < nose.y: 307 | return True 308 | 309 | if np.average([l_wrist.visibility, l_elbow.visibility]) > self.visibility_threshold: 310 | if l_wrist.y < l_elbow.y < nose.y : 311 | return True 312 | return False 313 | 314 | def move_tag(self, landmark_list): 315 | """ 316 | angle(left_elbow<-wirsts->right_elbow) should be b/w 169.5° and 190.5° 317 | TLDR: forearms should parallel to ground 318 | 319 | dist(wrists) : dist(elbows) < 3 : 10 320 | """ 321 | 322 | r_wrist = landmark_list[LandmarkIndexEnum.RIGHT_WRIST] 323 | r_elbow = landmark_list[LandmarkIndexEnum.RIGHT_ELBOW] 324 | l_wrist = landmark_list[LandmarkIndexEnum.LEFT_WRIST] 325 | l_elbow = landmark_list[LandmarkIndexEnum.LEFT_ELBOW] 326 | 327 | if np.average([ 328 | r_wrist.visibility, l_wrist.visibility, 329 | r_elbow.visibility, l_elbow.visibility 330 | ]) < self.visibility_threshold: 331 | return False 332 | 333 | wirst_x = (l_wrist.x + r_wrist.x)/2 334 | wirst_y = (l_wrist.y + r_wrist.y)/2 335 | 336 | l_elbow_to_wrist = np.complex(wirst_x - l_elbow.x, wirst_y - l_elbow.y) 337 | r_elbow_to_wrist = np.complex(wirst_x - r_elbow.x, wirst_y - r_elbow.y) 338 | 339 | if abs(np.sin( 340 | np.angle(l_elbow_to_wrist) + np.angle(r_elbow_to_wrist) 341 | )) < 0.18: 342 | if abs(r_wrist.x - l_wrist.x) / abs(r_elbow.x - l_elbow.x) < 0.3: 343 | return True 344 | return False 345 | 346 | def move_block(self, landmark_list): 347 | """ 348 | wrist should be above nose, they both should be above elbow of same arm 349 | """ 350 | 351 | nose = landmark_list[LandmarkIndexEnum.NOSE] 352 | if nose.visibility < self.visibility_threshold: 353 | return False 354 | 355 | r_wrist = landmark_list[LandmarkIndexEnum.RIGHT_WRIST] 356 | r_elbow = landmark_list[LandmarkIndexEnum.RIGHT_ELBOW] 357 | l_wrist = landmark_list[LandmarkIndexEnum.LEFT_WRIST] 358 | l_elbow = landmark_list[LandmarkIndexEnum.LEFT_ELBOW] 359 | 360 | if np.average([r_wrist.visibility, r_elbow.visibility]) > self.visibility_threshold: 361 | if r_wrist.y < nose.y < r_elbow.y: 362 | return True 363 | 364 | if np.average([l_wrist.visibility, l_elbow.visibility]) > self.visibility_threshold: 365 | if l_wrist.y < nose.y < l_elbow.y: 366 | return True 367 | return False 368 | 369 | # def move_flip_stance(self, landmark_list): 370 | # """ 371 | # Not Gonna Implement 372 | # """ 373 | # return False 374 | 375 | # def move_pause(self, landmark_list): 376 | # """ 377 | # Not Gonna Implement 378 | # """ 379 | # return False 380 | 381 | # def move_back(self, landmark_list): 382 | # """ 383 | # Not Gonna Implement 384 | # """ 385 | # return False 386 | --------------------------------------------------------------------------------